2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: iser_verbs.c 7051 2006-05-10 12:29:11Z ogerlitz $
36 #include <linux/kernel.h>
37 #include <linux/module.h>
38 #include <linux/smp_lock.h>
39 #include <linux/delay.h>
40 #include <linux/version.h>
42 #include "iscsi_iser.h"
44 #define ISCSI_ISER_MAX_CONN 8
45 #define ISER_MAX_CQ_LEN ((ISER_QP_MAX_RECV_DTOS + \
46 ISER_QP_MAX_REQ_DTOS) * \
49 static void iser_cq_tasklet_fn(unsigned long data
);
50 static void iser_cq_callback(struct ib_cq
*cq
, void *cq_context
);
51 static void iser_comp_error_worker(void *data
);
53 static void iser_cq_event_callback(struct ib_event
*cause
, void *context
)
55 iser_err("got cq event %d \n", cause
->event
);
58 static void iser_qp_event_callback(struct ib_event
*cause
, void *context
)
60 iser_err("got qp event %d\n",cause
->event
);
64 * iser_create_device_ib_res - creates Protection Domain (PD), Completion
65 * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
68 * returns 0 on success, -1 on failure
70 static int iser_create_device_ib_res(struct iser_device
*device
)
72 device
->pd
= ib_alloc_pd(device
->ib_device
);
73 if (IS_ERR(device
->pd
))
76 device
->cq
= ib_create_cq(device
->ib_device
,
78 iser_cq_event_callback
,
81 if (IS_ERR(device
->cq
))
84 if (ib_req_notify_cq(device
->cq
, IB_CQ_NEXT_COMP
))
87 tasklet_init(&device
->cq_tasklet
,
89 (unsigned long)device
);
91 device
->mr
= ib_get_dma_mr(device
->pd
, IB_ACCESS_LOCAL_WRITE
|
92 IB_ACCESS_REMOTE_WRITE
|
93 IB_ACCESS_REMOTE_READ
);
94 if (IS_ERR(device
->mr
))
100 tasklet_kill(&device
->cq_tasklet
);
102 ib_destroy_cq(device
->cq
);
104 ib_dealloc_pd(device
->pd
);
106 iser_err("failed to allocate an IB resource\n");
111 * iser_free_device_ib_res - destory/dealloc/dereg the DMA MR,
112 * CQ and PD created with the device associated with the adapator.
114 static void iser_free_device_ib_res(struct iser_device
*device
)
116 BUG_ON(device
->mr
== NULL
);
118 tasklet_kill(&device
->cq_tasklet
);
120 (void)ib_dereg_mr(device
->mr
);
121 (void)ib_destroy_cq(device
->cq
);
122 (void)ib_dealloc_pd(device
->pd
);
130 * iser_create_ib_conn_res - Creates FMR pool and Queue-Pair (QP)
132 * returns 0 on success, -1 on failure
134 static int iser_create_ib_conn_res(struct iser_conn
*ib_conn
)
136 struct iser_device
*device
;
137 struct ib_qp_init_attr init_attr
;
139 struct ib_fmr_pool_param params
;
141 BUG_ON(ib_conn
->device
== NULL
);
143 device
= ib_conn
->device
;
145 ib_conn
->page_vec
= kmalloc(sizeof(struct iser_page_vec
) +
146 (sizeof(u64
) * (ISCSI_ISER_SG_TABLESIZE
+1)),
148 if (!ib_conn
->page_vec
) {
152 ib_conn
->page_vec
->pages
= (u64
*) (ib_conn
->page_vec
+ 1);
154 params
.page_shift
= SHIFT_4K
;
155 /* when the first/last SG element are not start/end *
156 * page aligned, the map whould be of N+1 pages */
157 params
.max_pages_per_fmr
= ISCSI_ISER_SG_TABLESIZE
+ 1;
158 /* make the pool size twice the max number of SCSI commands *
159 * the ML is expected to queue, watermark for unmap at 50% */
160 params
.pool_size
= ISCSI_XMIT_CMDS_MAX
* 2;
161 params
.dirty_watermark
= ISCSI_XMIT_CMDS_MAX
;
163 params
.flush_function
= NULL
;
164 params
.access
= (IB_ACCESS_LOCAL_WRITE
|
165 IB_ACCESS_REMOTE_WRITE
|
166 IB_ACCESS_REMOTE_READ
);
168 ib_conn
->fmr_pool
= ib_create_fmr_pool(device
->pd
, ¶ms
);
169 if (IS_ERR(ib_conn
->fmr_pool
)) {
170 ret
= PTR_ERR(ib_conn
->fmr_pool
);
174 memset(&init_attr
, 0, sizeof init_attr
);
176 init_attr
.event_handler
= iser_qp_event_callback
;
177 init_attr
.qp_context
= (void *)ib_conn
;
178 init_attr
.send_cq
= device
->cq
;
179 init_attr
.recv_cq
= device
->cq
;
180 init_attr
.cap
.max_send_wr
= ISER_QP_MAX_REQ_DTOS
;
181 init_attr
.cap
.max_recv_wr
= ISER_QP_MAX_RECV_DTOS
;
182 init_attr
.cap
.max_send_sge
= MAX_REGD_BUF_VECTOR_LEN
;
183 init_attr
.cap
.max_recv_sge
= 2;
184 init_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
185 init_attr
.qp_type
= IB_QPT_RC
;
187 ret
= rdma_create_qp(ib_conn
->cma_id
, device
->pd
, &init_attr
);
191 ib_conn
->qp
= ib_conn
->cma_id
->qp
;
192 iser_err("setting conn %p cma_id %p: fmr_pool %p qp %p\n",
193 ib_conn
, ib_conn
->cma_id
,
194 ib_conn
->fmr_pool
, ib_conn
->cma_id
->qp
);
198 (void)ib_destroy_fmr_pool(ib_conn
->fmr_pool
);
200 kfree(ib_conn
->page_vec
);
202 iser_err("unable to alloc mem or create resource, err %d\n", ret
);
207 * releases the FMR pool, QP and CMA ID objects, returns 0 on success,
210 static int iser_free_ib_conn_res(struct iser_conn
*ib_conn
)
212 BUG_ON(ib_conn
== NULL
);
214 iser_err("freeing conn %p cma_id %p fmr pool %p qp %p\n",
215 ib_conn
, ib_conn
->cma_id
,
216 ib_conn
->fmr_pool
, ib_conn
->qp
);
218 /* qp is created only once both addr & route are resolved */
219 if (ib_conn
->fmr_pool
!= NULL
)
220 ib_destroy_fmr_pool(ib_conn
->fmr_pool
);
222 if (ib_conn
->qp
!= NULL
)
223 rdma_destroy_qp(ib_conn
->cma_id
);
225 if (ib_conn
->cma_id
!= NULL
)
226 rdma_destroy_id(ib_conn
->cma_id
);
228 ib_conn
->fmr_pool
= NULL
;
230 ib_conn
->cma_id
= NULL
;
231 kfree(ib_conn
->page_vec
);
237 * based on the resolved device node GUID see if there already allocated
238 * device for this device. If there's no such, create one.
241 struct iser_device
*iser_device_find_by_ib_device(struct rdma_cm_id
*cma_id
)
243 struct list_head
*p_list
;
244 struct iser_device
*device
= NULL
;
246 mutex_lock(&ig
.device_list_mutex
);
248 p_list
= ig
.device_list
.next
;
249 while (p_list
!= &ig
.device_list
) {
250 device
= list_entry(p_list
, struct iser_device
, ig_list
);
251 /* find if there's a match using the node GUID */
252 if (device
->ib_device
->node_guid
== cma_id
->device
->node_guid
)
256 if (device
== NULL
) {
257 device
= kzalloc(sizeof *device
, GFP_KERNEL
);
260 /* assign this device to the device */
261 device
->ib_device
= cma_id
->device
;
262 /* init the device and link it into ig device list */
263 if (iser_create_device_ib_res(device
)) {
268 list_add(&device
->ig_list
, &ig
.device_list
);
271 BUG_ON(device
== NULL
);
273 mutex_unlock(&ig
.device_list_mutex
);
277 /* if there's no demand for this device, release it */
278 static void iser_device_try_release(struct iser_device
*device
)
280 mutex_lock(&ig
.device_list_mutex
);
282 iser_err("device %p refcount %d\n",device
,device
->refcount
);
283 if (!device
->refcount
) {
284 iser_free_device_ib_res(device
);
285 list_del(&device
->ig_list
);
288 mutex_unlock(&ig
.device_list_mutex
);
291 int iser_conn_state_comp(struct iser_conn
*ib_conn
,
292 enum iser_ib_conn_state comp
)
296 spin_lock_bh(&ib_conn
->lock
);
297 ret
= (ib_conn
->state
== comp
);
298 spin_unlock_bh(&ib_conn
->lock
);
302 static int iser_conn_state_comp_exch(struct iser_conn
*ib_conn
,
303 enum iser_ib_conn_state comp
,
304 enum iser_ib_conn_state exch
)
308 spin_lock_bh(&ib_conn
->lock
);
309 if ((ret
= (ib_conn
->state
== comp
)))
310 ib_conn
->state
= exch
;
311 spin_unlock_bh(&ib_conn
->lock
);
316 * triggers start of the disconnect procedures and wait for them to be done
318 void iser_conn_terminate(struct iser_conn
*ib_conn
)
322 /* change the ib conn state only if the conn is UP, however always call
323 * rdma_disconnect since this is the only way to cause the CMA to change
324 * the QP state to ERROR
327 iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
, ISER_CONN_TERMINATING
);
328 err
= rdma_disconnect(ib_conn
->cma_id
);
330 iser_err("Failed to disconnect, conn: 0x%p err %d\n",
333 wait_event_interruptible(ib_conn
->wait
,
334 ib_conn
->state
== ISER_CONN_DOWN
);
336 iser_conn_release(ib_conn
);
339 static void iser_connect_error(struct rdma_cm_id
*cma_id
)
341 struct iser_conn
*ib_conn
;
342 ib_conn
= (struct iser_conn
*)cma_id
->context
;
344 ib_conn
->state
= ISER_CONN_DOWN
;
345 wake_up_interruptible(&ib_conn
->wait
);
348 static void iser_addr_handler(struct rdma_cm_id
*cma_id
)
350 struct iser_device
*device
;
351 struct iser_conn
*ib_conn
;
354 device
= iser_device_find_by_ib_device(cma_id
);
355 ib_conn
= (struct iser_conn
*)cma_id
->context
;
356 ib_conn
->device
= device
;
358 ret
= rdma_resolve_route(cma_id
, 1000);
360 iser_err("resolve route failed: %d\n", ret
);
361 iser_connect_error(cma_id
);
366 static void iser_route_handler(struct rdma_cm_id
*cma_id
)
368 struct rdma_conn_param conn_param
;
371 ret
= iser_create_ib_conn_res((struct iser_conn
*)cma_id
->context
);
375 iser_dbg("path.mtu is %d setting it to %d\n",
376 cma_id
->route
.path_rec
->mtu
, IB_MTU_1024
);
378 /* we must set the MTU to 1024 as this is what the target is assuming */
379 if (cma_id
->route
.path_rec
->mtu
> IB_MTU_1024
)
380 cma_id
->route
.path_rec
->mtu
= IB_MTU_1024
;
382 memset(&conn_param
, 0, sizeof conn_param
);
383 conn_param
.responder_resources
= 4;
384 conn_param
.initiator_depth
= 1;
385 conn_param
.retry_count
= 7;
386 conn_param
.rnr_retry_count
= 6;
388 ret
= rdma_connect(cma_id
, &conn_param
);
390 iser_err("failure connecting: %d\n", ret
);
396 iser_connect_error(cma_id
);
399 static void iser_connected_handler(struct rdma_cm_id
*cma_id
)
401 struct iser_conn
*ib_conn
;
403 ib_conn
= (struct iser_conn
*)cma_id
->context
;
404 ib_conn
->state
= ISER_CONN_UP
;
405 wake_up_interruptible(&ib_conn
->wait
);
408 static void iser_disconnected_handler(struct rdma_cm_id
*cma_id
)
410 struct iser_conn
*ib_conn
;
412 ib_conn
= (struct iser_conn
*)cma_id
->context
;
413 ib_conn
->disc_evt_flag
= 1;
415 /* getting here when the state is UP means that the conn is being *
416 * terminated asynchronously from the iSCSI layer's perspective. */
417 if (iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
,
418 ISER_CONN_TERMINATING
))
419 iscsi_conn_failure(ib_conn
->iser_conn
->iscsi_conn
,
420 ISCSI_ERR_CONN_FAILED
);
422 /* Complete the termination process if no posts are pending */
423 if ((atomic_read(&ib_conn
->post_recv_buf_count
) == 0) &&
424 (atomic_read(&ib_conn
->post_send_buf_count
) == 0)) {
425 ib_conn
->state
= ISER_CONN_DOWN
;
426 wake_up_interruptible(&ib_conn
->wait
);
430 static int iser_cma_handler(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
434 iser_err("event %d conn %p id %p\n",event
->event
,cma_id
->context
,cma_id
);
436 switch (event
->event
) {
437 case RDMA_CM_EVENT_ADDR_RESOLVED
:
438 iser_addr_handler(cma_id
);
440 case RDMA_CM_EVENT_ROUTE_RESOLVED
:
441 iser_route_handler(cma_id
);
443 case RDMA_CM_EVENT_ESTABLISHED
:
444 iser_connected_handler(cma_id
);
446 case RDMA_CM_EVENT_ADDR_ERROR
:
447 case RDMA_CM_EVENT_ROUTE_ERROR
:
448 case RDMA_CM_EVENT_CONNECT_ERROR
:
449 case RDMA_CM_EVENT_UNREACHABLE
:
450 case RDMA_CM_EVENT_REJECTED
:
451 iser_err("event: %d, error: %d\n", event
->event
, event
->status
);
452 iser_connect_error(cma_id
);
454 case RDMA_CM_EVENT_DISCONNECTED
:
455 iser_disconnected_handler(cma_id
);
457 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
460 case RDMA_CM_EVENT_CONNECT_RESPONSE
:
463 case RDMA_CM_EVENT_CONNECT_REQUEST
:
470 int iser_conn_init(struct iser_conn
**ibconn
)
472 struct iser_conn
*ib_conn
;
474 ib_conn
= kzalloc(sizeof *ib_conn
, GFP_KERNEL
);
476 iser_err("can't alloc memory for struct iser_conn\n");
479 ib_conn
->state
= ISER_CONN_INIT
;
480 init_waitqueue_head(&ib_conn
->wait
);
481 atomic_set(&ib_conn
->post_recv_buf_count
, 0);
482 atomic_set(&ib_conn
->post_send_buf_count
, 0);
483 INIT_WORK(&ib_conn
->comperror_work
, iser_comp_error_worker
,
485 INIT_LIST_HEAD(&ib_conn
->conn_list
);
486 spin_lock_init(&ib_conn
->lock
);
493 * starts the process of connecting to the target
494 * sleeps untill the connection is established or rejected
496 int iser_connect(struct iser_conn
*ib_conn
,
497 struct sockaddr_in
*src_addr
,
498 struct sockaddr_in
*dst_addr
,
501 struct sockaddr
*src
, *dst
;
504 sprintf(ib_conn
->name
,"%d.%d.%d.%d:%d",
505 NIPQUAD(dst_addr
->sin_addr
.s_addr
), dst_addr
->sin_port
);
507 /* the device is known only --after-- address resolution */
508 ib_conn
->device
= NULL
;
510 iser_err("connecting to: %d.%d.%d.%d, port 0x%x\n",
511 NIPQUAD(dst_addr
->sin_addr
), dst_addr
->sin_port
);
513 ib_conn
->state
= ISER_CONN_PENDING
;
515 ib_conn
->cma_id
= rdma_create_id(iser_cma_handler
,
518 if (IS_ERR(ib_conn
->cma_id
)) {
519 err
= PTR_ERR(ib_conn
->cma_id
);
520 iser_err("rdma_create_id failed: %d\n", err
);
524 src
= (struct sockaddr
*)src_addr
;
525 dst
= (struct sockaddr
*)dst_addr
;
526 err
= rdma_resolve_addr(ib_conn
->cma_id
, src
, dst
, 1000);
528 iser_err("rdma_resolve_addr failed: %d\n", err
);
533 wait_event_interruptible(ib_conn
->wait
,
534 (ib_conn
->state
!= ISER_CONN_PENDING
));
536 if (ib_conn
->state
!= ISER_CONN_UP
) {
538 goto connect_failure
;
542 mutex_lock(&ig
.connlist_mutex
);
543 list_add(&ib_conn
->conn_list
, &ig
.connlist
);
544 mutex_unlock(&ig
.connlist_mutex
);
548 ib_conn
->cma_id
= NULL
;
550 ib_conn
->state
= ISER_CONN_DOWN
;
552 iser_conn_release(ib_conn
);
557 * Frees all conn objects and deallocs conn descriptor
559 void iser_conn_release(struct iser_conn
*ib_conn
)
561 struct iser_device
*device
= ib_conn
->device
;
563 BUG_ON(ib_conn
->state
!= ISER_CONN_DOWN
);
565 mutex_lock(&ig
.connlist_mutex
);
566 list_del(&ib_conn
->conn_list
);
567 mutex_unlock(&ig
.connlist_mutex
);
569 iser_free_ib_conn_res(ib_conn
);
570 ib_conn
->device
= NULL
;
571 /* on EVENT_ADDR_ERROR there's no device yet for this conn */
573 iser_device_try_release(device
);
579 * iser_reg_page_vec - Register physical memory
581 * returns: 0 on success, errno code on failure
583 int iser_reg_page_vec(struct iser_conn
*ib_conn
,
584 struct iser_page_vec
*page_vec
,
585 struct iser_mem_reg
*mem_reg
)
587 struct ib_pool_fmr
*mem
;
592 page_list
= page_vec
->pages
;
593 io_addr
= page_list
[0];
595 mem
= ib_fmr_pool_map_phys(ib_conn
->fmr_pool
,
601 status
= (int)PTR_ERR(mem
);
602 iser_err("ib_fmr_pool_map_phys failed: %d\n", status
);
606 mem_reg
->lkey
= mem
->fmr
->lkey
;
607 mem_reg
->rkey
= mem
->fmr
->rkey
;
608 mem_reg
->len
= page_vec
->length
* SIZE_4K
;
609 mem_reg
->va
= io_addr
;
611 mem_reg
->mem_h
= (void *)mem
;
613 mem_reg
->va
+= page_vec
->offset
;
614 mem_reg
->len
= page_vec
->data_size
;
616 iser_dbg("PHYSICAL Mem.register, [PHYS p_array: 0x%p, sz: %d, "
617 "entry[0]: (0x%08lx,%ld)] -> "
618 "[lkey: 0x%08X mem_h: 0x%p va: 0x%08lX sz: %ld]\n",
619 page_vec
, page_vec
->length
,
620 (unsigned long)page_vec
->pages
[0],
621 (unsigned long)page_vec
->data_size
,
622 (unsigned int)mem_reg
->lkey
, mem_reg
->mem_h
,
623 (unsigned long)mem_reg
->va
, (unsigned long)mem_reg
->len
);
628 * Unregister (previosuly registered) memory.
630 void iser_unreg_mem(struct iser_mem_reg
*reg
)
634 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n",reg
->mem_h
);
636 ret
= ib_fmr_pool_unmap((struct ib_pool_fmr
*)reg
->mem_h
);
638 iser_err("ib_fmr_pool_unmap failed %d\n", ret
);
644 * iser_dto_to_iov - builds IOV from a dto descriptor
646 static void iser_dto_to_iov(struct iser_dto
*dto
, struct ib_sge
*iov
, int iov_len
)
650 struct iser_regd_buf
*regd_buf
;
652 if (dto
->regd_vector_len
> iov_len
) {
653 iser_err("iov size %d too small for posting dto of len %d\n",
654 iov_len
, dto
->regd_vector_len
);
658 for (i
= 0; i
< dto
->regd_vector_len
; i
++) {
660 regd_buf
= dto
->regd
[i
];
662 sge
->addr
= regd_buf
->reg
.va
;
663 sge
->length
= regd_buf
->reg
.len
;
664 sge
->lkey
= regd_buf
->reg
.lkey
;
666 if (dto
->used_sz
[i
] > 0) /* Adjust size */
667 sge
->length
= dto
->used_sz
[i
];
669 /* offset and length should not exceed the regd buf length */
670 if (sge
->length
+ dto
->offset
[i
] > regd_buf
->reg
.len
) {
671 iser_err("Used len:%ld + offset:%d, exceed reg.buf.len:"
672 "%ld in dto:0x%p [%d], va:0x%08lX\n",
673 (unsigned long)sge
->length
, dto
->offset
[i
],
674 (unsigned long)regd_buf
->reg
.len
, dto
, i
,
675 (unsigned long)sge
->addr
);
679 sge
->addr
+= dto
->offset
[i
]; /* Adjust offset */
684 * iser_post_recv - Posts a receive buffer.
686 * returns 0 on success, -1 on failure
688 int iser_post_recv(struct iser_desc
*rx_desc
)
690 int ib_ret
, ret_val
= 0;
691 struct ib_recv_wr recv_wr
, *recv_wr_failed
;
692 struct ib_sge iov
[2];
693 struct iser_conn
*ib_conn
;
694 struct iser_dto
*recv_dto
= &rx_desc
->dto
;
697 ib_conn
= recv_dto
->conn
->ib_conn
;
699 iser_dto_to_iov(recv_dto
, iov
, 2);
702 recv_wr
.sg_list
= iov
;
703 recv_wr
.num_sge
= recv_dto
->regd_vector_len
;
704 recv_wr
.wr_id
= (unsigned long)rx_desc
;
706 atomic_inc(&ib_conn
->post_recv_buf_count
);
707 ib_ret
= ib_post_recv(ib_conn
->qp
, &recv_wr
, &recv_wr_failed
);
709 iser_err("ib_post_recv failed ret=%d\n", ib_ret
);
710 atomic_dec(&ib_conn
->post_recv_buf_count
);
718 * iser_start_send - Initiate a Send DTO operation
720 * returns 0 on success, -1 on failure
722 int iser_post_send(struct iser_desc
*tx_desc
)
724 int ib_ret
, ret_val
= 0;
725 struct ib_send_wr send_wr
, *send_wr_failed
;
726 struct ib_sge iov
[MAX_REGD_BUF_VECTOR_LEN
];
727 struct iser_conn
*ib_conn
;
728 struct iser_dto
*dto
= &tx_desc
->dto
;
730 ib_conn
= dto
->conn
->ib_conn
;
732 iser_dto_to_iov(dto
, iov
, MAX_REGD_BUF_VECTOR_LEN
);
735 send_wr
.wr_id
= (unsigned long)tx_desc
;
736 send_wr
.sg_list
= iov
;
737 send_wr
.num_sge
= dto
->regd_vector_len
;
738 send_wr
.opcode
= IB_WR_SEND
;
739 send_wr
.send_flags
= dto
->notify_enable
? IB_SEND_SIGNALED
: 0;
741 atomic_inc(&ib_conn
->post_send_buf_count
);
743 ib_ret
= ib_post_send(ib_conn
->qp
, &send_wr
, &send_wr_failed
);
745 iser_err("Failed to start SEND DTO, dto: 0x%p, IOV len: %d\n",
746 dto
, dto
->regd_vector_len
);
747 iser_err("ib_post_send failed, ret:%d\n", ib_ret
);
748 atomic_dec(&ib_conn
->post_send_buf_count
);
755 static void iser_comp_error_worker(void *data
)
757 struct iser_conn
*ib_conn
= data
;
759 /* getting here when the state is UP means that the conn is being *
760 * terminated asynchronously from the iSCSI layer's perspective. */
761 if (iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
,
762 ISER_CONN_TERMINATING
))
763 iscsi_conn_failure(ib_conn
->iser_conn
->iscsi_conn
,
764 ISCSI_ERR_CONN_FAILED
);
766 /* complete the termination process if disconnect event was delivered *
767 * note there are no more non completed posts to the QP */
768 if (ib_conn
->disc_evt_flag
) {
769 ib_conn
->state
= ISER_CONN_DOWN
;
770 wake_up_interruptible(&ib_conn
->wait
);
774 static void iser_handle_comp_error(struct iser_desc
*desc
)
776 struct iser_dto
*dto
= &desc
->dto
;
777 struct iser_conn
*ib_conn
= dto
->conn
->ib_conn
;
779 iser_dto_buffs_release(dto
);
781 if (desc
->type
== ISCSI_RX
) {
783 kmem_cache_free(ig
.desc_cache
, desc
);
784 atomic_dec(&ib_conn
->post_recv_buf_count
);
785 } else { /* type is TX control/command/dataout */
786 if (desc
->type
== ISCSI_TX_DATAOUT
)
787 kmem_cache_free(ig
.desc_cache
, desc
);
788 atomic_dec(&ib_conn
->post_send_buf_count
);
791 if (atomic_read(&ib_conn
->post_recv_buf_count
) == 0 &&
792 atomic_read(&ib_conn
->post_send_buf_count
) == 0)
793 schedule_work(&ib_conn
->comperror_work
);
796 static void iser_cq_tasklet_fn(unsigned long data
)
798 struct iser_device
*device
= (struct iser_device
*)data
;
799 struct ib_cq
*cq
= device
->cq
;
801 struct iser_desc
*desc
;
802 unsigned long xfer_len
;
804 while (ib_poll_cq(cq
, 1, &wc
) == 1) {
805 desc
= (struct iser_desc
*) (unsigned long) wc
.wr_id
;
806 BUG_ON(desc
== NULL
);
808 if (wc
.status
== IB_WC_SUCCESS
) {
809 if (desc
->type
== ISCSI_RX
) {
810 xfer_len
= (unsigned long)wc
.byte_len
;
811 iser_rcv_completion(desc
, xfer_len
);
812 } else /* type == ISCSI_TX_CONTROL/SCSI_CMD/DOUT */
813 iser_snd_completion(desc
);
815 iser_err("comp w. error op %d status %d\n",desc
->type
,wc
.status
);
816 iser_handle_comp_error(desc
);
819 /* #warning "it is assumed here that arming CQ only once its empty" *
820 * " would not cause interrupts to be missed" */
821 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
824 static void iser_cq_callback(struct ib_cq
*cq
, void *cq_context
)
826 struct iser_device
*device
= (struct iser_device
*)cq_context
;
828 tasklet_schedule(&device
->cq_tasklet
);