2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
17 #include <linux/version.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <scsi/scsi_host.h>
25 #include "cxgb3_defs.h"
26 #include "cxgb3_ctl_defs.h"
27 #include "cxgb3_offload.h"
28 #include "firmware_exports.h"
31 static unsigned int dbg_level
;
32 #include "../libcxgbi.h"
34 #define DRV_MODULE_NAME "cxgb3i"
35 #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"
36 #define DRV_MODULE_VERSION "2.0.0"
37 #define DRV_MODULE_RELDATE "Jun. 2010"
39 static char version
[] =
40 DRV_MODULE_DESC
" " DRV_MODULE_NAME
41 " v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
43 MODULE_AUTHOR("Chelsio Communications, Inc.");
44 MODULE_DESCRIPTION(DRV_MODULE_DESC
);
45 MODULE_VERSION(DRV_MODULE_VERSION
);
46 MODULE_LICENSE("GPL");
48 module_param(dbg_level
, uint
, 0644);
49 MODULE_PARM_DESC(dbg_level
, "debug flag (default=0)");
51 static int cxgb3i_rcv_win
= 256 * 1024;
52 module_param(cxgb3i_rcv_win
, int, 0644);
53 MODULE_PARM_DESC(cxgb3i_rcv_win
, "TCP receive window in bytes (default=256KB)");
55 static int cxgb3i_snd_win
= 128 * 1024;
56 module_param(cxgb3i_snd_win
, int, 0644);
57 MODULE_PARM_DESC(cxgb3i_snd_win
, "TCP send window in bytes (default=128KB)");
59 static int cxgb3i_rx_credit_thres
= 10 * 1024;
60 module_param(cxgb3i_rx_credit_thres
, int, 0644);
61 MODULE_PARM_DESC(rx_credit_thres
,
62 "RX credits return threshold in bytes (default=10KB)");
64 static unsigned int cxgb3i_max_connect
= 8 * 1024;
65 module_param(cxgb3i_max_connect
, uint
, 0644);
66 MODULE_PARM_DESC(cxgb3i_max_connect
, "Max. # of connections (default=8092)");
68 static unsigned int cxgb3i_sport_base
= 20000;
69 module_param(cxgb3i_sport_base
, uint
, 0644);
70 MODULE_PARM_DESC(cxgb3i_sport_base
, "starting port number (default=20000)");
72 static void cxgb3i_dev_open(struct t3cdev
*);
73 static void cxgb3i_dev_close(struct t3cdev
*);
74 static void cxgb3i_dev_event_handler(struct t3cdev
*, u32
, u32
);
76 static struct cxgb3_client t3_client
= {
77 .name
= DRV_MODULE_NAME
,
78 .handlers
= cxgb3i_cpl_handlers
,
79 .add
= cxgb3i_dev_open
,
80 .remove
= cxgb3i_dev_close
,
81 .event_handler
= cxgb3i_dev_event_handler
,
84 static struct scsi_host_template cxgb3i_host_template
= {
85 .module
= THIS_MODULE
,
86 .name
= DRV_MODULE_NAME
,
87 .proc_name
= DRV_MODULE_NAME
,
88 .can_queue
= CXGB3I_SCSI_HOST_QDEPTH
,
89 .queuecommand
= iscsi_queuecommand
,
90 .change_queue_depth
= iscsi_change_queue_depth
,
91 .sg_tablesize
= SG_ALL
,
92 .max_sectors
= 0xFFFF,
93 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
94 .eh_abort_handler
= iscsi_eh_abort
,
95 .eh_device_reset_handler
= iscsi_eh_device_reset
,
96 .eh_target_reset_handler
= iscsi_eh_recover_target
,
97 .target_alloc
= iscsi_target_alloc
,
98 .use_clustering
= DISABLE_CLUSTERING
,
102 static struct iscsi_transport cxgb3i_iscsi_transport
= {
103 .owner
= THIS_MODULE
,
104 .name
= DRV_MODULE_NAME
,
105 /* owner and name should be set already */
106 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
107 | CAP_DATADGST
| CAP_DIGEST_OFFLOAD
|
108 CAP_PADDING_OFFLOAD
| CAP_TEXT_NEGO
,
110 ISCSI_INITIAL_R2T_EN
| ISCSI_MAX_R2T
|
111 ISCSI_IMM_DATA_EN
| ISCSI_FIRST_BURST
|
112 ISCSI_MAX_BURST
| ISCSI_PDU_INORDER_EN
|
113 ISCSI_DATASEQ_INORDER_EN
| ISCSI_ERL
|
114 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
115 ISCSI_USERNAME
| ISCSI_PASSWORD
|
116 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
117 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
118 ISCSI_LU_RESET_TMO
| ISCSI_TGT_RESET_TMO
|
119 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
120 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
121 ISCSI_HOST_INITIATOR_NAME
|
122 ISCSI_HOST_NETDEV_NAME
,
123 .attr_is_visible
= cxgbi_attr_is_visible
,
124 .get_host_param
= cxgbi_get_host_param
,
125 .set_host_param
= cxgbi_set_host_param
,
126 /* session management */
127 .create_session
= cxgbi_create_session
,
128 .destroy_session
= cxgbi_destroy_session
,
129 .get_session_param
= iscsi_session_get_param
,
130 /* connection management */
131 .create_conn
= cxgbi_create_conn
,
132 .bind_conn
= cxgbi_bind_conn
,
133 .destroy_conn
= iscsi_tcp_conn_teardown
,
134 .start_conn
= iscsi_conn_start
,
135 .stop_conn
= iscsi_conn_stop
,
136 .get_conn_param
= iscsi_conn_get_param
,
137 .set_param
= cxgbi_set_conn_param
,
138 .get_stats
= cxgbi_get_conn_stats
,
139 /* pdu xmit req from user space */
140 .send_pdu
= iscsi_conn_send_pdu
,
142 .init_task
= iscsi_tcp_task_init
,
143 .xmit_task
= iscsi_tcp_task_xmit
,
144 .cleanup_task
= cxgbi_cleanup_task
,
146 .alloc_pdu
= cxgbi_conn_alloc_pdu
,
147 .init_pdu
= cxgbi_conn_init_pdu
,
148 .xmit_pdu
= cxgbi_conn_xmit_pdu
,
149 .parse_pdu_itt
= cxgbi_parse_pdu_itt
,
150 /* TCP connect/disconnect */
151 .get_ep_param
= cxgbi_get_ep_param
,
152 .ep_connect
= cxgbi_ep_connect
,
153 .ep_poll
= cxgbi_ep_poll
,
154 .ep_disconnect
= cxgbi_ep_disconnect
,
155 /* Error recovery timeout call */
156 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
159 static struct scsi_transport_template
*cxgb3i_stt
;
162 * CPL (Chelsio Protocol Language) defines a message passing interface between
163 * the host driver and Chelsio asic.
164 * The section below implments CPLs that related to iscsi tcp connection
165 * open/close/abort and data send/receive.
168 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
);
170 static void send_act_open_req(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
171 const struct l2t_entry
*e
)
173 unsigned int wscale
= cxgbi_sock_compute_wscale(cxgb3i_rcv_win
);
174 struct cpl_act_open_req
*req
= (struct cpl_act_open_req
*)skb
->head
;
176 skb
->priority
= CPL_PRIORITY_SETUP
;
178 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
179 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
, csk
->atid
));
180 req
->local_port
= csk
->saddr
.sin_port
;
181 req
->peer_port
= csk
->daddr
.sin_port
;
182 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
183 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
185 req
->opt0h
= htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS
|
186 V_WND_SCALE(wscale
) | V_MSS_IDX(csk
->mss_idx
) |
187 V_L2T_IDX(e
->idx
) | V_TX_CHANNEL(e
->smt_idx
));
188 req
->opt0l
= htonl(V_ULP_MODE(ULP2_MODE_ISCSI
) |
189 V_RCV_BUFSIZ(cxgb3i_rcv_win
>>10));
191 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
192 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
193 csk
, csk
->state
, csk
->flags
, csk
->atid
,
194 &req
->local_ip
, ntohs(req
->local_port
),
195 &req
->peer_ip
, ntohs(req
->peer_port
),
196 csk
->mss_idx
, e
->idx
, e
->smt_idx
);
198 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
201 static inline void act_open_arp_failure(struct t3cdev
*dev
, struct sk_buff
*skb
)
203 cxgbi_sock_act_open_req_arp_failure(NULL
, skb
);
207 * CPL connection close request: host ->
209 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
210 * the write queue (i.e., after any unsent txt data).
212 static void send_close_req(struct cxgbi_sock
*csk
)
214 struct sk_buff
*skb
= csk
->cpl_close
;
215 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
216 unsigned int tid
= csk
->tid
;
218 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
219 "csk 0x%p,%u,0x%lx,%u.\n",
220 csk
, csk
->state
, csk
->flags
, csk
->tid
);
222 csk
->cpl_close
= NULL
;
223 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON
));
224 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
225 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
226 req
->rsvd
= htonl(csk
->write_seq
);
228 cxgbi_sock_skb_entail(csk
, skb
);
229 if (csk
->state
>= CTP_ESTABLISHED
)
230 push_tx_frames(csk
, 1);
234 * CPL connection abort request: host ->
236 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
237 * for the same connection and also that we do not try to send a message
238 * after the connection has closed.
240 static void abort_arp_failure(struct t3cdev
*tdev
, struct sk_buff
*skb
)
242 struct cpl_abort_req
*req
= cplhdr(skb
);
244 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
245 "t3dev 0x%p, tid %u, skb 0x%p.\n",
246 tdev
, GET_TID(req
), skb
);
247 req
->cmd
= CPL_ABORT_NO_RST
;
248 cxgb3_ofld_send(tdev
, skb
);
251 static void send_abort_req(struct cxgbi_sock
*csk
)
253 struct sk_buff
*skb
= csk
->cpl_abort_req
;
254 struct cpl_abort_req
*req
;
256 if (unlikely(csk
->state
== CTP_ABORTING
|| !skb
))
258 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
259 cxgbi_sock_set_flag(csk
, CTPF_ABORT_RPL_PENDING
);
260 /* Purge the send queue so we don't send anything after an abort. */
261 cxgbi_sock_purge_write_queue(csk
);
263 csk
->cpl_abort_req
= NULL
;
264 req
= (struct cpl_abort_req
*)skb
->head
;
265 skb
->priority
= CPL_PRIORITY_DATA
;
266 set_arp_failure_handler(skb
, abort_arp_failure
);
267 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ
));
268 req
->wr
.wr_lo
= htonl(V_WR_TID(csk
->tid
));
269 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ
, csk
->tid
));
270 req
->rsvd0
= htonl(csk
->snd_nxt
);
271 req
->rsvd1
= !cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
);
272 req
->cmd
= CPL_ABORT_SEND_RST
;
274 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
275 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
276 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->snd_nxt
,
279 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
283 * CPL connection abort reply: host ->
285 * Send an ABORT_RPL message in response of the ABORT_REQ received.
287 static void send_abort_rpl(struct cxgbi_sock
*csk
, int rst_status
)
289 struct sk_buff
*skb
= csk
->cpl_abort_rpl
;
290 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
292 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
293 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
294 csk
, csk
->state
, csk
->flags
, csk
->tid
, rst_status
);
296 csk
->cpl_abort_rpl
= NULL
;
297 skb
->priority
= CPL_PRIORITY_DATA
;
298 rpl
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
299 rpl
->wr
.wr_lo
= htonl(V_WR_TID(csk
->tid
));
300 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, csk
->tid
));
301 rpl
->cmd
= rst_status
;
302 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
306 * CPL connection rx data ack: host ->
307 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
310 static u32
send_rx_credits(struct cxgbi_sock
*csk
, u32 credits
)
313 struct cpl_rx_data_ack
*req
;
314 u32 dack
= F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1);
316 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
317 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
318 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
, dack
);
320 skb
= alloc_wr(sizeof(*req
), 0, GFP_ATOMIC
);
322 pr_info("csk 0x%p, credit %u, OOM.\n", csk
, credits
);
325 req
= (struct cpl_rx_data_ack
*)skb
->head
;
326 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
327 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK
, csk
->tid
));
328 req
->credit_dack
= htonl(F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1) |
329 V_RX_CREDITS(credits
));
330 skb
->priority
= CPL_PRIORITY_ACK
;
331 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
336 * CPL connection tx data: host ->
338 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
340 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
341 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
344 static unsigned int wrlen __read_mostly
;
345 static unsigned int skb_wrs
[SKB_WR_LIST_SIZE
] __read_mostly
;
347 static void init_wr_tab(unsigned int wr_len
)
351 if (skb_wrs
[1]) /* already initialized */
353 for (i
= 1; i
< SKB_WR_LIST_SIZE
; i
++) {
354 int sgl_len
= (3 * i
) / 2 + (i
& 1);
357 skb_wrs
[i
] = (sgl_len
<= wr_len
358 ? 1 : 1 + (sgl_len
- 2) / (wr_len
- 1));
363 static inline void make_tx_data_wr(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
364 int len
, int req_completion
)
366 struct tx_data_wr
*req
;
367 struct l2t_entry
*l2t
= csk
->l2t
;
369 skb_reset_transport_header(skb
);
370 req
= (struct tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
371 req
->wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA
) |
372 (req_completion
? F_WR_COMPL
: 0));
373 req
->wr_lo
= htonl(V_WR_TID(csk
->tid
));
374 /* len includes the length of any HW ULP additions */
375 req
->len
= htonl(len
);
376 /* V_TX_ULP_SUBMODE sets both the mode and submode */
377 req
->flags
= htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb
)) |
378 V_TX_SHOVE((skb_peek(&csk
->write_queue
) ? 0 : 1)));
379 req
->sndseq
= htonl(csk
->snd_nxt
);
380 req
->param
= htonl(V_TX_PORT(l2t
->smt_idx
));
382 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
383 req
->flags
|= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT
|
384 V_TX_CPU_IDX(csk
->rss_qid
));
385 /* sendbuffer is in units of 32KB. */
386 req
->param
|= htonl(V_TX_SNDBUF(cxgb3i_snd_win
>> 15));
387 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
392 * push_tx_frames -- start transmit
393 * @c3cn: the offloaded connection
394 * @req_completion: request wr_ack or not
396 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
397 * connection's send queue and sends them on to T3. Must be called with the
398 * connection's lock held. Returns the amount of send buffer space that was
399 * freed as a result of sending queued data to T3.
402 static void arp_failure_skb_discard(struct t3cdev
*dev
, struct sk_buff
*skb
)
407 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
)
412 if (unlikely(csk
->state
< CTP_ESTABLISHED
||
413 csk
->state
== CTP_CLOSE_WAIT_1
|| csk
->state
>= CTP_ABORTING
)) {
414 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
415 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
416 csk
, csk
->state
, csk
->flags
, csk
->tid
);
420 while (csk
->wr_cred
&& (skb
= skb_peek(&csk
->write_queue
)) != NULL
) {
421 int len
= skb
->len
; /* length before skb_push */
422 int frags
= skb_shinfo(skb
)->nr_frags
+ (len
!= skb
->data_len
);
423 int wrs_needed
= skb_wrs
[frags
];
425 if (wrs_needed
> 1 && len
+ sizeof(struct tx_data_wr
) <= wrlen
)
428 WARN_ON(frags
>= SKB_WR_LIST_SIZE
|| wrs_needed
< 1);
430 if (csk
->wr_cred
< wrs_needed
) {
431 log_debug(1 << CXGBI_DBG_PDU_TX
,
432 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
433 csk
, skb
->len
, skb
->data_len
, frags
,
434 wrs_needed
, csk
->wr_cred
);
438 __skb_unlink(skb
, &csk
->write_queue
);
439 skb
->priority
= CPL_PRIORITY_DATA
;
440 skb
->csum
= wrs_needed
; /* remember this until the WR_ACK */
441 csk
->wr_cred
-= wrs_needed
;
442 csk
->wr_una_cred
+= wrs_needed
;
443 cxgbi_sock_enqueue_wr(csk
, skb
);
445 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
446 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
447 "left %u, unack %u.\n",
448 csk
, skb
->len
, skb
->data_len
, frags
, skb
->csum
,
449 csk
->wr_cred
, csk
->wr_una_cred
);
451 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
))) {
452 if ((req_completion
&&
453 csk
->wr_una_cred
== wrs_needed
) ||
454 csk
->wr_una_cred
>= csk
->wr_max_cred
/ 2) {
456 csk
->wr_una_cred
= 0;
458 len
+= cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb
));
459 make_tx_data_wr(csk
, skb
, len
, req_completion
);
461 cxgbi_skcb_clear_flag(skb
, SKCBF_TX_NEED_HDR
);
463 total_size
+= skb
->truesize
;
464 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
465 "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
467 set_arp_failure_handler(skb
, arp_failure_skb_discard
);
468 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
474 * Process a CPL_ACT_ESTABLISH message: -> host
475 * Updates connection state from an active establish CPL message. Runs with
476 * the connection lock held.
479 static inline void free_atid(struct cxgbi_sock
*csk
)
481 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
)) {
482 cxgb3_free_atid(csk
->cdev
->lldev
, csk
->atid
);
483 cxgbi_sock_clear_flag(csk
, CTPF_HAS_ATID
);
488 static int do_act_establish(struct t3cdev
*tdev
, struct sk_buff
*skb
, void *ctx
)
490 struct cxgbi_sock
*csk
= ctx
;
491 struct cpl_act_establish
*req
= cplhdr(skb
);
492 unsigned int tid
= GET_TID(req
);
493 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
494 u32 rcv_isn
= ntohl(req
->rcv_isn
); /* real RCV_ISN + 1 */
496 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
497 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
498 atid
, atid
, csk
, csk
->state
, csk
->flags
, rcv_isn
);
501 cxgbi_sock_set_flag(csk
, CTPF_HAS_TID
);
503 cxgb3_insert_tid(csk
->cdev
->lldev
, &t3_client
, csk
, tid
);
507 csk
->rss_qid
= G_QNUM(ntohs(skb
->csum
));
509 spin_lock_bh(&csk
->lock
);
510 if (csk
->retry_timer
.function
) {
511 del_timer(&csk
->retry_timer
);
512 csk
->retry_timer
.function
= NULL
;
515 if (unlikely(csk
->state
!= CTP_ACTIVE_OPEN
))
516 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
517 csk
, csk
->state
, csk
->flags
, csk
->tid
);
519 csk
->copied_seq
= csk
->rcv_wup
= csk
->rcv_nxt
= rcv_isn
;
520 if (cxgb3i_rcv_win
> (M_RCV_BUFSIZ
<< 10))
521 csk
->rcv_wup
-= cxgb3i_rcv_win
- (M_RCV_BUFSIZ
<< 10);
523 cxgbi_sock_established(csk
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
525 if (unlikely(cxgbi_sock_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
)))
526 /* upper layer has requested closing */
529 if (skb_queue_len(&csk
->write_queue
))
530 push_tx_frames(csk
, 1);
531 cxgbi_conn_tx_open(csk
);
534 spin_unlock_bh(&csk
->lock
);
540 * Process a CPL_ACT_OPEN_RPL message: -> host
541 * Handle active open failures.
543 static int act_open_rpl_status_to_errno(int status
)
546 case CPL_ERR_CONN_RESET
:
547 return -ECONNREFUSED
;
548 case CPL_ERR_ARP_MISS
:
549 return -EHOSTUNREACH
;
550 case CPL_ERR_CONN_TIMEDOUT
:
552 case CPL_ERR_TCAM_FULL
:
554 case CPL_ERR_CONN_EXIST
:
561 static void act_open_retry_timer(unsigned long data
)
564 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)data
;
566 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
567 "csk 0x%p,%u,0x%lx,%u.\n",
568 csk
, csk
->state
, csk
->flags
, csk
->tid
);
571 spin_lock_bh(&csk
->lock
);
572 skb
= alloc_wr(sizeof(struct cpl_act_open_req
), 0, GFP_ATOMIC
);
574 cxgbi_sock_fail_act_open(csk
, -ENOMEM
);
576 skb
->sk
= (struct sock
*)csk
;
577 set_arp_failure_handler(skb
, act_open_arp_failure
);
578 send_act_open_req(csk
, skb
, csk
->l2t
);
580 spin_unlock_bh(&csk
->lock
);
584 static int do_act_open_rpl(struct t3cdev
*tdev
, struct sk_buff
*skb
, void *ctx
)
586 struct cxgbi_sock
*csk
= ctx
;
587 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
589 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
590 csk
, csk
->state
, csk
->flags
, csk
->atid
, rpl
->status
,
591 &csk
->saddr
.sin_addr
.s_addr
, ntohs(csk
->saddr
.sin_port
),
592 &csk
->daddr
.sin_addr
.s_addr
, ntohs(csk
->daddr
.sin_port
));
594 if (rpl
->status
!= CPL_ERR_TCAM_FULL
&&
595 rpl
->status
!= CPL_ERR_CONN_EXIST
&&
596 rpl
->status
!= CPL_ERR_ARP_MISS
)
597 cxgb3_queue_tid_release(tdev
, GET_TID(rpl
));
600 spin_lock_bh(&csk
->lock
);
601 if (rpl
->status
== CPL_ERR_CONN_EXIST
&&
602 csk
->retry_timer
.function
!= act_open_retry_timer
) {
603 csk
->retry_timer
.function
= act_open_retry_timer
;
604 mod_timer(&csk
->retry_timer
, jiffies
+ HZ
/ 2);
606 cxgbi_sock_fail_act_open(csk
,
607 act_open_rpl_status_to_errno(rpl
->status
));
609 spin_unlock_bh(&csk
->lock
);
616 * Process PEER_CLOSE CPL messages: -> host
619 static int do_peer_close(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
621 struct cxgbi_sock
*csk
= ctx
;
623 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
624 "csk 0x%p,%u,0x%lx,%u.\n",
625 csk
, csk
->state
, csk
->flags
, csk
->tid
);
627 cxgbi_sock_rcv_peer_close(csk
);
633 * Process CLOSE_CONN_RPL CPL message: -> host
634 * Process a peer ACK to our FIN.
636 static int do_close_con_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
,
639 struct cxgbi_sock
*csk
= ctx
;
640 struct cpl_close_con_rpl
*rpl
= cplhdr(skb
);
642 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
643 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
644 csk
, csk
->state
, csk
->flags
, csk
->tid
, ntohl(rpl
->snd_nxt
));
646 cxgbi_sock_rcv_close_conn_rpl(csk
, ntohl(rpl
->snd_nxt
));
652 * Process ABORT_REQ_RSS CPL message: -> host
653 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
654 * request except that we need to reply to it.
657 static int abort_status_to_errno(struct cxgbi_sock
*csk
, int abort_reason
,
660 switch (abort_reason
) {
661 case CPL_ERR_BAD_SYN
: /* fall through */
662 case CPL_ERR_CONN_RESET
:
663 return csk
->state
> CTP_ESTABLISHED
? -EPIPE
: -ECONNRESET
;
664 case CPL_ERR_XMIT_TIMEDOUT
:
665 case CPL_ERR_PERSIST_TIMEDOUT
:
666 case CPL_ERR_FINWAIT2_TIMEDOUT
:
667 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
674 static int do_abort_req(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
676 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
677 struct cxgbi_sock
*csk
= ctx
;
678 int rst_status
= CPL_ABORT_NO_RST
;
680 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
681 "csk 0x%p,%u,0x%lx,%u.\n",
682 csk
, csk
->state
, csk
->flags
, csk
->tid
);
684 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
685 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
) {
690 spin_lock_bh(&csk
->lock
);
692 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_REQ_RCVD
)) {
693 cxgbi_sock_set_flag(csk
, CTPF_ABORT_REQ_RCVD
);
694 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
698 cxgbi_sock_clear_flag(csk
, CTPF_ABORT_REQ_RCVD
);
699 send_abort_rpl(csk
, rst_status
);
701 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
)) {
702 csk
->err
= abort_status_to_errno(csk
, req
->status
, &rst_status
);
703 cxgbi_sock_closed(csk
);
707 spin_unlock_bh(&csk
->lock
);
715 * Process ABORT_RPL_RSS CPL message: -> host
716 * Process abort replies. We only process these messages if we anticipate
717 * them as the coordination between SW and HW in this area is somewhat lacking
718 * and sometimes we get ABORT_RPLs after we are done with the connection that
719 * originated the ABORT_REQ.
721 static int do_abort_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
723 struct cpl_abort_rpl_rss
*rpl
= cplhdr(skb
);
724 struct cxgbi_sock
*csk
= ctx
;
726 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
727 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
728 rpl
->status
, csk
, csk
? csk
->state
: 0,
729 csk
? csk
->flags
: 0UL);
731 * Ignore replies to post-close aborts indicating that the abort was
732 * requested too late. These connections are terminated when we get
733 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
734 * arrives the TID is either no longer used or it has been recycled.
736 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
739 * Sometimes we've already closed the connection, e.g., a post-close
740 * abort races with ABORT_REQ_RSS, the latter frees the connection
741 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
742 * but FW turns the ABORT_REQ into a regular one and so we get
743 * ABORT_RPL_RSS with status 0 and no connection.
746 cxgbi_sock_rcv_abort_rpl(csk
);
753 * Process RX_ISCSI_HDR CPL message: -> host
754 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
755 * follow after the bhs.
757 static int do_iscsi_hdr(struct t3cdev
*t3dev
, struct sk_buff
*skb
, void *ctx
)
759 struct cxgbi_sock
*csk
= ctx
;
760 struct cpl_iscsi_hdr
*hdr_cpl
= cplhdr(skb
);
761 struct cpl_iscsi_hdr_norss data_cpl
;
762 struct cpl_rx_data_ddp_norss ddp_cpl
;
763 unsigned int hdr_len
, data_len
, status
;
767 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
768 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
769 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, skb
->len
);
771 spin_lock_bh(&csk
->lock
);
773 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
774 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
775 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
776 csk
, csk
->state
, csk
->flags
, csk
->tid
);
777 if (csk
->state
!= CTP_ABORTING
)
783 cxgbi_skcb_tcp_seq(skb
) = ntohl(hdr_cpl
->seq
);
784 cxgbi_skcb_flags(skb
) = 0;
786 skb_reset_transport_header(skb
);
787 __skb_pull(skb
, sizeof(struct cpl_iscsi_hdr
));
789 len
= hdr_len
= ntohs(hdr_cpl
->len
);
790 /* msg coalesce is off or not enough data received */
791 if (skb
->len
<= hdr_len
) {
792 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
793 csk
->cdev
->ports
[csk
->port_id
]->name
, csk
->tid
,
797 cxgbi_skcb_set_flag(skb
, SKCBF_RX_COALESCED
);
799 err
= skb_copy_bits(skb
, skb
->len
- sizeof(ddp_cpl
), &ddp_cpl
,
802 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
803 csk
->cdev
->ports
[csk
->port_id
]->name
, csk
->tid
,
804 skb
->len
, sizeof(ddp_cpl
), err
);
808 cxgbi_skcb_set_flag(skb
, SKCBF_RX_STATUS
);
809 cxgbi_skcb_rx_pdulen(skb
) = ntohs(ddp_cpl
.len
);
810 cxgbi_skcb_rx_ddigest(skb
) = ntohl(ddp_cpl
.ulp_crc
);
811 status
= ntohl(ddp_cpl
.ddp_status
);
813 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
814 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
815 csk
, skb
, skb
->len
, cxgbi_skcb_rx_pdulen(skb
), status
);
817 if (status
& (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT
))
818 cxgbi_skcb_set_flag(skb
, SKCBF_RX_HCRC_ERR
);
819 if (status
& (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT
))
820 cxgbi_skcb_set_flag(skb
, SKCBF_RX_DCRC_ERR
);
821 if (status
& (1 << CPL_RX_DDP_STATUS_PAD_SHIFT
))
822 cxgbi_skcb_set_flag(skb
, SKCBF_RX_PAD_ERR
);
824 if (skb
->len
> (hdr_len
+ sizeof(ddp_cpl
))) {
825 err
= skb_copy_bits(skb
, hdr_len
, &data_cpl
, sizeof(data_cpl
));
827 pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
828 csk
->cdev
->ports
[csk
->port_id
]->name
,
829 csk
->tid
, sizeof(data_cpl
), skb
->len
, err
);
832 data_len
= ntohs(data_cpl
.len
);
833 log_debug(1 << CXGBI_DBG_DDP
| 1 << CXGBI_DBG_PDU_RX
,
834 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
835 skb
, data_len
, cxgbi_skcb_rx_pdulen(skb
), status
);
836 len
+= sizeof(data_cpl
) + data_len
;
837 } else if (status
& (1 << CPL_RX_DDP_STATUS_DDP_SHIFT
))
838 cxgbi_skcb_set_flag(skb
, SKCBF_RX_DATA_DDPD
);
840 csk
->rcv_nxt
= ntohl(ddp_cpl
.seq
) + cxgbi_skcb_rx_pdulen(skb
);
841 __pskb_trim(skb
, len
);
842 __skb_queue_tail(&csk
->receive_queue
, skb
);
843 cxgbi_conn_pdu_ready(csk
);
845 spin_unlock_bh(&csk
->lock
);
851 spin_unlock_bh(&csk
->lock
);
857 * Process TX_DATA_ACK CPL messages: -> host
858 * Process an acknowledgment of WR completion. Advance snd_una and send the
859 * next batch of work requests from the write queue.
861 static int do_wr_ack(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
863 struct cxgbi_sock
*csk
= ctx
;
864 struct cpl_wr_ack
*hdr
= cplhdr(skb
);
866 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
867 "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
868 csk
, csk
->state
, csk
->flags
, csk
->tid
, ntohs(hdr
->credits
));
870 cxgbi_sock_rcv_wr_ack(csk
, ntohs(hdr
->credits
), ntohl(hdr
->snd_una
), 1);
876 * for each connection, pre-allocate skbs needed for close/abort requests. So
877 * that we can service the request right away.
879 static int alloc_cpls(struct cxgbi_sock
*csk
)
881 csk
->cpl_close
= alloc_wr(sizeof(struct cpl_close_con_req
), 0,
885 csk
->cpl_abort_req
= alloc_wr(sizeof(struct cpl_abort_req
), 0,
887 if (!csk
->cpl_abort_req
)
890 csk
->cpl_abort_rpl
= alloc_wr(sizeof(struct cpl_abort_rpl
), 0,
892 if (!csk
->cpl_abort_rpl
)
898 cxgbi_sock_free_cpl_skbs(csk
);
903 * release_offload_resources - release offload resource
904 * @c3cn: the offloaded iscsi tcp connection.
905 * Release resources held by an offload connection (TID, L2T entry, etc.)
907 static void l2t_put(struct cxgbi_sock
*csk
)
909 struct t3cdev
*t3dev
= (struct t3cdev
*)csk
->cdev
->lldev
;
912 l2t_release(L2DATA(t3dev
), csk
->l2t
);
918 static void release_offload_resources(struct cxgbi_sock
*csk
)
920 struct t3cdev
*t3dev
= (struct t3cdev
*)csk
->cdev
->lldev
;
922 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
923 "csk 0x%p,%u,0x%lx,%u.\n",
924 csk
, csk
->state
, csk
->flags
, csk
->tid
);
927 cxgbi_sock_free_cpl_skbs(csk
);
929 if (csk
->wr_cred
!= csk
->wr_max_cred
) {
930 cxgbi_sock_purge_wr_queue(csk
);
931 cxgbi_sock_reset_wr_list(csk
);
934 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
))
936 else if (cxgbi_sock_flag(csk
, CTPF_HAS_TID
)) {
937 cxgb3_remove_tid(t3dev
, (void *)csk
, csk
->tid
);
938 cxgbi_sock_clear_flag(csk
, CTPF_HAS_TID
);
945 static void update_address(struct cxgbi_hba
*chba
)
947 if (chba
->ipv4addr
) {
949 chba
->ipv4addr
!= cxgb3i_get_private_ipv4addr(chba
->vdev
)) {
950 cxgb3i_set_private_ipv4addr(chba
->vdev
, chba
->ipv4addr
);
951 cxgb3i_set_private_ipv4addr(chba
->ndev
, 0);
952 pr_info("%s set %pI4.\n",
953 chba
->vdev
->name
, &chba
->ipv4addr
);
954 } else if (chba
->ipv4addr
!=
955 cxgb3i_get_private_ipv4addr(chba
->ndev
)) {
956 cxgb3i_set_private_ipv4addr(chba
->ndev
, chba
->ipv4addr
);
957 pr_info("%s set %pI4.\n",
958 chba
->ndev
->name
, &chba
->ipv4addr
);
960 } else if (cxgb3i_get_private_ipv4addr(chba
->ndev
)) {
962 cxgb3i_set_private_ipv4addr(chba
->vdev
, 0);
963 cxgb3i_set_private_ipv4addr(chba
->ndev
, 0);
967 static int init_act_open(struct cxgbi_sock
*csk
)
969 struct dst_entry
*dst
= csk
->dst
;
970 struct cxgbi_device
*cdev
= csk
->cdev
;
971 struct t3cdev
*t3dev
= (struct t3cdev
*)cdev
->lldev
;
972 struct net_device
*ndev
= cdev
->ports
[csk
->port_id
];
973 struct cxgbi_hba
*chba
= cdev
->hbas
[csk
->port_id
];
974 struct sk_buff
*skb
= NULL
;
976 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
977 "csk 0x%p,%u,0x%lx.\n", csk
, csk
->state
, csk
->flags
);
979 update_address(chba
);
981 csk
->saddr
.sin_addr
.s_addr
= chba
->ipv4addr
;
984 csk
->l2t
= t3_l2t_get(t3dev
, dst_get_neighbour(dst
), ndev
);
986 pr_err("NO l2t available.\n");
991 csk
->atid
= cxgb3_alloc_atid(t3dev
, &t3_client
, csk
);
993 pr_err("NO atid available.\n");
996 cxgbi_sock_set_flag(csk
, CTPF_HAS_ATID
);
999 skb
= alloc_wr(sizeof(struct cpl_act_open_req
), 0, GFP_KERNEL
);
1002 skb
->sk
= (struct sock
*)csk
;
1003 set_arp_failure_handler(skb
, act_open_arp_failure
);
1005 csk
->wr_max_cred
= csk
->wr_cred
= T3C_DATA(t3dev
)->max_wrs
- 1;
1006 csk
->wr_una_cred
= 0;
1007 csk
->mss_idx
= cxgbi_sock_select_mss(csk
, dst_mtu(dst
));
1008 cxgbi_sock_reset_wr_list(csk
);
1011 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1012 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1013 csk
, csk
->state
, csk
->flags
,
1014 &csk
->saddr
.sin_addr
.s_addr
, ntohs(csk
->saddr
.sin_port
),
1015 &csk
->daddr
.sin_addr
.s_addr
, ntohs(csk
->daddr
.sin_port
));
1017 cxgbi_sock_set_state(csk
, CTP_ACTIVE_OPEN
);
1018 send_act_open_req(csk
, skb
, csk
->l2t
);
1027 cxgb3_cpl_handler_func cxgb3i_cpl_handlers
[NUM_CPL_CMDS
] = {
1028 [CPL_ACT_ESTABLISH
] = do_act_establish
,
1029 [CPL_ACT_OPEN_RPL
] = do_act_open_rpl
,
1030 [CPL_PEER_CLOSE
] = do_peer_close
,
1031 [CPL_ABORT_REQ_RSS
] = do_abort_req
,
1032 [CPL_ABORT_RPL_RSS
] = do_abort_rpl
,
1033 [CPL_CLOSE_CON_RPL
] = do_close_con_rpl
,
1034 [CPL_TX_DMA_ACK
] = do_wr_ack
,
1035 [CPL_ISCSI_HDR
] = do_iscsi_hdr
,
1039 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1040 * @cdev: cxgbi adapter
1042 int cxgb3i_ofld_init(struct cxgbi_device
*cdev
)
1044 struct t3cdev
*t3dev
= (struct t3cdev
*)cdev
->lldev
;
1045 struct adap_ports port
;
1046 struct ofld_page_info rx_page_info
;
1047 unsigned int wr_len
;
1050 if (t3dev
->ctl(t3dev
, GET_WR_LEN
, &wr_len
) < 0 ||
1051 t3dev
->ctl(t3dev
, GET_PORTS
, &port
) < 0 ||
1052 t3dev
->ctl(t3dev
, GET_RX_PAGE_INFO
, &rx_page_info
) < 0) {
1053 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev
);
1057 if (cxgb3i_max_connect
> CXGBI_MAX_CONN
)
1058 cxgb3i_max_connect
= CXGBI_MAX_CONN
;
1060 rc
= cxgbi_device_portmap_create(cdev
, cxgb3i_sport_base
,
1061 cxgb3i_max_connect
);
1065 init_wr_tab(wr_len
);
1066 cdev
->csk_release_offload_resources
= release_offload_resources
;
1067 cdev
->csk_push_tx_frames
= push_tx_frames
;
1068 cdev
->csk_send_abort_req
= send_abort_req
;
1069 cdev
->csk_send_close_req
= send_close_req
;
1070 cdev
->csk_send_rx_credits
= send_rx_credits
;
1071 cdev
->csk_alloc_cpls
= alloc_cpls
;
1072 cdev
->csk_init_act_open
= init_act_open
;
1074 pr_info("cdev 0x%p, offload up, added.\n", cdev
);
1079 * functions to program the pagepod in h/w
1081 static inline void ulp_mem_io_set_hdr(struct sk_buff
*skb
, unsigned int addr
)
1083 struct ulp_mem_io
*req
= (struct ulp_mem_io
*)skb
->head
;
1085 memset(req
, 0, sizeof(*req
));
1087 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_BYPASS
));
1088 req
->cmd_lock_addr
= htonl(V_ULP_MEMIO_ADDR(addr
>> 5) |
1089 V_ULPTX_CMD(ULP_MEM_WRITE
));
1090 req
->len
= htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE
>> 5) |
1091 V_ULPTX_NFLITS((PPOD_SIZE
>> 3) + 1));
1094 static int ddp_set_map(struct cxgbi_sock
*csk
, struct cxgbi_pagepod_hdr
*hdr
,
1095 unsigned int idx
, unsigned int npods
,
1096 struct cxgbi_gather_list
*gl
)
1098 struct cxgbi_device
*cdev
= csk
->cdev
;
1099 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1100 unsigned int pm_addr
= (idx
<< PPOD_SIZE_SHIFT
) + ddp
->llimit
;
1103 log_debug(1 << CXGBI_DBG_DDP
,
1104 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
1105 csk
, idx
, npods
, gl
);
1107 for (i
= 0; i
< npods
; i
++, idx
++, pm_addr
+= PPOD_SIZE
) {
1108 struct sk_buff
*skb
= alloc_wr(sizeof(struct ulp_mem_io
) +
1109 PPOD_SIZE
, 0, GFP_ATOMIC
);
1114 ulp_mem_io_set_hdr(skb
, pm_addr
);
1115 cxgbi_ddp_ppod_set((struct cxgbi_pagepod
*)(skb
->head
+
1116 sizeof(struct ulp_mem_io
)),
1117 hdr
, gl
, i
* PPOD_PAGES_MAX
);
1118 skb
->priority
= CPL_PRIORITY_CONTROL
;
1119 cxgb3_ofld_send(cdev
->lldev
, skb
);
1124 static void ddp_clear_map(struct cxgbi_hba
*chba
, unsigned int tag
,
1125 unsigned int idx
, unsigned int npods
)
1127 struct cxgbi_device
*cdev
= chba
->cdev
;
1128 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1129 unsigned int pm_addr
= (idx
<< PPOD_SIZE_SHIFT
) + ddp
->llimit
;
1132 log_debug(1 << CXGBI_DBG_DDP
,
1133 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
1134 cdev
, idx
, npods
, tag
);
1136 for (i
= 0; i
< npods
; i
++, idx
++, pm_addr
+= PPOD_SIZE
) {
1137 struct sk_buff
*skb
= alloc_wr(sizeof(struct ulp_mem_io
) +
1138 PPOD_SIZE
, 0, GFP_ATOMIC
);
1141 pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
1142 tag
, idx
, i
, npods
);
1145 ulp_mem_io_set_hdr(skb
, pm_addr
);
1146 skb
->priority
= CPL_PRIORITY_CONTROL
;
1147 cxgb3_ofld_send(cdev
->lldev
, skb
);
1151 static int ddp_setup_conn_pgidx(struct cxgbi_sock
*csk
,
1152 unsigned int tid
, int pg_idx
, bool reply
)
1154 struct sk_buff
*skb
= alloc_wr(sizeof(struct cpl_set_tcb_field
), 0,
1156 struct cpl_set_tcb_field
*req
;
1157 u64 val
= pg_idx
< DDP_PGIDX_MAX
? pg_idx
: 0;
1159 log_debug(1 << CXGBI_DBG_DDP
,
1160 "csk 0x%p, tid %u, pg_idx %d.\n", csk
, tid
, pg_idx
);
1164 /* set up ulp submode and page size */
1165 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1166 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1167 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1168 req
->reply
= V_NO_REPLY(reply
? 0 : 1);
1170 req
->word
= htons(31);
1171 req
->mask
= cpu_to_be64(0xF0000000);
1172 req
->val
= cpu_to_be64(val
<< 28);
1173 skb
->priority
= CPL_PRIORITY_CONTROL
;
1175 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
1180 * cxgb3i_setup_conn_digest - setup conn. digest setting
1181 * @csk: cxgb tcp socket
1182 * @tid: connection id
1183 * @hcrc: header digest enabled
1184 * @dcrc: data digest enabled
1185 * @reply: request reply from h/w
1186 * set up the iscsi digest settings for a connection identified by tid
1188 static int ddp_setup_conn_digest(struct cxgbi_sock
*csk
, unsigned int tid
,
1189 int hcrc
, int dcrc
, int reply
)
1191 struct sk_buff
*skb
= alloc_wr(sizeof(struct cpl_set_tcb_field
), 0,
1193 struct cpl_set_tcb_field
*req
;
1194 u64 val
= (hcrc
? 1 : 0) | (dcrc
? 2 : 0);
1196 log_debug(1 << CXGBI_DBG_DDP
,
1197 "csk 0x%p, tid %u, crc %d,%d.\n", csk
, tid
, hcrc
, dcrc
);
1201 /* set up ulp submode and page size */
1202 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1203 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1204 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1205 req
->reply
= V_NO_REPLY(reply
? 0 : 1);
1207 req
->word
= htons(31);
1208 req
->mask
= cpu_to_be64(0x0F000000);
1209 req
->val
= cpu_to_be64(val
<< 24);
1210 skb
->priority
= CPL_PRIORITY_CONTROL
;
1212 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
1217 * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource
1218 * @cdev: cxgb3i adapter
1219 * release all the resource held by the ddp pagepod manager for a given
1223 static void t3_ddp_cleanup(struct cxgbi_device
*cdev
)
1225 struct t3cdev
*tdev
= (struct t3cdev
*)cdev
->lldev
;
1227 if (cxgbi_ddp_cleanup(cdev
)) {
1228 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev
);
1229 tdev
->ulp_iscsi
= NULL
;
1234 * ddp_init - initialize the cxgb3 adapter's ddp resource
1235 * @cdev: cxgb3i adapter
1236 * initialize the ddp pagepod manager for a given adapter
1238 static int cxgb3i_ddp_init(struct cxgbi_device
*cdev
)
1240 struct t3cdev
*tdev
= (struct t3cdev
*)cdev
->lldev
;
1241 struct cxgbi_ddp_info
*ddp
= tdev
->ulp_iscsi
;
1242 struct ulp_iscsi_info uinfo
;
1243 unsigned int pgsz_factor
[4];
1247 kref_get(&ddp
->refcnt
);
1248 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
1249 tdev
, tdev
->ulp_iscsi
);
1254 err
= tdev
->ctl(tdev
, ULP_ISCSI_GET_PARAMS
, &uinfo
);
1256 pr_err("%s, failed to get iscsi param err=%d.\n",
1261 err
= cxgbi_ddp_init(cdev
, uinfo
.llimit
, uinfo
.ulimit
,
1262 uinfo
.max_txsz
, uinfo
.max_rxsz
);
1268 uinfo
.tagmask
= ddp
->idx_mask
<< PPOD_IDX_SHIFT
;
1269 cxgbi_ddp_page_size_factor(pgsz_factor
);
1270 for (i
= 0; i
< 4; i
++)
1271 uinfo
.pgsz_factor
[i
] = pgsz_factor
[i
];
1272 uinfo
.ulimit
= uinfo
.llimit
+ (ddp
->nppods
<< PPOD_SIZE_SHIFT
);
1274 err
= tdev
->ctl(tdev
, ULP_ISCSI_SET_PARAMS
, &uinfo
);
1276 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
1278 cxgbi_ddp_cleanup(cdev
);
1281 tdev
->ulp_iscsi
= ddp
;
1283 cdev
->csk_ddp_setup_digest
= ddp_setup_conn_digest
;
1284 cdev
->csk_ddp_setup_pgidx
= ddp_setup_conn_pgidx
;
1285 cdev
->csk_ddp_set
= ddp_set_map
;
1286 cdev
->csk_ddp_clear
= ddp_clear_map
;
1288 pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1290 tdev
, ddp
->nppods
, ddp
->idx_bits
, ddp
->idx_mask
,
1291 ddp
->rsvd_tag_mask
, ddp
->max_txsz
, uinfo
.max_txsz
,
1292 ddp
->max_rxsz
, uinfo
.max_rxsz
);
1296 static void cxgb3i_dev_close(struct t3cdev
*t3dev
)
1298 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1300 if (!cdev
|| cdev
->flags
& CXGBI_FLAG_ADAPTER_RESET
) {
1301 pr_info("0x%p close, f 0x%x.\n", cdev
, cdev
? cdev
->flags
: 0);
1305 cxgbi_device_unregister(cdev
);
1309 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1310 * @t3dev: t3cdev adapter
1312 static void cxgb3i_dev_open(struct t3cdev
*t3dev
)
1314 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1315 struct adapter
*adapter
= tdev2adap(t3dev
);
1319 pr_info("0x%p, updating.\n", cdev
);
1323 cdev
= cxgbi_device_register(0, adapter
->params
.nports
);
1325 pr_warn("device 0x%p register failed.\n", t3dev
);
1329 cdev
->flags
= CXGBI_FLAG_DEV_T3
| CXGBI_FLAG_IPV4_SET
;
1330 cdev
->lldev
= t3dev
;
1331 cdev
->pdev
= adapter
->pdev
;
1332 cdev
->ports
= adapter
->port
;
1333 cdev
->nports
= adapter
->params
.nports
;
1334 cdev
->mtus
= adapter
->params
.mtus
;
1335 cdev
->nmtus
= NMTUS
;
1336 cdev
->snd_win
= cxgb3i_snd_win
;
1337 cdev
->rcv_win
= cxgb3i_rcv_win
;
1338 cdev
->rx_credit_thres
= cxgb3i_rx_credit_thres
;
1339 cdev
->skb_tx_rsvd
= CXGB3I_TX_HEADER_LEN
;
1340 cdev
->skb_rx_extra
= sizeof(struct cpl_iscsi_hdr_norss
);
1341 cdev
->dev_ddp_cleanup
= t3_ddp_cleanup
;
1342 cdev
->itp
= &cxgb3i_iscsi_transport
;
1344 err
= cxgb3i_ddp_init(cdev
);
1346 pr_info("0x%p ddp init failed\n", cdev
);
1350 err
= cxgb3i_ofld_init(cdev
);
1352 pr_info("0x%p offload init failed\n", cdev
);
1356 err
= cxgbi_hbas_add(cdev
, CXGB3I_MAX_LUN
, CXGBI_MAX_CONN
,
1357 &cxgb3i_host_template
, cxgb3i_stt
);
1361 for (i
= 0; i
< cdev
->nports
; i
++)
1362 cdev
->hbas
[i
]->ipv4addr
=
1363 cxgb3i_get_private_ipv4addr(cdev
->ports
[i
]);
1365 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1366 cdev
, cdev
? cdev
->flags
: 0, t3dev
, err
);
1370 cxgbi_device_unregister(cdev
);
1373 static void cxgb3i_dev_event_handler(struct t3cdev
*t3dev
, u32 event
, u32 port
)
1375 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1377 log_debug(1 << CXGBI_DBG_TOE
,
1378 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1379 t3dev
, cdev
, event
, port
);
1384 case OFFLOAD_STATUS_DOWN
:
1385 cdev
->flags
|= CXGBI_FLAG_ADAPTER_RESET
;
1387 case OFFLOAD_STATUS_UP
:
1388 cdev
->flags
&= ~CXGBI_FLAG_ADAPTER_RESET
;
1394 * cxgb3i_init_module - module init entry point
1396 * initialize any driver wide global data structures and register itself
1397 * with the cxgb3 module
1399 static int __init
cxgb3i_init_module(void)
1403 printk(KERN_INFO
"%s", version
);
1405 rc
= cxgbi_iscsi_init(&cxgb3i_iscsi_transport
, &cxgb3i_stt
);
1409 cxgb3_register_client(&t3_client
);
1414 * cxgb3i_exit_module - module cleanup/exit entry point
1416 * go through the driver hba list and for each hba, release any resource held.
1417 * and unregisters iscsi transport and the cxgb3 module
1419 static void __exit
cxgb3i_exit_module(void)
1421 cxgb3_unregister_client(&t3_client
);
1422 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3
);
1423 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport
, &cxgb3i_stt
);
1426 module_init(cxgb3i_init_module
);
1427 module_exit(cxgb3i_exit_module
);