2 * cxgb4i.c: Chelsio T4 iSCSI driver.
4 * Copyright (c) 2010 Chelsio Communications, Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Rakesh Ranjan (rranjan@chelsio.com)
14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16 #include <linux/module.h>
17 #include <linux/moduleparam.h>
18 #include <scsi/scsi_host.h>
21 #include <linux/netdevice.h>
22 #include <net/addrconf.h>
27 #include "cxgb4_uld.h"
32 static unsigned int dbg_level
;
34 #include "../libcxgbi.h"
36 #define DRV_MODULE_NAME "cxgb4i"
37 #define DRV_MODULE_DESC "Chelsio T4/T5 iSCSI Driver"
38 #define DRV_MODULE_VERSION "0.9.4"
40 static char version
[] =
41 DRV_MODULE_DESC
" " DRV_MODULE_NAME
42 " v" DRV_MODULE_VERSION
"\n";
44 MODULE_AUTHOR("Chelsio Communications, Inc.");
45 MODULE_DESCRIPTION(DRV_MODULE_DESC
);
46 MODULE_VERSION(DRV_MODULE_VERSION
);
47 MODULE_LICENSE("GPL");
49 module_param(dbg_level
, uint
, 0644);
50 MODULE_PARM_DESC(dbg_level
, "Debug flag (default=0)");
52 static int cxgb4i_rcv_win
= 256 * 1024;
53 module_param(cxgb4i_rcv_win
, int, 0644);
54 MODULE_PARM_DESC(cxgb4i_rcv_win
, "TCP reveive window in bytes");
56 static int cxgb4i_snd_win
= 128 * 1024;
57 module_param(cxgb4i_snd_win
, int, 0644);
58 MODULE_PARM_DESC(cxgb4i_snd_win
, "TCP send window in bytes");
60 static int cxgb4i_rx_credit_thres
= 10 * 1024;
61 module_param(cxgb4i_rx_credit_thres
, int, 0644);
62 MODULE_PARM_DESC(cxgb4i_rx_credit_thres
,
63 "RX credits return threshold in bytes (default=10KB)");
65 static unsigned int cxgb4i_max_connect
= (8 * 1024);
66 module_param(cxgb4i_max_connect
, uint
, 0644);
67 MODULE_PARM_DESC(cxgb4i_max_connect
, "Maximum number of connections");
69 static unsigned short cxgb4i_sport_base
= 20000;
70 module_param(cxgb4i_sport_base
, ushort
, 0644);
71 MODULE_PARM_DESC(cxgb4i_sport_base
, "Starting port number (default 20000)");
73 typedef void (*cxgb4i_cplhandler_func
)(struct cxgbi_device
*, struct sk_buff
*);
75 static void *t4_uld_add(const struct cxgb4_lld_info
*);
76 static int t4_uld_rx_handler(void *, const __be64
*, const struct pkt_gl
*);
77 static int t4_uld_state_change(void *, enum cxgb4_state state
);
79 static const struct cxgb4_uld_info cxgb4i_uld_info
= {
80 .name
= DRV_MODULE_NAME
,
82 .rx_handler
= t4_uld_rx_handler
,
83 .state_change
= t4_uld_state_change
,
86 static struct scsi_host_template cxgb4i_host_template
= {
87 .module
= THIS_MODULE
,
88 .name
= DRV_MODULE_NAME
,
89 .proc_name
= DRV_MODULE_NAME
,
90 .can_queue
= CXGB4I_SCSI_HOST_QDEPTH
,
91 .queuecommand
= iscsi_queuecommand
,
92 .change_queue_depth
= iscsi_change_queue_depth
,
93 .sg_tablesize
= SG_ALL
,
94 .max_sectors
= 0xFFFF,
95 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
96 .eh_abort_handler
= iscsi_eh_abort
,
97 .eh_device_reset_handler
= iscsi_eh_device_reset
,
98 .eh_target_reset_handler
= iscsi_eh_recover_target
,
99 .target_alloc
= iscsi_target_alloc
,
100 .use_clustering
= DISABLE_CLUSTERING
,
104 static struct iscsi_transport cxgb4i_iscsi_transport
= {
105 .owner
= THIS_MODULE
,
106 .name
= DRV_MODULE_NAME
,
107 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
|
108 CAP_DATADGST
| CAP_DIGEST_OFFLOAD
|
109 CAP_PADDING_OFFLOAD
| CAP_TEXT_NEGO
,
110 .attr_is_visible
= cxgbi_attr_is_visible
,
111 .get_host_param
= cxgbi_get_host_param
,
112 .set_host_param
= cxgbi_set_host_param
,
113 /* session management */
114 .create_session
= cxgbi_create_session
,
115 .destroy_session
= cxgbi_destroy_session
,
116 .get_session_param
= iscsi_session_get_param
,
117 /* connection management */
118 .create_conn
= cxgbi_create_conn
,
119 .bind_conn
= cxgbi_bind_conn
,
120 .destroy_conn
= iscsi_tcp_conn_teardown
,
121 .start_conn
= iscsi_conn_start
,
122 .stop_conn
= iscsi_conn_stop
,
123 .get_conn_param
= iscsi_conn_get_param
,
124 .set_param
= cxgbi_set_conn_param
,
125 .get_stats
= cxgbi_get_conn_stats
,
126 /* pdu xmit req from user space */
127 .send_pdu
= iscsi_conn_send_pdu
,
129 .init_task
= iscsi_tcp_task_init
,
130 .xmit_task
= iscsi_tcp_task_xmit
,
131 .cleanup_task
= cxgbi_cleanup_task
,
133 .alloc_pdu
= cxgbi_conn_alloc_pdu
,
134 .init_pdu
= cxgbi_conn_init_pdu
,
135 .xmit_pdu
= cxgbi_conn_xmit_pdu
,
136 .parse_pdu_itt
= cxgbi_parse_pdu_itt
,
137 /* TCP connect/disconnect */
138 .get_ep_param
= cxgbi_get_ep_param
,
139 .ep_connect
= cxgbi_ep_connect
,
140 .ep_poll
= cxgbi_ep_poll
,
141 .ep_disconnect
= cxgbi_ep_disconnect
,
142 /* Error recovery timeout call */
143 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
146 static struct scsi_transport_template
*cxgb4i_stt
;
149 * CPL (Chelsio Protocol Language) defines a message passing interface between
150 * the host driver and Chelsio asic.
151 * The section below implments CPLs that related to iscsi tcp connection
152 * open/close/abort and data send/receive.
155 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
156 #define RCV_BUFSIZ_MASK 0x3FFU
157 #define MAX_IMM_TX_PKT_LEN 128
159 static inline void set_queue(struct sk_buff
*skb
, unsigned int queue
,
160 const struct cxgbi_sock
*csk
)
162 skb
->queue_mapping
= queue
;
165 static int push_tx_frames(struct cxgbi_sock
*, int);
168 * is_ofld_imm - check whether a packet can be sent as immediate data
171 * Returns true if a packet can be sent as an offload WR with immediate
172 * data. We currently use the same limit as for Ethernet packets.
174 static inline int is_ofld_imm(const struct sk_buff
*skb
)
176 return skb
->len
<= (MAX_IMM_TX_PKT_LEN
-
177 sizeof(struct fw_ofld_tx_data_wr
));
180 static void send_act_open_req(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
183 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
184 int t4
= is_t4(lldi
->adapter_type
);
185 int wscale
= cxgbi_sock_compute_wscale(csk
->mss_idx
);
186 unsigned long long opt0
;
188 unsigned int qid_atid
= ((unsigned int)csk
->atid
) |
189 (((unsigned int)csk
->rss_qid
) << 14);
191 opt0
= KEEP_ALIVE(1) |
193 MSS_IDX(csk
->mss_idx
) |
194 L2T_IDX(((struct l2t_entry
*)csk
->l2t
)->idx
) |
195 TX_CHAN(csk
->tx_chan
) |
196 SMAC_SEL(csk
->smac_idx
) |
197 ULP_MODE(ULP_MODE_ISCSI
) |
198 RCV_BUFSIZ(cxgb4i_rcv_win
>> 10);
199 opt2
= RX_CHANNEL(0) |
202 RSS_QUEUE(csk
->rss_qid
);
204 if (is_t4(lldi
->adapter_type
)) {
205 struct cpl_act_open_req
*req
=
206 (struct cpl_act_open_req
*)skb
->head
;
209 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
,
211 req
->local_port
= csk
->saddr
.sin_port
;
212 req
->peer_port
= csk
->daddr
.sin_port
;
213 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
214 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
215 req
->opt0
= cpu_to_be64(opt0
);
216 req
->params
= cpu_to_be32(cxgb4_select_ntuple(
217 csk
->cdev
->ports
[csk
->port_id
],
220 req
->opt2
= cpu_to_be32(opt2
);
222 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
223 "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
224 csk
, &req
->local_ip
, ntohs(req
->local_port
),
225 &req
->peer_ip
, ntohs(req
->peer_port
),
226 csk
->atid
, csk
->rss_qid
);
228 struct cpl_t5_act_open_req
*req
=
229 (struct cpl_t5_act_open_req
*)skb
->head
;
232 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
,
234 req
->local_port
= csk
->saddr
.sin_port
;
235 req
->peer_port
= csk
->daddr
.sin_port
;
236 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
237 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
238 req
->opt0
= cpu_to_be64(opt0
);
239 req
->params
= cpu_to_be64(V_FILTER_TUPLE(
241 csk
->cdev
->ports
[csk
->port_id
],
244 req
->opt2
= cpu_to_be32(opt2
);
246 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
247 "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
248 csk
, &req
->local_ip
, ntohs(req
->local_port
),
249 &req
->peer_ip
, ntohs(req
->peer_port
),
250 csk
->atid
, csk
->rss_qid
);
253 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->port_id
);
255 pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n",
256 (&csk
->saddr
), (&csk
->daddr
), t4
? 4 : 5, csk
,
257 csk
->state
, csk
->flags
, csk
->atid
, csk
->rss_qid
);
259 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
262 static void send_act_open_req6(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
265 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
266 int t4
= is_t4(lldi
->adapter_type
);
267 int wscale
= cxgbi_sock_compute_wscale(csk
->mss_idx
);
268 unsigned long long opt0
;
270 unsigned int qid_atid
= ((unsigned int)csk
->atid
) |
271 (((unsigned int)csk
->rss_qid
) << 14);
273 opt0
= KEEP_ALIVE(1) |
275 MSS_IDX(csk
->mss_idx
) |
276 L2T_IDX(((struct l2t_entry
*)csk
->l2t
)->idx
) |
277 TX_CHAN(csk
->tx_chan
) |
278 SMAC_SEL(csk
->smac_idx
) |
279 ULP_MODE(ULP_MODE_ISCSI
) |
280 RCV_BUFSIZ(cxgb4i_rcv_win
>> 10);
282 opt2
= RX_CHANNEL(0) |
285 RSS_QUEUE(csk
->rss_qid
);
288 struct cpl_act_open_req6
*req
=
289 (struct cpl_act_open_req6
*)skb
->head
;
292 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
,
294 req
->local_port
= csk
->saddr6
.sin6_port
;
295 req
->peer_port
= csk
->daddr6
.sin6_port
;
297 req
->local_ip_hi
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
);
298 req
->local_ip_lo
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
+
300 req
->peer_ip_hi
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
);
301 req
->peer_ip_lo
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
+
304 req
->opt0
= cpu_to_be64(opt0
);
307 req
->opt2
= cpu_to_be32(opt2
);
309 req
->params
= cpu_to_be32(cxgb4_select_ntuple(
310 csk
->cdev
->ports
[csk
->port_id
],
313 struct cpl_t5_act_open_req6
*req
=
314 (struct cpl_t5_act_open_req6
*)skb
->head
;
317 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
,
319 req
->local_port
= csk
->saddr6
.sin6_port
;
320 req
->peer_port
= csk
->daddr6
.sin6_port
;
321 req
->local_ip_hi
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
);
322 req
->local_ip_lo
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
+
324 req
->peer_ip_hi
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
);
325 req
->peer_ip_lo
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
+
327 req
->opt0
= cpu_to_be64(opt0
);
329 opt2
|= T5_OPT_2_VALID
;
330 req
->opt2
= cpu_to_be32(opt2
);
332 req
->params
= cpu_to_be64(V_FILTER_TUPLE(cxgb4_select_ntuple(
333 csk
->cdev
->ports
[csk
->port_id
],
337 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->port_id
);
339 pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n",
340 t4
? 4 : 5, csk
, csk
->state
, csk
->flags
, csk
->atid
,
341 &csk
->saddr6
.sin6_addr
, ntohs(csk
->saddr
.sin_port
),
342 &csk
->daddr6
.sin6_addr
, ntohs(csk
->daddr
.sin_port
),
345 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
348 static void send_close_req(struct cxgbi_sock
*csk
)
350 struct sk_buff
*skb
= csk
->cpl_close
;
351 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
352 unsigned int tid
= csk
->tid
;
354 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
355 "csk 0x%p,%u,0x%lx, tid %u.\n",
356 csk
, csk
->state
, csk
->flags
, csk
->tid
);
357 csk
->cpl_close
= NULL
;
358 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
359 INIT_TP_WR(req
, tid
);
360 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
363 cxgbi_sock_skb_entail(csk
, skb
);
364 if (csk
->state
>= CTP_ESTABLISHED
)
365 push_tx_frames(csk
, 1);
368 static void abort_arp_failure(void *handle
, struct sk_buff
*skb
)
370 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)handle
;
371 struct cpl_abort_req
*req
;
373 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
374 "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
375 csk
, csk
->state
, csk
->flags
, csk
->tid
);
376 req
= (struct cpl_abort_req
*)skb
->data
;
377 req
->cmd
= CPL_ABORT_NO_RST
;
378 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
381 static void send_abort_req(struct cxgbi_sock
*csk
)
383 struct cpl_abort_req
*req
;
384 struct sk_buff
*skb
= csk
->cpl_abort_req
;
386 if (unlikely(csk
->state
== CTP_ABORTING
) || !skb
|| !csk
->cdev
)
388 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
389 cxgbi_sock_set_flag(csk
, CTPF_ABORT_RPL_PENDING
);
390 cxgbi_sock_purge_write_queue(csk
);
392 csk
->cpl_abort_req
= NULL
;
393 req
= (struct cpl_abort_req
*)skb
->head
;
394 set_queue(skb
, CPL_PRIORITY_DATA
, csk
);
395 req
->cmd
= CPL_ABORT_SEND_RST
;
396 t4_set_arp_err_handler(skb
, csk
, abort_arp_failure
);
397 INIT_TP_WR(req
, csk
->tid
);
398 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ
, csk
->tid
));
399 req
->rsvd0
= htonl(csk
->snd_nxt
);
400 req
->rsvd1
= !cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
);
402 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
403 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
404 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->snd_nxt
,
407 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
410 static void send_abort_rpl(struct cxgbi_sock
*csk
, int rst_status
)
412 struct sk_buff
*skb
= csk
->cpl_abort_rpl
;
413 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
415 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
416 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
417 csk
, csk
->state
, csk
->flags
, csk
->tid
, rst_status
);
419 csk
->cpl_abort_rpl
= NULL
;
420 set_queue(skb
, CPL_PRIORITY_DATA
, csk
);
421 INIT_TP_WR(rpl
, csk
->tid
);
422 OPCODE_TID(rpl
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL
, csk
->tid
));
423 rpl
->cmd
= rst_status
;
424 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
428 * CPL connection rx data ack: host ->
429 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
432 static u32
send_rx_credits(struct cxgbi_sock
*csk
, u32 credits
)
435 struct cpl_rx_data_ack
*req
;
437 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
438 "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
439 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
);
441 skb
= alloc_wr(sizeof(*req
), 0, GFP_ATOMIC
);
443 pr_info("csk 0x%p, credit %u, OOM.\n", csk
, credits
);
446 req
= (struct cpl_rx_data_ack
*)skb
->head
;
448 set_wr_txq(skb
, CPL_PRIORITY_ACK
, csk
->port_id
);
449 INIT_TP_WR(req
, csk
->tid
);
450 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK
,
452 req
->credit_dack
= cpu_to_be32(RX_CREDITS(credits
) | RX_FORCE_ACK(1));
453 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
458 * sgl_len - calculates the size of an SGL of the given capacity
459 * @n: the number of SGL entries
460 * Calculates the number of flits needed for a scatter/gather list that
461 * can hold the given number of entries.
463 static inline unsigned int sgl_len(unsigned int n
)
466 return (3 * n
) / 2 + (n
& 1) + 2;
470 * calc_tx_flits_ofld - calculate # of flits for an offload packet
473 * Returns the number of flits needed for the given offload packet.
474 * These packets are already fully constructed and no additional headers
477 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff
*skb
)
479 unsigned int flits
, cnt
;
481 if (is_ofld_imm(skb
))
482 return DIV_ROUND_UP(skb
->len
, 8);
483 flits
= skb_transport_offset(skb
) / 8;
484 cnt
= skb_shinfo(skb
)->nr_frags
;
485 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
487 return flits
+ sgl_len(cnt
);
490 static inline void send_tx_flowc_wr(struct cxgbi_sock
*csk
)
493 struct fw_flowc_wr
*flowc
;
497 skb
= alloc_wr(flowclen
, 0, GFP_ATOMIC
);
498 flowc
= (struct fw_flowc_wr
*)skb
->head
;
499 flowc
->op_to_nparams
=
500 htonl(FW_WR_OP(FW_FLOWC_WR
) | FW_FLOWC_WR_NPARAMS(8));
501 flowc
->flowid_len16
=
502 htonl(FW_WR_LEN16(DIV_ROUND_UP(72, 16)) |
503 FW_WR_FLOWID(csk
->tid
));
504 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
505 flowc
->mnemval
[0].val
= htonl(csk
->cdev
->pfvf
);
506 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
507 flowc
->mnemval
[1].val
= htonl(csk
->tx_chan
);
508 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
509 flowc
->mnemval
[2].val
= htonl(csk
->tx_chan
);
510 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
511 flowc
->mnemval
[3].val
= htonl(csk
->rss_qid
);
512 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
513 flowc
->mnemval
[4].val
= htonl(csk
->snd_nxt
);
514 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
515 flowc
->mnemval
[5].val
= htonl(csk
->rcv_nxt
);
516 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
517 flowc
->mnemval
[6].val
= htonl(cxgb4i_snd_win
);
518 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
519 flowc
->mnemval
[7].val
= htonl(csk
->advmss
);
520 flowc
->mnemval
[8].mnemonic
= 0;
521 flowc
->mnemval
[8].val
= 0;
522 for (i
= 0; i
< 9; i
++) {
523 flowc
->mnemval
[i
].r4
[0] = 0;
524 flowc
->mnemval
[i
].r4
[1] = 0;
525 flowc
->mnemval
[i
].r4
[2] = 0;
527 set_queue(skb
, CPL_PRIORITY_DATA
, csk
);
529 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
530 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
531 csk
, csk
->tid
, 0, csk
->tx_chan
, csk
->rss_qid
,
532 csk
->snd_nxt
, csk
->rcv_nxt
, cxgb4i_snd_win
,
535 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
538 static inline void make_tx_data_wr(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
539 int dlen
, int len
, u32 credits
, int compl)
541 struct fw_ofld_tx_data_wr
*req
;
542 unsigned int submode
= cxgbi_skcb_ulp_mode(skb
) & 3;
543 unsigned int wr_ulp_mode
= 0;
545 req
= (struct fw_ofld_tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
547 if (is_ofld_imm(skb
)) {
548 req
->op_to_immdlen
= htonl(FW_WR_OP(FW_OFLD_TX_DATA_WR
) |
550 FW_WR_IMMDLEN(dlen
));
551 req
->flowid_len16
= htonl(FW_WR_FLOWID(csk
->tid
) |
552 FW_WR_LEN16(credits
));
555 cpu_to_be32(FW_WR_OP(FW_OFLD_TX_DATA_WR
) |
559 cpu_to_be32(FW_WR_FLOWID(csk
->tid
) |
560 FW_WR_LEN16(credits
));
563 wr_ulp_mode
= FW_OFLD_TX_DATA_WR_ULPMODE(ULP2_MODE_ISCSI
) |
564 FW_OFLD_TX_DATA_WR_ULPSUBMODE(submode
);
565 req
->tunnel_to_proxy
= htonl(wr_ulp_mode
|
566 FW_OFLD_TX_DATA_WR_SHOVE(skb_peek(&csk
->write_queue
) ? 0 : 1));
567 req
->plen
= htonl(len
);
568 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
))
569 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
572 static void arp_failure_skb_discard(void *handle
, struct sk_buff
*skb
)
577 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
)
582 if (unlikely(csk
->state
< CTP_ESTABLISHED
||
583 csk
->state
== CTP_CLOSE_WAIT_1
|| csk
->state
>= CTP_ABORTING
)) {
584 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
|
585 1 << CXGBI_DBG_PDU_TX
,
586 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
587 csk
, csk
->state
, csk
->flags
, csk
->tid
);
591 while (csk
->wr_cred
&& (skb
= skb_peek(&csk
->write_queue
)) != NULL
) {
594 unsigned int credits_needed
;
596 skb_reset_transport_header(skb
);
597 if (is_ofld_imm(skb
))
598 credits_needed
= DIV_ROUND_UP(dlen
+
599 sizeof(struct fw_ofld_tx_data_wr
), 16);
601 credits_needed
= DIV_ROUND_UP(8*calc_tx_flits_ofld(skb
)
602 + sizeof(struct fw_ofld_tx_data_wr
),
605 if (csk
->wr_cred
< credits_needed
) {
606 log_debug(1 << CXGBI_DBG_PDU_TX
,
607 "csk 0x%p, skb %u/%u, wr %d < %u.\n",
608 csk
, skb
->len
, skb
->data_len
,
609 credits_needed
, csk
->wr_cred
);
612 __skb_unlink(skb
, &csk
->write_queue
);
613 set_queue(skb
, CPL_PRIORITY_DATA
, csk
);
614 skb
->csum
= credits_needed
;
615 csk
->wr_cred
-= credits_needed
;
616 csk
->wr_una_cred
+= credits_needed
;
617 cxgbi_sock_enqueue_wr(csk
, skb
);
619 log_debug(1 << CXGBI_DBG_PDU_TX
,
620 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
621 csk
, skb
->len
, skb
->data_len
, credits_needed
,
622 csk
->wr_cred
, csk
->wr_una_cred
);
624 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
))) {
625 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
626 send_tx_flowc_wr(csk
);
629 csk
->wr_una_cred
+= 5;
631 len
+= cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb
));
632 make_tx_data_wr(csk
, skb
, dlen
, len
, credits_needed
,
635 cxgbi_skcb_clear_flag(skb
, SKCBF_TX_NEED_HDR
);
637 total_size
+= skb
->truesize
;
638 t4_set_arp_err_handler(skb
, csk
, arp_failure_skb_discard
);
640 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
641 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
642 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, len
);
644 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
649 static inline void free_atid(struct cxgbi_sock
*csk
)
651 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
653 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
)) {
654 cxgb4_free_atid(lldi
->tids
, csk
->atid
);
655 cxgbi_sock_clear_flag(csk
, CTPF_HAS_ATID
);
660 static void do_act_establish(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
662 struct cxgbi_sock
*csk
;
663 struct cpl_act_establish
*req
= (struct cpl_act_establish
*)skb
->data
;
664 unsigned short tcp_opt
= ntohs(req
->tcp_opt
);
665 unsigned int tid
= GET_TID(req
);
666 unsigned int atid
= GET_TID_TID(ntohl(req
->tos_atid
));
667 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
668 struct tid_info
*t
= lldi
->tids
;
669 u32 rcv_isn
= be32_to_cpu(req
->rcv_isn
);
671 csk
= lookup_atid(t
, atid
);
672 if (unlikely(!csk
)) {
673 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid
, cdev
);
677 if (csk
->atid
!= atid
) {
678 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
679 atid
, csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->atid
);
683 pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
684 (&csk
->saddr
), (&csk
->daddr
),
685 atid
, tid
, csk
, csk
->state
, csk
->flags
, rcv_isn
);
687 module_put(THIS_MODULE
);
691 cxgb4_insert_tid(lldi
->tids
, csk
, tid
);
692 cxgbi_sock_set_flag(csk
, CTPF_HAS_TID
);
696 spin_lock_bh(&csk
->lock
);
697 if (unlikely(csk
->state
!= CTP_ACTIVE_OPEN
))
698 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
699 csk
, csk
->state
, csk
->flags
, csk
->tid
);
701 if (csk
->retry_timer
.function
) {
702 del_timer(&csk
->retry_timer
);
703 csk
->retry_timer
.function
= NULL
;
706 csk
->copied_seq
= csk
->rcv_wup
= csk
->rcv_nxt
= rcv_isn
;
708 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
711 if (cxgb4i_rcv_win
> (RCV_BUFSIZ_MASK
<< 10))
712 csk
->rcv_wup
-= cxgb4i_rcv_win
- (RCV_BUFSIZ_MASK
<< 10);
714 csk
->advmss
= lldi
->mtus
[GET_TCPOPT_MSS(tcp_opt
)] - 40;
715 if (GET_TCPOPT_TSTAMP(tcp_opt
))
717 if (csk
->advmss
< 128)
720 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
721 "csk 0x%p, mss_idx %u, advmss %u.\n",
722 csk
, GET_TCPOPT_MSS(tcp_opt
), csk
->advmss
);
724 cxgbi_sock_established(csk
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
726 if (unlikely(cxgbi_sock_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
)))
729 if (skb_queue_len(&csk
->write_queue
))
730 push_tx_frames(csk
, 0);
731 cxgbi_conn_tx_open(csk
);
733 spin_unlock_bh(&csk
->lock
);
739 static int act_open_rpl_status_to_errno(int status
)
742 case CPL_ERR_CONN_RESET
:
743 return -ECONNREFUSED
;
744 case CPL_ERR_ARP_MISS
:
745 return -EHOSTUNREACH
;
746 case CPL_ERR_CONN_TIMEDOUT
:
748 case CPL_ERR_TCAM_FULL
:
750 case CPL_ERR_CONN_EXIST
:
757 static void csk_act_open_retry_timer(unsigned long data
)
760 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)data
;
761 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
762 void (*send_act_open_func
)(struct cxgbi_sock
*, struct sk_buff
*,
764 int t4
= is_t4(lldi
->adapter_type
), size
, size6
;
766 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
767 "csk 0x%p,%u,0x%lx,%u.\n",
768 csk
, csk
->state
, csk
->flags
, csk
->tid
);
771 spin_lock_bh(&csk
->lock
);
774 size
= sizeof(struct cpl_act_open_req
);
775 size6
= sizeof(struct cpl_act_open_req6
);
777 size
= sizeof(struct cpl_t5_act_open_req
);
778 size6
= sizeof(struct cpl_t5_act_open_req6
);
781 if (csk
->csk_family
== AF_INET
) {
782 send_act_open_func
= send_act_open_req
;
783 skb
= alloc_wr(size
, 0, GFP_ATOMIC
);
785 send_act_open_func
= send_act_open_req6
;
786 skb
= alloc_wr(size6
, 0, GFP_ATOMIC
);
790 cxgbi_sock_fail_act_open(csk
, -ENOMEM
);
792 skb
->sk
= (struct sock
*)csk
;
793 t4_set_arp_err_handler(skb
, csk
,
794 cxgbi_sock_act_open_req_arp_failure
);
795 send_act_open_func(csk
, skb
, csk
->l2t
);
798 spin_unlock_bh(&csk
->lock
);
803 static void do_act_open_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
805 struct cxgbi_sock
*csk
;
806 struct cpl_act_open_rpl
*rpl
= (struct cpl_act_open_rpl
*)skb
->data
;
807 unsigned int tid
= GET_TID(rpl
);
809 GET_TID_TID(GET_AOPEN_ATID(be32_to_cpu(rpl
->atid_status
)));
810 unsigned int status
= GET_AOPEN_STATUS(be32_to_cpu(rpl
->atid_status
));
811 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
812 struct tid_info
*t
= lldi
->tids
;
814 csk
= lookup_atid(t
, atid
);
815 if (unlikely(!csk
)) {
816 pr_err("NO matching conn. atid %u, tid %u.\n", atid
, tid
);
820 pr_info_ipaddr("tid %u/%u, status %u.\n"
821 "csk 0x%p,%u,0x%lx. ", (&csk
->saddr
), (&csk
->daddr
),
822 atid
, tid
, status
, csk
, csk
->state
, csk
->flags
);
824 if (status
== CPL_ERR_RTX_NEG_ADVICE
)
827 if (status
&& status
!= CPL_ERR_TCAM_FULL
&&
828 status
!= CPL_ERR_CONN_EXIST
&&
829 status
!= CPL_ERR_ARP_MISS
)
830 cxgb4_remove_tid(lldi
->tids
, csk
->port_id
, GET_TID(rpl
));
833 spin_lock_bh(&csk
->lock
);
835 if (status
== CPL_ERR_CONN_EXIST
&&
836 csk
->retry_timer
.function
!= csk_act_open_retry_timer
) {
837 csk
->retry_timer
.function
= csk_act_open_retry_timer
;
838 mod_timer(&csk
->retry_timer
, jiffies
+ HZ
/ 2);
840 cxgbi_sock_fail_act_open(csk
,
841 act_open_rpl_status_to_errno(status
));
843 spin_unlock_bh(&csk
->lock
);
849 static void do_peer_close(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
851 struct cxgbi_sock
*csk
;
852 struct cpl_peer_close
*req
= (struct cpl_peer_close
*)skb
->data
;
853 unsigned int tid
= GET_TID(req
);
854 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
855 struct tid_info
*t
= lldi
->tids
;
857 csk
= lookup_tid(t
, tid
);
858 if (unlikely(!csk
)) {
859 pr_err("can't find connection for tid %u.\n", tid
);
862 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
863 (&csk
->saddr
), (&csk
->daddr
),
864 csk
, csk
->state
, csk
->flags
, csk
->tid
);
865 cxgbi_sock_rcv_peer_close(csk
);
870 static void do_close_con_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
872 struct cxgbi_sock
*csk
;
873 struct cpl_close_con_rpl
*rpl
= (struct cpl_close_con_rpl
*)skb
->data
;
874 unsigned int tid
= GET_TID(rpl
);
875 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
876 struct tid_info
*t
= lldi
->tids
;
878 csk
= lookup_tid(t
, tid
);
879 if (unlikely(!csk
)) {
880 pr_err("can't find connection for tid %u.\n", tid
);
883 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
884 (&csk
->saddr
), (&csk
->daddr
),
885 csk
, csk
->state
, csk
->flags
, csk
->tid
);
886 cxgbi_sock_rcv_close_conn_rpl(csk
, ntohl(rpl
->snd_nxt
));
891 static int abort_status_to_errno(struct cxgbi_sock
*csk
, int abort_reason
,
894 switch (abort_reason
) {
895 case CPL_ERR_BAD_SYN
: /* fall through */
896 case CPL_ERR_CONN_RESET
:
897 return csk
->state
> CTP_ESTABLISHED
?
898 -EPIPE
: -ECONNRESET
;
899 case CPL_ERR_XMIT_TIMEDOUT
:
900 case CPL_ERR_PERSIST_TIMEDOUT
:
901 case CPL_ERR_FINWAIT2_TIMEDOUT
:
902 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
909 static void do_abort_req_rss(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
911 struct cxgbi_sock
*csk
;
912 struct cpl_abort_req_rss
*req
= (struct cpl_abort_req_rss
*)skb
->data
;
913 unsigned int tid
= GET_TID(req
);
914 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
915 struct tid_info
*t
= lldi
->tids
;
916 int rst_status
= CPL_ABORT_NO_RST
;
918 csk
= lookup_tid(t
, tid
);
919 if (unlikely(!csk
)) {
920 pr_err("can't find connection for tid %u.\n", tid
);
924 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
925 (&csk
->saddr
), (&csk
->daddr
),
926 csk
, csk
->state
, csk
->flags
, csk
->tid
, req
->status
);
928 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
929 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
)
933 spin_lock_bh(&csk
->lock
);
935 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_REQ_RCVD
)) {
936 cxgbi_sock_set_flag(csk
, CTPF_ABORT_REQ_RCVD
);
937 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
941 cxgbi_sock_clear_flag(csk
, CTPF_ABORT_REQ_RCVD
);
942 send_abort_rpl(csk
, rst_status
);
944 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
)) {
945 csk
->err
= abort_status_to_errno(csk
, req
->status
, &rst_status
);
946 cxgbi_sock_closed(csk
);
949 spin_unlock_bh(&csk
->lock
);
955 static void do_abort_rpl_rss(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
957 struct cxgbi_sock
*csk
;
958 struct cpl_abort_rpl_rss
*rpl
= (struct cpl_abort_rpl_rss
*)skb
->data
;
959 unsigned int tid
= GET_TID(rpl
);
960 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
961 struct tid_info
*t
= lldi
->tids
;
963 csk
= lookup_tid(t
, tid
);
968 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
969 (&csk
->saddr
), (&csk
->daddr
), csk
,
970 csk
->state
, csk
->flags
, csk
->tid
, rpl
->status
);
972 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
975 cxgbi_sock_rcv_abort_rpl(csk
);
980 static void do_rx_iscsi_hdr(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
982 struct cxgbi_sock
*csk
;
983 struct cpl_iscsi_hdr
*cpl
= (struct cpl_iscsi_hdr
*)skb
->data
;
984 unsigned short pdu_len_ddp
= be16_to_cpu(cpl
->pdu_len_ddp
);
985 unsigned int tid
= GET_TID(cpl
);
986 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
987 struct tid_info
*t
= lldi
->tids
;
989 csk
= lookup_tid(t
, tid
);
990 if (unlikely(!csk
)) {
991 pr_err("can't find conn. for tid %u.\n", tid
);
995 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
996 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
997 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, skb
->len
,
1000 spin_lock_bh(&csk
->lock
);
1002 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
1003 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1004 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1005 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1006 if (csk
->state
!= CTP_ABORTING
)
1012 cxgbi_skcb_tcp_seq(skb
) = ntohl(cpl
->seq
);
1013 cxgbi_skcb_flags(skb
) = 0;
1015 skb_reset_transport_header(skb
);
1016 __skb_pull(skb
, sizeof(*cpl
));
1017 __pskb_trim(skb
, ntohs(cpl
->len
));
1019 if (!csk
->skb_ulp_lhdr
) {
1021 unsigned int hlen
, dlen
, plen
;
1023 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1024 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
1025 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
);
1026 csk
->skb_ulp_lhdr
= skb
;
1027 cxgbi_skcb_set_flag(skb
, SKCBF_RX_HDR
);
1029 if (cxgbi_skcb_tcp_seq(skb
) != csk
->rcv_nxt
) {
1030 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
1031 csk
->tid
, cxgbi_skcb_tcp_seq(skb
),
1037 hlen
= ntohs(cpl
->len
);
1038 dlen
= ntohl(*(unsigned int *)(bhs
+ 4)) & 0xFFFFFF;
1040 plen
= ISCSI_PDU_LEN(pdu_len_ddp
);
1041 if (is_t4(lldi
->adapter_type
))
1044 if ((hlen
+ dlen
) != plen
) {
1045 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
1046 "mismatch %u != %u + %u, seq 0x%x.\n",
1047 csk
->tid
, plen
, hlen
, dlen
,
1048 cxgbi_skcb_tcp_seq(skb
));
1052 cxgbi_skcb_rx_pdulen(skb
) = (hlen
+ dlen
+ 3) & (~0x3);
1054 cxgbi_skcb_rx_pdulen(skb
) += csk
->dcrc_len
;
1055 csk
->rcv_nxt
+= cxgbi_skcb_rx_pdulen(skb
);
1057 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1058 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
1059 csk
, skb
, *bhs
, hlen
, dlen
,
1060 ntohl(*((unsigned int *)(bhs
+ 16))),
1061 ntohl(*((unsigned int *)(bhs
+ 24))));
1064 struct sk_buff
*lskb
= csk
->skb_ulp_lhdr
;
1066 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DATA
);
1067 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1068 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
1069 csk
, csk
->state
, csk
->flags
, skb
, lskb
);
1072 __skb_queue_tail(&csk
->receive_queue
, skb
);
1073 spin_unlock_bh(&csk
->lock
);
1077 send_abort_req(csk
);
1079 spin_unlock_bh(&csk
->lock
);
1084 static void do_rx_data_ddp(struct cxgbi_device
*cdev
,
1085 struct sk_buff
*skb
)
1087 struct cxgbi_sock
*csk
;
1088 struct sk_buff
*lskb
;
1089 struct cpl_rx_data_ddp
*rpl
= (struct cpl_rx_data_ddp
*)skb
->data
;
1090 unsigned int tid
= GET_TID(rpl
);
1091 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1092 struct tid_info
*t
= lldi
->tids
;
1093 unsigned int status
= ntohl(rpl
->ddpvld
);
1095 csk
= lookup_tid(t
, tid
);
1096 if (unlikely(!csk
)) {
1097 pr_err("can't find connection for tid %u.\n", tid
);
1101 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1102 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
1103 csk
, csk
->state
, csk
->flags
, skb
, status
, csk
->skb_ulp_lhdr
);
1105 spin_lock_bh(&csk
->lock
);
1107 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
1108 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1109 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1110 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1111 if (csk
->state
!= CTP_ABORTING
)
1117 if (!csk
->skb_ulp_lhdr
) {
1118 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk
->tid
);
1122 lskb
= csk
->skb_ulp_lhdr
;
1123 csk
->skb_ulp_lhdr
= NULL
;
1125 cxgbi_skcb_rx_ddigest(lskb
) = ntohl(rpl
->ulp_crc
);
1127 if (ntohs(rpl
->len
) != cxgbi_skcb_rx_pdulen(lskb
))
1128 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
1129 csk
->tid
, ntohs(rpl
->len
), cxgbi_skcb_rx_pdulen(lskb
));
1131 if (status
& (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT
)) {
1132 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
1133 csk
, lskb
, status
, cxgbi_skcb_flags(lskb
));
1134 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_HCRC_ERR
);
1136 if (status
& (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT
)) {
1137 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
1138 csk
, lskb
, status
, cxgbi_skcb_flags(lskb
));
1139 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DCRC_ERR
);
1141 if (status
& (1 << CPL_RX_DDP_STATUS_PAD_SHIFT
)) {
1142 log_debug(1 << CXGBI_DBG_PDU_RX
,
1143 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
1145 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_PAD_ERR
);
1147 if ((status
& (1 << CPL_RX_DDP_STATUS_DDP_SHIFT
)) &&
1148 !cxgbi_skcb_test_flag(lskb
, SKCBF_RX_DATA
)) {
1149 log_debug(1 << CXGBI_DBG_PDU_RX
,
1150 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
1152 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DATA_DDPD
);
1154 log_debug(1 << CXGBI_DBG_PDU_RX
,
1155 "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
1156 csk
, lskb
, cxgbi_skcb_flags(lskb
));
1158 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_STATUS
);
1159 cxgbi_conn_pdu_ready(csk
);
1160 spin_unlock_bh(&csk
->lock
);
1164 send_abort_req(csk
);
1166 spin_unlock_bh(&csk
->lock
);
1171 static void do_fw4_ack(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1173 struct cxgbi_sock
*csk
;
1174 struct cpl_fw4_ack
*rpl
= (struct cpl_fw4_ack
*)skb
->data
;
1175 unsigned int tid
= GET_TID(rpl
);
1176 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1177 struct tid_info
*t
= lldi
->tids
;
1179 csk
= lookup_tid(t
, tid
);
1181 pr_err("can't find connection for tid %u.\n", tid
);
1183 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1184 "csk 0x%p,%u,0x%lx,%u.\n",
1185 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1186 cxgbi_sock_rcv_wr_ack(csk
, rpl
->credits
, ntohl(rpl
->snd_una
),
1192 static void do_set_tcb_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1194 struct cpl_set_tcb_rpl
*rpl
= (struct cpl_set_tcb_rpl
*)skb
->data
;
1195 unsigned int tid
= GET_TID(rpl
);
1196 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1197 struct tid_info
*t
= lldi
->tids
;
1198 struct cxgbi_sock
*csk
;
1200 csk
= lookup_tid(t
, tid
);
1202 pr_err("can't find conn. for tid %u.\n", tid
);
1204 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1205 "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
1206 csk
, csk
->state
, csk
->flags
, csk
->tid
, rpl
->status
);
1208 if (rpl
->status
!= CPL_ERR_NONE
)
1209 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
1210 csk
, tid
, rpl
->status
);
1215 static int alloc_cpls(struct cxgbi_sock
*csk
)
1217 csk
->cpl_close
= alloc_wr(sizeof(struct cpl_close_con_req
),
1219 if (!csk
->cpl_close
)
1222 csk
->cpl_abort_req
= alloc_wr(sizeof(struct cpl_abort_req
),
1224 if (!csk
->cpl_abort_req
)
1227 csk
->cpl_abort_rpl
= alloc_wr(sizeof(struct cpl_abort_rpl
),
1229 if (!csk
->cpl_abort_rpl
)
1234 cxgbi_sock_free_cpl_skbs(csk
);
1238 static inline void l2t_put(struct cxgbi_sock
*csk
)
1241 cxgb4_l2t_release(csk
->l2t
);
1243 cxgbi_sock_put(csk
);
1247 static void release_offload_resources(struct cxgbi_sock
*csk
)
1249 struct cxgb4_lld_info
*lldi
;
1251 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1252 "csk 0x%p,%u,0x%lx,%u.\n",
1253 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1255 cxgbi_sock_free_cpl_skbs(csk
);
1256 if (csk
->wr_cred
!= csk
->wr_max_cred
) {
1257 cxgbi_sock_purge_wr_queue(csk
);
1258 cxgbi_sock_reset_wr_list(csk
);
1262 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
))
1264 else if (cxgbi_sock_flag(csk
, CTPF_HAS_TID
)) {
1265 lldi
= cxgbi_cdev_priv(csk
->cdev
);
1266 cxgb4_remove_tid(lldi
->tids
, 0, csk
->tid
);
1267 cxgbi_sock_clear_flag(csk
, CTPF_HAS_TID
);
1268 cxgbi_sock_put(csk
);
1274 static int init_act_open(struct cxgbi_sock
*csk
)
1276 struct cxgbi_device
*cdev
= csk
->cdev
;
1277 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1278 struct net_device
*ndev
= cdev
->ports
[csk
->port_id
];
1279 struct sk_buff
*skb
= NULL
;
1280 struct neighbour
*n
= NULL
;
1283 unsigned int size
, size6
;
1284 int t4
= is_t4(lldi
->adapter_type
);
1286 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1287 "csk 0x%p,%u,0x%lx,%u.\n",
1288 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1290 if (csk
->csk_family
== AF_INET
)
1291 daddr
= &csk
->daddr
.sin_addr
.s_addr
;
1292 #if IS_ENABLED(CONFIG_IPV6)
1293 else if (csk
->csk_family
== AF_INET6
)
1294 daddr
= &csk
->daddr6
.sin6_addr
;
1297 pr_err("address family 0x%x not supported\n", csk
->csk_family
);
1301 n
= dst_neigh_lookup(csk
->dst
, daddr
);
1304 pr_err("%s, can't get neighbour of csk->dst.\n", ndev
->name
);
1308 csk
->atid
= cxgb4_alloc_atid(lldi
->tids
, csk
);
1309 if (csk
->atid
< 0) {
1310 pr_err("%s, NO atid available.\n", ndev
->name
);
1313 cxgbi_sock_set_flag(csk
, CTPF_HAS_ATID
);
1314 cxgbi_sock_get(csk
);
1316 n
= dst_neigh_lookup(csk
->dst
, &csk
->daddr
.sin_addr
.s_addr
);
1318 pr_err("%s, can't get neighbour of csk->dst.\n", ndev
->name
);
1321 csk
->l2t
= cxgb4_l2t_get(lldi
->l2t
, n
, ndev
, 0);
1323 pr_err("%s, cannot alloc l2t.\n", ndev
->name
);
1326 cxgbi_sock_get(csk
);
1329 size
= sizeof(struct cpl_act_open_req
);
1330 size6
= sizeof(struct cpl_act_open_req6
);
1332 size
= sizeof(struct cpl_t5_act_open_req
);
1333 size6
= sizeof(struct cpl_t5_act_open_req6
);
1336 if (csk
->csk_family
== AF_INET
)
1337 skb
= alloc_wr(size
, 0, GFP_NOIO
);
1339 skb
= alloc_wr(size6
, 0, GFP_NOIO
);
1343 skb
->sk
= (struct sock
*)csk
;
1344 t4_set_arp_err_handler(skb
, csk
, cxgbi_sock_act_open_req_arp_failure
);
1347 csk
->mtu
= dst_mtu(csk
->dst
);
1348 cxgb4_best_mtu(lldi
->mtus
, csk
->mtu
, &csk
->mss_idx
);
1349 csk
->tx_chan
= cxgb4_port_chan(ndev
);
1350 /* SMT two entries per row */
1351 csk
->smac_idx
= ((cxgb4_port_viid(ndev
) & 0x7F)) << 1;
1352 step
= lldi
->ntxq
/ lldi
->nchan
;
1353 csk
->txq_idx
= cxgb4_port_idx(ndev
) * step
;
1354 step
= lldi
->nrxq
/ lldi
->nchan
;
1355 csk
->rss_qid
= lldi
->rxq_ids
[cxgb4_port_idx(ndev
) * step
];
1356 csk
->wr_cred
= lldi
->wr_cred
-
1357 DIV_ROUND_UP(sizeof(struct cpl_abort_req
), 16);
1358 csk
->wr_max_cred
= csk
->wr_cred
;
1359 csk
->wr_una_cred
= 0;
1360 cxgbi_sock_reset_wr_list(csk
);
1363 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n",
1364 (&csk
->saddr
), (&csk
->daddr
), csk
, csk
->state
,
1365 csk
->flags
, csk
->tx_chan
, csk
->txq_idx
, csk
->rss_qid
,
1366 csk
->mtu
, csk
->mss_idx
, csk
->smac_idx
);
1368 /* must wait for either a act_open_rpl or act_open_establish */
1369 try_module_get(THIS_MODULE
);
1370 cxgbi_sock_set_state(csk
, CTP_ACTIVE_OPEN
);
1371 if (csk
->csk_family
== AF_INET
)
1372 send_act_open_req(csk
, skb
, csk
->l2t
);
1374 send_act_open_req6(csk
, skb
, csk
->l2t
);
1387 #define CPL_ISCSI_DATA 0xB2
1388 #define CPL_RX_ISCSI_DDP 0x49
1389 cxgb4i_cplhandler_func cxgb4i_cplhandlers
[NUM_CPL_CMDS
] = {
1390 [CPL_ACT_ESTABLISH
] = do_act_establish
,
1391 [CPL_ACT_OPEN_RPL
] = do_act_open_rpl
,
1392 [CPL_PEER_CLOSE
] = do_peer_close
,
1393 [CPL_ABORT_REQ_RSS
] = do_abort_req_rss
,
1394 [CPL_ABORT_RPL_RSS
] = do_abort_rpl_rss
,
1395 [CPL_CLOSE_CON_RPL
] = do_close_con_rpl
,
1396 [CPL_FW4_ACK
] = do_fw4_ack
,
1397 [CPL_ISCSI_HDR
] = do_rx_iscsi_hdr
,
1398 [CPL_ISCSI_DATA
] = do_rx_iscsi_hdr
,
1399 [CPL_SET_TCB_RPL
] = do_set_tcb_rpl
,
1400 [CPL_RX_DATA_DDP
] = do_rx_data_ddp
,
1401 [CPL_RX_ISCSI_DDP
] = do_rx_data_ddp
,
1404 int cxgb4i_ofld_init(struct cxgbi_device
*cdev
)
1408 if (cxgb4i_max_connect
> CXGB4I_MAX_CONN
)
1409 cxgb4i_max_connect
= CXGB4I_MAX_CONN
;
1411 rc
= cxgbi_device_portmap_create(cdev
, cxgb4i_sport_base
,
1412 cxgb4i_max_connect
);
1416 cdev
->csk_release_offload_resources
= release_offload_resources
;
1417 cdev
->csk_push_tx_frames
= push_tx_frames
;
1418 cdev
->csk_send_abort_req
= send_abort_req
;
1419 cdev
->csk_send_close_req
= send_close_req
;
1420 cdev
->csk_send_rx_credits
= send_rx_credits
;
1421 cdev
->csk_alloc_cpls
= alloc_cpls
;
1422 cdev
->csk_init_act_open
= init_act_open
;
1424 pr_info("cdev 0x%p, offload up, added.\n", cdev
);
1429 * functions to program the pagepod in h/w
1431 #define ULPMEM_IDATA_MAX_NPPODS 4 /* 256/PPOD_SIZE */
1432 static inline void ulp_mem_io_set_hdr(struct cxgb4_lld_info
*lldi
,
1433 struct ulp_mem_io
*req
,
1434 unsigned int wr_len
, unsigned int dlen
,
1435 unsigned int pm_addr
)
1437 struct ulptx_idata
*idata
= (struct ulptx_idata
*)(req
+ 1);
1439 INIT_ULPTX_WR(req
, wr_len
, 0, 0);
1440 if (is_t4(lldi
->adapter_type
))
1441 req
->cmd
= htonl(ULPTX_CMD(ULP_TX_MEM_WRITE
) |
1442 (ULP_MEMIO_ORDER(1)));
1444 req
->cmd
= htonl(ULPTX_CMD(ULP_TX_MEM_WRITE
) |
1445 (V_T5_ULP_MEMIO_IMM(1)));
1446 req
->dlen
= htonl(ULP_MEMIO_DATA_LEN(dlen
>> 5));
1447 req
->lock_addr
= htonl(ULP_MEMIO_ADDR(pm_addr
>> 5));
1448 req
->len16
= htonl(DIV_ROUND_UP(wr_len
- sizeof(req
->wr
), 16));
1450 idata
->cmd_more
= htonl(ULPTX_CMD(ULP_TX_SC_IMM
));
1451 idata
->len
= htonl(dlen
);
1454 static int ddp_ppod_write_idata(struct cxgbi_device
*cdev
, unsigned int port_id
,
1455 struct cxgbi_pagepod_hdr
*hdr
, unsigned int idx
,
1457 struct cxgbi_gather_list
*gl
,
1458 unsigned int gl_pidx
)
1460 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1461 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1462 struct sk_buff
*skb
;
1463 struct ulp_mem_io
*req
;
1464 struct ulptx_idata
*idata
;
1465 struct cxgbi_pagepod
*ppod
;
1466 unsigned int pm_addr
= idx
* PPOD_SIZE
+ ddp
->llimit
;
1467 unsigned int dlen
= PPOD_SIZE
* npods
;
1468 unsigned int wr_len
= roundup(sizeof(struct ulp_mem_io
) +
1469 sizeof(struct ulptx_idata
) + dlen
, 16);
1472 skb
= alloc_wr(wr_len
, 0, GFP_ATOMIC
);
1474 pr_err("cdev 0x%p, idx %u, npods %u, OOM.\n",
1478 req
= (struct ulp_mem_io
*)skb
->head
;
1479 set_queue(skb
, CPL_PRIORITY_CONTROL
, NULL
);
1481 ulp_mem_io_set_hdr(lldi
, req
, wr_len
, dlen
, pm_addr
);
1482 idata
= (struct ulptx_idata
*)(req
+ 1);
1483 ppod
= (struct cxgbi_pagepod
*)(idata
+ 1);
1485 for (i
= 0; i
< npods
; i
++, ppod
++, gl_pidx
+= PPOD_PAGES_MAX
) {
1487 cxgbi_ddp_ppod_clear(ppod
);
1489 cxgbi_ddp_ppod_set(ppod
, hdr
, gl
, gl_pidx
);
1492 cxgb4_ofld_send(cdev
->ports
[port_id
], skb
);
1496 static int ddp_set_map(struct cxgbi_sock
*csk
, struct cxgbi_pagepod_hdr
*hdr
,
1497 unsigned int idx
, unsigned int npods
,
1498 struct cxgbi_gather_list
*gl
)
1500 unsigned int i
, cnt
;
1503 for (i
= 0; i
< npods
; i
+= cnt
, idx
+= cnt
) {
1505 if (cnt
> ULPMEM_IDATA_MAX_NPPODS
)
1506 cnt
= ULPMEM_IDATA_MAX_NPPODS
;
1507 err
= ddp_ppod_write_idata(csk
->cdev
, csk
->port_id
, hdr
,
1508 idx
, cnt
, gl
, 4 * i
);
1515 static void ddp_clear_map(struct cxgbi_hba
*chba
, unsigned int tag
,
1516 unsigned int idx
, unsigned int npods
)
1518 unsigned int i
, cnt
;
1521 for (i
= 0; i
< npods
; i
+= cnt
, idx
+= cnt
) {
1523 if (cnt
> ULPMEM_IDATA_MAX_NPPODS
)
1524 cnt
= ULPMEM_IDATA_MAX_NPPODS
;
1525 err
= ddp_ppod_write_idata(chba
->cdev
, chba
->port_id
, NULL
,
1532 static int ddp_setup_conn_pgidx(struct cxgbi_sock
*csk
, unsigned int tid
,
1533 int pg_idx
, bool reply
)
1535 struct sk_buff
*skb
;
1536 struct cpl_set_tcb_field
*req
;
1538 if (!pg_idx
|| pg_idx
>= DDP_PGIDX_MAX
)
1541 skb
= alloc_wr(sizeof(*req
), 0, GFP_KERNEL
);
1545 /* set up ulp page size */
1546 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1547 INIT_TP_WR(req
, csk
->tid
);
1548 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1549 req
->reply_ctrl
= htons(NO_REPLY(reply
) | QUEUENO(csk
->rss_qid
));
1550 req
->word_cookie
= htons(0);
1551 req
->mask
= cpu_to_be64(0x3 << 8);
1552 req
->val
= cpu_to_be64(pg_idx
<< 8);
1553 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->port_id
);
1555 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1556 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk
, csk
->tid
, pg_idx
);
1558 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
1562 static int ddp_setup_conn_digest(struct cxgbi_sock
*csk
, unsigned int tid
,
1563 int hcrc
, int dcrc
, int reply
)
1565 struct sk_buff
*skb
;
1566 struct cpl_set_tcb_field
*req
;
1571 skb
= alloc_wr(sizeof(*req
), 0, GFP_KERNEL
);
1575 csk
->hcrc_len
= (hcrc
? 4 : 0);
1576 csk
->dcrc_len
= (dcrc
? 4 : 0);
1577 /* set up ulp submode */
1578 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1579 INIT_TP_WR(req
, tid
);
1580 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1581 req
->reply_ctrl
= htons(NO_REPLY(reply
) | QUEUENO(csk
->rss_qid
));
1582 req
->word_cookie
= htons(0);
1583 req
->mask
= cpu_to_be64(0x3 << 4);
1584 req
->val
= cpu_to_be64(((hcrc
? ULP_CRC_HEADER
: 0) |
1585 (dcrc
? ULP_CRC_DATA
: 0)) << 4);
1586 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->port_id
);
1588 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1589 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk
, csk
->tid
, hcrc
, dcrc
);
1591 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
1595 static int cxgb4i_ddp_init(struct cxgbi_device
*cdev
)
1597 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1598 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1599 unsigned int tagmask
, pgsz_factor
[4];
1603 kref_get(&ddp
->refcnt
);
1604 pr_warn("cdev 0x%p, ddp 0x%p already set up.\n",
1609 err
= cxgbi_ddp_init(cdev
, lldi
->vr
->iscsi
.start
,
1610 lldi
->vr
->iscsi
.start
+ lldi
->vr
->iscsi
.size
- 1,
1611 lldi
->iscsi_iolen
, lldi
->iscsi_iolen
);
1617 tagmask
= ddp
->idx_mask
<< PPOD_IDX_SHIFT
;
1618 cxgbi_ddp_page_size_factor(pgsz_factor
);
1619 cxgb4_iscsi_init(lldi
->ports
[0], tagmask
, pgsz_factor
);
1621 cdev
->csk_ddp_setup_digest
= ddp_setup_conn_digest
;
1622 cdev
->csk_ddp_setup_pgidx
= ddp_setup_conn_pgidx
;
1623 cdev
->csk_ddp_set
= ddp_set_map
;
1624 cdev
->csk_ddp_clear
= ddp_clear_map
;
1626 pr_info("cxgb4i 0x%p tag: sw %u, rsvd %u,%u, mask 0x%x.\n",
1627 cdev
, cdev
->tag_format
.sw_bits
, cdev
->tag_format
.rsvd_bits
,
1628 cdev
->tag_format
.rsvd_shift
, cdev
->tag_format
.rsvd_mask
);
1629 pr_info("cxgb4i 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1631 cdev
, ddp
->nppods
, ddp
->idx_bits
, ddp
->idx_mask
,
1632 ddp
->rsvd_tag_mask
, ddp
->max_txsz
, lldi
->iscsi_iolen
,
1633 ddp
->max_rxsz
, lldi
->iscsi_iolen
);
1634 pr_info("cxgb4i 0x%p max payload size: %u/%u, %u/%u.\n",
1635 cdev
, cdev
->tx_max_size
, ddp
->max_txsz
, cdev
->rx_max_size
,
1640 #if IS_ENABLED(CONFIG_IPV6)
1641 static int cxgbi_inet6addr_handler(struct notifier_block
*this,
1642 unsigned long event
, void *data
)
1644 struct inet6_ifaddr
*ifa
= data
;
1645 struct net_device
*event_dev
= ifa
->idev
->dev
;
1646 struct cxgbi_device
*cdev
;
1647 int ret
= NOTIFY_DONE
;
1649 if (event_dev
->priv_flags
& IFF_802_1Q_VLAN
)
1650 event_dev
= vlan_dev_real_dev(event_dev
);
1652 cdev
= cxgbi_device_find_by_netdev(event_dev
, NULL
);
1659 ret
= cxgb4_clip_get(event_dev
,
1660 (const struct in6_addr
*)
1661 ((ifa
)->addr
.s6_addr
));
1669 cxgb4_clip_release(event_dev
,
1670 (const struct in6_addr
*)
1671 ((ifa
)->addr
.s6_addr
));
1682 static struct notifier_block cxgbi_inet6addr_notifier
= {
1683 .notifier_call
= cxgbi_inet6addr_handler
1686 /* Retrieve IPv6 addresses from a root device (bond, vlan) associated with
1687 * a physical device.
1688 * The physical device reference is needed to send the actual CLIP command.
1690 static int update_dev_clip(struct net_device
*root_dev
, struct net_device
*dev
)
1692 struct inet6_dev
*idev
= NULL
;
1693 struct inet6_ifaddr
*ifa
;
1696 idev
= __in6_dev_get(root_dev
);
1700 read_lock_bh(&idev
->lock
);
1701 list_for_each_entry(ifa
, &idev
->addr_list
, if_list
) {
1702 pr_info("updating the clip for addr %pI6\n",
1704 ret
= cxgb4_clip_get(dev
, (const struct in6_addr
*)
1710 read_unlock_bh(&idev
->lock
);
1714 static int update_root_dev_clip(struct net_device
*dev
)
1716 struct net_device
*root_dev
= NULL
;
1719 /* First populate the real net device's IPv6 address */
1720 ret
= update_dev_clip(dev
, dev
);
1724 /* Parse all bond and vlan devices layered on top of the physical dev */
1725 root_dev
= netdev_master_upper_dev_get(dev
);
1727 ret
= update_dev_clip(root_dev
, dev
);
1732 for (i
= 0; i
< VLAN_N_VID
; i
++) {
1733 root_dev
= __vlan_find_dev_deep_rcu(dev
, htons(ETH_P_8021Q
), i
);
1737 ret
= update_dev_clip(root_dev
, dev
);
1744 static void cxgbi_update_clip(struct cxgbi_device
*cdev
)
1750 for (i
= 0; i
< cdev
->nports
; i
++) {
1751 struct net_device
*dev
= cdev
->ports
[i
];
1755 ret
= update_root_dev_clip(dev
);
1761 #endif /* IS_ENABLED(CONFIG_IPV6) */
1763 static void *t4_uld_add(const struct cxgb4_lld_info
*lldi
)
1765 struct cxgbi_device
*cdev
;
1766 struct port_info
*pi
;
1769 cdev
= cxgbi_device_register(sizeof(*lldi
), lldi
->nports
);
1771 pr_info("t4 device 0x%p, register failed.\n", lldi
);
1774 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
1775 cdev
, lldi
->adapter_type
, lldi
->nports
,
1776 lldi
->ports
[0]->name
, lldi
->nchan
, lldi
->ntxq
,
1777 lldi
->nrxq
, lldi
->wr_cred
);
1778 for (i
= 0; i
< lldi
->nrxq
; i
++)
1779 log_debug(1 << CXGBI_DBG_DEV
,
1780 "t4 0x%p, rxq id #%d: %u.\n",
1781 cdev
, i
, lldi
->rxq_ids
[i
]);
1783 memcpy(cxgbi_cdev_priv(cdev
), lldi
, sizeof(*lldi
));
1784 cdev
->flags
= CXGBI_FLAG_DEV_T4
;
1785 cdev
->pdev
= lldi
->pdev
;
1786 cdev
->ports
= lldi
->ports
;
1787 cdev
->nports
= lldi
->nports
;
1788 cdev
->mtus
= lldi
->mtus
;
1789 cdev
->nmtus
= NMTUS
;
1790 cdev
->snd_win
= cxgb4i_snd_win
;
1791 cdev
->rcv_win
= cxgb4i_rcv_win
;
1792 cdev
->rx_credit_thres
= cxgb4i_rx_credit_thres
;
1793 cdev
->skb_tx_rsvd
= CXGB4I_TX_HEADER_LEN
;
1794 cdev
->skb_rx_extra
= sizeof(struct cpl_iscsi_hdr
);
1795 cdev
->itp
= &cxgb4i_iscsi_transport
;
1797 cdev
->pfvf
= FW_VIID_PFN_GET(cxgb4_port_viid(lldi
->ports
[0])) << 8;
1798 pr_info("cdev 0x%p,%s, pfvf %u.\n",
1799 cdev
, lldi
->ports
[0]->name
, cdev
->pfvf
);
1801 rc
= cxgb4i_ddp_init(cdev
);
1803 pr_info("t4 0x%p ddp init failed.\n", cdev
);
1806 rc
= cxgb4i_ofld_init(cdev
);
1808 pr_info("t4 0x%p ofld init failed.\n", cdev
);
1812 rc
= cxgbi_hbas_add(cdev
, CXGB4I_MAX_LUN
, CXGBI_MAX_CONN
,
1813 &cxgb4i_host_template
, cxgb4i_stt
);
1817 for (i
= 0; i
< cdev
->nports
; i
++) {
1818 pi
= netdev_priv(lldi
->ports
[i
]);
1819 cdev
->hbas
[i
]->port_id
= pi
->port_id
;
1824 cxgbi_device_unregister(cdev
);
1825 return ERR_PTR(-ENOMEM
);
1828 #define RX_PULL_LEN 128
1829 static int t4_uld_rx_handler(void *handle
, const __be64
*rsp
,
1830 const struct pkt_gl
*pgl
)
1832 const struct cpl_act_establish
*rpl
;
1833 struct sk_buff
*skb
;
1835 struct cxgbi_device
*cdev
= handle
;
1838 unsigned int len
= 64 - sizeof(struct rsp_ctrl
) - 8;
1840 skb
= alloc_wr(len
, 0, GFP_ATOMIC
);
1843 skb_copy_to_linear_data(skb
, &rsp
[1], len
);
1845 if (unlikely(*(u8
*)rsp
!= *(u8
*)pgl
->va
)) {
1846 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
1847 pgl
->va
, be64_to_cpu(*rsp
),
1848 be64_to_cpu(*(u64
*)pgl
->va
),
1852 skb
= cxgb4_pktgl_to_skb(pgl
, RX_PULL_LEN
, RX_PULL_LEN
);
1857 rpl
= (struct cpl_act_establish
*)skb
->data
;
1858 opc
= rpl
->ot
.opcode
;
1859 log_debug(1 << CXGBI_DBG_TOE
,
1860 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
1861 cdev
, opc
, rpl
->ot
.opcode_tid
, ntohl(rpl
->ot
.opcode_tid
), skb
);
1862 if (cxgb4i_cplhandlers
[opc
])
1863 cxgb4i_cplhandlers
[opc
](cdev
, skb
);
1865 pr_err("No handler for opcode 0x%x.\n", opc
);
1870 log_debug(1 << CXGBI_DBG_TOE
, "OOM bailing out.\n");
1874 static int t4_uld_state_change(void *handle
, enum cxgb4_state state
)
1876 struct cxgbi_device
*cdev
= handle
;
1879 case CXGB4_STATE_UP
:
1880 pr_info("cdev 0x%p, UP.\n", cdev
);
1881 #if IS_ENABLED(CONFIG_IPV6)
1882 cxgbi_update_clip(cdev
);
1886 case CXGB4_STATE_START_RECOVERY
:
1887 pr_info("cdev 0x%p, RECOVERY.\n", cdev
);
1888 /* close all connections */
1890 case CXGB4_STATE_DOWN
:
1891 pr_info("cdev 0x%p, DOWN.\n", cdev
);
1893 case CXGB4_STATE_DETACH
:
1894 pr_info("cdev 0x%p, DETACH.\n", cdev
);
1895 cxgbi_device_unregister(cdev
);
1898 pr_info("cdev 0x%p, unknown state %d.\n", cdev
, state
);
1904 static int __init
cxgb4i_init_module(void)
1908 printk(KERN_INFO
"%s", version
);
1910 rc
= cxgbi_iscsi_init(&cxgb4i_iscsi_transport
, &cxgb4i_stt
);
1913 cxgb4_register_uld(CXGB4_ULD_ISCSI
, &cxgb4i_uld_info
);
1915 #if IS_ENABLED(CONFIG_IPV6)
1916 register_inet6addr_notifier(&cxgbi_inet6addr_notifier
);
1921 static void __exit
cxgb4i_exit_module(void)
1923 #if IS_ENABLED(CONFIG_IPV6)
1924 unregister_inet6addr_notifier(&cxgbi_inet6addr_notifier
);
1926 cxgb4_unregister_uld(CXGB4_ULD_ISCSI
);
1927 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4
);
1928 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport
, &cxgb4i_stt
);
1931 module_init(cxgb4i_init_module
);
1932 module_exit(cxgb4i_exit_module
);