2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
15 #include <linux/if_vlan.h>
16 #include <linux/version.h>
18 #include "cxgb3_defs.h"
19 #include "cxgb3_ctl_defs.h"
20 #include "firmware_exports.h"
21 #include "cxgb3i_offload.h"
22 #include "cxgb3i_pdu.h"
23 #include "cxgb3i_ddp.h"
25 #ifdef __DEBUG_C3CN_CONN__
26 #define c3cn_conn_debug cxgb3i_log_debug
28 #define c3cn_conn_debug(fmt...)
31 #ifdef __DEBUG_C3CN_TX__
32 #define c3cn_tx_debug cxgb3i_log_debug
34 #define c3cn_tx_debug(fmt...)
37 #ifdef __DEBUG_C3CN_RX__
38 #define c3cn_rx_debug cxgb3i_log_debug
40 #define c3cn_rx_debug(fmt...)
44 * module parameters releated to offloaded iscsi connection
46 static int cxgb3_rcv_win
= 256 * 1024;
47 module_param(cxgb3_rcv_win
, int, 0644);
48 MODULE_PARM_DESC(cxgb3_rcv_win
, "TCP receive window in bytes (default=256KB)");
50 static int cxgb3_snd_win
= 128 * 1024;
51 module_param(cxgb3_snd_win
, int, 0644);
52 MODULE_PARM_DESC(cxgb3_snd_win
, "TCP send window in bytes (default=128KB)");
54 static int cxgb3_rx_credit_thres
= 10 * 1024;
55 module_param(cxgb3_rx_credit_thres
, int, 0644);
56 MODULE_PARM_DESC(rx_credit_thres
,
57 "RX credits return threshold in bytes (default=10KB)");
59 static unsigned int cxgb3_max_connect
= 8 * 1024;
60 module_param(cxgb3_max_connect
, uint
, 0644);
61 MODULE_PARM_DESC(cxgb3_max_connect
, "Max. # of connections (default=8092)");
63 static unsigned int cxgb3_sport_base
= 20000;
64 module_param(cxgb3_sport_base
, uint
, 0644);
65 MODULE_PARM_DESC(cxgb3_sport_base
, "starting port number (default=20000)");
68 * cxgb3i tcp connection data(per adapter) list
70 static LIST_HEAD(cdata_list
);
71 static DEFINE_RWLOCK(cdata_rwlock
);
73 static int c3cn_push_tx_frames(struct s3_conn
*c3cn
, int req_completion
);
74 static void c3cn_release_offload_resources(struct s3_conn
*c3cn
);
77 * iscsi source port management
79 * Find a free source port in the port allocation map. We use a very simple
80 * rotor scheme to look for the next free port.
82 * If a source port has been specified make sure that it doesn't collide with
83 * our normal source port allocation map. If it's outside the range of our
84 * allocation/deallocation scheme just let them use it.
86 * If the source port is outside our allocation range, the caller is
87 * responsible for keeping track of their port usage.
89 static int c3cn_get_port(struct s3_conn
*c3cn
, struct cxgb3i_sdev_data
*cdata
)
97 if (c3cn
->saddr
.sin_port
!= 0) {
98 idx
= ntohs(c3cn
->saddr
.sin_port
) - cxgb3_sport_base
;
99 if (idx
< 0 || idx
>= cxgb3_max_connect
)
101 if (!test_and_set_bit(idx
, cdata
->sport_map
))
105 /* the sport_map_next may not be accurate but that is okay, sport_map
107 start
= idx
= cdata
->sport_map_next
;
109 if (++idx
>= cxgb3_max_connect
)
111 if (!(test_and_set_bit(idx
, cdata
->sport_map
))) {
112 c3cn
->saddr
.sin_port
= htons(cxgb3_sport_base
+ idx
);
113 cdata
->sport_map_next
= idx
;
114 c3cn_conn_debug("%s reserve port %u.\n",
116 cxgb3_sport_base
+ idx
);
119 } while (idx
!= start
);
122 return -EADDRNOTAVAIL
;
125 static void c3cn_put_port(struct s3_conn
*c3cn
)
127 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(c3cn
->cdev
);
129 if (c3cn
->saddr
.sin_port
) {
130 int idx
= ntohs(c3cn
->saddr
.sin_port
) - cxgb3_sport_base
;
132 c3cn
->saddr
.sin_port
= 0;
133 if (idx
< 0 || idx
>= cxgb3_max_connect
)
135 clear_bit(idx
, cdata
->sport_map
);
136 c3cn_conn_debug("%s, release port %u.\n",
137 cdata
->cdev
->name
, cxgb3_sport_base
+ idx
);
141 static inline void c3cn_set_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
143 __set_bit(flag
, &c3cn
->flags
);
144 c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
145 c3cn
, flag
, c3cn
->state
, c3cn
->flags
);
148 static inline void c3cn_clear_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
150 __clear_bit(flag
, &c3cn
->flags
);
151 c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
152 c3cn
, flag
, c3cn
->state
, c3cn
->flags
);
155 static inline int c3cn_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
159 return test_bit(flag
, &c3cn
->flags
);
162 static void c3cn_set_state(struct s3_conn
*c3cn
, int state
)
164 c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn
, state
);
168 static inline void c3cn_hold(struct s3_conn
*c3cn
)
170 atomic_inc(&c3cn
->refcnt
);
173 static inline void c3cn_put(struct s3_conn
*c3cn
)
175 if (atomic_dec_and_test(&c3cn
->refcnt
)) {
176 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
177 c3cn
, c3cn
->state
, c3cn
->flags
);
182 static void c3cn_closed(struct s3_conn
*c3cn
)
184 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
185 c3cn
, c3cn
->state
, c3cn
->flags
);
188 c3cn_release_offload_resources(c3cn
);
189 c3cn_set_state(c3cn
, C3CN_STATE_CLOSED
);
190 cxgb3i_conn_closing(c3cn
);
194 * CPL (Chelsio Protocol Language) defines a message passing interface between
195 * the host driver and T3 asic.
196 * The section below implments CPLs that related to iscsi tcp connection
197 * open/close/abort and data send/receive.
201 * CPL connection active open request: host ->
203 static unsigned int find_best_mtu(const struct t3c_data
*d
, unsigned short mtu
)
207 while (i
< d
->nmtus
- 1 && d
->mtus
[i
+ 1] <= mtu
)
212 static unsigned int select_mss(struct s3_conn
*c3cn
, unsigned int pmtu
)
215 struct dst_entry
*dst
= c3cn
->dst_cache
;
216 struct t3cdev
*cdev
= c3cn
->cdev
;
217 const struct t3c_data
*td
= T3C_DATA(cdev
);
218 u16 advmss
= dst_metric(dst
, RTAX_ADVMSS
);
220 if (advmss
> pmtu
- 40)
222 if (advmss
< td
->mtus
[0] - 40)
223 advmss
= td
->mtus
[0] - 40;
224 idx
= find_best_mtu(td
, advmss
+ 40);
228 static inline int compute_wscale(int win
)
231 while (wscale
< 14 && (65535<<wscale
) < win
)
236 static inline unsigned int calc_opt0h(struct s3_conn
*c3cn
)
238 int wscale
= compute_wscale(cxgb3_rcv_win
);
239 return V_KEEP_ALIVE(1) |
241 V_WND_SCALE(wscale
) |
242 V_MSS_IDX(c3cn
->mss_idx
);
245 static inline unsigned int calc_opt0l(struct s3_conn
*c3cn
)
247 return V_ULP_MODE(ULP_MODE_ISCSI
) |
248 V_RCV_BUFSIZ(cxgb3_rcv_win
>>10);
251 static void make_act_open_req(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
252 unsigned int atid
, const struct l2t_entry
*e
)
254 struct cpl_act_open_req
*req
;
256 c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn
, atid
);
258 skb
->priority
= CPL_PRIORITY_SETUP
;
259 req
= (struct cpl_act_open_req
*)__skb_put(skb
, sizeof(*req
));
260 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
261 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
, atid
));
262 req
->local_port
= c3cn
->saddr
.sin_port
;
263 req
->peer_port
= c3cn
->daddr
.sin_port
;
264 req
->local_ip
= c3cn
->saddr
.sin_addr
.s_addr
;
265 req
->peer_ip
= c3cn
->daddr
.sin_addr
.s_addr
;
266 req
->opt0h
= htonl(calc_opt0h(c3cn
) | V_L2T_IDX(e
->idx
) |
267 V_TX_CHANNEL(e
->smt_idx
));
268 req
->opt0l
= htonl(calc_opt0l(c3cn
));
272 static void fail_act_open(struct s3_conn
*c3cn
, int errno
)
274 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
275 c3cn
, c3cn
->state
, c3cn
->flags
);
280 static void act_open_req_arp_failure(struct t3cdev
*dev
, struct sk_buff
*skb
)
282 struct s3_conn
*c3cn
= (struct s3_conn
*)skb
->sk
;
284 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
287 spin_lock_bh(&c3cn
->lock
);
288 if (c3cn
->state
== C3CN_STATE_CONNECTING
)
289 fail_act_open(c3cn
, EHOSTUNREACH
);
290 spin_unlock_bh(&c3cn
->lock
);
296 * CPL connection close request: host ->
298 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
299 * the write queue (i.e., after any unsent txt data).
301 static void skb_entail(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
304 skb_tcp_seq(skb
) = c3cn
->write_seq
;
305 skb_flags(skb
) = flags
;
306 __skb_queue_tail(&c3cn
->write_queue
, skb
);
309 static void send_close_req(struct s3_conn
*c3cn
)
311 struct sk_buff
*skb
= c3cn
->cpl_close
;
312 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
313 unsigned int tid
= c3cn
->tid
;
315 c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
316 c3cn
, c3cn
->state
, c3cn
->flags
);
318 c3cn
->cpl_close
= NULL
;
320 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON
));
321 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
322 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
323 req
->rsvd
= htonl(c3cn
->write_seq
);
325 skb_entail(c3cn
, skb
, C3CB_FLAG_NO_APPEND
);
326 if (c3cn
->state
!= C3CN_STATE_CONNECTING
)
327 c3cn_push_tx_frames(c3cn
, 1);
331 * CPL connection abort request: host ->
333 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
334 * for the same connection and also that we do not try to send a message
335 * after the connection has closed.
337 static void abort_arp_failure(struct t3cdev
*cdev
, struct sk_buff
*skb
)
339 struct cpl_abort_req
*req
= cplhdr(skb
);
341 c3cn_conn_debug("tdev 0x%p.\n", cdev
);
343 req
->cmd
= CPL_ABORT_NO_RST
;
344 cxgb3_ofld_send(cdev
, skb
);
347 static inline void c3cn_purge_write_queue(struct s3_conn
*c3cn
)
351 while ((skb
= __skb_dequeue(&c3cn
->write_queue
)))
355 static void send_abort_req(struct s3_conn
*c3cn
)
357 struct sk_buff
*skb
= c3cn
->cpl_abort_req
;
358 struct cpl_abort_req
*req
;
359 unsigned int tid
= c3cn
->tid
;
361 if (unlikely(c3cn
->state
== C3CN_STATE_ABORTING
) || !skb
||
365 c3cn_set_state(c3cn
, C3CN_STATE_ABORTING
);
367 c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn
);
369 c3cn_set_flag(c3cn
, C3CN_ABORT_RPL_PENDING
);
371 /* Purge the send queue so we don't send anything after an abort. */
372 c3cn_purge_write_queue(c3cn
);
374 c3cn
->cpl_abort_req
= NULL
;
375 req
= (struct cpl_abort_req
*)skb
->head
;
377 skb
->priority
= CPL_PRIORITY_DATA
;
378 set_arp_failure_handler(skb
, abort_arp_failure
);
380 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ
));
381 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
382 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ
, tid
));
383 req
->rsvd0
= htonl(c3cn
->snd_nxt
);
384 req
->rsvd1
= !c3cn_flag(c3cn
, C3CN_TX_DATA_SENT
);
385 req
->cmd
= CPL_ABORT_SEND_RST
;
387 l2t_send(c3cn
->cdev
, skb
, c3cn
->l2t
);
391 * CPL connection abort reply: host ->
393 * Send an ABORT_RPL message in response of the ABORT_REQ received.
395 static void send_abort_rpl(struct s3_conn
*c3cn
, int rst_status
)
397 struct sk_buff
*skb
= c3cn
->cpl_abort_rpl
;
398 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
400 c3cn
->cpl_abort_rpl
= NULL
;
402 skb
->priority
= CPL_PRIORITY_DATA
;
403 rpl
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
404 rpl
->wr
.wr_lo
= htonl(V_WR_TID(c3cn
->tid
));
405 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, c3cn
->tid
));
406 rpl
->cmd
= rst_status
;
408 cxgb3_ofld_send(c3cn
->cdev
, skb
);
412 * CPL connection rx data ack: host ->
413 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
416 static u32
send_rx_credits(struct s3_conn
*c3cn
, u32 credits
, u32 dack
)
419 struct cpl_rx_data_ack
*req
;
421 skb
= alloc_skb(sizeof(*req
), GFP_ATOMIC
);
425 req
= (struct cpl_rx_data_ack
*)__skb_put(skb
, sizeof(*req
));
426 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
427 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK
, c3cn
->tid
));
428 req
->credit_dack
= htonl(dack
| V_RX_CREDITS(credits
));
429 skb
->priority
= CPL_PRIORITY_ACK
;
430 cxgb3_ofld_send(c3cn
->cdev
, skb
);
435 * CPL connection tx data: host ->
437 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
439 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
440 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
444 * For ULP connections HW may inserts digest bytes into the pdu. Those digest
445 * bytes are not sent by the host but are part of the TCP payload and therefore
446 * consume TCP sequence space.
448 static const unsigned int cxgb3_ulp_extra_len
[] = { 0, 4, 4, 8 };
449 static inline unsigned int ulp_extra_len(const struct sk_buff
*skb
)
451 return cxgb3_ulp_extra_len
[skb_ulp_mode(skb
) & 3];
454 static unsigned int wrlen __read_mostly
;
457 * The number of WRs needed for an skb depends on the number of fragments
458 * in the skb and whether it has any payload in its main body. This maps the
459 * length of the gather list represented by an skb into the # of necessary WRs.
460 * The extra two fragments are for iscsi bhs and payload padding.
462 #define SKB_WR_LIST_SIZE (MAX_SKB_FRAGS + 2)
463 static unsigned int skb_wrs
[SKB_WR_LIST_SIZE
] __read_mostly
;
465 static void s3_init_wr_tab(unsigned int wr_len
)
469 if (skb_wrs
[1]) /* already initialized */
472 for (i
= 1; i
< SKB_WR_LIST_SIZE
; i
++) {
473 int sgl_len
= (3 * i
) / 2 + (i
& 1);
476 skb_wrs
[i
] = (sgl_len
<= wr_len
477 ? 1 : 1 + (sgl_len
- 2) / (wr_len
- 1));
483 static inline void reset_wr_list(struct s3_conn
*c3cn
)
485 c3cn
->wr_pending_head
= c3cn
->wr_pending_tail
= NULL
;
489 * Add a WR to a connections's list of pending WRs. This is a singly-linked
490 * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
491 * and the tail in wr_pending_tail.
493 static inline void enqueue_wr(struct s3_conn
*c3cn
,
496 skb_tx_wr_next(skb
) = NULL
;
499 * We want to take an extra reference since both us and the driver
500 * need to free the packet before it's really freed. We know there's
501 * just one user currently so we use atomic_set rather than skb_get
502 * to avoid the atomic op.
504 atomic_set(&skb
->users
, 2);
506 if (!c3cn
->wr_pending_head
)
507 c3cn
->wr_pending_head
= skb
;
509 skb_tx_wr_next(c3cn
->wr_pending_tail
) = skb
;
510 c3cn
->wr_pending_tail
= skb
;
513 static int count_pending_wrs(struct s3_conn
*c3cn
)
516 const struct sk_buff
*skb
= c3cn
->wr_pending_head
;
520 skb
= skb_tx_wr_next(skb
);
525 static inline struct sk_buff
*peek_wr(const struct s3_conn
*c3cn
)
527 return c3cn
->wr_pending_head
;
530 static inline void free_wr_skb(struct sk_buff
*skb
)
535 static inline struct sk_buff
*dequeue_wr(struct s3_conn
*c3cn
)
537 struct sk_buff
*skb
= c3cn
->wr_pending_head
;
540 /* Don't bother clearing the tail */
541 c3cn
->wr_pending_head
= skb_tx_wr_next(skb
);
542 skb_tx_wr_next(skb
) = NULL
;
547 static void purge_wr_queue(struct s3_conn
*c3cn
)
550 while ((skb
= dequeue_wr(c3cn
)) != NULL
)
554 static inline void make_tx_data_wr(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
555 int len
, int req_completion
)
557 struct tx_data_wr
*req
;
559 skb_reset_transport_header(skb
);
560 req
= (struct tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
561 req
->wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA
) |
562 (req_completion
? F_WR_COMPL
: 0));
563 req
->wr_lo
= htonl(V_WR_TID(c3cn
->tid
));
564 req
->sndseq
= htonl(c3cn
->snd_nxt
);
565 /* len includes the length of any HW ULP additions */
566 req
->len
= htonl(len
);
567 req
->param
= htonl(V_TX_PORT(c3cn
->l2t
->smt_idx
));
568 /* V_TX_ULP_SUBMODE sets both the mode and submode */
569 req
->flags
= htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb
)) |
570 V_TX_SHOVE((skb_peek(&c3cn
->write_queue
) ? 0 : 1)));
572 if (!c3cn_flag(c3cn
, C3CN_TX_DATA_SENT
)) {
573 req
->flags
|= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT
|
574 V_TX_CPU_IDX(c3cn
->qset
));
575 /* Sendbuffer is in units of 32KB. */
576 req
->param
|= htonl(V_TX_SNDBUF(cxgb3_snd_win
>> 15));
577 c3cn_set_flag(c3cn
, C3CN_TX_DATA_SENT
);
582 * c3cn_push_tx_frames -- start transmit
583 * @c3cn: the offloaded connection
584 * @req_completion: request wr_ack or not
586 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
587 * connection's send queue and sends them on to T3. Must be called with the
588 * connection's lock held. Returns the amount of send buffer space that was
589 * freed as a result of sending queued data to T3.
591 static void arp_failure_discard(struct t3cdev
*cdev
, struct sk_buff
*skb
)
596 static int c3cn_push_tx_frames(struct s3_conn
*c3cn
, int req_completion
)
601 struct cxgb3i_sdev_data
*cdata
;
603 if (unlikely(c3cn
->state
== C3CN_STATE_CONNECTING
||
604 c3cn
->state
== C3CN_STATE_CLOSE_WAIT_1
||
605 c3cn
->state
>= C3CN_STATE_ABORTING
)) {
606 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
612 cdata
= CXGB3_SDEV_DATA(cdev
);
614 while (c3cn
->wr_avail
615 && (skb
= skb_peek(&c3cn
->write_queue
)) != NULL
) {
616 int len
= skb
->len
; /* length before skb_push */
617 int frags
= skb_shinfo(skb
)->nr_frags
+ (len
!= skb
->data_len
);
618 int wrs_needed
= skb_wrs
[frags
];
620 if (wrs_needed
> 1 && len
+ sizeof(struct tx_data_wr
) <= wrlen
)
623 WARN_ON(frags
>= SKB_WR_LIST_SIZE
|| wrs_needed
< 1);
625 if (c3cn
->wr_avail
< wrs_needed
) {
626 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
628 c3cn
, skb
->len
, skb
->data_len
, frags
,
629 wrs_needed
, c3cn
->wr_avail
);
633 __skb_unlink(skb
, &c3cn
->write_queue
);
634 skb
->priority
= CPL_PRIORITY_DATA
;
635 skb
->csum
= wrs_needed
; /* remember this until the WR_ACK */
636 c3cn
->wr_avail
-= wrs_needed
;
637 c3cn
->wr_unacked
+= wrs_needed
;
638 enqueue_wr(c3cn
, skb
);
640 c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
641 "wr %d, left %u, unack %u.\n",
642 c3cn
, skb
->len
, skb
->data_len
, frags
,
643 wrs_needed
, c3cn
->wr_avail
, c3cn
->wr_unacked
);
646 if (likely(skb_flags(skb
) & C3CB_FLAG_NEED_HDR
)) {
647 if ((req_completion
&&
648 c3cn
->wr_unacked
== wrs_needed
) ||
649 (skb_flags(skb
) & C3CB_FLAG_COMPL
) ||
650 c3cn
->wr_unacked
>= c3cn
->wr_max
/ 2) {
652 c3cn
->wr_unacked
= 0;
654 len
+= ulp_extra_len(skb
);
655 make_tx_data_wr(c3cn
, skb
, len
, req_completion
);
656 c3cn
->snd_nxt
+= len
;
657 skb_flags(skb
) &= ~C3CB_FLAG_NEED_HDR
;
660 total_size
+= skb
->truesize
;
661 set_arp_failure_handler(skb
, arp_failure_discard
);
662 l2t_send(cdev
, skb
, c3cn
->l2t
);
668 * process_cpl_msg: -> host
669 * Top-level CPL message processing used by most CPL messages that
670 * pertain to connections.
672 static inline void process_cpl_msg(void (*fn
)(struct s3_conn
*,
674 struct s3_conn
*c3cn
,
677 spin_lock_bh(&c3cn
->lock
);
679 spin_unlock_bh(&c3cn
->lock
);
683 * process_cpl_msg_ref: -> host
684 * Similar to process_cpl_msg() but takes an extra connection reference around
685 * the call to the handler. Should be used if the handler may drop a
686 * connection reference.
688 static inline void process_cpl_msg_ref(void (*fn
) (struct s3_conn
*,
690 struct s3_conn
*c3cn
,
694 process_cpl_msg(fn
, c3cn
, skb
);
699 * Process a CPL_ACT_ESTABLISH message: -> host
700 * Updates connection state from an active establish CPL message. Runs with
701 * the connection lock held.
704 static inline void s3_free_atid(struct t3cdev
*cdev
, unsigned int tid
)
706 struct s3_conn
*c3cn
= cxgb3_free_atid(cdev
, tid
);
711 static void c3cn_established(struct s3_conn
*c3cn
, u32 snd_isn
,
714 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
716 c3cn
->write_seq
= c3cn
->snd_nxt
= c3cn
->snd_una
= snd_isn
;
719 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
722 if (cxgb3_rcv_win
> (M_RCV_BUFSIZ
<< 10))
723 c3cn
->rcv_wup
-= cxgb3_rcv_win
- (M_RCV_BUFSIZ
<< 10);
725 dst_confirm(c3cn
->dst_cache
);
729 c3cn_set_state(c3cn
, C3CN_STATE_ESTABLISHED
);
732 static void process_act_establish(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
734 struct cpl_act_establish
*req
= cplhdr(skb
);
735 u32 rcv_isn
= ntohl(req
->rcv_isn
); /* real RCV_ISN + 1 */
737 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
738 c3cn
, c3cn
->state
, c3cn
->flags
);
740 if (unlikely(c3cn
->state
!= C3CN_STATE_CONNECTING
))
741 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
742 c3cn
->tid
, c3cn
->state
);
744 c3cn
->copied_seq
= c3cn
->rcv_wup
= c3cn
->rcv_nxt
= rcv_isn
;
745 c3cn_established(c3cn
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
749 if (unlikely(c3cn_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
)))
750 /* upper layer has requested closing */
751 send_abort_req(c3cn
);
753 if (skb_queue_len(&c3cn
->write_queue
))
754 c3cn_push_tx_frames(c3cn
, 1);
755 cxgb3i_conn_tx_open(c3cn
);
759 static int do_act_establish(struct t3cdev
*cdev
, struct sk_buff
*skb
,
762 struct cpl_act_establish
*req
= cplhdr(skb
);
763 unsigned int tid
= GET_TID(req
);
764 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
765 struct s3_conn
*c3cn
= ctx
;
766 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(cdev
);
768 c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
769 tid
, c3cn
, c3cn
->state
, c3cn
->flags
);
773 cxgb3_insert_tid(cdata
->cdev
, cdata
->client
, c3cn
, tid
);
774 s3_free_atid(cdev
, atid
);
776 c3cn
->qset
= G_QNUM(ntohl(skb
->csum
));
778 process_cpl_msg(process_act_establish
, c3cn
, skb
);
783 * Process a CPL_ACT_OPEN_RPL message: -> host
784 * Handle active open failures.
786 static int act_open_rpl_status_to_errno(int status
)
789 case CPL_ERR_CONN_RESET
:
791 case CPL_ERR_ARP_MISS
:
793 case CPL_ERR_CONN_TIMEDOUT
:
795 case CPL_ERR_TCAM_FULL
:
797 case CPL_ERR_CONN_EXIST
:
798 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
805 static void act_open_retry_timer(unsigned long data
)
808 struct s3_conn
*c3cn
= (struct s3_conn
*)data
;
810 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
812 spin_lock_bh(&c3cn
->lock
);
813 skb
= alloc_skb(sizeof(struct cpl_act_open_req
), GFP_ATOMIC
);
815 fail_act_open(c3cn
, ENOMEM
);
817 skb
->sk
= (struct sock
*)c3cn
;
818 set_arp_failure_handler(skb
, act_open_req_arp_failure
);
819 make_act_open_req(c3cn
, skb
, c3cn
->tid
, c3cn
->l2t
);
820 l2t_send(c3cn
->cdev
, skb
, c3cn
->l2t
);
822 spin_unlock_bh(&c3cn
->lock
);
826 static void process_act_open_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
828 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
830 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
831 c3cn
, c3cn
->state
, c3cn
->flags
);
833 if (rpl
->status
== CPL_ERR_CONN_EXIST
&&
834 c3cn
->retry_timer
.function
!= act_open_retry_timer
) {
835 c3cn
->retry_timer
.function
= act_open_retry_timer
;
836 if (!mod_timer(&c3cn
->retry_timer
, jiffies
+ HZ
/ 2))
839 fail_act_open(c3cn
, act_open_rpl_status_to_errno(rpl
->status
));
843 static int do_act_open_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
845 struct s3_conn
*c3cn
= ctx
;
846 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
848 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
849 rpl
->status
, c3cn
, c3cn
->state
, c3cn
->flags
);
851 if (rpl
->status
!= CPL_ERR_TCAM_FULL
&&
852 rpl
->status
!= CPL_ERR_CONN_EXIST
&&
853 rpl
->status
!= CPL_ERR_ARP_MISS
)
854 cxgb3_queue_tid_release(cdev
, GET_TID(rpl
));
856 process_cpl_msg_ref(process_act_open_rpl
, c3cn
, skb
);
861 * Process PEER_CLOSE CPL messages: -> host
864 static void process_peer_close(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
866 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
867 c3cn
, c3cn
->state
, c3cn
->flags
);
869 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
))
872 switch (c3cn
->state
) {
873 case C3CN_STATE_ESTABLISHED
:
874 c3cn_set_state(c3cn
, C3CN_STATE_PASSIVE_CLOSE
);
876 case C3CN_STATE_ACTIVE_CLOSE
:
877 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_2
);
879 case C3CN_STATE_CLOSE_WAIT_1
:
882 case C3CN_STATE_ABORTING
:
885 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
886 c3cn
->cdev
->name
, c3cn
->tid
, c3cn
->state
);
889 cxgb3i_conn_closing(c3cn
);
894 static int do_peer_close(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
896 struct s3_conn
*c3cn
= ctx
;
898 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
899 c3cn
, c3cn
->state
, c3cn
->flags
);
900 process_cpl_msg_ref(process_peer_close
, c3cn
, skb
);
905 * Process CLOSE_CONN_RPL CPL message: -> host
906 * Process a peer ACK to our FIN.
908 static void process_close_con_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
910 struct cpl_close_con_rpl
*rpl
= cplhdr(skb
);
912 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
913 c3cn
, c3cn
->state
, c3cn
->flags
);
915 c3cn
->snd_una
= ntohl(rpl
->snd_nxt
) - 1; /* exclude FIN */
917 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
))
920 switch (c3cn
->state
) {
921 case C3CN_STATE_ACTIVE_CLOSE
:
922 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_1
);
924 case C3CN_STATE_CLOSE_WAIT_1
:
925 case C3CN_STATE_CLOSE_WAIT_2
:
928 case C3CN_STATE_ABORTING
:
931 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
932 c3cn
->cdev
->name
, c3cn
->tid
, c3cn
->state
);
939 static int do_close_con_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
,
942 struct s3_conn
*c3cn
= ctx
;
944 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
945 c3cn
, c3cn
->state
, c3cn
->flags
);
947 process_cpl_msg_ref(process_close_con_rpl
, c3cn
, skb
);
952 * Process ABORT_REQ_RSS CPL message: -> host
953 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
954 * request except that we need to reply to it.
957 static int abort_status_to_errno(struct s3_conn
*c3cn
, int abort_reason
,
960 switch (abort_reason
) {
961 case CPL_ERR_BAD_SYN
: /* fall through */
962 case CPL_ERR_CONN_RESET
:
963 return c3cn
->state
> C3CN_STATE_ESTABLISHED
?
965 case CPL_ERR_XMIT_TIMEDOUT
:
966 case CPL_ERR_PERSIST_TIMEDOUT
:
967 case CPL_ERR_FINWAIT2_TIMEDOUT
:
968 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
975 static void process_abort_req(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
977 int rst_status
= CPL_ABORT_NO_RST
;
978 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
980 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
981 c3cn
, c3cn
->state
, c3cn
->flags
);
983 if (!c3cn_flag(c3cn
, C3CN_ABORT_REQ_RCVD
)) {
984 c3cn_set_flag(c3cn
, C3CN_ABORT_REQ_RCVD
);
985 c3cn_set_state(c3cn
, C3CN_STATE_ABORTING
);
990 c3cn_clear_flag(c3cn
, C3CN_ABORT_REQ_RCVD
);
991 send_abort_rpl(c3cn
, rst_status
);
993 if (!c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
)) {
995 abort_status_to_errno(c3cn
, req
->status
, &rst_status
);
1000 static int do_abort_req(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1002 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
1003 struct s3_conn
*c3cn
= ctx
;
1005 c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
1006 c3cn
, c3cn
->state
, c3cn
->flags
);
1008 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
1009 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
) {
1014 process_cpl_msg_ref(process_abort_req
, c3cn
, skb
);
1019 * Process ABORT_RPL_RSS CPL message: -> host
1020 * Process abort replies. We only process these messages if we anticipate
1021 * them as the coordination between SW and HW in this area is somewhat lacking
1022 * and sometimes we get ABORT_RPLs after we are done with the connection that
1023 * originated the ABORT_REQ.
1025 static void process_abort_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1027 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1028 c3cn
, c3cn
->state
, c3cn
->flags
);
1030 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
)) {
1031 if (!c3cn_flag(c3cn
, C3CN_ABORT_RPL_RCVD
))
1032 c3cn_set_flag(c3cn
, C3CN_ABORT_RPL_RCVD
);
1034 c3cn_clear_flag(c3cn
, C3CN_ABORT_RPL_RCVD
);
1035 c3cn_clear_flag(c3cn
, C3CN_ABORT_RPL_PENDING
);
1036 if (c3cn_flag(c3cn
, C3CN_ABORT_REQ_RCVD
))
1037 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1038 c3cn
->cdev
->name
, c3cn
->tid
);
1045 static int do_abort_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1047 struct cpl_abort_rpl_rss
*rpl
= cplhdr(skb
);
1048 struct s3_conn
*c3cn
= ctx
;
1050 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1051 rpl
->status
, c3cn
, c3cn
? c3cn
->state
: 0,
1052 c3cn
? c3cn
->flags
: 0UL);
1055 * Ignore replies to post-close aborts indicating that the abort was
1056 * requested too late. These connections are terminated when we get
1057 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1058 * arrives the TID is either no longer used or it has been recycled.
1060 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
1064 * Sometimes we've already closed the connection, e.g., a post-close
1065 * abort races with ABORT_REQ_RSS, the latter frees the connection
1066 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1067 * but FW turns the ABORT_REQ into a regular one and so we get
1068 * ABORT_RPL_RSS with status 0 and no connection.
1073 process_cpl_msg_ref(process_abort_rpl
, c3cn
, skb
);
1082 * Process RX_ISCSI_HDR CPL message: -> host
1083 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1084 * follow after the bhs.
1086 static void process_rx_iscsi_hdr(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1088 struct cpl_iscsi_hdr
*hdr_cpl
= cplhdr(skb
);
1089 struct cpl_iscsi_hdr_norss data_cpl
;
1090 struct cpl_rx_data_ddp_norss ddp_cpl
;
1091 unsigned int hdr_len
, data_len
, status
;
1095 if (unlikely(c3cn
->state
>= C3CN_STATE_PASSIVE_CLOSE
)) {
1096 if (c3cn
->state
!= C3CN_STATE_ABORTING
)
1097 send_abort_req(c3cn
);
1102 skb_tcp_seq(skb
) = ntohl(hdr_cpl
->seq
);
1105 skb_reset_transport_header(skb
);
1106 __skb_pull(skb
, sizeof(struct cpl_iscsi_hdr
));
1108 len
= hdr_len
= ntohs(hdr_cpl
->len
);
1109 /* msg coalesce is off or not enough data received */
1110 if (skb
->len
<= hdr_len
) {
1111 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1112 c3cn
->cdev
->name
, c3cn
->tid
,
1117 err
= skb_copy_bits(skb
, skb
->len
- sizeof(ddp_cpl
), &ddp_cpl
,
1122 skb_ulp_mode(skb
) = ULP2_FLAG_DATA_READY
;
1123 skb_rx_pdulen(skb
) = ntohs(ddp_cpl
.len
);
1124 skb_rx_ddigest(skb
) = ntohl(ddp_cpl
.ulp_crc
);
1125 status
= ntohl(ddp_cpl
.ddp_status
);
1127 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1128 skb
, skb
->len
, skb_rx_pdulen(skb
), status
);
1130 if (status
& (1 << RX_DDP_STATUS_HCRC_SHIFT
))
1131 skb_ulp_mode(skb
) |= ULP2_FLAG_HCRC_ERROR
;
1132 if (status
& (1 << RX_DDP_STATUS_DCRC_SHIFT
))
1133 skb_ulp_mode(skb
) |= ULP2_FLAG_DCRC_ERROR
;
1134 if (status
& (1 << RX_DDP_STATUS_PAD_SHIFT
))
1135 skb_ulp_mode(skb
) |= ULP2_FLAG_PAD_ERROR
;
1137 if (skb
->len
> (hdr_len
+ sizeof(ddp_cpl
))) {
1138 err
= skb_copy_bits(skb
, hdr_len
, &data_cpl
, sizeof(data_cpl
));
1141 data_len
= ntohs(data_cpl
.len
);
1142 len
+= sizeof(data_cpl
) + data_len
;
1143 } else if (status
& (1 << RX_DDP_STATUS_DDP_SHIFT
))
1144 skb_ulp_mode(skb
) |= ULP2_FLAG_DATA_DDPED
;
1146 c3cn
->rcv_nxt
= ntohl(ddp_cpl
.seq
) + skb_rx_pdulen(skb
);
1147 __pskb_trim(skb
, len
);
1148 __skb_queue_tail(&c3cn
->receive_queue
, skb
);
1149 cxgb3i_conn_pdu_ready(c3cn
);
1154 send_abort_req(c3cn
);
1158 static int do_iscsi_hdr(struct t3cdev
*t3dev
, struct sk_buff
*skb
, void *ctx
)
1160 struct s3_conn
*c3cn
= ctx
;
1162 process_cpl_msg(process_rx_iscsi_hdr
, c3cn
, skb
);
1167 * Process TX_DATA_ACK CPL messages: -> host
1168 * Process an acknowledgment of WR completion. Advance snd_una and send the
1169 * next batch of work requests from the write queue.
1171 static void check_wr_invariants(struct s3_conn
*c3cn
)
1173 int pending
= count_pending_wrs(c3cn
);
1175 if (unlikely(c3cn
->wr_avail
+ pending
!= c3cn
->wr_max
))
1176 cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
1177 "pending %u, total should be %u\n",
1178 c3cn
->tid
, c3cn
->wr_avail
, pending
,
1182 static void process_wr_ack(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1184 struct cpl_wr_ack
*hdr
= cplhdr(skb
);
1185 unsigned int credits
= ntohs(hdr
->credits
);
1186 u32 snd_una
= ntohl(hdr
->snd_una
);
1188 c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
1189 credits
, c3cn
->wr_avail
, c3cn
->wr_unacked
,
1190 c3cn
->tid
, c3cn
->state
);
1192 c3cn
->wr_avail
+= credits
;
1193 if (c3cn
->wr_unacked
> c3cn
->wr_max
- c3cn
->wr_avail
)
1194 c3cn
->wr_unacked
= c3cn
->wr_max
- c3cn
->wr_avail
;
1197 struct sk_buff
*p
= peek_wr(c3cn
);
1200 cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1201 "nothing pending, state %u\n",
1202 credits
, c3cn
->tid
, c3cn
->state
);
1205 if (unlikely(credits
< p
->csum
)) {
1206 struct tx_data_wr
*w
= cplhdr(p
);
1207 cxgb3i_log_error("TID %u got %u WR credits need %u, "
1208 "len %u, main body %u, frags %u, "
1209 "seq # %u, ACK una %u, ACK nxt %u, "
1210 "WR_AVAIL %u, WRs pending %u\n",
1211 c3cn
->tid
, credits
, p
->csum
, p
->len
,
1212 p
->len
- p
->data_len
,
1213 skb_shinfo(p
)->nr_frags
,
1214 ntohl(w
->sndseq
), snd_una
,
1215 ntohl(hdr
->snd_nxt
), c3cn
->wr_avail
,
1216 count_pending_wrs(c3cn
) - credits
);
1226 check_wr_invariants(c3cn
);
1228 if (unlikely(before(snd_una
, c3cn
->snd_una
))) {
1229 cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
1231 c3cn
->tid
, snd_una
, c3cn
->snd_una
);
1235 if (c3cn
->snd_una
!= snd_una
) {
1236 c3cn
->snd_una
= snd_una
;
1237 dst_confirm(c3cn
->dst_cache
);
1240 if (skb_queue_len(&c3cn
->write_queue
)) {
1241 if (c3cn_push_tx_frames(c3cn
, 0))
1242 cxgb3i_conn_tx_open(c3cn
);
1244 cxgb3i_conn_tx_open(c3cn
);
1249 static int do_wr_ack(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1251 struct s3_conn
*c3cn
= ctx
;
1253 process_cpl_msg(process_wr_ack
, c3cn
, skb
);
1258 * for each connection, pre-allocate skbs needed for close/abort requests. So
1259 * that we can service the request right away.
1261 static void c3cn_free_cpl_skbs(struct s3_conn
*c3cn
)
1263 if (c3cn
->cpl_close
)
1264 kfree_skb(c3cn
->cpl_close
);
1265 if (c3cn
->cpl_abort_req
)
1266 kfree_skb(c3cn
->cpl_abort_req
);
1267 if (c3cn
->cpl_abort_rpl
)
1268 kfree_skb(c3cn
->cpl_abort_rpl
);
1271 static int c3cn_alloc_cpl_skbs(struct s3_conn
*c3cn
)
1273 c3cn
->cpl_close
= alloc_skb(sizeof(struct cpl_close_con_req
),
1275 if (!c3cn
->cpl_close
)
1277 skb_put(c3cn
->cpl_close
, sizeof(struct cpl_close_con_req
));
1279 c3cn
->cpl_abort_req
= alloc_skb(sizeof(struct cpl_abort_req
),
1281 if (!c3cn
->cpl_abort_req
)
1283 skb_put(c3cn
->cpl_abort_req
, sizeof(struct cpl_abort_req
));
1285 c3cn
->cpl_abort_rpl
= alloc_skb(sizeof(struct cpl_abort_rpl
),
1287 if (!c3cn
->cpl_abort_rpl
)
1289 skb_put(c3cn
->cpl_abort_rpl
, sizeof(struct cpl_abort_rpl
));
1294 c3cn_free_cpl_skbs(c3cn
);
1299 * c3cn_release_offload_resources - release offload resource
1300 * @c3cn: the offloaded iscsi tcp connection.
1301 * Release resources held by an offload connection (TID, L2T entry, etc.)
1303 static void c3cn_release_offload_resources(struct s3_conn
*c3cn
)
1305 struct t3cdev
*cdev
= c3cn
->cdev
;
1306 unsigned int tid
= c3cn
->tid
;
1313 c3cn_free_cpl_skbs(c3cn
);
1315 if (c3cn
->wr_avail
!= c3cn
->wr_max
) {
1316 purge_wr_queue(c3cn
);
1317 reset_wr_list(c3cn
);
1321 l2t_release(L2DATA(cdev
), c3cn
->l2t
);
1325 if (c3cn
->state
== C3CN_STATE_CONNECTING
) /* we have ATID */
1326 s3_free_atid(cdev
, tid
);
1327 else { /* we have TID */
1328 cxgb3_remove_tid(cdev
, (void *)c3cn
, tid
);
1336 * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1337 * returns the s3_conn structure allocated.
1339 struct s3_conn
*cxgb3i_c3cn_create(void)
1341 struct s3_conn
*c3cn
;
1343 c3cn
= kzalloc(sizeof(*c3cn
), GFP_KERNEL
);
1347 /* pre-allocate close/abort cpl, so we don't need to wait for memory
1348 when close/abort is requested. */
1349 if (c3cn_alloc_cpl_skbs(c3cn
) < 0)
1352 c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn
);
1355 spin_lock_init(&c3cn
->lock
);
1356 atomic_set(&c3cn
->refcnt
, 1);
1357 skb_queue_head_init(&c3cn
->receive_queue
);
1358 skb_queue_head_init(&c3cn
->write_queue
);
1359 setup_timer(&c3cn
->retry_timer
, NULL
, (unsigned long)c3cn
);
1360 rwlock_init(&c3cn
->callback_lock
);
1369 static void c3cn_active_close(struct s3_conn
*c3cn
)
1374 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1375 c3cn
, c3cn
->state
, c3cn
->flags
);
1377 dst_confirm(c3cn
->dst_cache
);
1380 spin_lock_bh(&c3cn
->lock
);
1382 data_lost
= skb_queue_len(&c3cn
->receive_queue
);
1383 __skb_queue_purge(&c3cn
->receive_queue
);
1385 switch (c3cn
->state
) {
1386 case C3CN_STATE_CLOSED
:
1387 case C3CN_STATE_ACTIVE_CLOSE
:
1388 case C3CN_STATE_CLOSE_WAIT_1
:
1389 case C3CN_STATE_CLOSE_WAIT_2
:
1390 case C3CN_STATE_ABORTING
:
1391 /* nothing need to be done */
1393 case C3CN_STATE_CONNECTING
:
1394 /* defer until cpl_act_open_rpl or cpl_act_establish */
1395 c3cn_set_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
);
1397 case C3CN_STATE_ESTABLISHED
:
1399 c3cn_set_state(c3cn
, C3CN_STATE_ACTIVE_CLOSE
);
1401 case C3CN_STATE_PASSIVE_CLOSE
:
1403 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_2
);
1409 /* Unread data was tossed, zap the connection. */
1410 send_abort_req(c3cn
);
1412 send_close_req(c3cn
);
1415 spin_unlock_bh(&c3cn
->lock
);
1420 * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1422 * @c3cn: the iscsi tcp connection
1424 void cxgb3i_c3cn_release(struct s3_conn
*c3cn
)
1426 c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1427 c3cn
, c3cn
->state
, c3cn
->flags
);
1428 if (likely(c3cn
->state
!= C3CN_STATE_CONNECTING
))
1429 c3cn_active_close(c3cn
);
1431 c3cn_set_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
);
1435 static int is_cxgb3_dev(struct net_device
*dev
)
1437 struct cxgb3i_sdev_data
*cdata
;
1439 write_lock(&cdata_rwlock
);
1440 list_for_each_entry(cdata
, &cdata_list
, list
) {
1441 struct adap_ports
*ports
= &cdata
->ports
;
1444 for (i
= 0; i
< ports
->nports
; i
++)
1445 if (dev
== ports
->lldevs
[i
]) {
1446 write_unlock(&cdata_rwlock
);
1450 write_unlock(&cdata_rwlock
);
1455 * cxgb3_egress_dev - return the cxgb3 egress device
1456 * @root_dev: the root device anchoring the search
1457 * @c3cn: the connection used to determine egress port in bonding mode
1458 * @context: in bonding mode, indicates a connection set up or failover
1460 * Return egress device or NULL if the egress device isn't one of our ports.
1462 static struct net_device
*cxgb3_egress_dev(struct net_device
*root_dev
,
1463 struct s3_conn
*c3cn
,
1467 if (root_dev
->priv_flags
& IFF_802_1Q_VLAN
)
1468 root_dev
= vlan_dev_real_dev(root_dev
);
1469 else if (is_cxgb3_dev(root_dev
))
1477 static struct rtable
*find_route(__be32 saddr
, __be32 daddr
,
1478 __be16 sport
, __be16 dport
)
1488 .proto
= IPPROTO_TCP
,
1492 .dport
= dport
} } };
1494 if (ip_route_output_flow(&init_net
, &rt
, &fl
, NULL
, 0))
1500 * Assign offload parameters to some connection fields.
1502 static void init_offload_conn(struct s3_conn
*c3cn
,
1503 struct t3cdev
*cdev
,
1504 struct dst_entry
*dst
)
1506 BUG_ON(c3cn
->cdev
!= cdev
);
1507 c3cn
->wr_max
= c3cn
->wr_avail
= T3C_DATA(cdev
)->max_wrs
- 1;
1508 c3cn
->wr_unacked
= 0;
1509 c3cn
->mss_idx
= select_mss(c3cn
, dst_mtu(dst
));
1511 reset_wr_list(c3cn
);
1514 static int initiate_act_open(struct s3_conn
*c3cn
, struct net_device
*dev
)
1516 struct cxgb3i_sdev_data
*cdata
= NDEV2CDATA(dev
);
1517 struct t3cdev
*cdev
= cdata
->cdev
;
1518 struct dst_entry
*dst
= c3cn
->dst_cache
;
1519 struct sk_buff
*skb
;
1521 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1522 c3cn
, c3cn
->state
, c3cn
->flags
);
1524 * Initialize connection data. Note that the flags and ULP mode are
1525 * initialized higher up ...
1529 c3cn
->tid
= cxgb3_alloc_atid(cdev
, cdata
->client
, c3cn
);
1534 c3cn
->l2t
= t3_l2t_get(cdev
, dst
->neighbour
, dev
);
1538 skb
= alloc_skb(sizeof(struct cpl_act_open_req
), GFP_KERNEL
);
1542 skb
->sk
= (struct sock
*)c3cn
;
1543 set_arp_failure_handler(skb
, act_open_req_arp_failure
);
1547 init_offload_conn(c3cn
, cdev
, dst
);
1550 make_act_open_req(c3cn
, skb
, c3cn
->tid
, c3cn
->l2t
);
1551 l2t_send(cdev
, skb
, c3cn
->l2t
);
1555 l2t_release(L2DATA(cdev
), c3cn
->l2t
);
1557 s3_free_atid(cdev
, c3cn
->tid
);
1565 * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1566 * @c3cn: the iscsi tcp connection
1567 * @usin: destination address
1569 * return 0 if active open request is sent, < 0 otherwise.
1571 int cxgb3i_c3cn_connect(struct s3_conn
*c3cn
, struct sockaddr_in
*usin
)
1574 struct net_device
*dev
;
1575 struct cxgb3i_sdev_data
*cdata
;
1576 struct t3cdev
*cdev
;
1580 if (usin
->sin_family
!= AF_INET
)
1581 return -EAFNOSUPPORT
;
1583 c3cn
->daddr
.sin_port
= usin
->sin_port
;
1584 c3cn
->daddr
.sin_addr
.s_addr
= usin
->sin_addr
.s_addr
;
1586 rt
= find_route(c3cn
->saddr
.sin_addr
.s_addr
,
1587 c3cn
->daddr
.sin_addr
.s_addr
,
1588 c3cn
->saddr
.sin_port
,
1589 c3cn
->daddr
.sin_port
);
1591 c3cn_conn_debug("NO route to 0x%x, port %u.\n",
1592 c3cn
->daddr
.sin_addr
.s_addr
,
1593 ntohs(c3cn
->daddr
.sin_port
));
1594 return -ENETUNREACH
;
1597 if (rt
->rt_flags
& (RTCF_MULTICAST
| RTCF_BROADCAST
)) {
1598 c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
1599 c3cn
->daddr
.sin_addr
.s_addr
,
1600 ntohs(c3cn
->daddr
.sin_port
));
1602 return -ENETUNREACH
;
1605 if (!c3cn
->saddr
.sin_addr
.s_addr
)
1606 c3cn
->saddr
.sin_addr
.s_addr
= rt
->rt_src
;
1608 /* now commit destination to connection */
1609 c3cn
->dst_cache
= &rt
->u
.dst
;
1611 /* try to establish an offloaded connection */
1612 dev
= cxgb3_egress_dev(c3cn
->dst_cache
->dev
, c3cn
, 0);
1614 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn
);
1615 return -ENETUNREACH
;
1617 cdata
= NDEV2CDATA(dev
);
1620 /* get a source port if one hasn't been provided */
1621 err
= c3cn_get_port(c3cn
, cdata
);
1625 c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1626 c3cn
, ntohs(c3cn
->saddr
.sin_port
));
1628 sipv4
= cxgb3i_get_private_ipv4addr(dev
);
1630 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn
);
1631 sipv4
= c3cn
->saddr
.sin_addr
.s_addr
;
1632 cxgb3i_set_private_ipv4addr(dev
, sipv4
);
1634 c3cn
->saddr
.sin_addr
.s_addr
= sipv4
;
1636 c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
1637 c3cn
, NIPQUAD(c3cn
->saddr
.sin_addr
.s_addr
),
1638 ntohs(c3cn
->saddr
.sin_port
),
1639 NIPQUAD(c3cn
->daddr
.sin_addr
.s_addr
),
1640 ntohs(c3cn
->daddr
.sin_port
));
1642 c3cn_set_state(c3cn
, C3CN_STATE_CONNECTING
);
1643 if (!initiate_act_open(c3cn
, dev
))
1647 * If we get here, we don't have an offload connection so simply
1653 * This trashes the connection and releases the local port,
1656 c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn
);
1657 c3cn_set_state(c3cn
, C3CN_STATE_CLOSED
);
1659 c3cn_put_port(c3cn
);
1660 c3cn
->daddr
.sin_port
= 0;
1665 * cxgb3i_c3cn_rx_credits - ack received tcp data.
1666 * @c3cn: iscsi tcp connection
1667 * @copied: # of bytes processed
1669 * Called after some received data has been read. It returns RX credits
1670 * to the HW for the amount of data processed.
1672 void cxgb3i_c3cn_rx_credits(struct s3_conn
*c3cn
, int copied
)
1674 struct t3cdev
*cdev
;
1676 u32 credits
, dack
= 0;
1678 if (c3cn
->state
!= C3CN_STATE_ESTABLISHED
)
1681 credits
= c3cn
->copied_seq
- c3cn
->rcv_wup
;
1682 if (unlikely(!credits
))
1687 if (unlikely(cxgb3_rx_credit_thres
== 0))
1690 dack
= F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1);
1693 * For coalescing to work effectively ensure the receive window has
1694 * at least 16KB left.
1696 must_send
= credits
+ 16384 >= cxgb3_rcv_win
;
1698 if (must_send
|| credits
>= cxgb3_rx_credit_thres
)
1699 c3cn
->rcv_wup
+= send_rx_credits(c3cn
, credits
, dack
);
1703 * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1704 * @c3cn: iscsi tcp connection
1705 * @skb: skb contains the iscsi pdu
1707 * Add a list of skbs to a connection send queue. The skbs must comply with
1708 * the max size limit of the device and have a headroom of at least
1709 * TX_HEADER_LEN bytes.
1710 * Return # of bytes queued.
1712 int cxgb3i_c3cn_send_pdus(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1714 struct sk_buff
*next
;
1715 int err
, copied
= 0;
1717 spin_lock_bh(&c3cn
->lock
);
1719 if (c3cn
->state
!= C3CN_STATE_ESTABLISHED
) {
1720 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1727 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn
, c3cn
->err
);
1732 if (c3cn
->write_seq
- c3cn
->snd_una
>= cxgb3_snd_win
) {
1733 c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
1734 c3cn
, c3cn
->write_seq
, c3cn
->snd_una
,
1741 int frags
= skb_shinfo(skb
)->nr_frags
+
1742 (skb
->len
!= skb
->data_len
);
1744 if (unlikely(skb_headroom(skb
) < TX_HEADER_LEN
)) {
1745 c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn
);
1750 if (frags
>= SKB_WR_LIST_SIZE
) {
1751 cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1752 c3cn
, skb_shinfo(skb
)->nr_frags
,
1753 skb
->len
, skb
->data_len
);
1760 skb_entail(c3cn
, skb
, C3CB_FLAG_NO_APPEND
| C3CB_FLAG_NEED_HDR
);
1762 c3cn
->write_seq
+= skb
->len
+ ulp_extra_len(skb
);
1766 if (likely(skb_queue_len(&c3cn
->write_queue
)))
1767 c3cn_push_tx_frames(c3cn
, 1);
1768 spin_unlock_bh(&c3cn
->lock
);
1772 if (copied
== 0 && err
== -EPIPE
)
1773 copied
= c3cn
->err
? c3cn
->err
: -EPIPE
;
1777 static void sdev_data_cleanup(struct cxgb3i_sdev_data
*cdata
)
1779 struct adap_ports
*ports
= &cdata
->ports
;
1782 for (i
= 0; i
< ports
->nports
; i
++)
1783 NDEV2CDATA(ports
->lldevs
[i
]) = NULL
;
1784 cxgb3i_free_big_mem(cdata
);
1787 void cxgb3i_sdev_cleanup(void)
1789 struct cxgb3i_sdev_data
*cdata
;
1791 write_lock(&cdata_rwlock
);
1792 list_for_each_entry(cdata
, &cdata_list
, list
) {
1793 list_del(&cdata
->list
);
1794 sdev_data_cleanup(cdata
);
1796 write_unlock(&cdata_rwlock
);
1799 int cxgb3i_sdev_init(cxgb3_cpl_handler_func
*cpl_handlers
)
1801 cpl_handlers
[CPL_ACT_ESTABLISH
] = do_act_establish
;
1802 cpl_handlers
[CPL_ACT_OPEN_RPL
] = do_act_open_rpl
;
1803 cpl_handlers
[CPL_PEER_CLOSE
] = do_peer_close
;
1804 cpl_handlers
[CPL_ABORT_REQ_RSS
] = do_abort_req
;
1805 cpl_handlers
[CPL_ABORT_RPL_RSS
] = do_abort_rpl
;
1806 cpl_handlers
[CPL_CLOSE_CON_RPL
] = do_close_con_rpl
;
1807 cpl_handlers
[CPL_TX_DMA_ACK
] = do_wr_ack
;
1808 cpl_handlers
[CPL_ISCSI_HDR
] = do_iscsi_hdr
;
1810 if (cxgb3_max_connect
> CXGB3I_MAX_CONN
)
1811 cxgb3_max_connect
= CXGB3I_MAX_CONN
;
1816 * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1817 * @cdev: t3cdev adapter
1818 * @client: cxgb3 driver client
1820 void cxgb3i_sdev_add(struct t3cdev
*cdev
, struct cxgb3_client
*client
)
1822 struct cxgb3i_sdev_data
*cdata
;
1823 struct ofld_page_info rx_page_info
;
1824 unsigned int wr_len
;
1825 int mapsize
= DIV_ROUND_UP(cxgb3_max_connect
,
1826 8 * sizeof(unsigned long));
1829 cdata
= cxgb3i_alloc_big_mem(sizeof(*cdata
) + mapsize
, GFP_KERNEL
);
1833 if (cdev
->ctl(cdev
, GET_WR_LEN
, &wr_len
) < 0 ||
1834 cdev
->ctl(cdev
, GET_PORTS
, &cdata
->ports
) < 0 ||
1835 cdev
->ctl(cdev
, GET_RX_PAGE_INFO
, &rx_page_info
) < 0)
1838 s3_init_wr_tab(wr_len
);
1840 INIT_LIST_HEAD(&cdata
->list
);
1842 cdata
->client
= client
;
1844 for (i
= 0; i
< cdata
->ports
.nports
; i
++)
1845 NDEV2CDATA(cdata
->ports
.lldevs
[i
]) = cdata
;
1847 write_lock(&cdata_rwlock
);
1848 list_add_tail(&cdata
->list
, &cdata_list
);
1849 write_unlock(&cdata_rwlock
);
1854 cxgb3i_free_big_mem(cdata
);
1858 * cxgb3i_sdev_remove - free the allocated resources for the adapter
1859 * @cdev: t3cdev adapter
1861 void cxgb3i_sdev_remove(struct t3cdev
*cdev
)
1863 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(cdev
);
1865 write_lock(&cdata_rwlock
);
1866 list_del(&cdata
->list
);
1867 write_unlock(&cdata_rwlock
);
1869 sdev_data_cleanup(cdata
);