2 * Copyright (c) 2009-2014 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <linux/list.h>
34 #include <linux/workqueue.h>
35 #include <linux/skbuff.h>
36 #include <linux/timer.h>
37 #include <linux/notifier.h>
38 #include <linux/inetdevice.h>
40 #include <linux/tcp.h>
41 #include <linux/if_vlan.h>
43 #include <net/neighbour.h>
44 #include <net/netevent.h>
45 #include <net/route.h>
47 #include <net/ip6_route.h>
48 #include <net/addrconf.h>
50 #include <rdma/ib_addr.h>
55 static char *states
[] = {
72 module_param(nocong
, int, 0644);
73 MODULE_PARM_DESC(nocong
, "Turn of congestion control (default=0)");
75 static int enable_ecn
;
76 module_param(enable_ecn
, int, 0644);
77 MODULE_PARM_DESC(enable_ecn
, "Enable ECN (default=0/disabled)");
79 static int dack_mode
= 1;
80 module_param(dack_mode
, int, 0644);
81 MODULE_PARM_DESC(dack_mode
, "Delayed ack mode (default=1)");
83 uint c4iw_max_read_depth
= 32;
84 module_param(c4iw_max_read_depth
, int, 0644);
85 MODULE_PARM_DESC(c4iw_max_read_depth
,
86 "Per-connection max ORD/IRD (default=32)");
88 static int enable_tcp_timestamps
;
89 module_param(enable_tcp_timestamps
, int, 0644);
90 MODULE_PARM_DESC(enable_tcp_timestamps
, "Enable tcp timestamps (default=0)");
92 static int enable_tcp_sack
;
93 module_param(enable_tcp_sack
, int, 0644);
94 MODULE_PARM_DESC(enable_tcp_sack
, "Enable tcp SACK (default=0)");
96 static int enable_tcp_window_scaling
= 1;
97 module_param(enable_tcp_window_scaling
, int, 0644);
98 MODULE_PARM_DESC(enable_tcp_window_scaling
,
99 "Enable tcp window scaling (default=1)");
102 module_param(c4iw_debug
, int, 0644);
103 MODULE_PARM_DESC(c4iw_debug
, "Enable debug logging (default=0)");
105 static int peer2peer
= 1;
106 module_param(peer2peer
, int, 0644);
107 MODULE_PARM_DESC(peer2peer
, "Support peer2peer ULPs (default=1)");
109 static int p2p_type
= FW_RI_INIT_P2PTYPE_READ_REQ
;
110 module_param(p2p_type
, int, 0644);
111 MODULE_PARM_DESC(p2p_type
, "RDMAP opcode to use for the RTR message: "
112 "1=RDMA_READ 0=RDMA_WRITE (default 1)");
114 static int ep_timeout_secs
= 60;
115 module_param(ep_timeout_secs
, int, 0644);
116 MODULE_PARM_DESC(ep_timeout_secs
, "CM Endpoint operation timeout "
117 "in seconds (default=60)");
119 static int mpa_rev
= 2;
120 module_param(mpa_rev
, int, 0644);
121 MODULE_PARM_DESC(mpa_rev
, "MPA Revision, 0 supports amso1100, "
122 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft"
123 " compliant (default=2)");
125 static int markers_enabled
;
126 module_param(markers_enabled
, int, 0644);
127 MODULE_PARM_DESC(markers_enabled
, "Enable MPA MARKERS (default(0)=disabled)");
129 static int crc_enabled
= 1;
130 module_param(crc_enabled
, int, 0644);
131 MODULE_PARM_DESC(crc_enabled
, "Enable MPA CRC (default(1)=enabled)");
133 static int rcv_win
= 256 * 1024;
134 module_param(rcv_win
, int, 0644);
135 MODULE_PARM_DESC(rcv_win
, "TCP receive window in bytes (default=256KB)");
137 static int snd_win
= 128 * 1024;
138 module_param(snd_win
, int, 0644);
139 MODULE_PARM_DESC(snd_win
, "TCP send window in bytes (default=128KB)");
141 static struct workqueue_struct
*workq
;
143 static struct sk_buff_head rxq
;
145 static struct sk_buff
*get_skb(struct sk_buff
*skb
, int len
, gfp_t gfp
);
146 static void ep_timeout(unsigned long arg
);
147 static void connect_reply_upcall(struct c4iw_ep
*ep
, int status
);
148 static int sched(struct c4iw_dev
*dev
, struct sk_buff
*skb
);
150 static LIST_HEAD(timeout_list
);
151 static spinlock_t timeout_lock
;
153 static void deref_qp(struct c4iw_ep
*ep
)
155 c4iw_qp_rem_ref(&ep
->com
.qp
->ibqp
);
156 clear_bit(QP_REFERENCED
, &ep
->com
.flags
);
159 static void ref_qp(struct c4iw_ep
*ep
)
161 set_bit(QP_REFERENCED
, &ep
->com
.flags
);
162 c4iw_qp_add_ref(&ep
->com
.qp
->ibqp
);
165 static void start_ep_timer(struct c4iw_ep
*ep
)
167 PDBG("%s ep %p\n", __func__
, ep
);
168 if (timer_pending(&ep
->timer
)) {
169 pr_err("%s timer already started! ep %p\n",
173 clear_bit(TIMEOUT
, &ep
->com
.flags
);
174 c4iw_get_ep(&ep
->com
);
175 ep
->timer
.expires
= jiffies
+ ep_timeout_secs
* HZ
;
176 ep
->timer
.data
= (unsigned long)ep
;
177 ep
->timer
.function
= ep_timeout
;
178 add_timer(&ep
->timer
);
181 static int stop_ep_timer(struct c4iw_ep
*ep
)
183 PDBG("%s ep %p stopping\n", __func__
, ep
);
184 del_timer_sync(&ep
->timer
);
185 if (!test_and_set_bit(TIMEOUT
, &ep
->com
.flags
)) {
186 c4iw_put_ep(&ep
->com
);
192 static int c4iw_l2t_send(struct c4iw_rdev
*rdev
, struct sk_buff
*skb
,
193 struct l2t_entry
*l2e
)
197 if (c4iw_fatal_error(rdev
)) {
199 PDBG("%s - device in error state - dropping\n", __func__
);
202 error
= cxgb4_l2t_send(rdev
->lldi
.ports
[0], skb
, l2e
);
205 return error
< 0 ? error
: 0;
208 int c4iw_ofld_send(struct c4iw_rdev
*rdev
, struct sk_buff
*skb
)
212 if (c4iw_fatal_error(rdev
)) {
214 PDBG("%s - device in error state - dropping\n", __func__
);
217 error
= cxgb4_ofld_send(rdev
->lldi
.ports
[0], skb
);
220 return error
< 0 ? error
: 0;
223 static void release_tid(struct c4iw_rdev
*rdev
, u32 hwtid
, struct sk_buff
*skb
)
225 struct cpl_tid_release
*req
;
227 skb
= get_skb(skb
, sizeof *req
, GFP_KERNEL
);
230 req
= (struct cpl_tid_release
*) skb_put(skb
, sizeof(*req
));
231 INIT_TP_WR(req
, hwtid
);
232 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE
, hwtid
));
233 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, 0);
234 c4iw_ofld_send(rdev
, skb
);
238 static void set_emss(struct c4iw_ep
*ep
, u16 opt
)
240 ep
->emss
= ep
->com
.dev
->rdev
.lldi
.mtus
[TCPOPT_MSS_G(opt
)] -
241 ((AF_INET
== ep
->com
.remote_addr
.ss_family
) ?
242 sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) -
243 sizeof(struct tcphdr
);
245 if (TCPOPT_TSTAMP_G(opt
))
246 ep
->emss
-= round_up(TCPOLEN_TIMESTAMP
, 4);
250 PDBG("Warning: misaligned mtu idx %u mss %u emss=%u\n",
251 TCPOPT_MSS_G(opt
), ep
->mss
, ep
->emss
);
252 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__
, TCPOPT_MSS_G(opt
),
256 static enum c4iw_ep_state
state_read(struct c4iw_ep_common
*epc
)
258 enum c4iw_ep_state state
;
260 mutex_lock(&epc
->mutex
);
262 mutex_unlock(&epc
->mutex
);
266 static void __state_set(struct c4iw_ep_common
*epc
, enum c4iw_ep_state
new)
271 static void state_set(struct c4iw_ep_common
*epc
, enum c4iw_ep_state
new)
273 mutex_lock(&epc
->mutex
);
274 PDBG("%s - %s -> %s\n", __func__
, states
[epc
->state
], states
[new]);
275 __state_set(epc
, new);
276 mutex_unlock(&epc
->mutex
);
280 static void *alloc_ep(int size
, gfp_t gfp
)
282 struct c4iw_ep_common
*epc
;
284 epc
= kzalloc(size
, gfp
);
286 kref_init(&epc
->kref
);
287 mutex_init(&epc
->mutex
);
288 c4iw_init_wr_wait(&epc
->wr_wait
);
290 PDBG("%s alloc ep %p\n", __func__
, epc
);
294 void _c4iw_free_ep(struct kref
*kref
)
298 ep
= container_of(kref
, struct c4iw_ep
, com
.kref
);
299 PDBG("%s ep %p state %s\n", __func__
, ep
, states
[ep
->com
.state
]);
300 if (test_bit(QP_REFERENCED
, &ep
->com
.flags
))
302 if (test_bit(RELEASE_RESOURCES
, &ep
->com
.flags
)) {
303 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
304 struct sockaddr_in6
*sin6
=
305 (struct sockaddr_in6
*)
309 ep
->com
.dev
->rdev
.lldi
.ports
[0],
310 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
313 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->hwtid_idr
, ep
->hwtid
);
314 cxgb4_remove_tid(ep
->com
.dev
->rdev
.lldi
.tids
, 0, ep
->hwtid
);
315 dst_release(ep
->dst
);
316 cxgb4_l2t_release(ep
->l2t
);
321 static void release_ep_resources(struct c4iw_ep
*ep
)
323 set_bit(RELEASE_RESOURCES
, &ep
->com
.flags
);
324 c4iw_put_ep(&ep
->com
);
327 static int status2errno(int status
)
332 case CPL_ERR_CONN_RESET
:
334 case CPL_ERR_ARP_MISS
:
335 return -EHOSTUNREACH
;
336 case CPL_ERR_CONN_TIMEDOUT
:
338 case CPL_ERR_TCAM_FULL
:
340 case CPL_ERR_CONN_EXIST
:
348 * Try and reuse skbs already allocated...
350 static struct sk_buff
*get_skb(struct sk_buff
*skb
, int len
, gfp_t gfp
)
352 if (skb
&& !skb_is_nonlinear(skb
) && !skb_cloned(skb
)) {
355 skb_reset_transport_header(skb
);
357 skb
= alloc_skb(len
, gfp
);
359 t4_set_arp_err_handler(skb
, NULL
, NULL
);
363 static struct net_device
*get_real_dev(struct net_device
*egress_dev
)
365 return rdma_vlan_dev_real_dev(egress_dev
) ? : egress_dev
;
368 static int our_interface(struct c4iw_dev
*dev
, struct net_device
*egress_dev
)
372 egress_dev
= get_real_dev(egress_dev
);
373 for (i
= 0; i
< dev
->rdev
.lldi
.nports
; i
++)
374 if (dev
->rdev
.lldi
.ports
[i
] == egress_dev
)
379 static struct dst_entry
*find_route6(struct c4iw_dev
*dev
, __u8
*local_ip
,
380 __u8
*peer_ip
, __be16 local_port
,
381 __be16 peer_port
, u8 tos
,
384 struct dst_entry
*dst
= NULL
;
386 if (IS_ENABLED(CONFIG_IPV6
)) {
389 memset(&fl6
, 0, sizeof(fl6
));
390 memcpy(&fl6
.daddr
, peer_ip
, 16);
391 memcpy(&fl6
.saddr
, local_ip
, 16);
392 if (ipv6_addr_type(&fl6
.daddr
) & IPV6_ADDR_LINKLOCAL
)
393 fl6
.flowi6_oif
= sin6_scope_id
;
394 dst
= ip6_route_output(&init_net
, NULL
, &fl6
);
397 if (!our_interface(dev
, ip6_dst_idev(dst
)->dev
) &&
398 !(ip6_dst_idev(dst
)->dev
->flags
& IFF_LOOPBACK
)) {
408 static struct dst_entry
*find_route(struct c4iw_dev
*dev
, __be32 local_ip
,
409 __be32 peer_ip
, __be16 local_port
,
410 __be16 peer_port
, u8 tos
)
416 rt
= ip_route_output_ports(&init_net
, &fl4
, NULL
, peer_ip
, local_ip
,
417 peer_port
, local_port
, IPPROTO_TCP
,
421 n
= dst_neigh_lookup(&rt
->dst
, &peer_ip
);
424 if (!our_interface(dev
, n
->dev
) &&
425 !(n
->dev
->flags
& IFF_LOOPBACK
)) {
427 dst_release(&rt
->dst
);
434 static void arp_failure_discard(void *handle
, struct sk_buff
*skb
)
436 pr_err(MOD
"ARP failure\n");
442 FAKE_CPL_PUT_EP_SAFE
= NUM_CPL_CMDS
+ 0,
445 static int _put_ep_safe(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
449 ep
= *((struct c4iw_ep
**)(skb
->cb
+ 2 * sizeof(void *)));
450 release_ep_resources(ep
);
455 * Fake up a special CPL opcode and call sched() so process_work() will call
456 * _put_ep_safe() in a safe context to free the ep resources. This is needed
457 * because ARP error handlers are called in an ATOMIC context, and
458 * _c4iw_free_ep() needs to block.
460 static void queue_arp_failure_cpl(struct c4iw_ep
*ep
, struct sk_buff
*skb
)
462 struct cpl_act_establish
*rpl
= cplhdr(skb
);
464 /* Set our special ARP_FAILURE opcode */
465 rpl
->ot
.opcode
= FAKE_CPL_PUT_EP_SAFE
;
468 * Save ep in the skb->cb area, after where sched() will save the dev
471 *((struct c4iw_ep
**)(skb
->cb
+ 2 * sizeof(void *))) = ep
;
472 sched(ep
->com
.dev
, skb
);
475 /* Handle an ARP failure for an accept */
476 static void pass_accept_rpl_arp_failure(void *handle
, struct sk_buff
*skb
)
478 struct c4iw_ep
*ep
= handle
;
480 pr_err(MOD
"ARP failure during accept - tid %u -dropping connection\n",
483 __state_set(&ep
->com
, DEAD
);
484 queue_arp_failure_cpl(ep
, skb
);
488 * Handle an ARP failure for an active open.
490 static void act_open_req_arp_failure(void *handle
, struct sk_buff
*skb
)
492 struct c4iw_ep
*ep
= handle
;
494 printk(KERN_ERR MOD
"ARP failure during connect\n");
495 connect_reply_upcall(ep
, -EHOSTUNREACH
);
496 __state_set(&ep
->com
, DEAD
);
497 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
498 struct sockaddr_in6
*sin6
=
499 (struct sockaddr_in6
*)&ep
->com
.local_addr
;
500 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
501 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
503 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, ep
->atid
);
504 cxgb4_free_atid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
->atid
);
505 queue_arp_failure_cpl(ep
, skb
);
509 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
512 static void abort_arp_failure(void *handle
, struct sk_buff
*skb
)
514 struct c4iw_rdev
*rdev
= handle
;
515 struct cpl_abort_req
*req
= cplhdr(skb
);
517 PDBG("%s rdev %p\n", __func__
, rdev
);
518 req
->cmd
= CPL_ABORT_NO_RST
;
519 c4iw_ofld_send(rdev
, skb
);
522 static int send_flowc(struct c4iw_ep
*ep
, struct sk_buff
*skb
)
524 unsigned int flowclen
= 80;
525 struct fw_flowc_wr
*flowc
;
527 u16 vlan
= ep
->l2t
->vlan
;
530 if (vlan
== CPL_L2T_VLAN_NONE
)
535 skb
= get_skb(skb
, flowclen
, GFP_KERNEL
);
536 flowc
= (struct fw_flowc_wr
*)__skb_put(skb
, flowclen
);
538 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
539 FW_FLOWC_WR_NPARAMS_V(nparams
));
540 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(flowclen
,
541 16)) | FW_WR_FLOWID_V(ep
->hwtid
));
543 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
544 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V
545 (ep
->com
.dev
->rdev
.lldi
.pf
));
546 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
547 flowc
->mnemval
[1].val
= cpu_to_be32(ep
->tx_chan
);
548 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
549 flowc
->mnemval
[2].val
= cpu_to_be32(ep
->tx_chan
);
550 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
551 flowc
->mnemval
[3].val
= cpu_to_be32(ep
->rss_qid
);
552 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
553 flowc
->mnemval
[4].val
= cpu_to_be32(ep
->snd_seq
);
554 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
555 flowc
->mnemval
[5].val
= cpu_to_be32(ep
->rcv_seq
);
556 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
557 flowc
->mnemval
[6].val
= cpu_to_be32(ep
->snd_win
);
558 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
559 flowc
->mnemval
[7].val
= cpu_to_be32(ep
->emss
);
563 pri
= (vlan
& VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
;
564 flowc
->mnemval
[8].mnemonic
= FW_FLOWC_MNEM_SCHEDCLASS
;
565 flowc
->mnemval
[8].val
= cpu_to_be32(pri
);
567 /* Pad WR to 16 byte boundary */
568 flowc
->mnemval
[8].mnemonic
= 0;
569 flowc
->mnemval
[8].val
= 0;
571 for (i
= 0; i
< 9; i
++) {
572 flowc
->mnemval
[i
].r4
[0] = 0;
573 flowc
->mnemval
[i
].r4
[1] = 0;
574 flowc
->mnemval
[i
].r4
[2] = 0;
577 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
578 return c4iw_ofld_send(&ep
->com
.dev
->rdev
, skb
);
581 static int send_halfclose(struct c4iw_ep
*ep
, gfp_t gfp
)
583 struct cpl_close_con_req
*req
;
585 int wrlen
= roundup(sizeof *req
, 16);
587 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
588 skb
= get_skb(NULL
, wrlen
, gfp
);
590 printk(KERN_ERR MOD
"%s - failed to alloc skb\n", __func__
);
593 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
594 t4_set_arp_err_handler(skb
, NULL
, arp_failure_discard
);
595 req
= (struct cpl_close_con_req
*) skb_put(skb
, wrlen
);
596 memset(req
, 0, wrlen
);
597 INIT_TP_WR(req
, ep
->hwtid
);
598 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
,
600 return c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
603 static int send_abort(struct c4iw_ep
*ep
, struct sk_buff
*skb
, gfp_t gfp
)
605 struct cpl_abort_req
*req
;
606 int wrlen
= roundup(sizeof *req
, 16);
608 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
609 skb
= get_skb(skb
, wrlen
, gfp
);
611 printk(KERN_ERR MOD
"%s - failed to alloc skb.\n",
615 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
616 t4_set_arp_err_handler(skb
, &ep
->com
.dev
->rdev
, abort_arp_failure
);
617 req
= (struct cpl_abort_req
*) skb_put(skb
, wrlen
);
618 memset(req
, 0, wrlen
);
619 INIT_TP_WR(req
, ep
->hwtid
);
620 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ
, ep
->hwtid
));
621 req
->cmd
= CPL_ABORT_SEND_RST
;
622 return c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
625 static void best_mtu(const unsigned short *mtus
, unsigned short mtu
,
626 unsigned int *idx
, int use_ts
, int ipv6
)
628 unsigned short hdr_size
= (ipv6
?
629 sizeof(struct ipv6hdr
) :
630 sizeof(struct iphdr
)) +
631 sizeof(struct tcphdr
) +
633 round_up(TCPOLEN_TIMESTAMP
, 4) : 0);
634 unsigned short data_size
= mtu
- hdr_size
;
636 cxgb4_best_aligned_mtu(mtus
, hdr_size
, data_size
, 8, idx
);
639 static int send_connect(struct c4iw_ep
*ep
)
641 struct cpl_act_open_req
*req
= NULL
;
642 struct cpl_t5_act_open_req
*t5req
= NULL
;
643 struct cpl_t6_act_open_req
*t6req
= NULL
;
644 struct cpl_act_open_req6
*req6
= NULL
;
645 struct cpl_t5_act_open_req6
*t5req6
= NULL
;
646 struct cpl_t6_act_open_req6
*t6req6
= NULL
;
650 unsigned int mtu_idx
;
652 int win
, sizev4
, sizev6
, wrlen
;
653 struct sockaddr_in
*la
= (struct sockaddr_in
*)
655 struct sockaddr_in
*ra
= (struct sockaddr_in
*)
656 &ep
->com
.remote_addr
;
657 struct sockaddr_in6
*la6
= (struct sockaddr_in6
*)
659 struct sockaddr_in6
*ra6
= (struct sockaddr_in6
*)
660 &ep
->com
.remote_addr
;
662 enum chip_type adapter_type
= ep
->com
.dev
->rdev
.lldi
.adapter_type
;
663 u32 isn
= (prandom_u32() & ~7UL) - 1;
665 switch (CHELSIO_CHIP_VERSION(adapter_type
)) {
667 sizev4
= sizeof(struct cpl_act_open_req
);
668 sizev6
= sizeof(struct cpl_act_open_req6
);
671 sizev4
= sizeof(struct cpl_t5_act_open_req
);
672 sizev6
= sizeof(struct cpl_t5_act_open_req6
);
675 sizev4
= sizeof(struct cpl_t6_act_open_req
);
676 sizev6
= sizeof(struct cpl_t6_act_open_req6
);
679 pr_err("T%d Chip is not supported\n",
680 CHELSIO_CHIP_VERSION(adapter_type
));
684 wrlen
= (ep
->com
.remote_addr
.ss_family
== AF_INET
) ?
685 roundup(sizev4
, 16) :
688 PDBG("%s ep %p atid %u\n", __func__
, ep
, ep
->atid
);
690 skb
= get_skb(NULL
, wrlen
, GFP_KERNEL
);
692 printk(KERN_ERR MOD
"%s - failed to alloc skb.\n",
696 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, ep
->ctrlq_idx
);
698 best_mtu(ep
->com
.dev
->rdev
.lldi
.mtus
, ep
->mtu
, &mtu_idx
,
699 enable_tcp_timestamps
,
700 (AF_INET
== ep
->com
.remote_addr
.ss_family
) ? 0 : 1);
701 wscale
= compute_wscale(rcv_win
);
704 * Specify the largest window that will fit in opt0. The
705 * remainder will be specified in the rx_data_ack.
707 win
= ep
->rcv_win
>> 10;
708 if (win
> RCV_BUFSIZ_M
)
711 opt0
= (nocong
? NO_CONG_F
: 0) |
714 WND_SCALE_V(wscale
) |
716 L2T_IDX_V(ep
->l2t
->idx
) |
717 TX_CHAN_V(ep
->tx_chan
) |
718 SMAC_SEL_V(ep
->smac_idx
) |
719 DSCP_V(ep
->tos
>> 2) |
720 ULP_MODE_V(ULP_MODE_TCPDDP
) |
722 opt2
= RX_CHANNEL_V(0) |
723 CCTRL_ECN_V(enable_ecn
) |
724 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(ep
->rss_qid
);
725 if (enable_tcp_timestamps
)
726 opt2
|= TSTAMPS_EN_F
;
729 if (wscale
&& enable_tcp_window_scaling
)
730 opt2
|= WND_SCALE_EN_F
;
731 if (CHELSIO_CHIP_VERSION(adapter_type
) > CHELSIO_T4
) {
735 opt2
|= T5_OPT_2_VALID_F
;
736 opt2
|= CONG_CNTRL_V(CONG_ALG_TAHOE
);
740 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
)
741 cxgb4_clip_get(ep
->com
.dev
->rdev
.lldi
.ports
[0],
742 (const u32
*)&la6
->sin6_addr
.s6_addr
, 1);
744 t4_set_arp_err_handler(skb
, ep
, act_open_req_arp_failure
);
746 if (ep
->com
.remote_addr
.ss_family
== AF_INET
) {
747 switch (CHELSIO_CHIP_VERSION(adapter_type
)) {
749 req
= (struct cpl_act_open_req
*)skb_put(skb
, wrlen
);
753 t5req
= (struct cpl_t5_act_open_req
*)skb_put(skb
,
755 INIT_TP_WR(t5req
, 0);
756 req
= (struct cpl_act_open_req
*)t5req
;
759 t6req
= (struct cpl_t6_act_open_req
*)skb_put(skb
,
761 INIT_TP_WR(t6req
, 0);
762 req
= (struct cpl_act_open_req
*)t6req
;
763 t5req
= (struct cpl_t5_act_open_req
*)t6req
;
766 pr_err("T%d Chip is not supported\n",
767 CHELSIO_CHIP_VERSION(adapter_type
));
772 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
,
773 ((ep
->rss_qid
<<14) | ep
->atid
)));
774 req
->local_port
= la
->sin_port
;
775 req
->peer_port
= ra
->sin_port
;
776 req
->local_ip
= la
->sin_addr
.s_addr
;
777 req
->peer_ip
= ra
->sin_addr
.s_addr
;
778 req
->opt0
= cpu_to_be64(opt0
);
780 if (is_t4(ep
->com
.dev
->rdev
.lldi
.adapter_type
)) {
781 req
->params
= cpu_to_be32(cxgb4_select_ntuple(
782 ep
->com
.dev
->rdev
.lldi
.ports
[0],
784 req
->opt2
= cpu_to_be32(opt2
);
786 t5req
->params
= cpu_to_be64(FILTER_TUPLE_V(
788 ep
->com
.dev
->rdev
.lldi
.ports
[0],
790 t5req
->rsvd
= cpu_to_be32(isn
);
791 PDBG("%s snd_isn %u\n", __func__
, t5req
->rsvd
);
792 t5req
->opt2
= cpu_to_be32(opt2
);
795 switch (CHELSIO_CHIP_VERSION(adapter_type
)) {
797 req6
= (struct cpl_act_open_req6
*)skb_put(skb
, wrlen
);
801 t5req6
= (struct cpl_t5_act_open_req6
*)skb_put(skb
,
803 INIT_TP_WR(t5req6
, 0);
804 req6
= (struct cpl_act_open_req6
*)t5req6
;
807 t6req6
= (struct cpl_t6_act_open_req6
*)skb_put(skb
,
809 INIT_TP_WR(t6req6
, 0);
810 req6
= (struct cpl_act_open_req6
*)t6req6
;
811 t5req6
= (struct cpl_t5_act_open_req6
*)t6req6
;
814 pr_err("T%d Chip is not supported\n",
815 CHELSIO_CHIP_VERSION(adapter_type
));
820 OPCODE_TID(req6
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
,
821 ((ep
->rss_qid
<<14)|ep
->atid
)));
822 req6
->local_port
= la6
->sin6_port
;
823 req6
->peer_port
= ra6
->sin6_port
;
824 req6
->local_ip_hi
= *((__be64
*)(la6
->sin6_addr
.s6_addr
));
825 req6
->local_ip_lo
= *((__be64
*)(la6
->sin6_addr
.s6_addr
+ 8));
826 req6
->peer_ip_hi
= *((__be64
*)(ra6
->sin6_addr
.s6_addr
));
827 req6
->peer_ip_lo
= *((__be64
*)(ra6
->sin6_addr
.s6_addr
+ 8));
828 req6
->opt0
= cpu_to_be64(opt0
);
830 if (is_t4(ep
->com
.dev
->rdev
.lldi
.adapter_type
)) {
831 req6
->params
= cpu_to_be32(cxgb4_select_ntuple(
832 ep
->com
.dev
->rdev
.lldi
.ports
[0],
834 req6
->opt2
= cpu_to_be32(opt2
);
836 t5req6
->params
= cpu_to_be64(FILTER_TUPLE_V(
838 ep
->com
.dev
->rdev
.lldi
.ports
[0],
840 t5req6
->rsvd
= cpu_to_be32(isn
);
841 PDBG("%s snd_isn %u\n", __func__
, t5req6
->rsvd
);
842 t5req6
->opt2
= cpu_to_be32(opt2
);
846 set_bit(ACT_OPEN_REQ
, &ep
->com
.history
);
847 ret
= c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
849 if (ret
&& ep
->com
.remote_addr
.ss_family
== AF_INET6
)
850 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
851 (const u32
*)&la6
->sin6_addr
.s6_addr
, 1);
855 static void send_mpa_req(struct c4iw_ep
*ep
, struct sk_buff
*skb
,
859 struct fw_ofld_tx_data_wr
*req
;
860 struct mpa_message
*mpa
;
861 struct mpa_v2_conn_params mpa_v2_params
;
863 PDBG("%s ep %p tid %u pd_len %d\n", __func__
, ep
, ep
->hwtid
, ep
->plen
);
865 BUG_ON(skb_cloned(skb
));
867 mpalen
= sizeof(*mpa
) + ep
->plen
;
868 if (mpa_rev_to_use
== 2)
869 mpalen
+= sizeof(struct mpa_v2_conn_params
);
870 wrlen
= roundup(mpalen
+ sizeof *req
, 16);
871 skb
= get_skb(skb
, wrlen
, GFP_KERNEL
);
873 connect_reply_upcall(ep
, -ENOMEM
);
876 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
878 req
= (struct fw_ofld_tx_data_wr
*)skb_put(skb
, wrlen
);
879 memset(req
, 0, wrlen
);
880 req
->op_to_immdlen
= cpu_to_be32(
881 FW_WR_OP_V(FW_OFLD_TX_DATA_WR
) |
883 FW_WR_IMMDLEN_V(mpalen
));
884 req
->flowid_len16
= cpu_to_be32(
885 FW_WR_FLOWID_V(ep
->hwtid
) |
886 FW_WR_LEN16_V(wrlen
>> 4));
887 req
->plen
= cpu_to_be32(mpalen
);
888 req
->tunnel_to_proxy
= cpu_to_be32(
889 FW_OFLD_TX_DATA_WR_FLUSH_F
|
890 FW_OFLD_TX_DATA_WR_SHOVE_F
);
892 mpa
= (struct mpa_message
*)(req
+ 1);
893 memcpy(mpa
->key
, MPA_KEY_REQ
, sizeof(mpa
->key
));
894 mpa
->flags
= (crc_enabled
? MPA_CRC
: 0) |
895 (markers_enabled
? MPA_MARKERS
: 0) |
896 (mpa_rev_to_use
== 2 ? MPA_ENHANCED_RDMA_CONN
: 0);
897 mpa
->private_data_size
= htons(ep
->plen
);
898 mpa
->revision
= mpa_rev_to_use
;
899 if (mpa_rev_to_use
== 1) {
900 ep
->tried_with_mpa_v1
= 1;
901 ep
->retry_with_mpa_v1
= 0;
904 if (mpa_rev_to_use
== 2) {
905 mpa
->private_data_size
= htons(ntohs(mpa
->private_data_size
) +
906 sizeof (struct mpa_v2_conn_params
));
907 PDBG("%s initiator ird %u ord %u\n", __func__
, ep
->ird
,
909 mpa_v2_params
.ird
= htons((u16
)ep
->ird
);
910 mpa_v2_params
.ord
= htons((u16
)ep
->ord
);
913 mpa_v2_params
.ird
|= htons(MPA_V2_PEER2PEER_MODEL
);
914 if (p2p_type
== FW_RI_INIT_P2PTYPE_RDMA_WRITE
)
916 htons(MPA_V2_RDMA_WRITE_RTR
);
917 else if (p2p_type
== FW_RI_INIT_P2PTYPE_READ_REQ
)
919 htons(MPA_V2_RDMA_READ_RTR
);
921 memcpy(mpa
->private_data
, &mpa_v2_params
,
922 sizeof(struct mpa_v2_conn_params
));
925 memcpy(mpa
->private_data
+
926 sizeof(struct mpa_v2_conn_params
),
927 ep
->mpa_pkt
+ sizeof(*mpa
), ep
->plen
);
930 memcpy(mpa
->private_data
,
931 ep
->mpa_pkt
+ sizeof(*mpa
), ep
->plen
);
934 * Reference the mpa skb. This ensures the data area
935 * will remain in memory until the hw acks the tx.
936 * Function fw4_ack() will deref it.
939 t4_set_arp_err_handler(skb
, NULL
, arp_failure_discard
);
942 c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
944 __state_set(&ep
->com
, MPA_REQ_SENT
);
945 ep
->mpa_attr
.initiator
= 1;
946 ep
->snd_seq
+= mpalen
;
950 static int send_mpa_reject(struct c4iw_ep
*ep
, const void *pdata
, u8 plen
)
953 struct fw_ofld_tx_data_wr
*req
;
954 struct mpa_message
*mpa
;
956 struct mpa_v2_conn_params mpa_v2_params
;
958 PDBG("%s ep %p tid %u pd_len %d\n", __func__
, ep
, ep
->hwtid
, ep
->plen
);
960 mpalen
= sizeof(*mpa
) + plen
;
961 if (ep
->mpa_attr
.version
== 2 && ep
->mpa_attr
.enhanced_rdma_conn
)
962 mpalen
+= sizeof(struct mpa_v2_conn_params
);
963 wrlen
= roundup(mpalen
+ sizeof *req
, 16);
965 skb
= get_skb(NULL
, wrlen
, GFP_KERNEL
);
967 printk(KERN_ERR MOD
"%s - cannot alloc skb!\n", __func__
);
970 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
972 req
= (struct fw_ofld_tx_data_wr
*)skb_put(skb
, wrlen
);
973 memset(req
, 0, wrlen
);
974 req
->op_to_immdlen
= cpu_to_be32(
975 FW_WR_OP_V(FW_OFLD_TX_DATA_WR
) |
977 FW_WR_IMMDLEN_V(mpalen
));
978 req
->flowid_len16
= cpu_to_be32(
979 FW_WR_FLOWID_V(ep
->hwtid
) |
980 FW_WR_LEN16_V(wrlen
>> 4));
981 req
->plen
= cpu_to_be32(mpalen
);
982 req
->tunnel_to_proxy
= cpu_to_be32(
983 FW_OFLD_TX_DATA_WR_FLUSH_F
|
984 FW_OFLD_TX_DATA_WR_SHOVE_F
);
986 mpa
= (struct mpa_message
*)(req
+ 1);
987 memset(mpa
, 0, sizeof(*mpa
));
988 memcpy(mpa
->key
, MPA_KEY_REP
, sizeof(mpa
->key
));
989 mpa
->flags
= MPA_REJECT
;
990 mpa
->revision
= ep
->mpa_attr
.version
;
991 mpa
->private_data_size
= htons(plen
);
993 if (ep
->mpa_attr
.version
== 2 && ep
->mpa_attr
.enhanced_rdma_conn
) {
994 mpa
->flags
|= MPA_ENHANCED_RDMA_CONN
;
995 mpa
->private_data_size
= htons(ntohs(mpa
->private_data_size
) +
996 sizeof (struct mpa_v2_conn_params
));
997 mpa_v2_params
.ird
= htons(((u16
)ep
->ird
) |
998 (peer2peer
? MPA_V2_PEER2PEER_MODEL
:
1000 mpa_v2_params
.ord
= htons(((u16
)ep
->ord
) | (peer2peer
?
1002 FW_RI_INIT_P2PTYPE_RDMA_WRITE
?
1003 MPA_V2_RDMA_WRITE_RTR
: p2p_type
==
1004 FW_RI_INIT_P2PTYPE_READ_REQ
?
1005 MPA_V2_RDMA_READ_RTR
: 0) : 0));
1006 memcpy(mpa
->private_data
, &mpa_v2_params
,
1007 sizeof(struct mpa_v2_conn_params
));
1010 memcpy(mpa
->private_data
+
1011 sizeof(struct mpa_v2_conn_params
), pdata
, plen
);
1014 memcpy(mpa
->private_data
, pdata
, plen
);
1017 * Reference the mpa skb again. This ensures the data area
1018 * will remain in memory until the hw acks the tx.
1019 * Function fw4_ack() will deref it.
1022 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
1023 t4_set_arp_err_handler(skb
, NULL
, arp_failure_discard
);
1024 BUG_ON(ep
->mpa_skb
);
1026 ep
->snd_seq
+= mpalen
;
1027 return c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
1030 static int send_mpa_reply(struct c4iw_ep
*ep
, const void *pdata
, u8 plen
)
1033 struct fw_ofld_tx_data_wr
*req
;
1034 struct mpa_message
*mpa
;
1035 struct sk_buff
*skb
;
1036 struct mpa_v2_conn_params mpa_v2_params
;
1038 PDBG("%s ep %p tid %u pd_len %d\n", __func__
, ep
, ep
->hwtid
, ep
->plen
);
1040 mpalen
= sizeof(*mpa
) + plen
;
1041 if (ep
->mpa_attr
.version
== 2 && ep
->mpa_attr
.enhanced_rdma_conn
)
1042 mpalen
+= sizeof(struct mpa_v2_conn_params
);
1043 wrlen
= roundup(mpalen
+ sizeof *req
, 16);
1045 skb
= get_skb(NULL
, wrlen
, GFP_KERNEL
);
1047 printk(KERN_ERR MOD
"%s - cannot alloc skb!\n", __func__
);
1050 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
1052 req
= (struct fw_ofld_tx_data_wr
*) skb_put(skb
, wrlen
);
1053 memset(req
, 0, wrlen
);
1054 req
->op_to_immdlen
= cpu_to_be32(
1055 FW_WR_OP_V(FW_OFLD_TX_DATA_WR
) |
1057 FW_WR_IMMDLEN_V(mpalen
));
1058 req
->flowid_len16
= cpu_to_be32(
1059 FW_WR_FLOWID_V(ep
->hwtid
) |
1060 FW_WR_LEN16_V(wrlen
>> 4));
1061 req
->plen
= cpu_to_be32(mpalen
);
1062 req
->tunnel_to_proxy
= cpu_to_be32(
1063 FW_OFLD_TX_DATA_WR_FLUSH_F
|
1064 FW_OFLD_TX_DATA_WR_SHOVE_F
);
1066 mpa
= (struct mpa_message
*)(req
+ 1);
1067 memset(mpa
, 0, sizeof(*mpa
));
1068 memcpy(mpa
->key
, MPA_KEY_REP
, sizeof(mpa
->key
));
1069 mpa
->flags
= (ep
->mpa_attr
.crc_enabled
? MPA_CRC
: 0) |
1070 (markers_enabled
? MPA_MARKERS
: 0);
1071 mpa
->revision
= ep
->mpa_attr
.version
;
1072 mpa
->private_data_size
= htons(plen
);
1074 if (ep
->mpa_attr
.version
== 2 && ep
->mpa_attr
.enhanced_rdma_conn
) {
1075 mpa
->flags
|= MPA_ENHANCED_RDMA_CONN
;
1076 mpa
->private_data_size
= htons(ntohs(mpa
->private_data_size
) +
1077 sizeof (struct mpa_v2_conn_params
));
1078 mpa_v2_params
.ird
= htons((u16
)ep
->ird
);
1079 mpa_v2_params
.ord
= htons((u16
)ep
->ord
);
1080 if (peer2peer
&& (ep
->mpa_attr
.p2p_type
!=
1081 FW_RI_INIT_P2PTYPE_DISABLED
)) {
1082 mpa_v2_params
.ird
|= htons(MPA_V2_PEER2PEER_MODEL
);
1084 if (p2p_type
== FW_RI_INIT_P2PTYPE_RDMA_WRITE
)
1085 mpa_v2_params
.ord
|=
1086 htons(MPA_V2_RDMA_WRITE_RTR
);
1087 else if (p2p_type
== FW_RI_INIT_P2PTYPE_READ_REQ
)
1088 mpa_v2_params
.ord
|=
1089 htons(MPA_V2_RDMA_READ_RTR
);
1092 memcpy(mpa
->private_data
, &mpa_v2_params
,
1093 sizeof(struct mpa_v2_conn_params
));
1096 memcpy(mpa
->private_data
+
1097 sizeof(struct mpa_v2_conn_params
), pdata
, plen
);
1100 memcpy(mpa
->private_data
, pdata
, plen
);
1103 * Reference the mpa skb. This ensures the data area
1104 * will remain in memory until the hw acks the tx.
1105 * Function fw4_ack() will deref it.
1108 t4_set_arp_err_handler(skb
, NULL
, arp_failure_discard
);
1110 __state_set(&ep
->com
, MPA_REP_SENT
);
1111 ep
->snd_seq
+= mpalen
;
1112 return c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
1115 static int act_establish(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
1118 struct cpl_act_establish
*req
= cplhdr(skb
);
1119 unsigned int tid
= GET_TID(req
);
1120 unsigned int atid
= TID_TID_G(ntohl(req
->tos_atid
));
1121 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
1124 ep
= lookup_atid(t
, atid
);
1126 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__
, ep
, tid
,
1127 be32_to_cpu(req
->snd_isn
), be32_to_cpu(req
->rcv_isn
));
1129 mutex_lock(&ep
->com
.mutex
);
1130 dst_confirm(ep
->dst
);
1132 /* setup the hwtid for this connection */
1134 cxgb4_insert_tid(t
, ep
, tid
);
1135 insert_handle(dev
, &dev
->hwtid_idr
, ep
, ep
->hwtid
);
1137 ep
->snd_seq
= be32_to_cpu(req
->snd_isn
);
1138 ep
->rcv_seq
= be32_to_cpu(req
->rcv_isn
);
1140 set_emss(ep
, ntohs(req
->tcp_opt
));
1142 /* dealloc the atid */
1143 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, atid
);
1144 cxgb4_free_atid(t
, atid
);
1145 set_bit(ACT_ESTAB
, &ep
->com
.history
);
1147 /* start MPA negotiation */
1148 ret
= send_flowc(ep
, NULL
);
1151 if (ep
->retry_with_mpa_v1
)
1152 send_mpa_req(ep
, skb
, 1);
1154 send_mpa_req(ep
, skb
, mpa_rev
);
1155 mutex_unlock(&ep
->com
.mutex
);
1158 mutex_unlock(&ep
->com
.mutex
);
1159 connect_reply_upcall(ep
, -ENOMEM
);
1160 c4iw_ep_disconnect(ep
, 0, GFP_KERNEL
);
1164 static void close_complete_upcall(struct c4iw_ep
*ep
, int status
)
1166 struct iw_cm_event event
;
1168 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1169 memset(&event
, 0, sizeof(event
));
1170 event
.event
= IW_CM_EVENT_CLOSE
;
1171 event
.status
= status
;
1172 if (ep
->com
.cm_id
) {
1173 PDBG("close complete delivered ep %p cm_id %p tid %u\n",
1174 ep
, ep
->com
.cm_id
, ep
->hwtid
);
1175 ep
->com
.cm_id
->event_handler(ep
->com
.cm_id
, &event
);
1176 ep
->com
.cm_id
->rem_ref(ep
->com
.cm_id
);
1177 ep
->com
.cm_id
= NULL
;
1178 set_bit(CLOSE_UPCALL
, &ep
->com
.history
);
1182 static int abort_connection(struct c4iw_ep
*ep
, struct sk_buff
*skb
, gfp_t gfp
)
1184 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1185 __state_set(&ep
->com
, ABORTING
);
1186 set_bit(ABORT_CONN
, &ep
->com
.history
);
1187 return send_abort(ep
, skb
, gfp
);
1190 static void peer_close_upcall(struct c4iw_ep
*ep
)
1192 struct iw_cm_event event
;
1194 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1195 memset(&event
, 0, sizeof(event
));
1196 event
.event
= IW_CM_EVENT_DISCONNECT
;
1197 if (ep
->com
.cm_id
) {
1198 PDBG("peer close delivered ep %p cm_id %p tid %u\n",
1199 ep
, ep
->com
.cm_id
, ep
->hwtid
);
1200 ep
->com
.cm_id
->event_handler(ep
->com
.cm_id
, &event
);
1201 set_bit(DISCONN_UPCALL
, &ep
->com
.history
);
1205 static void peer_abort_upcall(struct c4iw_ep
*ep
)
1207 struct iw_cm_event event
;
1209 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1210 memset(&event
, 0, sizeof(event
));
1211 event
.event
= IW_CM_EVENT_CLOSE
;
1212 event
.status
= -ECONNRESET
;
1213 if (ep
->com
.cm_id
) {
1214 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep
,
1215 ep
->com
.cm_id
, ep
->hwtid
);
1216 ep
->com
.cm_id
->event_handler(ep
->com
.cm_id
, &event
);
1217 ep
->com
.cm_id
->rem_ref(ep
->com
.cm_id
);
1218 ep
->com
.cm_id
= NULL
;
1219 set_bit(ABORT_UPCALL
, &ep
->com
.history
);
1223 static void connect_reply_upcall(struct c4iw_ep
*ep
, int status
)
1225 struct iw_cm_event event
;
1227 PDBG("%s ep %p tid %u status %d\n", __func__
, ep
, ep
->hwtid
, status
);
1228 memset(&event
, 0, sizeof(event
));
1229 event
.event
= IW_CM_EVENT_CONNECT_REPLY
;
1230 event
.status
= status
;
1231 memcpy(&event
.local_addr
, &ep
->com
.local_addr
,
1232 sizeof(ep
->com
.local_addr
));
1233 memcpy(&event
.remote_addr
, &ep
->com
.remote_addr
,
1234 sizeof(ep
->com
.remote_addr
));
1236 if ((status
== 0) || (status
== -ECONNREFUSED
)) {
1237 if (!ep
->tried_with_mpa_v1
) {
1238 /* this means MPA_v2 is used */
1239 event
.ord
= ep
->ird
;
1240 event
.ird
= ep
->ord
;
1241 event
.private_data_len
= ep
->plen
-
1242 sizeof(struct mpa_v2_conn_params
);
1243 event
.private_data
= ep
->mpa_pkt
+
1244 sizeof(struct mpa_message
) +
1245 sizeof(struct mpa_v2_conn_params
);
1247 /* this means MPA_v1 is used */
1248 event
.ord
= cur_max_read_depth(ep
->com
.dev
);
1249 event
.ird
= cur_max_read_depth(ep
->com
.dev
);
1250 event
.private_data_len
= ep
->plen
;
1251 event
.private_data
= ep
->mpa_pkt
+
1252 sizeof(struct mpa_message
);
1256 PDBG("%s ep %p tid %u status %d\n", __func__
, ep
,
1258 set_bit(CONN_RPL_UPCALL
, &ep
->com
.history
);
1259 ep
->com
.cm_id
->event_handler(ep
->com
.cm_id
, &event
);
1262 ep
->com
.cm_id
->rem_ref(ep
->com
.cm_id
);
1263 ep
->com
.cm_id
= NULL
;
1267 static int connect_request_upcall(struct c4iw_ep
*ep
)
1269 struct iw_cm_event event
;
1272 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1273 memset(&event
, 0, sizeof(event
));
1274 event
.event
= IW_CM_EVENT_CONNECT_REQUEST
;
1275 memcpy(&event
.local_addr
, &ep
->com
.local_addr
,
1276 sizeof(ep
->com
.local_addr
));
1277 memcpy(&event
.remote_addr
, &ep
->com
.remote_addr
,
1278 sizeof(ep
->com
.remote_addr
));
1279 event
.provider_data
= ep
;
1280 if (!ep
->tried_with_mpa_v1
) {
1281 /* this means MPA_v2 is used */
1282 event
.ord
= ep
->ord
;
1283 event
.ird
= ep
->ird
;
1284 event
.private_data_len
= ep
->plen
-
1285 sizeof(struct mpa_v2_conn_params
);
1286 event
.private_data
= ep
->mpa_pkt
+ sizeof(struct mpa_message
) +
1287 sizeof(struct mpa_v2_conn_params
);
1289 /* this means MPA_v1 is used. Send max supported */
1290 event
.ord
= cur_max_read_depth(ep
->com
.dev
);
1291 event
.ird
= cur_max_read_depth(ep
->com
.dev
);
1292 event
.private_data_len
= ep
->plen
;
1293 event
.private_data
= ep
->mpa_pkt
+ sizeof(struct mpa_message
);
1295 c4iw_get_ep(&ep
->com
);
1296 ret
= ep
->parent_ep
->com
.cm_id
->event_handler(ep
->parent_ep
->com
.cm_id
,
1299 c4iw_put_ep(&ep
->com
);
1300 set_bit(CONNREQ_UPCALL
, &ep
->com
.history
);
1301 c4iw_put_ep(&ep
->parent_ep
->com
);
1305 static void established_upcall(struct c4iw_ep
*ep
)
1307 struct iw_cm_event event
;
1309 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1310 memset(&event
, 0, sizeof(event
));
1311 event
.event
= IW_CM_EVENT_ESTABLISHED
;
1312 event
.ird
= ep
->ord
;
1313 event
.ord
= ep
->ird
;
1314 if (ep
->com
.cm_id
) {
1315 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1316 ep
->com
.cm_id
->event_handler(ep
->com
.cm_id
, &event
);
1317 set_bit(ESTAB_UPCALL
, &ep
->com
.history
);
1321 static int update_rx_credits(struct c4iw_ep
*ep
, u32 credits
)
1323 struct cpl_rx_data_ack
*req
;
1324 struct sk_buff
*skb
;
1325 int wrlen
= roundup(sizeof *req
, 16);
1327 PDBG("%s ep %p tid %u credits %u\n", __func__
, ep
, ep
->hwtid
, credits
);
1328 skb
= get_skb(NULL
, wrlen
, GFP_KERNEL
);
1330 printk(KERN_ERR MOD
"update_rx_credits - cannot alloc skb!\n");
1335 * If we couldn't specify the entire rcv window at connection setup
1336 * due to the limit in the number of bits in the RCV_BUFSIZ field,
1337 * then add the overage in to the credits returned.
1339 if (ep
->rcv_win
> RCV_BUFSIZ_M
* 1024)
1340 credits
+= ep
->rcv_win
- RCV_BUFSIZ_M
* 1024;
1342 req
= (struct cpl_rx_data_ack
*) skb_put(skb
, wrlen
);
1343 memset(req
, 0, wrlen
);
1344 INIT_TP_WR(req
, ep
->hwtid
);
1345 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK
,
1347 req
->credit_dack
= cpu_to_be32(credits
| RX_FORCE_ACK_F
|
1349 RX_DACK_MODE_V(dack_mode
));
1350 set_wr_txq(skb
, CPL_PRIORITY_ACK
, ep
->ctrlq_idx
);
1351 c4iw_ofld_send(&ep
->com
.dev
->rdev
, skb
);
1355 #define RELAXED_IRD_NEGOTIATION 1
1358 * process_mpa_reply - process streaming mode MPA reply
1362 * 0 upon success indicating a connect request was delivered to the ULP
1363 * or the mpa request is incomplete but valid so far.
1365 * 1 if a failure requires the caller to close the connection.
1367 * 2 if a failure requires the caller to abort the connection.
1369 static int process_mpa_reply(struct c4iw_ep
*ep
, struct sk_buff
*skb
)
1371 struct mpa_message
*mpa
;
1372 struct mpa_v2_conn_params
*mpa_v2_params
;
1374 u16 resp_ird
, resp_ord
;
1375 u8 rtr_mismatch
= 0, insuff_ird
= 0;
1376 struct c4iw_qp_attributes attrs
;
1377 enum c4iw_qp_attr_mask mask
;
1381 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1384 * Stop mpa timer. If it expired, then
1385 * we ignore the MPA reply. process_timeout()
1386 * will abort the connection.
1388 if (stop_ep_timer(ep
))
1392 * If we get more than the supported amount of private data
1393 * then we must fail this connection.
1395 if (ep
->mpa_pkt_len
+ skb
->len
> sizeof(ep
->mpa_pkt
)) {
1401 * copy the new data into our accumulation buffer.
1403 skb_copy_from_linear_data(skb
, &(ep
->mpa_pkt
[ep
->mpa_pkt_len
]),
1405 ep
->mpa_pkt_len
+= skb
->len
;
1408 * if we don't even have the mpa message, then bail.
1410 if (ep
->mpa_pkt_len
< sizeof(*mpa
))
1412 mpa
= (struct mpa_message
*) ep
->mpa_pkt
;
1414 /* Validate MPA header. */
1415 if (mpa
->revision
> mpa_rev
) {
1416 printk(KERN_ERR MOD
"%s MPA version mismatch. Local = %d,"
1417 " Received = %d\n", __func__
, mpa_rev
, mpa
->revision
);
1421 if (memcmp(mpa
->key
, MPA_KEY_REP
, sizeof(mpa
->key
))) {
1426 plen
= ntohs(mpa
->private_data_size
);
1429 * Fail if there's too much private data.
1431 if (plen
> MPA_MAX_PRIVATE_DATA
) {
1437 * If plen does not account for pkt size
1439 if (ep
->mpa_pkt_len
> (sizeof(*mpa
) + plen
)) {
1444 ep
->plen
= (u8
) plen
;
1447 * If we don't have all the pdata yet, then bail.
1448 * We'll continue process when more data arrives.
1450 if (ep
->mpa_pkt_len
< (sizeof(*mpa
) + plen
))
1453 if (mpa
->flags
& MPA_REJECT
) {
1454 err
= -ECONNREFUSED
;
1459 * If we get here we have accumulated the entire mpa
1460 * start reply message including private data. And
1461 * the MPA header is valid.
1463 __state_set(&ep
->com
, FPDU_MODE
);
1464 ep
->mpa_attr
.crc_enabled
= (mpa
->flags
& MPA_CRC
) | crc_enabled
? 1 : 0;
1465 ep
->mpa_attr
.recv_marker_enabled
= markers_enabled
;
1466 ep
->mpa_attr
.xmit_marker_enabled
= mpa
->flags
& MPA_MARKERS
? 1 : 0;
1467 ep
->mpa_attr
.version
= mpa
->revision
;
1468 ep
->mpa_attr
.p2p_type
= FW_RI_INIT_P2PTYPE_DISABLED
;
1470 if (mpa
->revision
== 2) {
1471 ep
->mpa_attr
.enhanced_rdma_conn
=
1472 mpa
->flags
& MPA_ENHANCED_RDMA_CONN
? 1 : 0;
1473 if (ep
->mpa_attr
.enhanced_rdma_conn
) {
1474 mpa_v2_params
= (struct mpa_v2_conn_params
*)
1475 (ep
->mpa_pkt
+ sizeof(*mpa
));
1476 resp_ird
= ntohs(mpa_v2_params
->ird
) &
1477 MPA_V2_IRD_ORD_MASK
;
1478 resp_ord
= ntohs(mpa_v2_params
->ord
) &
1479 MPA_V2_IRD_ORD_MASK
;
1480 PDBG("%s responder ird %u ord %u ep ird %u ord %u\n",
1481 __func__
, resp_ird
, resp_ord
, ep
->ird
, ep
->ord
);
1484 * This is a double-check. Ideally, below checks are
1485 * not required since ird/ord stuff has been taken
1486 * care of in c4iw_accept_cr
1488 if (ep
->ird
< resp_ord
) {
1489 if (RELAXED_IRD_NEGOTIATION
&& resp_ord
<=
1490 ep
->com
.dev
->rdev
.lldi
.max_ordird_qp
)
1494 } else if (ep
->ird
> resp_ord
) {
1497 if (ep
->ord
> resp_ird
) {
1498 if (RELAXED_IRD_NEGOTIATION
)
1509 if (ntohs(mpa_v2_params
->ird
) &
1510 MPA_V2_PEER2PEER_MODEL
) {
1511 if (ntohs(mpa_v2_params
->ord
) &
1512 MPA_V2_RDMA_WRITE_RTR
)
1513 ep
->mpa_attr
.p2p_type
=
1514 FW_RI_INIT_P2PTYPE_RDMA_WRITE
;
1515 else if (ntohs(mpa_v2_params
->ord
) &
1516 MPA_V2_RDMA_READ_RTR
)
1517 ep
->mpa_attr
.p2p_type
=
1518 FW_RI_INIT_P2PTYPE_READ_REQ
;
1521 } else if (mpa
->revision
== 1)
1523 ep
->mpa_attr
.p2p_type
= p2p_type
;
1525 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
1526 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = "
1527 "%d\n", __func__
, ep
->mpa_attr
.crc_enabled
,
1528 ep
->mpa_attr
.recv_marker_enabled
,
1529 ep
->mpa_attr
.xmit_marker_enabled
, ep
->mpa_attr
.version
,
1530 ep
->mpa_attr
.p2p_type
, p2p_type
);
1533 * If responder's RTR does not match with that of initiator, assign
1534 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not
1535 * generated when moving QP to RTS state.
1536 * A TERM message will be sent after QP has moved to RTS state
1538 if ((ep
->mpa_attr
.version
== 2) && peer2peer
&&
1539 (ep
->mpa_attr
.p2p_type
!= p2p_type
)) {
1540 ep
->mpa_attr
.p2p_type
= FW_RI_INIT_P2PTYPE_DISABLED
;
1544 attrs
.mpa_attr
= ep
->mpa_attr
;
1545 attrs
.max_ird
= ep
->ird
;
1546 attrs
.max_ord
= ep
->ord
;
1547 attrs
.llp_stream_handle
= ep
;
1548 attrs
.next_state
= C4IW_QP_STATE_RTS
;
1550 mask
= C4IW_QP_ATTR_NEXT_STATE
|
1551 C4IW_QP_ATTR_LLP_STREAM_HANDLE
| C4IW_QP_ATTR_MPA_ATTR
|
1552 C4IW_QP_ATTR_MAX_IRD
| C4IW_QP_ATTR_MAX_ORD
;
1554 /* bind QP and TID with INIT_WR */
1555 err
= c4iw_modify_qp(ep
->com
.qp
->rhp
,
1556 ep
->com
.qp
, mask
, &attrs
, 1);
1561 * If responder's RTR requirement did not match with what initiator
1562 * supports, generate TERM message
1565 printk(KERN_ERR
"%s: RTR mismatch, sending TERM\n", __func__
);
1566 attrs
.layer_etype
= LAYER_MPA
| DDP_LLP
;
1567 attrs
.ecode
= MPA_NOMATCH_RTR
;
1568 attrs
.next_state
= C4IW_QP_STATE_TERMINATE
;
1569 attrs
.send_term
= 1;
1570 err
= c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
1571 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
1578 * Generate TERM if initiator IRD is not sufficient for responder
1579 * provided ORD. Currently, we do the same behaviour even when
1580 * responder provided IRD is also not sufficient as regards to
1584 printk(KERN_ERR
"%s: Insufficient IRD, sending TERM\n",
1586 attrs
.layer_etype
= LAYER_MPA
| DDP_LLP
;
1587 attrs
.ecode
= MPA_INSUFF_IRD
;
1588 attrs
.next_state
= C4IW_QP_STATE_TERMINATE
;
1589 attrs
.send_term
= 1;
1590 err
= c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
1591 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
1600 connect_reply_upcall(ep
, err
);
1604 static void process_mpa_request(struct c4iw_ep
*ep
, struct sk_buff
*skb
)
1606 struct mpa_message
*mpa
;
1607 struct mpa_v2_conn_params
*mpa_v2_params
;
1610 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1613 * If we get more than the supported amount of private data
1614 * then we must fail this connection.
1616 if (ep
->mpa_pkt_len
+ skb
->len
> sizeof(ep
->mpa_pkt
)) {
1617 (void)stop_ep_timer(ep
);
1618 abort_connection(ep
, skb
, GFP_KERNEL
);
1622 PDBG("%s enter (%s line %u)\n", __func__
, __FILE__
, __LINE__
);
1625 * Copy the new data into our accumulation buffer.
1627 skb_copy_from_linear_data(skb
, &(ep
->mpa_pkt
[ep
->mpa_pkt_len
]),
1629 ep
->mpa_pkt_len
+= skb
->len
;
1632 * If we don't even have the mpa message, then bail.
1633 * We'll continue process when more data arrives.
1635 if (ep
->mpa_pkt_len
< sizeof(*mpa
))
1638 PDBG("%s enter (%s line %u)\n", __func__
, __FILE__
, __LINE__
);
1639 mpa
= (struct mpa_message
*) ep
->mpa_pkt
;
1642 * Validate MPA Header.
1644 if (mpa
->revision
> mpa_rev
) {
1645 printk(KERN_ERR MOD
"%s MPA version mismatch. Local = %d,"
1646 " Received = %d\n", __func__
, mpa_rev
, mpa
->revision
);
1647 (void)stop_ep_timer(ep
);
1648 abort_connection(ep
, skb
, GFP_KERNEL
);
1652 if (memcmp(mpa
->key
, MPA_KEY_REQ
, sizeof(mpa
->key
))) {
1653 (void)stop_ep_timer(ep
);
1654 abort_connection(ep
, skb
, GFP_KERNEL
);
1658 plen
= ntohs(mpa
->private_data_size
);
1661 * Fail if there's too much private data.
1663 if (plen
> MPA_MAX_PRIVATE_DATA
) {
1664 (void)stop_ep_timer(ep
);
1665 abort_connection(ep
, skb
, GFP_KERNEL
);
1670 * If plen does not account for pkt size
1672 if (ep
->mpa_pkt_len
> (sizeof(*mpa
) + plen
)) {
1673 (void)stop_ep_timer(ep
);
1674 abort_connection(ep
, skb
, GFP_KERNEL
);
1677 ep
->plen
= (u8
) plen
;
1680 * If we don't have all the pdata yet, then bail.
1682 if (ep
->mpa_pkt_len
< (sizeof(*mpa
) + plen
))
1686 * If we get here we have accumulated the entire mpa
1687 * start reply message including private data.
1689 ep
->mpa_attr
.initiator
= 0;
1690 ep
->mpa_attr
.crc_enabled
= (mpa
->flags
& MPA_CRC
) | crc_enabled
? 1 : 0;
1691 ep
->mpa_attr
.recv_marker_enabled
= markers_enabled
;
1692 ep
->mpa_attr
.xmit_marker_enabled
= mpa
->flags
& MPA_MARKERS
? 1 : 0;
1693 ep
->mpa_attr
.version
= mpa
->revision
;
1694 if (mpa
->revision
== 1)
1695 ep
->tried_with_mpa_v1
= 1;
1696 ep
->mpa_attr
.p2p_type
= FW_RI_INIT_P2PTYPE_DISABLED
;
1698 if (mpa
->revision
== 2) {
1699 ep
->mpa_attr
.enhanced_rdma_conn
=
1700 mpa
->flags
& MPA_ENHANCED_RDMA_CONN
? 1 : 0;
1701 if (ep
->mpa_attr
.enhanced_rdma_conn
) {
1702 mpa_v2_params
= (struct mpa_v2_conn_params
*)
1703 (ep
->mpa_pkt
+ sizeof(*mpa
));
1704 ep
->ird
= ntohs(mpa_v2_params
->ird
) &
1705 MPA_V2_IRD_ORD_MASK
;
1706 ep
->ord
= ntohs(mpa_v2_params
->ord
) &
1707 MPA_V2_IRD_ORD_MASK
;
1708 PDBG("%s initiator ird %u ord %u\n", __func__
, ep
->ird
,
1710 if (ntohs(mpa_v2_params
->ird
) & MPA_V2_PEER2PEER_MODEL
)
1712 if (ntohs(mpa_v2_params
->ord
) &
1713 MPA_V2_RDMA_WRITE_RTR
)
1714 ep
->mpa_attr
.p2p_type
=
1715 FW_RI_INIT_P2PTYPE_RDMA_WRITE
;
1716 else if (ntohs(mpa_v2_params
->ord
) &
1717 MPA_V2_RDMA_READ_RTR
)
1718 ep
->mpa_attr
.p2p_type
=
1719 FW_RI_INIT_P2PTYPE_READ_REQ
;
1722 } else if (mpa
->revision
== 1)
1724 ep
->mpa_attr
.p2p_type
= p2p_type
;
1726 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
1727 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__
,
1728 ep
->mpa_attr
.crc_enabled
, ep
->mpa_attr
.recv_marker_enabled
,
1729 ep
->mpa_attr
.xmit_marker_enabled
, ep
->mpa_attr
.version
,
1730 ep
->mpa_attr
.p2p_type
);
1733 * If the endpoint timer already expired, then we ignore
1734 * the start request. process_timeout() will abort
1737 if (!stop_ep_timer(ep
)) {
1738 __state_set(&ep
->com
, MPA_REQ_RCVD
);
1741 mutex_lock_nested(&ep
->parent_ep
->com
.mutex
,
1742 SINGLE_DEPTH_NESTING
);
1743 if (ep
->parent_ep
->com
.state
!= DEAD
) {
1744 if (connect_request_upcall(ep
))
1745 abort_connection(ep
, skb
, GFP_KERNEL
);
1747 abort_connection(ep
, skb
, GFP_KERNEL
);
1749 mutex_unlock(&ep
->parent_ep
->com
.mutex
);
1754 static int rx_data(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
1757 struct cpl_rx_data
*hdr
= cplhdr(skb
);
1758 unsigned int dlen
= ntohs(hdr
->len
);
1759 unsigned int tid
= GET_TID(hdr
);
1760 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
1761 __u8 status
= hdr
->status
;
1764 ep
= lookup_tid(t
, tid
);
1767 PDBG("%s ep %p tid %u dlen %u\n", __func__
, ep
, ep
->hwtid
, dlen
);
1768 skb_pull(skb
, sizeof(*hdr
));
1769 skb_trim(skb
, dlen
);
1770 mutex_lock(&ep
->com
.mutex
);
1772 /* update RX credits */
1773 update_rx_credits(ep
, dlen
);
1775 switch (ep
->com
.state
) {
1777 ep
->rcv_seq
+= dlen
;
1778 disconnect
= process_mpa_reply(ep
, skb
);
1781 ep
->rcv_seq
+= dlen
;
1782 process_mpa_request(ep
, skb
);
1785 struct c4iw_qp_attributes attrs
;
1786 BUG_ON(!ep
->com
.qp
);
1788 pr_err("%s Unexpected streaming data." \
1789 " qpid %u ep %p state %d tid %u status %d\n",
1790 __func__
, ep
->com
.qp
->wq
.sq
.qid
, ep
,
1791 ep
->com
.state
, ep
->hwtid
, status
);
1792 attrs
.next_state
= C4IW_QP_STATE_TERMINATE
;
1793 c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
1794 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
1801 mutex_unlock(&ep
->com
.mutex
);
1803 c4iw_ep_disconnect(ep
, 0, GFP_KERNEL
);
1807 static int abort_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
1810 struct cpl_abort_rpl_rss
*rpl
= cplhdr(skb
);
1812 unsigned int tid
= GET_TID(rpl
);
1813 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
1815 ep
= lookup_tid(t
, tid
);
1817 printk(KERN_WARNING MOD
"Abort rpl to freed endpoint\n");
1820 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
1821 mutex_lock(&ep
->com
.mutex
);
1822 switch (ep
->com
.state
) {
1824 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
1825 __state_set(&ep
->com
, DEAD
);
1829 printk(KERN_ERR
"%s ep %p state %d\n",
1830 __func__
, ep
, ep
->com
.state
);
1833 mutex_unlock(&ep
->com
.mutex
);
1836 release_ep_resources(ep
);
1840 static void send_fw_act_open_req(struct c4iw_ep
*ep
, unsigned int atid
)
1842 struct sk_buff
*skb
;
1843 struct fw_ofld_connection_wr
*req
;
1844 unsigned int mtu_idx
;
1846 struct sockaddr_in
*sin
;
1849 skb
= get_skb(NULL
, sizeof(*req
), GFP_KERNEL
);
1850 req
= (struct fw_ofld_connection_wr
*)__skb_put(skb
, sizeof(*req
));
1851 memset(req
, 0, sizeof(*req
));
1852 req
->op_compl
= htonl(WR_OP_V(FW_OFLD_CONNECTION_WR
));
1853 req
->len16_pkd
= htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req
), 16)));
1854 req
->le
.filter
= cpu_to_be32(cxgb4_select_ntuple(
1855 ep
->com
.dev
->rdev
.lldi
.ports
[0],
1857 sin
= (struct sockaddr_in
*)&ep
->com
.local_addr
;
1858 req
->le
.lport
= sin
->sin_port
;
1859 req
->le
.u
.ipv4
.lip
= sin
->sin_addr
.s_addr
;
1860 sin
= (struct sockaddr_in
*)&ep
->com
.remote_addr
;
1861 req
->le
.pport
= sin
->sin_port
;
1862 req
->le
.u
.ipv4
.pip
= sin
->sin_addr
.s_addr
;
1863 req
->tcb
.t_state_to_astid
=
1864 htonl(FW_OFLD_CONNECTION_WR_T_STATE_V(TCP_SYN_SENT
) |
1865 FW_OFLD_CONNECTION_WR_ASTID_V(atid
));
1866 req
->tcb
.cplrxdataack_cplpassacceptrpl
=
1867 htons(FW_OFLD_CONNECTION_WR_CPLRXDATAACK_F
);
1868 req
->tcb
.tx_max
= (__force __be32
) jiffies
;
1869 req
->tcb
.rcv_adv
= htons(1);
1870 best_mtu(ep
->com
.dev
->rdev
.lldi
.mtus
, ep
->mtu
, &mtu_idx
,
1871 enable_tcp_timestamps
,
1872 (AF_INET
== ep
->com
.remote_addr
.ss_family
) ? 0 : 1);
1873 wscale
= compute_wscale(rcv_win
);
1876 * Specify the largest window that will fit in opt0. The
1877 * remainder will be specified in the rx_data_ack.
1879 win
= ep
->rcv_win
>> 10;
1880 if (win
> RCV_BUFSIZ_M
)
1883 req
->tcb
.opt0
= (__force __be64
) (TCAM_BYPASS_F
|
1884 (nocong
? NO_CONG_F
: 0) |
1887 WND_SCALE_V(wscale
) |
1888 MSS_IDX_V(mtu_idx
) |
1889 L2T_IDX_V(ep
->l2t
->idx
) |
1890 TX_CHAN_V(ep
->tx_chan
) |
1891 SMAC_SEL_V(ep
->smac_idx
) |
1892 DSCP_V(ep
->tos
>> 2) |
1893 ULP_MODE_V(ULP_MODE_TCPDDP
) |
1895 req
->tcb
.opt2
= (__force __be32
) (PACE_V(1) |
1896 TX_QUEUE_V(ep
->com
.dev
->rdev
.lldi
.tx_modq
[ep
->tx_chan
]) |
1898 CCTRL_ECN_V(enable_ecn
) |
1899 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(ep
->rss_qid
));
1900 if (enable_tcp_timestamps
)
1901 req
->tcb
.opt2
|= (__force __be32
)TSTAMPS_EN_F
;
1902 if (enable_tcp_sack
)
1903 req
->tcb
.opt2
|= (__force __be32
)SACK_EN_F
;
1904 if (wscale
&& enable_tcp_window_scaling
)
1905 req
->tcb
.opt2
|= (__force __be32
)WND_SCALE_EN_F
;
1906 req
->tcb
.opt0
= cpu_to_be64((__force u64
)req
->tcb
.opt0
);
1907 req
->tcb
.opt2
= cpu_to_be32((__force u32
)req
->tcb
.opt2
);
1908 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, ep
->ctrlq_idx
);
1909 set_bit(ACT_OFLD_CONN
, &ep
->com
.history
);
1910 c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
1914 * Return whether a failed active open has allocated a TID
1916 static inline int act_open_has_tid(int status
)
1918 return status
!= CPL_ERR_TCAM_FULL
&& status
!= CPL_ERR_CONN_EXIST
&&
1919 status
!= CPL_ERR_ARP_MISS
;
1922 /* Returns whether a CPL status conveys negative advice.
1924 static int is_neg_adv(unsigned int status
)
1926 return status
== CPL_ERR_RTX_NEG_ADVICE
||
1927 status
== CPL_ERR_PERSIST_NEG_ADVICE
||
1928 status
== CPL_ERR_KEEPALV_NEG_ADVICE
;
1931 static char *neg_adv_str(unsigned int status
)
1934 case CPL_ERR_RTX_NEG_ADVICE
:
1935 return "Retransmit timeout";
1936 case CPL_ERR_PERSIST_NEG_ADVICE
:
1937 return "Persist timeout";
1938 case CPL_ERR_KEEPALV_NEG_ADVICE
:
1939 return "Keepalive timeout";
1945 static void set_tcp_window(struct c4iw_ep
*ep
, struct port_info
*pi
)
1947 ep
->snd_win
= snd_win
;
1948 ep
->rcv_win
= rcv_win
;
1949 PDBG("%s snd_win %d rcv_win %d\n", __func__
, ep
->snd_win
, ep
->rcv_win
);
1952 #define ACT_OPEN_RETRY_COUNT 2
1954 static int import_ep(struct c4iw_ep
*ep
, int iptype
, __u8
*peer_ip
,
1955 struct dst_entry
*dst
, struct c4iw_dev
*cdev
,
1956 bool clear_mpa_v1
, enum chip_type adapter_type
, u8 tos
)
1958 struct neighbour
*n
;
1960 struct net_device
*pdev
;
1962 n
= dst_neigh_lookup(dst
, peer_ip
);
1968 if (n
->dev
->flags
& IFF_LOOPBACK
) {
1970 pdev
= ip_dev_find(&init_net
, *(__be32
*)peer_ip
);
1971 else if (IS_ENABLED(CONFIG_IPV6
))
1972 for_each_netdev(&init_net
, pdev
) {
1973 if (ipv6_chk_addr(&init_net
,
1974 (struct in6_addr
*)peer_ip
,
1985 ep
->l2t
= cxgb4_l2t_get(cdev
->rdev
.lldi
.l2t
,
1986 n
, pdev
, rt_tos2priority(tos
));
1989 ep
->mtu
= pdev
->mtu
;
1990 ep
->tx_chan
= cxgb4_port_chan(pdev
);
1991 ep
->smac_idx
= cxgb4_tp_smt_idx(adapter_type
,
1992 cxgb4_port_viid(pdev
));
1993 step
= cdev
->rdev
.lldi
.ntxq
/
1994 cdev
->rdev
.lldi
.nchan
;
1995 ep
->txq_idx
= cxgb4_port_idx(pdev
) * step
;
1996 step
= cdev
->rdev
.lldi
.nrxq
/
1997 cdev
->rdev
.lldi
.nchan
;
1998 ep
->ctrlq_idx
= cxgb4_port_idx(pdev
);
1999 ep
->rss_qid
= cdev
->rdev
.lldi
.rxq_ids
[
2000 cxgb4_port_idx(pdev
) * step
];
2001 set_tcp_window(ep
, (struct port_info
*)netdev_priv(pdev
));
2004 pdev
= get_real_dev(n
->dev
);
2005 ep
->l2t
= cxgb4_l2t_get(cdev
->rdev
.lldi
.l2t
,
2009 ep
->mtu
= dst_mtu(dst
);
2010 ep
->tx_chan
= cxgb4_port_chan(pdev
);
2011 ep
->smac_idx
= cxgb4_tp_smt_idx(adapter_type
,
2012 cxgb4_port_viid(pdev
));
2013 step
= cdev
->rdev
.lldi
.ntxq
/
2014 cdev
->rdev
.lldi
.nchan
;
2015 ep
->txq_idx
= cxgb4_port_idx(pdev
) * step
;
2016 ep
->ctrlq_idx
= cxgb4_port_idx(pdev
);
2017 step
= cdev
->rdev
.lldi
.nrxq
/
2018 cdev
->rdev
.lldi
.nchan
;
2019 ep
->rss_qid
= cdev
->rdev
.lldi
.rxq_ids
[
2020 cxgb4_port_idx(pdev
) * step
];
2021 set_tcp_window(ep
, (struct port_info
*)netdev_priv(pdev
));
2024 ep
->retry_with_mpa_v1
= 0;
2025 ep
->tried_with_mpa_v1
= 0;
2037 static int c4iw_reconnect(struct c4iw_ep
*ep
)
2040 struct sockaddr_in
*laddr
= (struct sockaddr_in
*)
2041 &ep
->com
.cm_id
->m_local_addr
;
2042 struct sockaddr_in
*raddr
= (struct sockaddr_in
*)
2043 &ep
->com
.cm_id
->m_remote_addr
;
2044 struct sockaddr_in6
*laddr6
= (struct sockaddr_in6
*)
2045 &ep
->com
.cm_id
->m_local_addr
;
2046 struct sockaddr_in6
*raddr6
= (struct sockaddr_in6
*)
2047 &ep
->com
.cm_id
->m_remote_addr
;
2051 PDBG("%s qp %p cm_id %p\n", __func__
, ep
->com
.qp
, ep
->com
.cm_id
);
2052 init_timer(&ep
->timer
);
2055 * Allocate an active TID to initiate a TCP connection.
2057 ep
->atid
= cxgb4_alloc_atid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
);
2058 if (ep
->atid
== -1) {
2059 pr_err("%s - cannot alloc atid.\n", __func__
);
2063 insert_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, ep
, ep
->atid
);
2066 if (ep
->com
.cm_id
->m_local_addr
.ss_family
== AF_INET
) {
2067 ep
->dst
= find_route(ep
->com
.dev
, laddr
->sin_addr
.s_addr
,
2068 raddr
->sin_addr
.s_addr
, laddr
->sin_port
,
2069 raddr
->sin_port
, ep
->com
.cm_id
->tos
);
2071 ra
= (__u8
*)&raddr
->sin_addr
;
2073 ep
->dst
= find_route6(ep
->com
.dev
, laddr6
->sin6_addr
.s6_addr
,
2074 raddr6
->sin6_addr
.s6_addr
,
2075 laddr6
->sin6_port
, raddr6
->sin6_port
, 0,
2076 raddr6
->sin6_scope_id
);
2078 ra
= (__u8
*)&raddr6
->sin6_addr
;
2081 pr_err("%s - cannot find route.\n", __func__
);
2082 err
= -EHOSTUNREACH
;
2085 err
= import_ep(ep
, iptype
, ra
, ep
->dst
, ep
->com
.dev
, false,
2086 ep
->com
.dev
->rdev
.lldi
.adapter_type
,
2087 ep
->com
.cm_id
->tos
);
2089 pr_err("%s - cannot alloc l2e.\n", __func__
);
2093 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n",
2094 __func__
, ep
->txq_idx
, ep
->tx_chan
, ep
->smac_idx
, ep
->rss_qid
,
2097 state_set(&ep
->com
, CONNECTING
);
2098 ep
->tos
= ep
->com
.cm_id
->tos
;
2100 /* send connect request to rnic */
2101 err
= send_connect(ep
);
2105 cxgb4_l2t_release(ep
->l2t
);
2107 dst_release(ep
->dst
);
2109 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, ep
->atid
);
2110 cxgb4_free_atid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
->atid
);
2113 * remember to send notification to upper layer.
2114 * We are in here so the upper layer is not aware that this is
2115 * re-connect attempt and so, upper layer is still waiting for
2116 * response of 1st connect request.
2118 connect_reply_upcall(ep
, -ECONNRESET
);
2119 c4iw_put_ep(&ep
->com
);
2124 static int act_open_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2127 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
2128 unsigned int atid
= TID_TID_G(AOPEN_ATID_G(
2129 ntohl(rpl
->atid_status
)));
2130 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2131 int status
= AOPEN_STATUS_G(ntohl(rpl
->atid_status
));
2132 struct sockaddr_in
*la
;
2133 struct sockaddr_in
*ra
;
2134 struct sockaddr_in6
*la6
;
2135 struct sockaddr_in6
*ra6
;
2137 ep
= lookup_atid(t
, atid
);
2138 la
= (struct sockaddr_in
*)&ep
->com
.local_addr
;
2139 ra
= (struct sockaddr_in
*)&ep
->com
.remote_addr
;
2140 la6
= (struct sockaddr_in6
*)&ep
->com
.local_addr
;
2141 ra6
= (struct sockaddr_in6
*)&ep
->com
.remote_addr
;
2143 PDBG("%s ep %p atid %u status %u errno %d\n", __func__
, ep
, atid
,
2144 status
, status2errno(status
));
2146 if (is_neg_adv(status
)) {
2147 PDBG("%s Connection problems for atid %u status %u (%s)\n",
2148 __func__
, atid
, status
, neg_adv_str(status
));
2149 ep
->stats
.connect_neg_adv
++;
2150 mutex_lock(&dev
->rdev
.stats
.lock
);
2151 dev
->rdev
.stats
.neg_adv
++;
2152 mutex_unlock(&dev
->rdev
.stats
.lock
);
2156 set_bit(ACT_OPEN_RPL
, &ep
->com
.history
);
2159 * Log interesting failures.
2162 case CPL_ERR_CONN_RESET
:
2163 case CPL_ERR_CONN_TIMEDOUT
:
2165 case CPL_ERR_TCAM_FULL
:
2166 mutex_lock(&dev
->rdev
.stats
.lock
);
2167 dev
->rdev
.stats
.tcam_full
++;
2168 mutex_unlock(&dev
->rdev
.stats
.lock
);
2169 if (ep
->com
.local_addr
.ss_family
== AF_INET
&&
2170 dev
->rdev
.lldi
.enable_fw_ofld_conn
) {
2171 send_fw_act_open_req(ep
,
2172 TID_TID_G(AOPEN_ATID_G(
2173 ntohl(rpl
->atid_status
))));
2177 case CPL_ERR_CONN_EXIST
:
2178 if (ep
->retry_count
++ < ACT_OPEN_RETRY_COUNT
) {
2179 set_bit(ACT_RETRY_INUSE
, &ep
->com
.history
);
2180 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
2181 struct sockaddr_in6
*sin6
=
2182 (struct sockaddr_in6
*)
2183 &ep
->com
.local_addr
;
2185 ep
->com
.dev
->rdev
.lldi
.ports
[0],
2187 &sin6
->sin6_addr
.s6_addr
, 1);
2189 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
,
2191 cxgb4_free_atid(t
, atid
);
2192 dst_release(ep
->dst
);
2193 cxgb4_l2t_release(ep
->l2t
);
2199 if (ep
->com
.local_addr
.ss_family
== AF_INET
) {
2200 pr_info("Active open failure - atid %u status %u errno %d %pI4:%u->%pI4:%u\n",
2201 atid
, status
, status2errno(status
),
2202 &la
->sin_addr
.s_addr
, ntohs(la
->sin_port
),
2203 &ra
->sin_addr
.s_addr
, ntohs(ra
->sin_port
));
2205 pr_info("Active open failure - atid %u status %u errno %d %pI6:%u->%pI6:%u\n",
2206 atid
, status
, status2errno(status
),
2207 la6
->sin6_addr
.s6_addr
, ntohs(la6
->sin6_port
),
2208 ra6
->sin6_addr
.s6_addr
, ntohs(ra6
->sin6_port
));
2213 connect_reply_upcall(ep
, status2errno(status
));
2214 state_set(&ep
->com
, DEAD
);
2216 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
2217 struct sockaddr_in6
*sin6
=
2218 (struct sockaddr_in6
*)&ep
->com
.local_addr
;
2219 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
2220 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
2222 if (status
&& act_open_has_tid(status
))
2223 cxgb4_remove_tid(ep
->com
.dev
->rdev
.lldi
.tids
, 0, GET_TID(rpl
));
2225 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, atid
);
2226 cxgb4_free_atid(t
, atid
);
2227 dst_release(ep
->dst
);
2228 cxgb4_l2t_release(ep
->l2t
);
2229 c4iw_put_ep(&ep
->com
);
2234 static int pass_open_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2236 struct cpl_pass_open_rpl
*rpl
= cplhdr(skb
);
2237 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2238 unsigned int stid
= GET_TID(rpl
);
2239 struct c4iw_listen_ep
*ep
= lookup_stid(t
, stid
);
2242 PDBG("%s stid %d lookup failure!\n", __func__
, stid
);
2245 PDBG("%s ep %p status %d error %d\n", __func__
, ep
,
2246 rpl
->status
, status2errno(rpl
->status
));
2247 c4iw_wake_up(&ep
->com
.wr_wait
, status2errno(rpl
->status
));
2253 static int close_listsrv_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2255 struct cpl_close_listsvr_rpl
*rpl
= cplhdr(skb
);
2256 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2257 unsigned int stid
= GET_TID(rpl
);
2258 struct c4iw_listen_ep
*ep
= lookup_stid(t
, stid
);
2260 PDBG("%s ep %p\n", __func__
, ep
);
2261 c4iw_wake_up(&ep
->com
.wr_wait
, status2errno(rpl
->status
));
2265 static int accept_cr(struct c4iw_ep
*ep
, struct sk_buff
*skb
,
2266 struct cpl_pass_accept_req
*req
)
2268 struct cpl_pass_accept_rpl
*rpl
;
2269 unsigned int mtu_idx
;
2273 struct cpl_t5_pass_accept_rpl
*rpl5
= NULL
;
2275 enum chip_type adapter_type
= ep
->com
.dev
->rdev
.lldi
.adapter_type
;
2277 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2278 BUG_ON(skb_cloned(skb
));
2282 if (!is_t4(adapter_type
)) {
2283 skb_trim(skb
, roundup(sizeof(*rpl5
), 16));
2285 INIT_TP_WR(rpl5
, ep
->hwtid
);
2287 skb_trim(skb
, sizeof(*rpl
));
2288 INIT_TP_WR(rpl
, ep
->hwtid
);
2290 OPCODE_TID(rpl
) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL
,
2293 best_mtu(ep
->com
.dev
->rdev
.lldi
.mtus
, ep
->mtu
, &mtu_idx
,
2294 enable_tcp_timestamps
&& req
->tcpopt
.tstamp
,
2295 (AF_INET
== ep
->com
.remote_addr
.ss_family
) ? 0 : 1);
2296 wscale
= compute_wscale(rcv_win
);
2299 * Specify the largest window that will fit in opt0. The
2300 * remainder will be specified in the rx_data_ack.
2302 win
= ep
->rcv_win
>> 10;
2303 if (win
> RCV_BUFSIZ_M
)
2305 opt0
= (nocong
? NO_CONG_F
: 0) |
2308 WND_SCALE_V(wscale
) |
2309 MSS_IDX_V(mtu_idx
) |
2310 L2T_IDX_V(ep
->l2t
->idx
) |
2311 TX_CHAN_V(ep
->tx_chan
) |
2312 SMAC_SEL_V(ep
->smac_idx
) |
2313 DSCP_V(ep
->tos
>> 2) |
2314 ULP_MODE_V(ULP_MODE_TCPDDP
) |
2316 opt2
= RX_CHANNEL_V(0) |
2317 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(ep
->rss_qid
);
2319 if (enable_tcp_timestamps
&& req
->tcpopt
.tstamp
)
2320 opt2
|= TSTAMPS_EN_F
;
2321 if (enable_tcp_sack
&& req
->tcpopt
.sack
)
2323 if (wscale
&& enable_tcp_window_scaling
)
2324 opt2
|= WND_SCALE_EN_F
;
2326 const struct tcphdr
*tcph
;
2327 u32 hlen
= ntohl(req
->hdr_len
);
2329 if (CHELSIO_CHIP_VERSION(adapter_type
) <= CHELSIO_T5
)
2330 tcph
= (const void *)(req
+ 1) + ETH_HDR_LEN_G(hlen
) +
2333 tcph
= (const void *)(req
+ 1) +
2334 T6_ETH_HDR_LEN_G(hlen
) + T6_IP_HDR_LEN_G(hlen
);
2335 if (tcph
->ece
&& tcph
->cwr
)
2336 opt2
|= CCTRL_ECN_V(1);
2338 if (CHELSIO_CHIP_VERSION(adapter_type
) > CHELSIO_T4
) {
2339 u32 isn
= (prandom_u32() & ~7UL) - 1;
2340 opt2
|= T5_OPT_2_VALID_F
;
2341 opt2
|= CONG_CNTRL_V(CONG_ALG_TAHOE
);
2344 memset(&rpl5
->iss
, 0, roundup(sizeof(*rpl5
)-sizeof(*rpl
), 16));
2347 rpl5
->iss
= cpu_to_be32(isn
);
2348 PDBG("%s iss %u\n", __func__
, be32_to_cpu(rpl5
->iss
));
2351 rpl
->opt0
= cpu_to_be64(opt0
);
2352 rpl
->opt2
= cpu_to_be32(opt2
);
2353 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, ep
->ctrlq_idx
);
2354 t4_set_arp_err_handler(skb
, ep
, pass_accept_rpl_arp_failure
);
2356 return c4iw_l2t_send(&ep
->com
.dev
->rdev
, skb
, ep
->l2t
);
2359 static void reject_cr(struct c4iw_dev
*dev
, u32 hwtid
, struct sk_buff
*skb
)
2361 PDBG("%s c4iw_dev %p tid %u\n", __func__
, dev
, hwtid
);
2362 BUG_ON(skb_cloned(skb
));
2363 skb_trim(skb
, sizeof(struct cpl_tid_release
));
2364 release_tid(&dev
->rdev
, hwtid
, skb
);
2368 static void get_4tuple(struct cpl_pass_accept_req
*req
, enum chip_type type
,
2369 int *iptype
, __u8
*local_ip
, __u8
*peer_ip
,
2370 __be16
*local_port
, __be16
*peer_port
)
2372 int eth_len
= (CHELSIO_CHIP_VERSION(type
) <= CHELSIO_T5
) ?
2373 ETH_HDR_LEN_G(be32_to_cpu(req
->hdr_len
)) :
2374 T6_ETH_HDR_LEN_G(be32_to_cpu(req
->hdr_len
));
2375 int ip_len
= (CHELSIO_CHIP_VERSION(type
) <= CHELSIO_T5
) ?
2376 IP_HDR_LEN_G(be32_to_cpu(req
->hdr_len
)) :
2377 T6_IP_HDR_LEN_G(be32_to_cpu(req
->hdr_len
));
2378 struct iphdr
*ip
= (struct iphdr
*)((u8
*)(req
+ 1) + eth_len
);
2379 struct ipv6hdr
*ip6
= (struct ipv6hdr
*)((u8
*)(req
+ 1) + eth_len
);
2380 struct tcphdr
*tcp
= (struct tcphdr
*)
2381 ((u8
*)(req
+ 1) + eth_len
+ ip_len
);
2383 if (ip
->version
== 4) {
2384 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__
,
2385 ntohl(ip
->saddr
), ntohl(ip
->daddr
), ntohs(tcp
->source
),
2388 memcpy(peer_ip
, &ip
->saddr
, 4);
2389 memcpy(local_ip
, &ip
->daddr
, 4);
2391 PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__
,
2392 ip6
->saddr
.s6_addr
, ip6
->daddr
.s6_addr
, ntohs(tcp
->source
),
2395 memcpy(peer_ip
, ip6
->saddr
.s6_addr
, 16);
2396 memcpy(local_ip
, ip6
->daddr
.s6_addr
, 16);
2398 *peer_port
= tcp
->source
;
2399 *local_port
= tcp
->dest
;
2404 static int pass_accept_req(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2406 struct c4iw_ep
*child_ep
= NULL
, *parent_ep
;
2407 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
2408 unsigned int stid
= PASS_OPEN_TID_G(ntohl(req
->tos_stid
));
2409 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2410 unsigned int hwtid
= GET_TID(req
);
2411 struct dst_entry
*dst
;
2412 __u8 local_ip
[16], peer_ip
[16];
2413 __be16 local_port
, peer_port
;
2414 struct sockaddr_in6
*sin6
;
2416 u16 peer_mss
= ntohs(req
->tcpopt
.mss
);
2418 unsigned short hdrs
;
2419 u8 tos
= PASS_OPEN_TOS_G(ntohl(req
->tos_stid
));
2421 parent_ep
= lookup_stid(t
, stid
);
2423 PDBG("%s connect request on invalid stid %d\n", __func__
, stid
);
2427 if (state_read(&parent_ep
->com
) != LISTEN
) {
2428 PDBG("%s - listening ep not in LISTEN\n", __func__
);
2432 get_4tuple(req
, parent_ep
->com
.dev
->rdev
.lldi
.adapter_type
, &iptype
,
2433 local_ip
, peer_ip
, &local_port
, &peer_port
);
2435 /* Find output route */
2437 PDBG("%s parent ep %p hwtid %u laddr %pI4 raddr %pI4 lport %d rport %d peer_mss %d\n"
2438 , __func__
, parent_ep
, hwtid
,
2439 local_ip
, peer_ip
, ntohs(local_port
),
2440 ntohs(peer_port
), peer_mss
);
2441 dst
= find_route(dev
, *(__be32
*)local_ip
, *(__be32
*)peer_ip
,
2442 local_port
, peer_port
,
2445 PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n"
2446 , __func__
, parent_ep
, hwtid
,
2447 local_ip
, peer_ip
, ntohs(local_port
),
2448 ntohs(peer_port
), peer_mss
);
2449 dst
= find_route6(dev
, local_ip
, peer_ip
, local_port
, peer_port
,
2450 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)),
2451 ((struct sockaddr_in6
*)
2452 &parent_ep
->com
.local_addr
)->sin6_scope_id
);
2455 printk(KERN_ERR MOD
"%s - failed to find dst entry!\n",
2460 child_ep
= alloc_ep(sizeof(*child_ep
), GFP_KERNEL
);
2462 printk(KERN_ERR MOD
"%s - failed to allocate ep entry!\n",
2468 err
= import_ep(child_ep
, iptype
, peer_ip
, dst
, dev
, false,
2469 parent_ep
->com
.dev
->rdev
.lldi
.adapter_type
, tos
);
2471 printk(KERN_ERR MOD
"%s - failed to allocate l2t entry!\n",
2478 hdrs
= sizeof(struct iphdr
) + sizeof(struct tcphdr
) +
2479 ((enable_tcp_timestamps
&& req
->tcpopt
.tstamp
) ? 12 : 0);
2480 if (peer_mss
&& child_ep
->mtu
> (peer_mss
+ hdrs
))
2481 child_ep
->mtu
= peer_mss
+ hdrs
;
2483 state_set(&child_ep
->com
, CONNECTING
);
2484 child_ep
->com
.dev
= dev
;
2485 child_ep
->com
.cm_id
= NULL
;
2488 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
2489 &child_ep
->com
.local_addr
;
2491 sin
->sin_family
= PF_INET
;
2492 sin
->sin_port
= local_port
;
2493 sin
->sin_addr
.s_addr
= *(__be32
*)local_ip
;
2495 sin
= (struct sockaddr_in
*)&child_ep
->com
.local_addr
;
2496 sin
->sin_family
= PF_INET
;
2497 sin
->sin_port
= ((struct sockaddr_in
*)
2498 &parent_ep
->com
.local_addr
)->sin_port
;
2499 sin
->sin_addr
.s_addr
= *(__be32
*)local_ip
;
2501 sin
= (struct sockaddr_in
*)&child_ep
->com
.remote_addr
;
2502 sin
->sin_family
= PF_INET
;
2503 sin
->sin_port
= peer_port
;
2504 sin
->sin_addr
.s_addr
= *(__be32
*)peer_ip
;
2506 sin6
= (struct sockaddr_in6
*)&child_ep
->com
.local_addr
;
2507 sin6
->sin6_family
= PF_INET6
;
2508 sin6
->sin6_port
= local_port
;
2509 memcpy(sin6
->sin6_addr
.s6_addr
, local_ip
, 16);
2511 sin6
= (struct sockaddr_in6
*)&child_ep
->com
.local_addr
;
2512 sin6
->sin6_family
= PF_INET6
;
2513 sin6
->sin6_port
= ((struct sockaddr_in6
*)
2514 &parent_ep
->com
.local_addr
)->sin6_port
;
2515 memcpy(sin6
->sin6_addr
.s6_addr
, local_ip
, 16);
2517 sin6
= (struct sockaddr_in6
*)&child_ep
->com
.remote_addr
;
2518 sin6
->sin6_family
= PF_INET6
;
2519 sin6
->sin6_port
= peer_port
;
2520 memcpy(sin6
->sin6_addr
.s6_addr
, peer_ip
, 16);
2523 c4iw_get_ep(&parent_ep
->com
);
2524 child_ep
->parent_ep
= parent_ep
;
2525 child_ep
->tos
= tos
;
2526 child_ep
->dst
= dst
;
2527 child_ep
->hwtid
= hwtid
;
2529 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__
,
2530 child_ep
->tx_chan
, child_ep
->smac_idx
, child_ep
->rss_qid
);
2532 init_timer(&child_ep
->timer
);
2533 cxgb4_insert_tid(t
, child_ep
, hwtid
);
2534 insert_handle(dev
, &dev
->hwtid_idr
, child_ep
, child_ep
->hwtid
);
2535 if (accept_cr(child_ep
, skb
, req
)) {
2536 c4iw_put_ep(&parent_ep
->com
);
2537 release_ep_resources(child_ep
);
2539 set_bit(PASS_ACCEPT_REQ
, &child_ep
->com
.history
);
2542 sin6
= (struct sockaddr_in6
*)&child_ep
->com
.local_addr
;
2543 cxgb4_clip_get(child_ep
->com
.dev
->rdev
.lldi
.ports
[0],
2544 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
2548 reject_cr(dev
, hwtid
, skb
);
2553 static int pass_establish(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2556 struct cpl_pass_establish
*req
= cplhdr(skb
);
2557 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2558 unsigned int tid
= GET_TID(req
);
2561 ep
= lookup_tid(t
, tid
);
2562 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2563 ep
->snd_seq
= be32_to_cpu(req
->snd_isn
);
2564 ep
->rcv_seq
= be32_to_cpu(req
->rcv_isn
);
2566 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__
, ep
, tid
,
2567 ntohs(req
->tcp_opt
));
2569 set_emss(ep
, ntohs(req
->tcp_opt
));
2571 dst_confirm(ep
->dst
);
2572 mutex_lock(&ep
->com
.mutex
);
2573 ep
->com
.state
= MPA_REQ_WAIT
;
2575 set_bit(PASS_ESTAB
, &ep
->com
.history
);
2576 ret
= send_flowc(ep
, skb
);
2577 mutex_unlock(&ep
->com
.mutex
);
2579 c4iw_ep_disconnect(ep
, 1, GFP_KERNEL
);
2584 static int peer_close(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2586 struct cpl_peer_close
*hdr
= cplhdr(skb
);
2588 struct c4iw_qp_attributes attrs
;
2591 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2592 unsigned int tid
= GET_TID(hdr
);
2595 ep
= lookup_tid(t
, tid
);
2596 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2597 dst_confirm(ep
->dst
);
2599 set_bit(PEER_CLOSE
, &ep
->com
.history
);
2600 mutex_lock(&ep
->com
.mutex
);
2601 switch (ep
->com
.state
) {
2603 __state_set(&ep
->com
, CLOSING
);
2606 __state_set(&ep
->com
, CLOSING
);
2607 connect_reply_upcall(ep
, -ECONNRESET
);
2612 * We're gonna mark this puppy DEAD, but keep
2613 * the reference on it until the ULP accepts or
2614 * rejects the CR. Also wake up anyone waiting
2615 * in rdma connection migration (see c4iw_accept_cr()).
2617 __state_set(&ep
->com
, CLOSING
);
2618 PDBG("waking up ep %p tid %u\n", ep
, ep
->hwtid
);
2619 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
2622 __state_set(&ep
->com
, CLOSING
);
2623 PDBG("waking up ep %p tid %u\n", ep
, ep
->hwtid
);
2624 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
2628 __state_set(&ep
->com
, CLOSING
);
2629 attrs
.next_state
= C4IW_QP_STATE_CLOSING
;
2630 ret
= c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
2631 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
2632 if (ret
!= -ECONNRESET
) {
2633 peer_close_upcall(ep
);
2641 __state_set(&ep
->com
, MORIBUND
);
2645 (void)stop_ep_timer(ep
);
2646 if (ep
->com
.cm_id
&& ep
->com
.qp
) {
2647 attrs
.next_state
= C4IW_QP_STATE_IDLE
;
2648 c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
2649 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
2651 close_complete_upcall(ep
, 0);
2652 __state_set(&ep
->com
, DEAD
);
2662 mutex_unlock(&ep
->com
.mutex
);
2664 c4iw_ep_disconnect(ep
, 0, GFP_KERNEL
);
2666 release_ep_resources(ep
);
2670 static int peer_abort(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2672 struct cpl_abort_req_rss
*req
= cplhdr(skb
);
2674 struct cpl_abort_rpl
*rpl
;
2675 struct sk_buff
*rpl_skb
;
2676 struct c4iw_qp_attributes attrs
;
2679 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2680 unsigned int tid
= GET_TID(req
);
2682 ep
= lookup_tid(t
, tid
);
2683 if (is_neg_adv(req
->status
)) {
2684 PDBG("%s Negative advice on abort- tid %u status %d (%s)\n",
2685 __func__
, ep
->hwtid
, req
->status
,
2686 neg_adv_str(req
->status
));
2687 ep
->stats
.abort_neg_adv
++;
2688 mutex_lock(&dev
->rdev
.stats
.lock
);
2689 dev
->rdev
.stats
.neg_adv
++;
2690 mutex_unlock(&dev
->rdev
.stats
.lock
);
2693 PDBG("%s ep %p tid %u state %u\n", __func__
, ep
, ep
->hwtid
,
2695 set_bit(PEER_ABORT
, &ep
->com
.history
);
2698 * Wake up any threads in rdma_init() or rdma_fini().
2699 * However, this is not needed if com state is just
2702 if (ep
->com
.state
!= MPA_REQ_SENT
)
2703 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
2705 mutex_lock(&ep
->com
.mutex
);
2706 switch (ep
->com
.state
) {
2708 c4iw_put_ep(&ep
->parent_ep
->com
);
2711 (void)stop_ep_timer(ep
);
2714 (void)stop_ep_timer(ep
);
2715 if (mpa_rev
== 1 || (mpa_rev
== 2 && ep
->tried_with_mpa_v1
))
2716 connect_reply_upcall(ep
, -ECONNRESET
);
2719 * we just don't send notification upwards because we
2720 * want to retry with mpa_v1 without upper layers even
2723 * do some housekeeping so as to re-initiate the
2726 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__
,
2728 ep
->retry_with_mpa_v1
= 1;
2740 if (ep
->com
.cm_id
&& ep
->com
.qp
) {
2741 attrs
.next_state
= C4IW_QP_STATE_ERROR
;
2742 ret
= c4iw_modify_qp(ep
->com
.qp
->rhp
,
2743 ep
->com
.qp
, C4IW_QP_ATTR_NEXT_STATE
,
2747 "%s - qp <- error failed!\n",
2750 peer_abort_upcall(ep
);
2755 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__
);
2756 mutex_unlock(&ep
->com
.mutex
);
2762 dst_confirm(ep
->dst
);
2763 if (ep
->com
.state
!= ABORTING
) {
2764 __state_set(&ep
->com
, DEAD
);
2765 /* we don't release if we want to retry with mpa_v1 */
2766 if (!ep
->retry_with_mpa_v1
)
2769 mutex_unlock(&ep
->com
.mutex
);
2771 rpl_skb
= get_skb(skb
, sizeof(*rpl
), GFP_KERNEL
);
2773 printk(KERN_ERR MOD
"%s - cannot allocate skb!\n",
2778 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
2779 rpl
= (struct cpl_abort_rpl
*) skb_put(rpl_skb
, sizeof(*rpl
));
2780 INIT_TP_WR(rpl
, ep
->hwtid
);
2781 OPCODE_TID(rpl
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL
, ep
->hwtid
));
2782 rpl
->cmd
= CPL_ABORT_NO_RST
;
2783 c4iw_ofld_send(&ep
->com
.dev
->rdev
, rpl_skb
);
2786 release_ep_resources(ep
);
2787 else if (ep
->retry_with_mpa_v1
) {
2788 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
2789 struct sockaddr_in6
*sin6
=
2790 (struct sockaddr_in6
*)
2791 &ep
->com
.local_addr
;
2793 ep
->com
.dev
->rdev
.lldi
.ports
[0],
2794 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
2797 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->hwtid_idr
, ep
->hwtid
);
2798 cxgb4_remove_tid(ep
->com
.dev
->rdev
.lldi
.tids
, 0, ep
->hwtid
);
2799 dst_release(ep
->dst
);
2800 cxgb4_l2t_release(ep
->l2t
);
2807 static int close_con_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2810 struct c4iw_qp_attributes attrs
;
2811 struct cpl_close_con_rpl
*rpl
= cplhdr(skb
);
2813 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2814 unsigned int tid
= GET_TID(rpl
);
2816 ep
= lookup_tid(t
, tid
);
2818 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2821 /* The cm_id may be null if we failed to connect */
2822 mutex_lock(&ep
->com
.mutex
);
2823 switch (ep
->com
.state
) {
2825 __state_set(&ep
->com
, MORIBUND
);
2828 (void)stop_ep_timer(ep
);
2829 if ((ep
->com
.cm_id
) && (ep
->com
.qp
)) {
2830 attrs
.next_state
= C4IW_QP_STATE_IDLE
;
2831 c4iw_modify_qp(ep
->com
.qp
->rhp
,
2833 C4IW_QP_ATTR_NEXT_STATE
,
2836 close_complete_upcall(ep
, 0);
2837 __state_set(&ep
->com
, DEAD
);
2847 mutex_unlock(&ep
->com
.mutex
);
2849 release_ep_resources(ep
);
2853 static int terminate(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2855 struct cpl_rdma_terminate
*rpl
= cplhdr(skb
);
2856 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2857 unsigned int tid
= GET_TID(rpl
);
2859 struct c4iw_qp_attributes attrs
;
2861 ep
= lookup_tid(t
, tid
);
2864 if (ep
&& ep
->com
.qp
) {
2865 printk(KERN_WARNING MOD
"TERM received tid %u qpid %u\n", tid
,
2866 ep
->com
.qp
->wq
.sq
.qid
);
2867 attrs
.next_state
= C4IW_QP_STATE_TERMINATE
;
2868 c4iw_modify_qp(ep
->com
.qp
->rhp
, ep
->com
.qp
,
2869 C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
2871 printk(KERN_WARNING MOD
"TERM received tid %u no ep/qp\n", tid
);
2877 * Upcall from the adapter indicating data has been transmitted.
2878 * For us its just the single MPA request or reply. We can now free
2879 * the skb holding the mpa message.
2881 static int fw4_ack(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
2884 struct cpl_fw4_ack
*hdr
= cplhdr(skb
);
2885 u8 credits
= hdr
->credits
;
2886 unsigned int tid
= GET_TID(hdr
);
2887 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
2890 ep
= lookup_tid(t
, tid
);
2891 PDBG("%s ep %p tid %u credits %u\n", __func__
, ep
, ep
->hwtid
, credits
);
2893 PDBG("%s 0 credit ack ep %p tid %u state %u\n",
2894 __func__
, ep
, ep
->hwtid
, state_read(&ep
->com
));
2898 dst_confirm(ep
->dst
);
2900 PDBG("%s last streaming msg ack ep %p tid %u state %u "
2901 "initiator %u freeing skb\n", __func__
, ep
, ep
->hwtid
,
2902 state_read(&ep
->com
), ep
->mpa_attr
.initiator
? 1 : 0);
2903 kfree_skb(ep
->mpa_skb
);
2909 int c4iw_reject_cr(struct iw_cm_id
*cm_id
, const void *pdata
, u8 pdata_len
)
2913 struct c4iw_ep
*ep
= to_ep(cm_id
);
2914 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2916 mutex_lock(&ep
->com
.mutex
);
2917 if (ep
->com
.state
== DEAD
) {
2918 mutex_unlock(&ep
->com
.mutex
);
2919 c4iw_put_ep(&ep
->com
);
2922 set_bit(ULP_REJECT
, &ep
->com
.history
);
2923 BUG_ON(ep
->com
.state
!= MPA_REQ_RCVD
);
2927 err
= send_mpa_reject(ep
, pdata
, pdata_len
);
2930 mutex_unlock(&ep
->com
.mutex
);
2932 err
= c4iw_ep_disconnect(ep
, disconnect
== 2, GFP_KERNEL
);
2933 c4iw_put_ep(&ep
->com
);
2937 int c4iw_accept_cr(struct iw_cm_id
*cm_id
, struct iw_cm_conn_param
*conn_param
)
2940 struct c4iw_qp_attributes attrs
;
2941 enum c4iw_qp_attr_mask mask
;
2942 struct c4iw_ep
*ep
= to_ep(cm_id
);
2943 struct c4iw_dev
*h
= to_c4iw_dev(cm_id
->device
);
2944 struct c4iw_qp
*qp
= get_qhp(h
, conn_param
->qpn
);
2947 PDBG("%s ep %p tid %u\n", __func__
, ep
, ep
->hwtid
);
2949 mutex_lock(&ep
->com
.mutex
);
2950 if (ep
->com
.state
== DEAD
) {
2955 BUG_ON(ep
->com
.state
!= MPA_REQ_RCVD
);
2958 set_bit(ULP_ACCEPT
, &ep
->com
.history
);
2959 if ((conn_param
->ord
> cur_max_read_depth(ep
->com
.dev
)) ||
2960 (conn_param
->ird
> cur_max_read_depth(ep
->com
.dev
))) {
2965 if (ep
->mpa_attr
.version
== 2 && ep
->mpa_attr
.enhanced_rdma_conn
) {
2966 if (conn_param
->ord
> ep
->ird
) {
2967 if (RELAXED_IRD_NEGOTIATION
) {
2970 ep
->ird
= conn_param
->ird
;
2971 ep
->ord
= conn_param
->ord
;
2972 send_mpa_reject(ep
, conn_param
->private_data
,
2973 conn_param
->private_data_len
);
2978 if (conn_param
->ird
< ep
->ord
) {
2979 if (RELAXED_IRD_NEGOTIATION
&&
2980 ep
->ord
<= h
->rdev
.lldi
.max_ordird_qp
) {
2981 conn_param
->ird
= ep
->ord
;
2988 ep
->ird
= conn_param
->ird
;
2989 ep
->ord
= conn_param
->ord
;
2991 if (ep
->mpa_attr
.version
== 1) {
2992 if (peer2peer
&& ep
->ird
== 0)
2996 (ep
->mpa_attr
.p2p_type
!= FW_RI_INIT_P2PTYPE_DISABLED
) &&
2997 (p2p_type
== FW_RI_INIT_P2PTYPE_READ_REQ
) && ep
->ird
== 0)
3001 PDBG("%s %d ird %d ord %d\n", __func__
, __LINE__
, ep
->ird
, ep
->ord
);
3003 cm_id
->add_ref(cm_id
);
3004 ep
->com
.cm_id
= cm_id
;
3008 /* bind QP to EP and move to RTS */
3009 attrs
.mpa_attr
= ep
->mpa_attr
;
3010 attrs
.max_ird
= ep
->ird
;
3011 attrs
.max_ord
= ep
->ord
;
3012 attrs
.llp_stream_handle
= ep
;
3013 attrs
.next_state
= C4IW_QP_STATE_RTS
;
3015 /* bind QP and TID with INIT_WR */
3016 mask
= C4IW_QP_ATTR_NEXT_STATE
|
3017 C4IW_QP_ATTR_LLP_STREAM_HANDLE
|
3018 C4IW_QP_ATTR_MPA_ATTR
|
3019 C4IW_QP_ATTR_MAX_IRD
|
3020 C4IW_QP_ATTR_MAX_ORD
;
3022 err
= c4iw_modify_qp(ep
->com
.qp
->rhp
,
3023 ep
->com
.qp
, mask
, &attrs
, 1);
3025 goto err_deref_cm_id
;
3026 err
= send_mpa_reply(ep
, conn_param
->private_data
,
3027 conn_param
->private_data_len
);
3029 goto err_deref_cm_id
;
3031 __state_set(&ep
->com
, FPDU_MODE
);
3032 established_upcall(ep
);
3033 mutex_unlock(&ep
->com
.mutex
);
3034 c4iw_put_ep(&ep
->com
);
3037 ep
->com
.cm_id
= NULL
;
3038 cm_id
->rem_ref(cm_id
);
3042 mutex_unlock(&ep
->com
.mutex
);
3044 c4iw_ep_disconnect(ep
, 1, GFP_KERNEL
);
3045 c4iw_put_ep(&ep
->com
);
3049 static int pick_local_ipaddrs(struct c4iw_dev
*dev
, struct iw_cm_id
*cm_id
)
3051 struct in_device
*ind
;
3053 struct sockaddr_in
*laddr
= (struct sockaddr_in
*)&cm_id
->m_local_addr
;
3054 struct sockaddr_in
*raddr
= (struct sockaddr_in
*)&cm_id
->m_remote_addr
;
3056 ind
= in_dev_get(dev
->rdev
.lldi
.ports
[0]);
3058 return -EADDRNOTAVAIL
;
3059 for_primary_ifa(ind
) {
3060 laddr
->sin_addr
.s_addr
= ifa
->ifa_address
;
3061 raddr
->sin_addr
.s_addr
= ifa
->ifa_address
;
3067 return found
? 0 : -EADDRNOTAVAIL
;
3070 static int get_lladdr(struct net_device
*dev
, struct in6_addr
*addr
,
3071 unsigned char banned_flags
)
3073 struct inet6_dev
*idev
;
3074 int err
= -EADDRNOTAVAIL
;
3077 idev
= __in6_dev_get(dev
);
3079 struct inet6_ifaddr
*ifp
;
3081 read_lock_bh(&idev
->lock
);
3082 list_for_each_entry(ifp
, &idev
->addr_list
, if_list
) {
3083 if (ifp
->scope
== IFA_LINK
&&
3084 !(ifp
->flags
& banned_flags
)) {
3085 memcpy(addr
, &ifp
->addr
, 16);
3090 read_unlock_bh(&idev
->lock
);
3096 static int pick_local_ip6addrs(struct c4iw_dev
*dev
, struct iw_cm_id
*cm_id
)
3098 struct in6_addr
uninitialized_var(addr
);
3099 struct sockaddr_in6
*la6
= (struct sockaddr_in6
*)&cm_id
->m_local_addr
;
3100 struct sockaddr_in6
*ra6
= (struct sockaddr_in6
*)&cm_id
->m_remote_addr
;
3102 if (!get_lladdr(dev
->rdev
.lldi
.ports
[0], &addr
, IFA_F_TENTATIVE
)) {
3103 memcpy(la6
->sin6_addr
.s6_addr
, &addr
, 16);
3104 memcpy(ra6
->sin6_addr
.s6_addr
, &addr
, 16);
3107 return -EADDRNOTAVAIL
;
3110 int c4iw_connect(struct iw_cm_id
*cm_id
, struct iw_cm_conn_param
*conn_param
)
3112 struct c4iw_dev
*dev
= to_c4iw_dev(cm_id
->device
);
3115 struct sockaddr_in
*laddr
;
3116 struct sockaddr_in
*raddr
;
3117 struct sockaddr_in6
*laddr6
;
3118 struct sockaddr_in6
*raddr6
;
3122 if ((conn_param
->ord
> cur_max_read_depth(dev
)) ||
3123 (conn_param
->ird
> cur_max_read_depth(dev
))) {
3127 ep
= alloc_ep(sizeof(*ep
), GFP_KERNEL
);
3129 printk(KERN_ERR MOD
"%s - cannot alloc ep.\n", __func__
);
3133 init_timer(&ep
->timer
);
3134 ep
->plen
= conn_param
->private_data_len
;
3136 memcpy(ep
->mpa_pkt
+ sizeof(struct mpa_message
),
3137 conn_param
->private_data
, ep
->plen
);
3138 ep
->ird
= conn_param
->ird
;
3139 ep
->ord
= conn_param
->ord
;
3141 if (peer2peer
&& ep
->ord
== 0)
3144 cm_id
->add_ref(cm_id
);
3146 ep
->com
.cm_id
= cm_id
;
3147 ep
->com
.qp
= get_qhp(dev
, conn_param
->qpn
);
3149 PDBG("%s qpn 0x%x not found!\n", __func__
, conn_param
->qpn
);
3154 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__
, conn_param
->qpn
,
3158 * Allocate an active TID to initiate a TCP connection.
3160 ep
->atid
= cxgb4_alloc_atid(dev
->rdev
.lldi
.tids
, ep
);
3161 if (ep
->atid
== -1) {
3162 printk(KERN_ERR MOD
"%s - cannot alloc atid.\n", __func__
);
3166 insert_handle(dev
, &dev
->atid_idr
, ep
, ep
->atid
);
3168 memcpy(&ep
->com
.local_addr
, &cm_id
->m_local_addr
,
3169 sizeof(ep
->com
.local_addr
));
3170 memcpy(&ep
->com
.remote_addr
, &cm_id
->m_remote_addr
,
3171 sizeof(ep
->com
.remote_addr
));
3173 laddr
= (struct sockaddr_in
*)&ep
->com
.local_addr
;
3174 raddr
= (struct sockaddr_in
*)&ep
->com
.remote_addr
;
3175 laddr6
= (struct sockaddr_in6
*)&ep
->com
.local_addr
;
3176 raddr6
= (struct sockaddr_in6
*) &ep
->com
.remote_addr
;
3178 if (cm_id
->m_remote_addr
.ss_family
== AF_INET
) {
3180 ra
= (__u8
*)&raddr
->sin_addr
;
3183 * Handle loopback requests to INADDR_ANY.
3185 if ((__force
int)raddr
->sin_addr
.s_addr
== INADDR_ANY
) {
3186 err
= pick_local_ipaddrs(dev
, cm_id
);
3192 PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n",
3193 __func__
, &laddr
->sin_addr
, ntohs(laddr
->sin_port
),
3194 ra
, ntohs(raddr
->sin_port
));
3195 ep
->dst
= find_route(dev
, laddr
->sin_addr
.s_addr
,
3196 raddr
->sin_addr
.s_addr
, laddr
->sin_port
,
3197 raddr
->sin_port
, cm_id
->tos
);
3200 ra
= (__u8
*)&raddr6
->sin6_addr
;
3203 * Handle loopback requests to INADDR_ANY.
3205 if (ipv6_addr_type(&raddr6
->sin6_addr
) == IPV6_ADDR_ANY
) {
3206 err
= pick_local_ip6addrs(dev
, cm_id
);
3212 PDBG("%s saddr %pI6 sport 0x%x raddr %pI6 rport 0x%x\n",
3213 __func__
, laddr6
->sin6_addr
.s6_addr
,
3214 ntohs(laddr6
->sin6_port
),
3215 raddr6
->sin6_addr
.s6_addr
, ntohs(raddr6
->sin6_port
));
3216 ep
->dst
= find_route6(dev
, laddr6
->sin6_addr
.s6_addr
,
3217 raddr6
->sin6_addr
.s6_addr
,
3218 laddr6
->sin6_port
, raddr6
->sin6_port
, 0,
3219 raddr6
->sin6_scope_id
);
3222 printk(KERN_ERR MOD
"%s - cannot find route.\n", __func__
);
3223 err
= -EHOSTUNREACH
;
3227 err
= import_ep(ep
, iptype
, ra
, ep
->dst
, ep
->com
.dev
, true,
3228 ep
->com
.dev
->rdev
.lldi
.adapter_type
, cm_id
->tos
);
3230 printk(KERN_ERR MOD
"%s - cannot alloc l2e.\n", __func__
);
3234 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n",
3235 __func__
, ep
->txq_idx
, ep
->tx_chan
, ep
->smac_idx
, ep
->rss_qid
,
3238 state_set(&ep
->com
, CONNECTING
);
3239 ep
->tos
= cm_id
->tos
;
3241 /* send connect request to rnic */
3242 err
= send_connect(ep
);
3246 cxgb4_l2t_release(ep
->l2t
);
3248 dst_release(ep
->dst
);
3250 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->atid_idr
, ep
->atid
);
3251 cxgb4_free_atid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
->atid
);
3253 cm_id
->rem_ref(cm_id
);
3254 c4iw_put_ep(&ep
->com
);
3259 static int create_server6(struct c4iw_dev
*dev
, struct c4iw_listen_ep
*ep
)
3262 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
3263 &ep
->com
.local_addr
;
3265 if (ipv6_addr_type(&sin6
->sin6_addr
) != IPV6_ADDR_ANY
) {
3266 err
= cxgb4_clip_get(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3267 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
3271 c4iw_init_wr_wait(&ep
->com
.wr_wait
);
3272 err
= cxgb4_create_server6(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3273 ep
->stid
, &sin6
->sin6_addr
,
3275 ep
->com
.dev
->rdev
.lldi
.rxq_ids
[0]);
3277 err
= c4iw_wait_for_reply(&ep
->com
.dev
->rdev
,
3281 err
= net_xmit_errno(err
);
3283 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3284 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
3285 pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n",
3287 sin6
->sin6_addr
.s6_addr
, ntohs(sin6
->sin6_port
));
3292 static int create_server4(struct c4iw_dev
*dev
, struct c4iw_listen_ep
*ep
)
3295 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
3296 &ep
->com
.local_addr
;
3298 if (dev
->rdev
.lldi
.enable_fw_ofld_conn
) {
3300 err
= cxgb4_create_server_filter(
3301 ep
->com
.dev
->rdev
.lldi
.ports
[0], ep
->stid
,
3302 sin
->sin_addr
.s_addr
, sin
->sin_port
, 0,
3303 ep
->com
.dev
->rdev
.lldi
.rxq_ids
[0], 0, 0);
3304 if (err
== -EBUSY
) {
3305 if (c4iw_fatal_error(&ep
->com
.dev
->rdev
)) {
3309 set_current_state(TASK_UNINTERRUPTIBLE
);
3310 schedule_timeout(usecs_to_jiffies(100));
3312 } while (err
== -EBUSY
);
3314 c4iw_init_wr_wait(&ep
->com
.wr_wait
);
3315 err
= cxgb4_create_server(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3316 ep
->stid
, sin
->sin_addr
.s_addr
, sin
->sin_port
,
3317 0, ep
->com
.dev
->rdev
.lldi
.rxq_ids
[0]);
3319 err
= c4iw_wait_for_reply(&ep
->com
.dev
->rdev
,
3323 err
= net_xmit_errno(err
);
3326 pr_err("cxgb4_create_server/filter failed err %d stid %d laddr %pI4 lport %d\n"
3328 &sin
->sin_addr
, ntohs(sin
->sin_port
));
3332 int c4iw_create_listen(struct iw_cm_id
*cm_id
, int backlog
)
3335 struct c4iw_dev
*dev
= to_c4iw_dev(cm_id
->device
);
3336 struct c4iw_listen_ep
*ep
;
3340 ep
= alloc_ep(sizeof(*ep
), GFP_KERNEL
);
3342 printk(KERN_ERR MOD
"%s - cannot alloc ep.\n", __func__
);
3346 PDBG("%s ep %p\n", __func__
, ep
);
3347 cm_id
->add_ref(cm_id
);
3348 ep
->com
.cm_id
= cm_id
;
3350 ep
->backlog
= backlog
;
3351 memcpy(&ep
->com
.local_addr
, &cm_id
->m_local_addr
,
3352 sizeof(ep
->com
.local_addr
));
3355 * Allocate a server TID.
3357 if (dev
->rdev
.lldi
.enable_fw_ofld_conn
&&
3358 ep
->com
.local_addr
.ss_family
== AF_INET
)
3359 ep
->stid
= cxgb4_alloc_sftid(dev
->rdev
.lldi
.tids
,
3360 cm_id
->m_local_addr
.ss_family
, ep
);
3362 ep
->stid
= cxgb4_alloc_stid(dev
->rdev
.lldi
.tids
,
3363 cm_id
->m_local_addr
.ss_family
, ep
);
3365 if (ep
->stid
== -1) {
3366 printk(KERN_ERR MOD
"%s - cannot alloc stid.\n", __func__
);
3370 insert_handle(dev
, &dev
->stid_idr
, ep
, ep
->stid
);
3372 memcpy(&ep
->com
.local_addr
, &cm_id
->m_local_addr
,
3373 sizeof(ep
->com
.local_addr
));
3375 state_set(&ep
->com
, LISTEN
);
3376 if (ep
->com
.local_addr
.ss_family
== AF_INET
)
3377 err
= create_server4(dev
, ep
);
3379 err
= create_server6(dev
, ep
);
3381 cm_id
->provider_data
= ep
;
3385 cxgb4_free_stid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
->stid
,
3386 ep
->com
.local_addr
.ss_family
);
3388 cm_id
->rem_ref(cm_id
);
3389 c4iw_put_ep(&ep
->com
);
3395 int c4iw_destroy_listen(struct iw_cm_id
*cm_id
)
3398 struct c4iw_listen_ep
*ep
= to_listen_ep(cm_id
);
3400 PDBG("%s ep %p\n", __func__
, ep
);
3403 state_set(&ep
->com
, DEAD
);
3404 if (ep
->com
.dev
->rdev
.lldi
.enable_fw_ofld_conn
&&
3405 ep
->com
.local_addr
.ss_family
== AF_INET
) {
3406 err
= cxgb4_remove_server_filter(
3407 ep
->com
.dev
->rdev
.lldi
.ports
[0], ep
->stid
,
3408 ep
->com
.dev
->rdev
.lldi
.rxq_ids
[0], 0);
3410 struct sockaddr_in6
*sin6
;
3411 c4iw_init_wr_wait(&ep
->com
.wr_wait
);
3412 err
= cxgb4_remove_server(
3413 ep
->com
.dev
->rdev
.lldi
.ports
[0], ep
->stid
,
3414 ep
->com
.dev
->rdev
.lldi
.rxq_ids
[0], 0);
3417 err
= c4iw_wait_for_reply(&ep
->com
.dev
->rdev
, &ep
->com
.wr_wait
,
3419 sin6
= (struct sockaddr_in6
*)&ep
->com
.local_addr
;
3420 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3421 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
3423 remove_handle(ep
->com
.dev
, &ep
->com
.dev
->stid_idr
, ep
->stid
);
3424 cxgb4_free_stid(ep
->com
.dev
->rdev
.lldi
.tids
, ep
->stid
,
3425 ep
->com
.local_addr
.ss_family
);
3427 cm_id
->rem_ref(cm_id
);
3428 c4iw_put_ep(&ep
->com
);
3432 int c4iw_ep_disconnect(struct c4iw_ep
*ep
, int abrupt
, gfp_t gfp
)
3437 struct c4iw_rdev
*rdev
;
3439 mutex_lock(&ep
->com
.mutex
);
3441 PDBG("%s ep %p state %s, abrupt %d\n", __func__
, ep
,
3442 states
[ep
->com
.state
], abrupt
);
3445 * Ref the ep here in case we have fatal errors causing the
3446 * ep to be released and freed.
3448 c4iw_get_ep(&ep
->com
);
3450 rdev
= &ep
->com
.dev
->rdev
;
3451 if (c4iw_fatal_error(rdev
)) {
3453 close_complete_upcall(ep
, -EIO
);
3454 ep
->com
.state
= DEAD
;
3456 switch (ep
->com
.state
) {
3464 ep
->com
.state
= ABORTING
;
3466 ep
->com
.state
= CLOSING
;
3469 set_bit(CLOSE_SENT
, &ep
->com
.flags
);
3472 if (!test_and_set_bit(CLOSE_SENT
, &ep
->com
.flags
)) {
3475 (void)stop_ep_timer(ep
);
3476 ep
->com
.state
= ABORTING
;
3478 ep
->com
.state
= MORIBUND
;
3484 PDBG("%s ignoring disconnect ep %p state %u\n",
3485 __func__
, ep
, ep
->com
.state
);
3494 set_bit(EP_DISC_ABORT
, &ep
->com
.history
);
3495 close_complete_upcall(ep
, -ECONNRESET
);
3496 ret
= send_abort(ep
, NULL
, gfp
);
3498 set_bit(EP_DISC_CLOSE
, &ep
->com
.history
);
3499 ret
= send_halfclose(ep
, gfp
);
3504 close_complete_upcall(ep
, -EIO
);
3509 mutex_unlock(&ep
->com
.mutex
);
3510 c4iw_put_ep(&ep
->com
);
3512 release_ep_resources(ep
);
3516 static void active_ofld_conn_reply(struct c4iw_dev
*dev
, struct sk_buff
*skb
,
3517 struct cpl_fw6_msg_ofld_connection_wr_rpl
*req
)
3520 int atid
= be32_to_cpu(req
->tid
);
3522 ep
= (struct c4iw_ep
*)lookup_atid(dev
->rdev
.lldi
.tids
,
3523 (__force u32
) req
->tid
);
3527 switch (req
->retval
) {
3529 set_bit(ACT_RETRY_NOMEM
, &ep
->com
.history
);
3530 if (ep
->retry_count
++ < ACT_OPEN_RETRY_COUNT
) {
3531 send_fw_act_open_req(ep
, atid
);
3535 set_bit(ACT_RETRY_INUSE
, &ep
->com
.history
);
3536 if (ep
->retry_count
++ < ACT_OPEN_RETRY_COUNT
) {
3537 send_fw_act_open_req(ep
, atid
);
3542 pr_info("%s unexpected ofld conn wr retval %d\n",
3543 __func__
, req
->retval
);
3546 pr_err("active ofld_connect_wr failure %d atid %d\n",
3548 mutex_lock(&dev
->rdev
.stats
.lock
);
3549 dev
->rdev
.stats
.act_ofld_conn_fails
++;
3550 mutex_unlock(&dev
->rdev
.stats
.lock
);
3551 connect_reply_upcall(ep
, status2errno(req
->retval
));
3552 state_set(&ep
->com
, DEAD
);
3553 if (ep
->com
.remote_addr
.ss_family
== AF_INET6
) {
3554 struct sockaddr_in6
*sin6
=
3555 (struct sockaddr_in6
*)&ep
->com
.local_addr
;
3556 cxgb4_clip_release(ep
->com
.dev
->rdev
.lldi
.ports
[0],
3557 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
3559 remove_handle(dev
, &dev
->atid_idr
, atid
);
3560 cxgb4_free_atid(dev
->rdev
.lldi
.tids
, atid
);
3561 dst_release(ep
->dst
);
3562 cxgb4_l2t_release(ep
->l2t
);
3563 c4iw_put_ep(&ep
->com
);
3566 static void passive_ofld_conn_reply(struct c4iw_dev
*dev
, struct sk_buff
*skb
,
3567 struct cpl_fw6_msg_ofld_connection_wr_rpl
*req
)
3569 struct sk_buff
*rpl_skb
;
3570 struct cpl_pass_accept_req
*cpl
;
3573 rpl_skb
= (struct sk_buff
*)(unsigned long)req
->cookie
;
3576 PDBG("%s passive open failure %d\n", __func__
, req
->retval
);
3577 mutex_lock(&dev
->rdev
.stats
.lock
);
3578 dev
->rdev
.stats
.pas_ofld_conn_fails
++;
3579 mutex_unlock(&dev
->rdev
.stats
.lock
);
3582 cpl
= (struct cpl_pass_accept_req
*)cplhdr(rpl_skb
);
3583 OPCODE_TID(cpl
) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ
,
3584 (__force u32
) htonl(
3585 (__force u32
) req
->tid
)));
3586 ret
= pass_accept_req(dev
, rpl_skb
);
3593 static int deferred_fw6_msg(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
3595 struct cpl_fw6_msg
*rpl
= cplhdr(skb
);
3596 struct cpl_fw6_msg_ofld_connection_wr_rpl
*req
;
3598 switch (rpl
->type
) {
3600 c4iw_ev_dispatch(dev
, (struct t4_cqe
*)&rpl
->data
[0]);
3602 case FW6_TYPE_OFLD_CONNECTION_WR_RPL
:
3603 req
= (struct cpl_fw6_msg_ofld_connection_wr_rpl
*)rpl
->data
;
3604 switch (req
->t_state
) {
3606 active_ofld_conn_reply(dev
, skb
, req
);
3609 passive_ofld_conn_reply(dev
, skb
, req
);
3612 pr_err("%s unexpected ofld conn wr state %d\n",
3613 __func__
, req
->t_state
);
3621 static void build_cpl_pass_accept_req(struct sk_buff
*skb
, int stid
, u8 tos
)
3624 __be16 hdr_len
, vlantag
, len
;
3626 int tcp_hdr_len
, ip_hdr_len
;
3628 struct cpl_rx_pkt
*cpl
= cplhdr(skb
);
3629 struct cpl_pass_accept_req
*req
;
3630 struct tcp_options_received tmp_opt
;
3631 struct c4iw_dev
*dev
;
3632 enum chip_type type
;
3634 dev
= *((struct c4iw_dev
**) (skb
->cb
+ sizeof(void *)));
3635 /* Store values from cpl_rx_pkt in temporary location. */
3636 vlantag
= cpl
->vlan
;
3638 l2info
= cpl
->l2info
;
3639 hdr_len
= cpl
->hdr_len
;
3642 __skb_pull(skb
, sizeof(*req
) + sizeof(struct rss_header
));
3645 * We need to parse the TCP options from SYN packet.
3646 * to generate cpl_pass_accept_req.
3648 memset(&tmp_opt
, 0, sizeof(tmp_opt
));
3649 tcp_clear_options(&tmp_opt
);
3650 tcp_parse_options(skb
, &tmp_opt
, 0, NULL
);
3652 req
= (struct cpl_pass_accept_req
*)__skb_push(skb
, sizeof(*req
));
3653 memset(req
, 0, sizeof(*req
));
3654 req
->l2info
= cpu_to_be16(SYN_INTF_V(intf
) |
3655 SYN_MAC_IDX_V(RX_MACIDX_G(
3656 be32_to_cpu(l2info
))) |
3658 type
= dev
->rdev
.lldi
.adapter_type
;
3659 tcp_hdr_len
= RX_TCPHDR_LEN_G(be16_to_cpu(hdr_len
));
3660 ip_hdr_len
= RX_IPHDR_LEN_G(be16_to_cpu(hdr_len
));
3662 cpu_to_be32(SYN_RX_CHAN_V(RX_CHAN_G(be32_to_cpu(l2info
))));
3663 if (CHELSIO_CHIP_VERSION(type
) <= CHELSIO_T5
) {
3664 eth_hdr_len
= is_t4(type
) ?
3665 RX_ETHHDR_LEN_G(be32_to_cpu(l2info
)) :
3666 RX_T5_ETHHDR_LEN_G(be32_to_cpu(l2info
));
3667 req
->hdr_len
|= cpu_to_be32(TCP_HDR_LEN_V(tcp_hdr_len
) |
3668 IP_HDR_LEN_V(ip_hdr_len
) |
3669 ETH_HDR_LEN_V(eth_hdr_len
));
3670 } else { /* T6 and later */
3671 eth_hdr_len
= RX_T6_ETHHDR_LEN_G(be32_to_cpu(l2info
));
3672 req
->hdr_len
|= cpu_to_be32(T6_TCP_HDR_LEN_V(tcp_hdr_len
) |
3673 T6_IP_HDR_LEN_V(ip_hdr_len
) |
3674 T6_ETH_HDR_LEN_V(eth_hdr_len
));
3676 req
->vlan
= vlantag
;
3678 req
->tos_stid
= cpu_to_be32(PASS_OPEN_TID_V(stid
) |
3679 PASS_OPEN_TOS_V(tos
));
3680 req
->tcpopt
.mss
= htons(tmp_opt
.mss_clamp
);
3681 if (tmp_opt
.wscale_ok
)
3682 req
->tcpopt
.wsf
= tmp_opt
.snd_wscale
;
3683 req
->tcpopt
.tstamp
= tmp_opt
.saw_tstamp
;
3684 if (tmp_opt
.sack_ok
)
3685 req
->tcpopt
.sack
= 1;
3686 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ
, 0));
3690 static void send_fw_pass_open_req(struct c4iw_dev
*dev
, struct sk_buff
*skb
,
3691 __be32 laddr
, __be16 lport
,
3692 __be32 raddr
, __be16 rport
,
3693 u32 rcv_isn
, u32 filter
, u16 window
,
3694 u32 rss_qid
, u8 port_id
)
3696 struct sk_buff
*req_skb
;
3697 struct fw_ofld_connection_wr
*req
;
3698 struct cpl_pass_accept_req
*cpl
= cplhdr(skb
);
3701 req_skb
= alloc_skb(sizeof(struct fw_ofld_connection_wr
), GFP_KERNEL
);
3702 req
= (struct fw_ofld_connection_wr
*)__skb_put(req_skb
, sizeof(*req
));
3703 memset(req
, 0, sizeof(*req
));
3704 req
->op_compl
= htonl(WR_OP_V(FW_OFLD_CONNECTION_WR
) | FW_WR_COMPL_F
);
3705 req
->len16_pkd
= htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req
), 16)));
3706 req
->le
.version_cpl
= htonl(FW_OFLD_CONNECTION_WR_CPL_F
);
3707 req
->le
.filter
= (__force __be32
) filter
;
3708 req
->le
.lport
= lport
;
3709 req
->le
.pport
= rport
;
3710 req
->le
.u
.ipv4
.lip
= laddr
;
3711 req
->le
.u
.ipv4
.pip
= raddr
;
3712 req
->tcb
.rcv_nxt
= htonl(rcv_isn
+ 1);
3713 req
->tcb
.rcv_adv
= htons(window
);
3714 req
->tcb
.t_state_to_astid
=
3715 htonl(FW_OFLD_CONNECTION_WR_T_STATE_V(TCP_SYN_RECV
) |
3716 FW_OFLD_CONNECTION_WR_RCV_SCALE_V(cpl
->tcpopt
.wsf
) |
3717 FW_OFLD_CONNECTION_WR_ASTID_V(
3718 PASS_OPEN_TID_G(ntohl(cpl
->tos_stid
))));
3721 * We store the qid in opt2 which will be used by the firmware
3722 * to send us the wr response.
3724 req
->tcb
.opt2
= htonl(RSS_QUEUE_V(rss_qid
));
3727 * We initialize the MSS index in TCB to 0xF.
3728 * So that when driver sends cpl_pass_accept_rpl
3729 * TCB picks up the correct value. If this was 0
3730 * TP will ignore any value > 0 for MSS index.
3732 req
->tcb
.opt0
= cpu_to_be64(MSS_IDX_V(0xF));
3733 req
->cookie
= (uintptr_t)skb
;
3735 set_wr_txq(req_skb
, CPL_PRIORITY_CONTROL
, port_id
);
3736 ret
= cxgb4_ofld_send(dev
->rdev
.lldi
.ports
[0], req_skb
);
3738 pr_err("%s - cxgb4_ofld_send error %d - dropping\n", __func__
,
3746 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt
3747 * messages when a filter is being used instead of server to
3748 * redirect a syn packet. When packets hit filter they are redirected
3749 * to the offload queue and driver tries to establish the connection
3750 * using firmware work request.
3752 static int rx_pkt(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
3755 unsigned int filter
;
3756 struct ethhdr
*eh
= NULL
;
3757 struct vlan_ethhdr
*vlan_eh
= NULL
;
3759 struct tcphdr
*tcph
;
3760 struct rss_header
*rss
= (void *)skb
->data
;
3761 struct cpl_rx_pkt
*cpl
= (void *)skb
->data
;
3762 struct cpl_pass_accept_req
*req
= (void *)(rss
+ 1);
3763 struct l2t_entry
*e
;
3764 struct dst_entry
*dst
;
3765 struct c4iw_ep
*lep
;
3767 struct port_info
*pi
;
3768 struct net_device
*pdev
;
3769 u16 rss_qid
, eth_hdr_len
;
3772 struct neighbour
*neigh
;
3774 /* Drop all non-SYN packets */
3775 if (!(cpl
->l2info
& cpu_to_be32(RXF_SYN_F
)))
3779 * Drop all packets which did not hit the filter.
3780 * Unlikely to happen.
3782 if (!(rss
->filter_hit
&& rss
->filter_tid
))
3786 * Calculate the server tid from filter hit index from cpl_rx_pkt.
3788 stid
= (__force
int) cpu_to_be32((__force u32
) rss
->hash_val
);
3790 lep
= (struct c4iw_ep
*)lookup_stid(dev
->rdev
.lldi
.tids
, stid
);
3792 PDBG("%s connect request on invalid stid %d\n", __func__
, stid
);
3796 switch (CHELSIO_CHIP_VERSION(dev
->rdev
.lldi
.adapter_type
)) {
3798 eth_hdr_len
= RX_ETHHDR_LEN_G(be32_to_cpu(cpl
->l2info
));
3801 eth_hdr_len
= RX_T5_ETHHDR_LEN_G(be32_to_cpu(cpl
->l2info
));
3804 eth_hdr_len
= RX_T6_ETHHDR_LEN_G(be32_to_cpu(cpl
->l2info
));
3807 pr_err("T%d Chip is not supported\n",
3808 CHELSIO_CHIP_VERSION(dev
->rdev
.lldi
.adapter_type
));
3812 if (eth_hdr_len
== ETH_HLEN
) {
3813 eh
= (struct ethhdr
*)(req
+ 1);
3814 iph
= (struct iphdr
*)(eh
+ 1);
3816 vlan_eh
= (struct vlan_ethhdr
*)(req
+ 1);
3817 iph
= (struct iphdr
*)(vlan_eh
+ 1);
3818 skb
->vlan_tci
= ntohs(cpl
->vlan
);
3821 if (iph
->version
!= 0x4)
3824 tcph
= (struct tcphdr
*)(iph
+ 1);
3825 skb_set_network_header(skb
, (void *)iph
- (void *)rss
);
3826 skb_set_transport_header(skb
, (void *)tcph
- (void *)rss
);
3829 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__
,
3830 ntohl(iph
->daddr
), ntohs(tcph
->dest
), ntohl(iph
->saddr
),
3831 ntohs(tcph
->source
), iph
->tos
);
3833 dst
= find_route(dev
, iph
->daddr
, iph
->saddr
, tcph
->dest
, tcph
->source
,
3836 pr_err("%s - failed to find dst entry!\n",
3840 neigh
= dst_neigh_lookup_skb(dst
, skb
);
3843 pr_err("%s - failed to allocate neigh!\n",
3848 if (neigh
->dev
->flags
& IFF_LOOPBACK
) {
3849 pdev
= ip_dev_find(&init_net
, iph
->daddr
);
3850 e
= cxgb4_l2t_get(dev
->rdev
.lldi
.l2t
, neigh
,
3852 pi
= (struct port_info
*)netdev_priv(pdev
);
3853 tx_chan
= cxgb4_port_chan(pdev
);
3856 pdev
= get_real_dev(neigh
->dev
);
3857 e
= cxgb4_l2t_get(dev
->rdev
.lldi
.l2t
, neigh
,
3859 pi
= (struct port_info
*)netdev_priv(pdev
);
3860 tx_chan
= cxgb4_port_chan(pdev
);
3862 neigh_release(neigh
);
3864 pr_err("%s - failed to allocate l2t entry!\n",
3869 step
= dev
->rdev
.lldi
.nrxq
/ dev
->rdev
.lldi
.nchan
;
3870 rss_qid
= dev
->rdev
.lldi
.rxq_ids
[pi
->port_id
* step
];
3871 window
= (__force u16
) htons((__force u16
)tcph
->window
);
3873 /* Calcuate filter portion for LE region. */
3874 filter
= (__force
unsigned int) cpu_to_be32(cxgb4_select_ntuple(
3875 dev
->rdev
.lldi
.ports
[0],
3879 * Synthesize the cpl_pass_accept_req. We have everything except the
3880 * TID. Once firmware sends a reply with TID we update the TID field
3881 * in cpl and pass it through the regular cpl_pass_accept_req path.
3883 build_cpl_pass_accept_req(skb
, stid
, iph
->tos
);
3884 send_fw_pass_open_req(dev
, skb
, iph
->daddr
, tcph
->dest
, iph
->saddr
,
3885 tcph
->source
, ntohl(tcph
->seq
), filter
, window
,
3886 rss_qid
, pi
->port_id
);
3887 cxgb4_l2t_release(e
);
3895 * These are the real handlers that are called from a
3898 static c4iw_handler_func work_handlers
[NUM_CPL_CMDS
+ NUM_FAKE_CPLS
] = {
3899 [CPL_ACT_ESTABLISH
] = act_establish
,
3900 [CPL_ACT_OPEN_RPL
] = act_open_rpl
,
3901 [CPL_RX_DATA
] = rx_data
,
3902 [CPL_ABORT_RPL_RSS
] = abort_rpl
,
3903 [CPL_ABORT_RPL
] = abort_rpl
,
3904 [CPL_PASS_OPEN_RPL
] = pass_open_rpl
,
3905 [CPL_CLOSE_LISTSRV_RPL
] = close_listsrv_rpl
,
3906 [CPL_PASS_ACCEPT_REQ
] = pass_accept_req
,
3907 [CPL_PASS_ESTABLISH
] = pass_establish
,
3908 [CPL_PEER_CLOSE
] = peer_close
,
3909 [CPL_ABORT_REQ_RSS
] = peer_abort
,
3910 [CPL_CLOSE_CON_RPL
] = close_con_rpl
,
3911 [CPL_RDMA_TERMINATE
] = terminate
,
3912 [CPL_FW4_ACK
] = fw4_ack
,
3913 [CPL_FW6_MSG
] = deferred_fw6_msg
,
3914 [CPL_RX_PKT
] = rx_pkt
,
3915 [FAKE_CPL_PUT_EP_SAFE
] = _put_ep_safe
3918 static void process_timeout(struct c4iw_ep
*ep
)
3920 struct c4iw_qp_attributes attrs
;
3923 mutex_lock(&ep
->com
.mutex
);
3924 PDBG("%s ep %p tid %u state %d\n", __func__
, ep
, ep
->hwtid
,
3926 set_bit(TIMEDOUT
, &ep
->com
.history
);
3927 switch (ep
->com
.state
) {
3929 __state_set(&ep
->com
, ABORTING
);
3930 connect_reply_upcall(ep
, -ETIMEDOUT
);
3933 __state_set(&ep
->com
, ABORTING
);
3937 if (ep
->com
.cm_id
&& ep
->com
.qp
) {
3938 attrs
.next_state
= C4IW_QP_STATE_ERROR
;
3939 c4iw_modify_qp(ep
->com
.qp
->rhp
,
3940 ep
->com
.qp
, C4IW_QP_ATTR_NEXT_STATE
,
3943 __state_set(&ep
->com
, ABORTING
);
3944 close_complete_upcall(ep
, -ETIMEDOUT
);
3950 * These states are expected if the ep timed out at the same
3951 * time as another thread was calling stop_ep_timer().
3952 * So we silently do nothing for these states.
3957 WARN(1, "%s unexpected state ep %p tid %u state %u\n",
3958 __func__
, ep
, ep
->hwtid
, ep
->com
.state
);
3962 abort_connection(ep
, NULL
, GFP_KERNEL
);
3963 mutex_unlock(&ep
->com
.mutex
);
3964 c4iw_put_ep(&ep
->com
);
3967 static void process_timedout_eps(void)
3971 spin_lock_irq(&timeout_lock
);
3972 while (!list_empty(&timeout_list
)) {
3973 struct list_head
*tmp
;
3975 tmp
= timeout_list
.next
;
3979 spin_unlock_irq(&timeout_lock
);
3980 ep
= list_entry(tmp
, struct c4iw_ep
, entry
);
3981 process_timeout(ep
);
3982 spin_lock_irq(&timeout_lock
);
3984 spin_unlock_irq(&timeout_lock
);
3987 static void process_work(struct work_struct
*work
)
3989 struct sk_buff
*skb
= NULL
;
3990 struct c4iw_dev
*dev
;
3991 struct cpl_act_establish
*rpl
;
3992 unsigned int opcode
;
3995 process_timedout_eps();
3996 while ((skb
= skb_dequeue(&rxq
))) {
3998 dev
= *((struct c4iw_dev
**) (skb
->cb
+ sizeof(void *)));
3999 opcode
= rpl
->ot
.opcode
;
4001 BUG_ON(!work_handlers
[opcode
]);
4002 ret
= work_handlers
[opcode
](dev
, skb
);
4005 process_timedout_eps();
4009 static DECLARE_WORK(skb_work
, process_work
);
4011 static void ep_timeout(unsigned long arg
)
4013 struct c4iw_ep
*ep
= (struct c4iw_ep
*)arg
;
4016 spin_lock(&timeout_lock
);
4017 if (!test_and_set_bit(TIMEOUT
, &ep
->com
.flags
)) {
4019 * Only insert if it is not already on the list.
4021 if (!ep
->entry
.next
) {
4022 list_add_tail(&ep
->entry
, &timeout_list
);
4026 spin_unlock(&timeout_lock
);
4028 queue_work(workq
, &skb_work
);
4032 * All the CM events are handled on a work queue to have a safe context.
4034 static int sched(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
4038 * Save dev in the skb->cb area.
4040 *((struct c4iw_dev
**) (skb
->cb
+ sizeof(void *))) = dev
;
4043 * Queue the skb and schedule the worker thread.
4045 skb_queue_tail(&rxq
, skb
);
4046 queue_work(workq
, &skb_work
);
4050 static int set_tcb_rpl(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
4052 struct cpl_set_tcb_rpl
*rpl
= cplhdr(skb
);
4054 if (rpl
->status
!= CPL_ERR_NONE
) {
4055 printk(KERN_ERR MOD
"Unexpected SET_TCB_RPL status %u "
4056 "for tid %u\n", rpl
->status
, GET_TID(rpl
));
4062 static int fw6_msg(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
4064 struct cpl_fw6_msg
*rpl
= cplhdr(skb
);
4065 struct c4iw_wr_wait
*wr_waitp
;
4068 PDBG("%s type %u\n", __func__
, rpl
->type
);
4070 switch (rpl
->type
) {
4071 case FW6_TYPE_WR_RPL
:
4072 ret
= (int)((be64_to_cpu(rpl
->data
[0]) >> 8) & 0xff);
4073 wr_waitp
= (struct c4iw_wr_wait
*)(__force
unsigned long) rpl
->data
[1];
4074 PDBG("%s wr_waitp %p ret %u\n", __func__
, wr_waitp
, ret
);
4076 c4iw_wake_up(wr_waitp
, ret
? -ret
: 0);
4080 case FW6_TYPE_OFLD_CONNECTION_WR_RPL
:
4084 printk(KERN_ERR MOD
"%s unexpected fw6 msg type %u\n", __func__
,
4092 static int peer_abort_intr(struct c4iw_dev
*dev
, struct sk_buff
*skb
)
4094 struct cpl_abort_req_rss
*req
= cplhdr(skb
);
4096 struct tid_info
*t
= dev
->rdev
.lldi
.tids
;
4097 unsigned int tid
= GET_TID(req
);
4099 ep
= lookup_tid(t
, tid
);
4101 printk(KERN_WARNING MOD
4102 "Abort on non-existent endpoint, tid %d\n", tid
);
4106 if (is_neg_adv(req
->status
)) {
4107 PDBG("%s Negative advice on abort- tid %u status %d (%s)\n",
4108 __func__
, ep
->hwtid
, req
->status
,
4109 neg_adv_str(req
->status
));
4110 ep
->stats
.abort_neg_adv
++;
4111 dev
->rdev
.stats
.neg_adv
++;
4115 PDBG("%s ep %p tid %u state %u\n", __func__
, ep
, ep
->hwtid
,
4119 * Wake up any threads in rdma_init() or rdma_fini().
4120 * However, if we are on MPAv2 and want to retry with MPAv1
4121 * then, don't wake up yet.
4123 if (mpa_rev
== 2 && !ep
->tried_with_mpa_v1
) {
4124 if (ep
->com
.state
!= MPA_REQ_SENT
)
4125 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
4127 c4iw_wake_up(&ep
->com
.wr_wait
, -ECONNRESET
);
4133 * Most upcalls from the T4 Core go to sched() to
4134 * schedule the processing on a work queue.
4136 c4iw_handler_func c4iw_handlers
[NUM_CPL_CMDS
] = {
4137 [CPL_ACT_ESTABLISH
] = sched
,
4138 [CPL_ACT_OPEN_RPL
] = sched
,
4139 [CPL_RX_DATA
] = sched
,
4140 [CPL_ABORT_RPL_RSS
] = sched
,
4141 [CPL_ABORT_RPL
] = sched
,
4142 [CPL_PASS_OPEN_RPL
] = sched
,
4143 [CPL_CLOSE_LISTSRV_RPL
] = sched
,
4144 [CPL_PASS_ACCEPT_REQ
] = sched
,
4145 [CPL_PASS_ESTABLISH
] = sched
,
4146 [CPL_PEER_CLOSE
] = sched
,
4147 [CPL_CLOSE_CON_RPL
] = sched
,
4148 [CPL_ABORT_REQ_RSS
] = peer_abort_intr
,
4149 [CPL_RDMA_TERMINATE
] = sched
,
4150 [CPL_FW4_ACK
] = sched
,
4151 [CPL_SET_TCB_RPL
] = set_tcb_rpl
,
4152 [CPL_FW6_MSG
] = fw6_msg
,
4153 [CPL_RX_PKT
] = sched
4156 int __init
c4iw_cm_init(void)
4158 spin_lock_init(&timeout_lock
);
4159 skb_queue_head_init(&rxq
);
4161 workq
= create_singlethread_workqueue("iw_cxgb4");
4168 void c4iw_cm_term(void)
4170 WARN_ON(!list_empty(&timeout_list
));
4171 flush_workqueue(workq
);
4172 destroy_workqueue(workq
);