2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <linux/moduleparam.h>
34 #include <linux/debugfs.h>
36 #include <rdma/ib_verbs.h>
40 #define DRV_VERSION "0.1"
42 MODULE_AUTHOR("Steve Wise");
43 MODULE_DESCRIPTION("Chelsio T4 RDMA Driver");
44 MODULE_LICENSE("Dual BSD/GPL");
45 MODULE_VERSION(DRV_VERSION
);
47 static LIST_HEAD(dev_list
);
48 static DEFINE_MUTEX(dev_mutex
);
50 static struct dentry
*c4iw_debugfs_root
;
52 struct debugfs_qp_data
{
53 struct c4iw_dev
*devp
;
59 static int count_qps(int id
, void *p
, void *data
)
61 struct c4iw_qp
*qp
= p
;
64 if (id
!= qp
->wq
.sq
.qid
)
67 *countp
= *countp
+ 1;
71 static int dump_qps(int id
, void *p
, void *data
)
73 struct c4iw_qp
*qp
= p
;
74 struct debugfs_qp_data
*qpd
= data
;
78 if (id
!= qp
->wq
.sq
.qid
)
81 space
= qpd
->bufsize
- qpd
->pos
- 1;
86 cc
= snprintf(qpd
->buf
+ qpd
->pos
, space
, "qp id %u state %u "
87 "ep tid %u state %u %pI4:%u->%pI4:%u\n",
88 qp
->wq
.sq
.qid
, (int)qp
->attr
.state
,
89 qp
->ep
->hwtid
, (int)qp
->ep
->com
.state
,
90 &qp
->ep
->com
.local_addr
.sin_addr
.s_addr
,
91 ntohs(qp
->ep
->com
.local_addr
.sin_port
),
92 &qp
->ep
->com
.remote_addr
.sin_addr
.s_addr
,
93 ntohs(qp
->ep
->com
.remote_addr
.sin_port
));
95 cc
= snprintf(qpd
->buf
+ qpd
->pos
, space
, "qp id %u state %u\n",
96 qp
->wq
.sq
.qid
, (int)qp
->attr
.state
);
102 static int qp_release(struct inode
*inode
, struct file
*file
)
104 struct debugfs_qp_data
*qpd
= file
->private_data
;
106 printk(KERN_INFO
"%s null qpd?\n", __func__
);
114 static int qp_open(struct inode
*inode
, struct file
*file
)
116 struct debugfs_qp_data
*qpd
;
120 qpd
= kmalloc(sizeof *qpd
, GFP_KERNEL
);
125 qpd
->devp
= inode
->i_private
;
128 spin_lock_irq(&qpd
->devp
->lock
);
129 idr_for_each(&qpd
->devp
->qpidr
, count_qps
, &count
);
130 spin_unlock_irq(&qpd
->devp
->lock
);
132 qpd
->bufsize
= count
* 128;
133 qpd
->buf
= kmalloc(qpd
->bufsize
, GFP_KERNEL
);
139 spin_lock_irq(&qpd
->devp
->lock
);
140 idr_for_each(&qpd
->devp
->qpidr
, dump_qps
, qpd
);
141 spin_unlock_irq(&qpd
->devp
->lock
);
143 qpd
->buf
[qpd
->pos
++] = 0;
144 file
->private_data
= qpd
;
152 static ssize_t
qp_read(struct file
*file
, char __user
*buf
, size_t count
,
155 struct debugfs_qp_data
*qpd
= file
->private_data
;
157 loff_t avail
= qpd
->pos
;
163 if (count
> avail
- pos
)
169 len
= min((int)count
, (int)qpd
->pos
- (int)pos
);
170 if (copy_to_user(buf
, qpd
->buf
+ pos
, len
))
184 static const struct file_operations qp_debugfs_fops
= {
185 .owner
= THIS_MODULE
,
187 .release
= qp_release
,
191 static int setup_debugfs(struct c4iw_dev
*devp
)
195 if (!devp
->debugfs_root
)
198 de
= debugfs_create_file("qps", S_IWUSR
, devp
->debugfs_root
,
199 (void *)devp
, &qp_debugfs_fops
);
200 if (de
&& de
->d_inode
)
201 de
->d_inode
->i_size
= 4096;
205 void c4iw_release_dev_ucontext(struct c4iw_rdev
*rdev
,
206 struct c4iw_dev_ucontext
*uctx
)
208 struct list_head
*pos
, *nxt
;
209 struct c4iw_qid_list
*entry
;
211 mutex_lock(&uctx
->lock
);
212 list_for_each_safe(pos
, nxt
, &uctx
->qpids
) {
213 entry
= list_entry(pos
, struct c4iw_qid_list
, entry
);
214 list_del_init(&entry
->entry
);
215 if (!(entry
->qid
& rdev
->qpmask
))
216 c4iw_put_resource(&rdev
->resource
.qid_fifo
, entry
->qid
,
217 &rdev
->resource
.qid_fifo_lock
);
221 list_for_each_safe(pos
, nxt
, &uctx
->qpids
) {
222 entry
= list_entry(pos
, struct c4iw_qid_list
, entry
);
223 list_del_init(&entry
->entry
);
226 mutex_unlock(&uctx
->lock
);
229 void c4iw_init_dev_ucontext(struct c4iw_rdev
*rdev
,
230 struct c4iw_dev_ucontext
*uctx
)
232 INIT_LIST_HEAD(&uctx
->qpids
);
233 INIT_LIST_HEAD(&uctx
->cqids
);
234 mutex_init(&uctx
->lock
);
237 /* Caller takes care of locking if needed */
238 static int c4iw_rdev_open(struct c4iw_rdev
*rdev
)
242 c4iw_init_dev_ucontext(rdev
, &rdev
->uctx
);
245 * qpshift is the number of bits to shift the qpid left in order
246 * to get the correct address of the doorbell for that qp.
248 rdev
->qpshift
= PAGE_SHIFT
- ilog2(rdev
->lldi
.udb_density
);
249 rdev
->qpmask
= rdev
->lldi
.udb_density
- 1;
250 rdev
->cqshift
= PAGE_SHIFT
- ilog2(rdev
->lldi
.ucq_density
);
251 rdev
->cqmask
= rdev
->lldi
.ucq_density
- 1;
252 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d "
253 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x "
254 "qp qid start %u size %u cq qid start %u size %u\n",
255 __func__
, pci_name(rdev
->lldi
.pdev
), rdev
->lldi
.vr
->stag
.start
,
256 rdev
->lldi
.vr
->stag
.size
, c4iw_num_stags(rdev
),
257 rdev
->lldi
.vr
->pbl
.start
,
258 rdev
->lldi
.vr
->pbl
.size
, rdev
->lldi
.vr
->rq
.start
,
259 rdev
->lldi
.vr
->rq
.size
,
260 rdev
->lldi
.vr
->qp
.start
,
261 rdev
->lldi
.vr
->qp
.size
,
262 rdev
->lldi
.vr
->cq
.start
,
263 rdev
->lldi
.vr
->cq
.size
);
264 PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu "
265 "qpmask 0x%x cqshift %lu cqmask 0x%x\n",
266 (unsigned)pci_resource_len(rdev
->lldi
.pdev
, 2),
267 (void *)pci_resource_start(rdev
->lldi
.pdev
, 2),
270 rdev
->qpshift
, rdev
->qpmask
,
271 rdev
->cqshift
, rdev
->cqmask
);
273 if (c4iw_num_stags(rdev
) == 0) {
278 err
= c4iw_init_resource(rdev
, c4iw_num_stags(rdev
), T4_MAX_NUM_PD
);
280 printk(KERN_ERR MOD
"error %d initializing resources\n", err
);
283 err
= c4iw_pblpool_create(rdev
);
285 printk(KERN_ERR MOD
"error %d initializing pbl pool\n", err
);
288 err
= c4iw_rqtpool_create(rdev
);
290 printk(KERN_ERR MOD
"error %d initializing rqt pool\n", err
);
295 c4iw_pblpool_destroy(rdev
);
297 c4iw_destroy_resource(&rdev
->resource
);
302 static void c4iw_rdev_close(struct c4iw_rdev
*rdev
)
304 c4iw_pblpool_destroy(rdev
);
305 c4iw_rqtpool_destroy(rdev
);
306 c4iw_destroy_resource(&rdev
->resource
);
309 static void c4iw_remove(struct c4iw_dev
*dev
)
311 PDBG("%s c4iw_dev %p\n", __func__
, dev
);
312 cancel_delayed_work_sync(&dev
->db_drop_task
);
313 list_del(&dev
->entry
);
315 c4iw_unregister_device(dev
);
316 c4iw_rdev_close(&dev
->rdev
);
317 idr_destroy(&dev
->cqidr
);
318 idr_destroy(&dev
->qpidr
);
319 idr_destroy(&dev
->mmidr
);
320 ib_dealloc_device(&dev
->ibdev
);
323 static struct c4iw_dev
*c4iw_alloc(const struct cxgb4_lld_info
*infop
)
325 struct c4iw_dev
*devp
;
328 devp
= (struct c4iw_dev
*)ib_alloc_device(sizeof(*devp
));
330 printk(KERN_ERR MOD
"Cannot allocate ib device\n");
333 devp
->rdev
.lldi
= *infop
;
335 mutex_lock(&dev_mutex
);
337 ret
= c4iw_rdev_open(&devp
->rdev
);
339 mutex_unlock(&dev_mutex
);
340 printk(KERN_ERR MOD
"Unable to open CXIO rdev err %d\n", ret
);
341 ib_dealloc_device(&devp
->ibdev
);
345 idr_init(&devp
->cqidr
);
346 idr_init(&devp
->qpidr
);
347 idr_init(&devp
->mmidr
);
348 spin_lock_init(&devp
->lock
);
349 list_add_tail(&devp
->entry
, &dev_list
);
350 mutex_unlock(&dev_mutex
);
352 if (c4iw_debugfs_root
) {
353 devp
->debugfs_root
= debugfs_create_dir(
354 pci_name(devp
->rdev
.lldi
.pdev
),
361 static void *c4iw_uld_add(const struct cxgb4_lld_info
*infop
)
363 struct c4iw_dev
*dev
;
364 static int vers_printed
;
368 printk(KERN_INFO MOD
"Chelsio T4 RDMA Driver - version %s\n",
371 dev
= c4iw_alloc(infop
);
375 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n",
376 __func__
, pci_name(dev
->rdev
.lldi
.pdev
),
377 dev
->rdev
.lldi
.nchan
, dev
->rdev
.lldi
.nrxq
,
378 dev
->rdev
.lldi
.ntxq
, dev
->rdev
.lldi
.nports
);
380 for (i
= 0; i
< dev
->rdev
.lldi
.nrxq
; i
++)
381 PDBG("rxqid[%u] %u\n", i
, dev
->rdev
.lldi
.rxq_ids
[i
]);
386 static struct sk_buff
*t4_pktgl_to_skb(const struct pkt_gl
*gl
,
387 unsigned int skb_len
,
388 unsigned int pull_len
)
391 struct skb_shared_info
*ssi
;
393 if (gl
->tot_len
<= 512) {
394 skb
= alloc_skb(gl
->tot_len
, GFP_ATOMIC
);
397 __skb_put(skb
, gl
->tot_len
);
398 skb_copy_to_linear_data(skb
, gl
->va
, gl
->tot_len
);
400 skb
= alloc_skb(skb_len
, GFP_ATOMIC
);
403 __skb_put(skb
, pull_len
);
404 skb_copy_to_linear_data(skb
, gl
->va
, pull_len
);
406 ssi
= skb_shinfo(skb
);
407 ssi
->frags
[0].page
= gl
->frags
[0].page
;
408 ssi
->frags
[0].page_offset
= gl
->frags
[0].page_offset
+ pull_len
;
409 ssi
->frags
[0].size
= gl
->frags
[0].size
- pull_len
;
411 memcpy(&ssi
->frags
[1], &gl
->frags
[1],
412 (gl
->nfrags
- 1) * sizeof(skb_frag_t
));
413 ssi
->nr_frags
= gl
->nfrags
;
415 skb
->len
= gl
->tot_len
;
416 skb
->data_len
= skb
->len
- pull_len
;
417 skb
->truesize
+= skb
->data_len
;
419 /* Get a reference for the last page, we don't own it */
420 get_page(gl
->frags
[gl
->nfrags
- 1].page
);
426 static int c4iw_uld_rx_handler(void *handle
, const __be64
*rsp
,
427 const struct pkt_gl
*gl
)
429 struct c4iw_dev
*dev
= handle
;
431 const struct cpl_act_establish
*rpl
;
435 /* omit RSS and rsp_ctrl at end of descriptor */
436 unsigned int len
= 64 - sizeof(struct rsp_ctrl
) - 8;
438 skb
= alloc_skb(256, GFP_ATOMIC
);
442 skb_copy_to_linear_data(skb
, &rsp
[1], len
);
443 } else if (gl
== CXGB4_MSG_AN
) {
444 const struct rsp_ctrl
*rc
= (void *)rsp
;
446 u32 qid
= be32_to_cpu(rc
->pldbuflen_qid
);
447 c4iw_ev_handler(dev
, qid
);
450 skb
= t4_pktgl_to_skb(gl
, 128, 128);
456 opcode
= rpl
->ot
.opcode
;
458 if (c4iw_handlers
[opcode
])
459 c4iw_handlers
[opcode
](dev
, skb
);
461 printk(KERN_INFO
"%s no handler opcode 0x%x...\n", __func__
,
469 static int c4iw_uld_state_change(void *handle
, enum cxgb4_state new_state
)
471 struct c4iw_dev
*dev
= handle
;
473 PDBG("%s new_state %u\n", __func__
, new_state
);
476 printk(KERN_INFO MOD
"%s: Up\n", pci_name(dev
->rdev
.lldi
.pdev
));
477 if (!dev
->registered
) {
479 ret
= c4iw_register_device(dev
);
482 "%s: RDMA registration failed: %d\n",
483 pci_name(dev
->rdev
.lldi
.pdev
), ret
);
486 case CXGB4_STATE_DOWN
:
487 printk(KERN_INFO MOD
"%s: Down\n",
488 pci_name(dev
->rdev
.lldi
.pdev
));
490 c4iw_unregister_device(dev
);
492 case CXGB4_STATE_START_RECOVERY
:
493 printk(KERN_INFO MOD
"%s: Fatal Error\n",
494 pci_name(dev
->rdev
.lldi
.pdev
));
496 c4iw_unregister_device(dev
);
498 case CXGB4_STATE_DETACH
:
499 printk(KERN_INFO MOD
"%s: Detach\n",
500 pci_name(dev
->rdev
.lldi
.pdev
));
501 mutex_lock(&dev_mutex
);
503 mutex_unlock(&dev_mutex
);
509 static struct cxgb4_uld_info c4iw_uld_info
= {
512 .rx_handler
= c4iw_uld_rx_handler
,
513 .state_change
= c4iw_uld_state_change
,
516 static int __init
c4iw_init_module(void)
520 err
= c4iw_cm_init();
524 c4iw_debugfs_root
= debugfs_create_dir(DRV_NAME
, NULL
);
525 if (!c4iw_debugfs_root
)
526 printk(KERN_WARNING MOD
527 "could not create debugfs entry, continuing\n");
529 cxgb4_register_uld(CXGB4_ULD_RDMA
, &c4iw_uld_info
);
534 static void __exit
c4iw_exit_module(void)
536 struct c4iw_dev
*dev
, *tmp
;
538 mutex_lock(&dev_mutex
);
539 list_for_each_entry_safe(dev
, tmp
, &dev_list
, entry
) {
542 mutex_unlock(&dev_mutex
);
543 cxgb4_unregister_uld(CXGB4_ULD_RDMA
);
545 debugfs_remove_recursive(c4iw_debugfs_root
);
548 module_init(c4iw_init_module
);
549 module_exit(c4iw_exit_module
);