2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <linux/moduleparam.h>
34 #include <linux/debugfs.h>
36 #include <rdma/ib_verbs.h>
40 #define DRV_VERSION "0.1"
42 MODULE_AUTHOR("Steve Wise");
43 MODULE_DESCRIPTION("Chelsio T4 RDMA Driver");
44 MODULE_LICENSE("Dual BSD/GPL");
45 MODULE_VERSION(DRV_VERSION
);
47 static LIST_HEAD(dev_list
);
48 static DEFINE_MUTEX(dev_mutex
);
50 static struct dentry
*c4iw_debugfs_root
;
52 struct debugfs_qp_data
{
53 struct c4iw_dev
*devp
;
59 static int count_qps(int id
, void *p
, void *data
)
61 struct c4iw_qp
*qp
= p
;
64 if (id
!= qp
->wq
.sq
.qid
)
67 *countp
= *countp
+ 1;
71 static int dump_qps(int id
, void *p
, void *data
)
73 struct c4iw_qp
*qp
= p
;
74 struct debugfs_qp_data
*qpd
= data
;
78 if (id
!= qp
->wq
.sq
.qid
)
81 space
= qpd
->bufsize
- qpd
->pos
- 1;
86 cc
= snprintf(qpd
->buf
+ qpd
->pos
, space
, "qp id %u state %u "
87 "ep tid %u state %u %pI4:%u->%pI4:%u\n",
88 qp
->wq
.sq
.qid
, (int)qp
->attr
.state
,
89 qp
->ep
->hwtid
, (int)qp
->ep
->com
.state
,
90 &qp
->ep
->com
.local_addr
.sin_addr
.s_addr
,
91 ntohs(qp
->ep
->com
.local_addr
.sin_port
),
92 &qp
->ep
->com
.remote_addr
.sin_addr
.s_addr
,
93 ntohs(qp
->ep
->com
.remote_addr
.sin_port
));
95 cc
= snprintf(qpd
->buf
+ qpd
->pos
, space
, "qp id %u state %u\n",
96 qp
->wq
.sq
.qid
, (int)qp
->attr
.state
);
102 static int qp_release(struct inode
*inode
, struct file
*file
)
104 struct debugfs_qp_data
*qpd
= file
->private_data
;
106 printk(KERN_INFO
"%s null qpd?\n", __func__
);
114 static int qp_open(struct inode
*inode
, struct file
*file
)
116 struct debugfs_qp_data
*qpd
;
120 qpd
= kmalloc(sizeof *qpd
, GFP_KERNEL
);
125 qpd
->devp
= inode
->i_private
;
128 spin_lock_irq(&qpd
->devp
->lock
);
129 idr_for_each(&qpd
->devp
->qpidr
, count_qps
, &count
);
130 spin_unlock_irq(&qpd
->devp
->lock
);
132 qpd
->bufsize
= count
* 128;
133 qpd
->buf
= kmalloc(qpd
->bufsize
, GFP_KERNEL
);
139 spin_lock_irq(&qpd
->devp
->lock
);
140 idr_for_each(&qpd
->devp
->qpidr
, dump_qps
, qpd
);
141 spin_unlock_irq(&qpd
->devp
->lock
);
143 qpd
->buf
[qpd
->pos
++] = 0;
144 file
->private_data
= qpd
;
152 static ssize_t
qp_read(struct file
*file
, char __user
*buf
, size_t count
,
155 struct debugfs_qp_data
*qpd
= file
->private_data
;
157 loff_t avail
= qpd
->pos
;
163 if (count
> avail
- pos
)
169 len
= min((int)count
, (int)qpd
->pos
- (int)pos
);
170 if (copy_to_user(buf
, qpd
->buf
+ pos
, len
))
184 static const struct file_operations qp_debugfs_fops
= {
185 .owner
= THIS_MODULE
,
187 .release
= qp_release
,
191 static int setup_debugfs(struct c4iw_dev
*devp
)
195 if (!devp
->debugfs_root
)
198 de
= debugfs_create_file("qps", S_IWUSR
, devp
->debugfs_root
,
199 (void *)devp
, &qp_debugfs_fops
);
200 if (de
&& de
->d_inode
)
201 de
->d_inode
->i_size
= 4096;
205 void c4iw_release_dev_ucontext(struct c4iw_rdev
*rdev
,
206 struct c4iw_dev_ucontext
*uctx
)
208 struct list_head
*pos
, *nxt
;
209 struct c4iw_qid_list
*entry
;
211 mutex_lock(&uctx
->lock
);
212 list_for_each_safe(pos
, nxt
, &uctx
->qpids
) {
213 entry
= list_entry(pos
, struct c4iw_qid_list
, entry
);
214 list_del_init(&entry
->entry
);
215 if (!(entry
->qid
& rdev
->qpmask
))
216 c4iw_put_resource(&rdev
->resource
.qid_fifo
, entry
->qid
,
217 &rdev
->resource
.qid_fifo_lock
);
221 list_for_each_safe(pos
, nxt
, &uctx
->qpids
) {
222 entry
= list_entry(pos
, struct c4iw_qid_list
, entry
);
223 list_del_init(&entry
->entry
);
226 mutex_unlock(&uctx
->lock
);
229 void c4iw_init_dev_ucontext(struct c4iw_rdev
*rdev
,
230 struct c4iw_dev_ucontext
*uctx
)
232 INIT_LIST_HEAD(&uctx
->qpids
);
233 INIT_LIST_HEAD(&uctx
->cqids
);
234 mutex_init(&uctx
->lock
);
237 /* Caller takes care of locking if needed */
238 static int c4iw_rdev_open(struct c4iw_rdev
*rdev
)
242 c4iw_init_dev_ucontext(rdev
, &rdev
->uctx
);
245 * qpshift is the number of bits to shift the qpid left in order
246 * to get the correct address of the doorbell for that qp.
248 rdev
->qpshift
= PAGE_SHIFT
- ilog2(rdev
->lldi
.udb_density
);
249 rdev
->qpmask
= rdev
->lldi
.udb_density
- 1;
250 rdev
->cqshift
= PAGE_SHIFT
- ilog2(rdev
->lldi
.ucq_density
);
251 rdev
->cqmask
= rdev
->lldi
.ucq_density
- 1;
252 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d "
253 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x\n",
254 __func__
, pci_name(rdev
->lldi
.pdev
), rdev
->lldi
.vr
->stag
.start
,
255 rdev
->lldi
.vr
->stag
.size
, c4iw_num_stags(rdev
),
256 rdev
->lldi
.vr
->pbl
.start
,
257 rdev
->lldi
.vr
->pbl
.size
, rdev
->lldi
.vr
->rq
.start
,
258 rdev
->lldi
.vr
->rq
.size
);
259 PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu "
260 "qpmask 0x%x cqshift %lu cqmask 0x%x\n",
261 (unsigned)pci_resource_len(rdev
->lldi
.pdev
, 2),
262 (void *)pci_resource_start(rdev
->lldi
.pdev
, 2),
265 rdev
->qpshift
, rdev
->qpmask
,
266 rdev
->cqshift
, rdev
->cqmask
);
268 if (c4iw_num_stags(rdev
) == 0) {
273 err
= c4iw_init_resource(rdev
, c4iw_num_stags(rdev
), T4_MAX_NUM_PD
);
275 printk(KERN_ERR MOD
"error %d initializing resources\n", err
);
278 err
= c4iw_pblpool_create(rdev
);
280 printk(KERN_ERR MOD
"error %d initializing pbl pool\n", err
);
283 err
= c4iw_rqtpool_create(rdev
);
285 printk(KERN_ERR MOD
"error %d initializing rqt pool\n", err
);
290 c4iw_pblpool_destroy(rdev
);
292 c4iw_destroy_resource(&rdev
->resource
);
297 static void c4iw_rdev_close(struct c4iw_rdev
*rdev
)
299 c4iw_pblpool_destroy(rdev
);
300 c4iw_rqtpool_destroy(rdev
);
301 c4iw_destroy_resource(&rdev
->resource
);
304 static void c4iw_remove(struct c4iw_dev
*dev
)
306 PDBG("%s c4iw_dev %p\n", __func__
, dev
);
307 cancel_delayed_work_sync(&dev
->db_drop_task
);
308 list_del(&dev
->entry
);
310 c4iw_unregister_device(dev
);
311 c4iw_rdev_close(&dev
->rdev
);
312 idr_destroy(&dev
->cqidr
);
313 idr_destroy(&dev
->qpidr
);
314 idr_destroy(&dev
->mmidr
);
315 ib_dealloc_device(&dev
->ibdev
);
318 static struct c4iw_dev
*c4iw_alloc(const struct cxgb4_lld_info
*infop
)
320 struct c4iw_dev
*devp
;
323 devp
= (struct c4iw_dev
*)ib_alloc_device(sizeof(*devp
));
325 printk(KERN_ERR MOD
"Cannot allocate ib device\n");
328 devp
->rdev
.lldi
= *infop
;
330 mutex_lock(&dev_mutex
);
332 ret
= c4iw_rdev_open(&devp
->rdev
);
334 mutex_unlock(&dev_mutex
);
335 printk(KERN_ERR MOD
"Unable to open CXIO rdev err %d\n", ret
);
336 ib_dealloc_device(&devp
->ibdev
);
340 idr_init(&devp
->cqidr
);
341 idr_init(&devp
->qpidr
);
342 idr_init(&devp
->mmidr
);
343 spin_lock_init(&devp
->lock
);
344 list_add_tail(&devp
->entry
, &dev_list
);
345 mutex_unlock(&dev_mutex
);
347 if (c4iw_debugfs_root
) {
348 devp
->debugfs_root
= debugfs_create_dir(
349 pci_name(devp
->rdev
.lldi
.pdev
),
356 static void *c4iw_uld_add(const struct cxgb4_lld_info
*infop
)
358 struct c4iw_dev
*dev
;
359 static int vers_printed
;
363 printk(KERN_INFO MOD
"Chelsio T4 RDMA Driver - version %s\n",
366 dev
= c4iw_alloc(infop
);
370 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n",
371 __func__
, pci_name(dev
->rdev
.lldi
.pdev
),
372 dev
->rdev
.lldi
.nchan
, dev
->rdev
.lldi
.nrxq
,
373 dev
->rdev
.lldi
.ntxq
, dev
->rdev
.lldi
.nports
);
375 for (i
= 0; i
< dev
->rdev
.lldi
.nrxq
; i
++)
376 PDBG("rxqid[%u] %u\n", i
, dev
->rdev
.lldi
.rxq_ids
[i
]);
381 static struct sk_buff
*t4_pktgl_to_skb(const struct pkt_gl
*gl
,
382 unsigned int skb_len
,
383 unsigned int pull_len
)
386 struct skb_shared_info
*ssi
;
388 if (gl
->tot_len
<= 512) {
389 skb
= alloc_skb(gl
->tot_len
, GFP_ATOMIC
);
392 __skb_put(skb
, gl
->tot_len
);
393 skb_copy_to_linear_data(skb
, gl
->va
, gl
->tot_len
);
395 skb
= alloc_skb(skb_len
, GFP_ATOMIC
);
398 __skb_put(skb
, pull_len
);
399 skb_copy_to_linear_data(skb
, gl
->va
, pull_len
);
401 ssi
= skb_shinfo(skb
);
402 ssi
->frags
[0].page
= gl
->frags
[0].page
;
403 ssi
->frags
[0].page_offset
= gl
->frags
[0].page_offset
+ pull_len
;
404 ssi
->frags
[0].size
= gl
->frags
[0].size
- pull_len
;
406 memcpy(&ssi
->frags
[1], &gl
->frags
[1],
407 (gl
->nfrags
- 1) * sizeof(skb_frag_t
));
408 ssi
->nr_frags
= gl
->nfrags
;
410 skb
->len
= gl
->tot_len
;
411 skb
->data_len
= skb
->len
- pull_len
;
412 skb
->truesize
+= skb
->data_len
;
414 /* Get a reference for the last page, we don't own it */
415 get_page(gl
->frags
[gl
->nfrags
- 1].page
);
421 static int c4iw_uld_rx_handler(void *handle
, const __be64
*rsp
,
422 const struct pkt_gl
*gl
)
424 struct c4iw_dev
*dev
= handle
;
426 const struct cpl_act_establish
*rpl
;
430 /* omit RSS and rsp_ctrl at end of descriptor */
431 unsigned int len
= 64 - sizeof(struct rsp_ctrl
) - 8;
433 skb
= alloc_skb(256, GFP_ATOMIC
);
437 skb_copy_to_linear_data(skb
, &rsp
[1], len
);
438 } else if (gl
== CXGB4_MSG_AN
) {
439 const struct rsp_ctrl
*rc
= (void *)rsp
;
441 u32 qid
= be32_to_cpu(rc
->pldbuflen_qid
);
442 c4iw_ev_handler(dev
, qid
);
445 skb
= t4_pktgl_to_skb(gl
, 128, 128);
451 opcode
= rpl
->ot
.opcode
;
453 if (c4iw_handlers
[opcode
])
454 c4iw_handlers
[opcode
](dev
, skb
);
456 printk(KERN_INFO
"%s no handler opcode 0x%x...\n", __func__
,
464 static int c4iw_uld_state_change(void *handle
, enum cxgb4_state new_state
)
466 struct c4iw_dev
*dev
= handle
;
468 PDBG("%s new_state %u\n", __func__
, new_state
);
471 printk(KERN_INFO MOD
"%s: Up\n", pci_name(dev
->rdev
.lldi
.pdev
));
472 if (!dev
->registered
) {
474 ret
= c4iw_register_device(dev
);
477 "%s: RDMA registration failed: %d\n",
478 pci_name(dev
->rdev
.lldi
.pdev
), ret
);
481 case CXGB4_STATE_DOWN
:
482 printk(KERN_INFO MOD
"%s: Down\n",
483 pci_name(dev
->rdev
.lldi
.pdev
));
485 c4iw_unregister_device(dev
);
487 case CXGB4_STATE_START_RECOVERY
:
488 printk(KERN_INFO MOD
"%s: Fatal Error\n",
489 pci_name(dev
->rdev
.lldi
.pdev
));
491 c4iw_unregister_device(dev
);
493 case CXGB4_STATE_DETACH
:
494 printk(KERN_INFO MOD
"%s: Detach\n",
495 pci_name(dev
->rdev
.lldi
.pdev
));
496 mutex_lock(&dev_mutex
);
498 mutex_unlock(&dev_mutex
);
504 static struct cxgb4_uld_info c4iw_uld_info
= {
507 .rx_handler
= c4iw_uld_rx_handler
,
508 .state_change
= c4iw_uld_state_change
,
511 static int __init
c4iw_init_module(void)
515 err
= c4iw_cm_init();
519 c4iw_debugfs_root
= debugfs_create_dir(DRV_NAME
, NULL
);
520 if (!c4iw_debugfs_root
)
521 printk(KERN_WARNING MOD
522 "could not create debugfs entry, continuing\n");
524 cxgb4_register_uld(CXGB4_ULD_RDMA
, &c4iw_uld_info
);
529 static void __exit
c4iw_exit_module(void)
531 struct c4iw_dev
*dev
, *tmp
;
533 mutex_lock(&dev_mutex
);
534 list_for_each_entry_safe(dev
, tmp
, &dev_list
, entry
) {
537 mutex_unlock(&dev_mutex
);
538 cxgb4_unregister_uld(CXGB4_ULD_RDMA
);
540 debugfs_remove_recursive(c4iw_debugfs_root
);
543 module_init(c4iw_init_module
);
544 module_exit(c4iw_exit_module
);