2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
15 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * Intel PCIe NTB Linux driver
45 * Contact Information:
46 * Jon Mason <jon.mason@intel.com>
48 #include <linux/debugfs.h>
49 #include <linux/delay.h>
50 #include <linux/dma-mapping.h>
51 #include <linux/errno.h>
52 #include <linux/export.h>
53 #include <linux/interrupt.h>
54 #include <linux/module.h>
55 #include <linux/pci.h>
56 #include <linux/slab.h>
57 #include <linux/types.h>
58 #include <linux/ntb.h>
61 #define NTB_TRANSPORT_VERSION 1
63 static int transport_mtu
= 0x401E;
64 module_param(transport_mtu
, uint
, 0644);
65 MODULE_PARM_DESC(transport_mtu
, "Maximum size of NTB transport packets");
67 static unsigned char max_num_clients
= 2;
68 module_param(max_num_clients
, byte
, 0644);
69 MODULE_PARM_DESC(max_num_clients
, "Maximum number of NTB transport clients");
71 struct ntb_queue_entry
{
72 /* ntb_queue list reference */
73 struct list_head entry
;
74 /* pointers to data to be transfered */
81 struct ntb_transport_qp
{
82 struct ntb_transport
*transport
;
83 struct ntb_device
*ndev
;
88 u8 qp_num
; /* Only 64 QP's are allowed. 0-63 */
90 void (*tx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
92 struct list_head tx_free_q
;
93 spinlock_t ntb_tx_free_q_lock
;
98 void (*rx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
100 struct tasklet_struct rx_work
;
101 struct list_head rx_pend_q
;
102 struct list_head rx_free_q
;
103 spinlock_t ntb_rx_pend_q_lock
;
104 spinlock_t ntb_rx_free_q_lock
;
109 void (*event_handler
) (void *data
, int status
);
110 struct delayed_work link_work
;
112 struct dentry
*debugfs_dir
;
113 struct dentry
*debugfs_stats
;
127 struct ntb_transport_mw
{
133 struct ntb_transport_client_dev
{
134 struct list_head entry
;
138 struct ntb_transport
{
139 struct list_head entry
;
140 struct list_head client_devs
;
142 struct ntb_device
*ndev
;
143 struct ntb_transport_mw mw
[NTB_NUM_MW
];
144 struct ntb_transport_qp
*qps
;
145 unsigned int max_qps
;
146 unsigned long qp_bitmap
;
148 struct delayed_work link_work
;
149 struct dentry
*debugfs_dir
;
153 DESC_DONE_FLAG
= 1 << 0,
154 LINK_DOWN_FLAG
= 1 << 1,
157 struct ntb_payload_header
{
172 #define QP_TO_MW(qp) ((qp) % NTB_NUM_MW)
173 #define NTB_QP_DEF_NUM_ENTRIES 100
174 #define NTB_LINK_DOWN_TIMEOUT 10
176 static int ntb_match_bus(struct device
*dev
, struct device_driver
*drv
)
178 return !strncmp(dev_name(dev
), drv
->name
, strlen(drv
->name
));
181 static int ntb_client_probe(struct device
*dev
)
183 const struct ntb_client
*drv
= container_of(dev
->driver
,
184 struct ntb_client
, driver
);
185 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
189 if (drv
&& drv
->probe
)
190 rc
= drv
->probe(pdev
);
197 static int ntb_client_remove(struct device
*dev
)
199 const struct ntb_client
*drv
= container_of(dev
->driver
,
200 struct ntb_client
, driver
);
201 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
203 if (drv
&& drv
->remove
)
211 struct bus_type ntb_bus_type
= {
213 .match
= ntb_match_bus
,
214 .probe
= ntb_client_probe
,
215 .remove
= ntb_client_remove
,
218 static LIST_HEAD(ntb_transport_list
);
220 static int ntb_bus_init(struct ntb_transport
*nt
)
222 if (list_empty(&ntb_transport_list
)) {
223 int rc
= bus_register(&ntb_bus_type
);
228 list_add(&nt
->entry
, &ntb_transport_list
);
233 static void ntb_bus_remove(struct ntb_transport
*nt
)
235 struct ntb_transport_client_dev
*client_dev
, *cd
;
237 list_for_each_entry_safe(client_dev
, cd
, &nt
->client_devs
, entry
) {
238 dev_err(client_dev
->dev
.parent
, "%s still attached to bus, removing\n",
239 dev_name(&client_dev
->dev
));
240 list_del(&client_dev
->entry
);
241 device_unregister(&client_dev
->dev
);
244 list_del(&nt
->entry
);
246 if (list_empty(&ntb_transport_list
))
247 bus_unregister(&ntb_bus_type
);
250 static void ntb_client_release(struct device
*dev
)
252 struct ntb_transport_client_dev
*client_dev
;
253 client_dev
= container_of(dev
, struct ntb_transport_client_dev
, dev
);
259 * ntb_unregister_client_dev - Unregister NTB client device
260 * @device_name: Name of NTB client device
262 * Unregister an NTB client device with the NTB transport layer
264 void ntb_unregister_client_dev(char *device_name
)
266 struct ntb_transport_client_dev
*client
, *cd
;
267 struct ntb_transport
*nt
;
269 list_for_each_entry(nt
, &ntb_transport_list
, entry
)
270 list_for_each_entry_safe(client
, cd
, &nt
->client_devs
, entry
)
271 if (!strncmp(dev_name(&client
->dev
), device_name
,
272 strlen(device_name
))) {
273 list_del(&client
->entry
);
274 device_unregister(&client
->dev
);
277 EXPORT_SYMBOL_GPL(ntb_unregister_client_dev
);
280 * ntb_register_client_dev - Register NTB client device
281 * @device_name: Name of NTB client device
283 * Register an NTB client device with the NTB transport layer
285 int ntb_register_client_dev(char *device_name
)
287 struct ntb_transport_client_dev
*client_dev
;
288 struct ntb_transport
*nt
;
291 list_for_each_entry(nt
, &ntb_transport_list
, entry
) {
294 client_dev
= kzalloc(sizeof(struct ntb_transport_client_dev
),
301 dev
= &client_dev
->dev
;
303 /* setup and register client devices */
304 dev_set_name(dev
, "%s", device_name
);
305 dev
->bus
= &ntb_bus_type
;
306 dev
->release
= ntb_client_release
;
307 dev
->parent
= &ntb_query_pdev(nt
->ndev
)->dev
;
309 rc
= device_register(dev
);
315 list_add_tail(&client_dev
->entry
, &nt
->client_devs
);
321 ntb_unregister_client_dev(device_name
);
325 EXPORT_SYMBOL_GPL(ntb_register_client_dev
);
328 * ntb_register_client - Register NTB client driver
329 * @drv: NTB client driver to be registered
331 * Register an NTB client driver with the NTB transport layer
333 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
335 int ntb_register_client(struct ntb_client
*drv
)
337 drv
->driver
.bus
= &ntb_bus_type
;
339 return driver_register(&drv
->driver
);
341 EXPORT_SYMBOL_GPL(ntb_register_client
);
344 * ntb_unregister_client - Unregister NTB client driver
345 * @drv: NTB client driver to be unregistered
347 * Unregister an NTB client driver with the NTB transport layer
349 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
351 void ntb_unregister_client(struct ntb_client
*drv
)
353 driver_unregister(&drv
->driver
);
355 EXPORT_SYMBOL_GPL(ntb_unregister_client
);
357 static int debugfs_open(struct inode
*inode
, struct file
*filp
)
359 filp
->private_data
= inode
->i_private
;
363 static ssize_t
debugfs_read(struct file
*filp
, char __user
*ubuf
, size_t count
,
366 struct ntb_transport_qp
*qp
;
368 ssize_t ret
, out_offset
, out_count
;
372 qp
= filp
->private_data
;
374 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
376 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
377 "rx_bytes - \t%llu\n", qp
->rx_bytes
);
378 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
379 "rx_pkts - \t%llu\n", qp
->rx_pkts
);
380 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
381 "rx_ring_empty - %llu\n", qp
->rx_ring_empty
);
382 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
383 "rx_err_no_buf - %llu\n", qp
->rx_err_no_buf
);
384 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
385 "rx_err_oflow - \t%llu\n", qp
->rx_err_oflow
);
386 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
387 "rx_err_ver - \t%llu\n", qp
->rx_err_ver
);
388 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
389 "rx_buff_begin - %p\n", qp
->rx_buff_begin
);
390 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
391 "rx_offset - \t%p\n", qp
->rx_offset
);
392 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
393 "rx_buff_end - \t%p\n", qp
->rx_buff_end
);
395 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
396 "tx_bytes - \t%llu\n", qp
->tx_bytes
);
397 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
398 "tx_pkts - \t%llu\n", qp
->tx_pkts
);
399 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
400 "tx_ring_full - \t%llu\n", qp
->tx_ring_full
);
401 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
402 "tx_mw_begin - \t%p\n", qp
->tx_mw_begin
);
403 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
404 "tx_offset - \t%p\n", qp
->tx_offset
);
405 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
406 "tx_mw_end - \t%p\n", qp
->tx_mw_end
);
408 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
409 "QP Link %s\n", (qp
->qp_link
== NTB_LINK_UP
) ?
412 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, out_offset
);
416 static const struct file_operations ntb_qp_debugfs_stats
= {
417 .owner
= THIS_MODULE
,
418 .open
= debugfs_open
,
419 .read
= debugfs_read
,
422 static void ntb_list_add(spinlock_t
*lock
, struct list_head
*entry
,
423 struct list_head
*list
)
427 spin_lock_irqsave(lock
, flags
);
428 list_add_tail(entry
, list
);
429 spin_unlock_irqrestore(lock
, flags
);
432 static struct ntb_queue_entry
*ntb_list_rm(spinlock_t
*lock
,
433 struct list_head
*list
)
435 struct ntb_queue_entry
*entry
;
438 spin_lock_irqsave(lock
, flags
);
439 if (list_empty(list
)) {
443 entry
= list_first_entry(list
, struct ntb_queue_entry
, entry
);
444 list_del(&entry
->entry
);
446 spin_unlock_irqrestore(lock
, flags
);
451 static void ntb_transport_setup_qp_mw(struct ntb_transport
*nt
,
454 struct ntb_transport_qp
*qp
= &nt
->qps
[qp_num
];
455 unsigned int size
, num_qps_mw
;
456 u8 mw_num
= QP_TO_MW(qp_num
);
458 WARN_ON(nt
->mw
[mw_num
].virt_addr
== 0);
460 if (nt
->max_qps
% NTB_NUM_MW
&& !mw_num
)
461 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
+
462 (nt
->max_qps
% NTB_NUM_MW
- mw_num
);
464 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
;
466 size
= nt
->mw
[mw_num
].size
/ num_qps_mw
;
468 qp
->rx_buff_begin
= nt
->mw
[mw_num
].virt_addr
+
469 (qp_num
/ NTB_NUM_MW
* size
);
470 qp
->rx_buff_end
= qp
->rx_buff_begin
+ size
;
471 qp
->rx_offset
= qp
->rx_buff_begin
;
473 qp
->tx_mw_begin
= ntb_get_mw_vbase(nt
->ndev
, mw_num
) +
474 (qp_num
/ NTB_NUM_MW
* size
);
475 qp
->tx_mw_end
= qp
->tx_mw_begin
+ size
;
476 qp
->tx_offset
= qp
->tx_mw_begin
;
482 static int ntb_set_mw(struct ntb_transport
*nt
, int num_mw
, unsigned int size
)
484 struct ntb_transport_mw
*mw
= &nt
->mw
[num_mw
];
485 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
488 /* Alloc memory for receiving data. Must be 4k aligned */
489 mw
->size
= ALIGN(size
, 4096);
491 mw
->virt_addr
= dma_alloc_coherent(&pdev
->dev
, mw
->size
, &mw
->dma_addr
,
493 if (!mw
->virt_addr
) {
494 dev_err(&pdev
->dev
, "Unable to allocate MW buffer of size %d\n",
499 /* setup the hdr offsets with 0's */
500 for (offset
= mw
->virt_addr
+ transport_mtu
-
501 sizeof(struct ntb_payload_header
);
502 offset
< mw
->virt_addr
+ size
; offset
+= transport_mtu
)
503 memset(offset
, 0, sizeof(struct ntb_payload_header
));
505 /* Notify HW the memory location of the receive buffer */
506 ntb_set_mw_addr(nt
->ndev
, num_mw
, mw
->dma_addr
);
511 static void ntb_qp_link_down(struct ntb_transport_qp
*qp
)
513 struct ntb_transport
*nt
= qp
->transport
;
514 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
516 if (qp
->qp_link
== NTB_LINK_DOWN
) {
517 cancel_delayed_work_sync(&qp
->link_work
);
521 if (qp
->event_handler
)
522 qp
->event_handler(qp
->cb_data
, NTB_LINK_DOWN
);
524 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
525 qp
->qp_link
= NTB_LINK_DOWN
;
527 if (nt
->transport_link
== NTB_LINK_UP
)
528 schedule_delayed_work(&qp
->link_work
,
529 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
532 static void ntb_transport_conn_down(struct ntb_transport
*nt
)
536 if (nt
->transport_link
== NTB_LINK_DOWN
)
537 cancel_delayed_work_sync(&nt
->link_work
);
539 nt
->transport_link
= NTB_LINK_DOWN
;
541 /* Pass along the info to any clients */
542 for (i
= 0; i
< nt
->max_qps
; i
++)
543 if (!test_bit(i
, &nt
->qp_bitmap
))
544 ntb_qp_link_down(&nt
->qps
[i
]);
546 /* The scratchpad registers keep the values if the remote side
547 * goes down, blast them now to give them a sane value the next
548 * time they are accessed
550 for (i
= 0; i
< MAX_SPAD
; i
++)
551 ntb_write_local_spad(nt
->ndev
, i
, 0);
554 static void ntb_transport_event_callback(void *data
, enum ntb_hw_event event
)
556 struct ntb_transport
*nt
= data
;
559 case NTB_EVENT_HW_LINK_UP
:
560 schedule_delayed_work(&nt
->link_work
, 0);
562 case NTB_EVENT_HW_LINK_DOWN
:
563 ntb_transport_conn_down(nt
);
570 static void ntb_transport_link_work(struct work_struct
*work
)
572 struct ntb_transport
*nt
= container_of(work
, struct ntb_transport
,
574 struct ntb_device
*ndev
= nt
->ndev
;
575 struct pci_dev
*pdev
= ntb_query_pdev(ndev
);
579 /* send the local info */
580 rc
= ntb_write_remote_spad(ndev
, VERSION
, NTB_TRANSPORT_VERSION
);
582 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
587 rc
= ntb_write_remote_spad(ndev
, MW0_SZ
, ntb_get_mw_size(ndev
, 0));
589 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
590 (u32
) ntb_get_mw_size(ndev
, 0), MW0_SZ
);
594 rc
= ntb_write_remote_spad(ndev
, MW1_SZ
, ntb_get_mw_size(ndev
, 1));
596 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
597 (u32
) ntb_get_mw_size(ndev
, 1), MW1_SZ
);
601 rc
= ntb_write_remote_spad(ndev
, NUM_QPS
, nt
->max_qps
);
603 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
604 nt
->max_qps
, NUM_QPS
);
608 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
610 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
614 rc
= ntb_write_remote_spad(ndev
, QP_LINKS
, val
);
616 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
621 /* Query the remote side for its info */
622 rc
= ntb_read_remote_spad(ndev
, VERSION
, &val
);
624 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", VERSION
);
628 if (val
!= NTB_TRANSPORT_VERSION
)
630 dev_dbg(&pdev
->dev
, "Remote version = %d\n", val
);
632 rc
= ntb_read_remote_spad(ndev
, NUM_QPS
, &val
);
634 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", NUM_QPS
);
638 if (val
!= nt
->max_qps
)
640 dev_dbg(&pdev
->dev
, "Remote max number of qps = %d\n", val
);
642 rc
= ntb_read_remote_spad(ndev
, MW0_SZ
, &val
);
644 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW0_SZ
);
650 dev_dbg(&pdev
->dev
, "Remote MW0 size = %d\n", val
);
652 rc
= ntb_set_mw(nt
, 0, val
);
656 rc
= ntb_read_remote_spad(ndev
, MW1_SZ
, &val
);
658 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW1_SZ
);
664 dev_dbg(&pdev
->dev
, "Remote MW1 size = %d\n", val
);
666 rc
= ntb_set_mw(nt
, 1, val
);
670 nt
->transport_link
= NTB_LINK_UP
;
672 for (i
= 0; i
< nt
->max_qps
; i
++) {
673 struct ntb_transport_qp
*qp
= &nt
->qps
[i
];
675 ntb_transport_setup_qp_mw(nt
, i
);
677 if (qp
->client_ready
== NTB_LINK_UP
)
678 schedule_delayed_work(&qp
->link_work
, 0);
684 if (ntb_hw_link_status(ndev
))
685 schedule_delayed_work(&nt
->link_work
,
686 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
689 static void ntb_qp_link_work(struct work_struct
*work
)
691 struct ntb_transport_qp
*qp
= container_of(work
,
692 struct ntb_transport_qp
,
694 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
695 struct ntb_transport
*nt
= qp
->transport
;
698 WARN_ON(nt
->transport_link
!= NTB_LINK_UP
);
700 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
702 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
706 rc
= ntb_write_remote_spad(nt
->ndev
, QP_LINKS
, val
| 1 << qp
->qp_num
);
708 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
709 val
| 1 << qp
->qp_num
, QP_LINKS
);
711 /* query remote spad for qp ready bits */
712 rc
= ntb_read_remote_spad(nt
->ndev
, QP_LINKS
, &val
);
714 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", QP_LINKS
);
716 dev_dbg(&pdev
->dev
, "Remote QP link status = %x\n", val
);
718 /* See if the remote side is up */
719 if (1 << qp
->qp_num
& val
) {
720 qp
->qp_link
= NTB_LINK_UP
;
722 dev_info(&pdev
->dev
, "qp %d: Link Up\n", qp
->qp_num
);
723 if (qp
->event_handler
)
724 qp
->event_handler(qp
->cb_data
, NTB_LINK_UP
);
725 } else if (nt
->transport_link
== NTB_LINK_UP
)
726 schedule_delayed_work(&qp
->link_work
,
727 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
730 static void ntb_transport_init_queue(struct ntb_transport
*nt
,
733 struct ntb_transport_qp
*qp
;
735 qp
= &nt
->qps
[qp_num
];
739 qp
->qp_link
= NTB_LINK_DOWN
;
740 qp
->client_ready
= NTB_LINK_DOWN
;
741 qp
->event_handler
= NULL
;
743 if (nt
->debugfs_dir
) {
744 char debugfs_name
[4];
746 snprintf(debugfs_name
, 4, "qp%d", qp_num
);
747 qp
->debugfs_dir
= debugfs_create_dir(debugfs_name
,
750 qp
->debugfs_stats
= debugfs_create_file("stats", S_IRUSR
,
752 &ntb_qp_debugfs_stats
);
755 INIT_DELAYED_WORK(&qp
->link_work
, ntb_qp_link_work
);
757 spin_lock_init(&qp
->ntb_rx_pend_q_lock
);
758 spin_lock_init(&qp
->ntb_rx_free_q_lock
);
759 spin_lock_init(&qp
->ntb_tx_free_q_lock
);
761 INIT_LIST_HEAD(&qp
->rx_pend_q
);
762 INIT_LIST_HEAD(&qp
->rx_free_q
);
763 INIT_LIST_HEAD(&qp
->tx_free_q
);
766 int ntb_transport_init(struct pci_dev
*pdev
)
768 struct ntb_transport
*nt
;
771 nt
= kzalloc(sizeof(struct ntb_transport
), GFP_KERNEL
);
775 if (debugfs_initialized())
776 nt
->debugfs_dir
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
778 nt
->debugfs_dir
= NULL
;
780 nt
->ndev
= ntb_register_transport(pdev
, nt
);
786 nt
->max_qps
= min(nt
->ndev
->max_cbs
, max_num_clients
);
788 nt
->qps
= kcalloc(nt
->max_qps
, sizeof(struct ntb_transport_qp
),
795 nt
->qp_bitmap
= ((u64
) 1 << nt
->max_qps
) - 1;
797 for (i
= 0; i
< nt
->max_qps
; i
++)
798 ntb_transport_init_queue(nt
, i
);
800 INIT_DELAYED_WORK(&nt
->link_work
, ntb_transport_link_work
);
802 rc
= ntb_register_event_callback(nt
->ndev
,
803 ntb_transport_event_callback
);
807 INIT_LIST_HEAD(&nt
->client_devs
);
808 rc
= ntb_bus_init(nt
);
812 if (ntb_hw_link_status(nt
->ndev
))
813 schedule_delayed_work(&nt
->link_work
, 0);
818 ntb_unregister_event_callback(nt
->ndev
);
822 ntb_unregister_transport(nt
->ndev
);
824 debugfs_remove_recursive(nt
->debugfs_dir
);
829 void ntb_transport_free(void *transport
)
831 struct ntb_transport
*nt
= transport
;
832 struct pci_dev
*pdev
;
835 nt
->transport_link
= NTB_LINK_DOWN
;
837 /* verify that all the qp's are freed */
838 for (i
= 0; i
< nt
->max_qps
; i
++)
839 if (!test_bit(i
, &nt
->qp_bitmap
))
840 ntb_transport_free_queue(&nt
->qps
[i
]);
844 cancel_delayed_work_sync(&nt
->link_work
);
846 debugfs_remove_recursive(nt
->debugfs_dir
);
848 ntb_unregister_event_callback(nt
->ndev
);
850 pdev
= ntb_query_pdev(nt
->ndev
);
852 for (i
= 0; i
< NTB_NUM_MW
; i
++)
853 if (nt
->mw
[i
].virt_addr
)
854 dma_free_coherent(&pdev
->dev
, nt
->mw
[i
].size
,
859 ntb_unregister_transport(nt
->ndev
);
863 static void ntb_rx_copy_task(struct ntb_transport_qp
*qp
,
864 struct ntb_queue_entry
*entry
, void *offset
)
867 struct ntb_payload_header
*hdr
;
869 BUG_ON(offset
< qp
->rx_buff_begin
||
870 offset
+ transport_mtu
>= qp
->rx_buff_end
);
872 hdr
= offset
+ transport_mtu
- sizeof(struct ntb_payload_header
);
873 entry
->len
= hdr
->len
;
875 memcpy(entry
->buf
, offset
, entry
->len
);
877 /* Ensure that the data is fully copied out before clearing the flag */
881 if (qp
->rx_handler
&& qp
->client_ready
== NTB_LINK_UP
)
882 qp
->rx_handler(qp
, qp
->cb_data
, entry
->cb_data
, entry
->len
);
884 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
, &qp
->rx_free_q
);
887 static int ntb_process_rxc(struct ntb_transport_qp
*qp
)
889 struct ntb_payload_header
*hdr
;
890 struct ntb_queue_entry
*entry
;
893 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
895 hdr
= offset
+ transport_mtu
-
896 sizeof(struct ntb_payload_header
);
897 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
898 "no buffer - HDR ver %llu, len %d, flags %x\n",
899 hdr
->ver
, hdr
->len
, hdr
->flags
);
904 offset
= qp
->rx_offset
;
905 hdr
= offset
+ transport_mtu
- sizeof(struct ntb_payload_header
);
907 if (!(hdr
->flags
& DESC_DONE_FLAG
)) {
908 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
914 if (hdr
->ver
!= qp
->rx_pkts
) {
915 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
916 "qp %d: version mismatch, expected %llu - got %llu\n",
917 qp
->qp_num
, qp
->rx_pkts
, hdr
->ver
);
918 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
924 if (hdr
->flags
& LINK_DOWN_FLAG
) {
925 ntb_qp_link_down(qp
);
927 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
930 /* Ensure that the data is fully copied out before clearing the
938 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
939 "rx offset %p, ver %llu - %d payload received, buf size %d\n",
940 qp
->rx_offset
, hdr
->ver
, hdr
->len
, entry
->len
);
942 if (hdr
->len
<= entry
->len
)
943 ntb_rx_copy_task(qp
, entry
, offset
);
945 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
948 /* Ensure that the data is fully copied out before clearing the
954 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
955 "RX overflow! Wanted %d got %d\n",
956 hdr
->len
, entry
->len
);
959 qp
->rx_bytes
+= hdr
->len
;
963 qp
->rx_offset
+= transport_mtu
;
964 if (qp
->rx_offset
+ transport_mtu
>= qp
->rx_buff_end
)
965 qp
->rx_offset
= qp
->rx_buff_begin
;
970 static void ntb_transport_rx(unsigned long data
)
972 struct ntb_transport_qp
*qp
= (struct ntb_transport_qp
*)data
;
976 rc
= ntb_process_rxc(qp
);
980 static void ntb_transport_rxc_db(void *data
, int db_num
)
982 struct ntb_transport_qp
*qp
= data
;
984 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%s: doorbell %d received\n",
987 tasklet_schedule(&qp
->rx_work
);
990 static void ntb_tx_copy_task(struct ntb_transport_qp
*qp
,
991 struct ntb_queue_entry
*entry
,
994 struct ntb_payload_header
*hdr
;
996 BUG_ON(offset
< qp
->tx_mw_begin
||
997 offset
+ transport_mtu
>= qp
->tx_mw_end
);
999 memcpy_toio(offset
, entry
->buf
, entry
->len
);
1001 hdr
= offset
+ transport_mtu
- sizeof(struct ntb_payload_header
);
1002 hdr
->len
= entry
->len
;
1003 hdr
->ver
= qp
->tx_pkts
;
1005 /* Ensure that the data is fully copied out before setting the flag */
1007 hdr
->flags
= entry
->flags
| DESC_DONE_FLAG
;
1009 ntb_ring_sdb(qp
->ndev
, qp
->qp_num
);
1011 /* The entry length can only be zero if the packet is intended to be a
1012 * "link down" or similar. Since no payload is being sent in these
1013 * cases, there is nothing to add to the completion queue.
1015 if (entry
->len
> 0) {
1016 qp
->tx_bytes
+= entry
->len
;
1019 qp
->tx_handler(qp
, qp
->cb_data
, entry
->cb_data
,
1023 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
, &qp
->tx_free_q
);
1026 static int ntb_process_tx(struct ntb_transport_qp
*qp
,
1027 struct ntb_queue_entry
*entry
)
1029 struct ntb_payload_header
*hdr
;
1032 offset
= qp
->tx_offset
;
1033 hdr
= offset
+ transport_mtu
- sizeof(struct ntb_payload_header
);
1035 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%lld - offset %p, tx %p, entry len %d flags %x buff %p\n",
1036 qp
->tx_pkts
, offset
, qp
->tx_offset
, entry
->len
, entry
->flags
,
1043 if (entry
->len
> transport_mtu
- sizeof(struct ntb_payload_header
)) {
1045 qp
->tx_handler(qp
->cb_data
, qp
, NULL
, -EIO
);
1047 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1052 ntb_tx_copy_task(qp
, entry
, offset
);
1054 qp
->tx_offset
+= transport_mtu
;
1055 if (qp
->tx_offset
+ transport_mtu
>= qp
->tx_mw_end
)
1056 qp
->tx_offset
= qp
->tx_mw_begin
;
1063 static void ntb_send_link_down(struct ntb_transport_qp
*qp
)
1065 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1066 struct ntb_queue_entry
*entry
;
1069 if (qp
->qp_link
== NTB_LINK_DOWN
)
1072 qp
->qp_link
= NTB_LINK_DOWN
;
1073 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
1075 for (i
= 0; i
< NTB_LINK_DOWN_TIMEOUT
; i
++) {
1076 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
,
1086 entry
->cb_data
= NULL
;
1089 entry
->flags
= LINK_DOWN_FLAG
;
1091 rc
= ntb_process_tx(qp
, entry
);
1093 dev_err(&pdev
->dev
, "ntb: QP%d unable to send linkdown msg\n",
1098 * ntb_transport_create_queue - Create a new NTB transport layer queue
1099 * @rx_handler: receive callback function
1100 * @tx_handler: transmit callback function
1101 * @event_handler: event callback function
1103 * Create a new NTB transport layer queue and provide the queue with a callback
1104 * routine for both transmit and receive. The receive callback routine will be
1105 * used to pass up data when the transport has received it on the queue. The
1106 * transmit callback routine will be called when the transport has completed the
1107 * transmission of the data on the queue and the data is ready to be freed.
1109 * RETURNS: pointer to newly created ntb_queue, NULL on error.
1111 struct ntb_transport_qp
*
1112 ntb_transport_create_queue(void *data
, struct pci_dev
*pdev
,
1113 const struct ntb_queue_handlers
*handlers
)
1115 struct ntb_queue_entry
*entry
;
1116 struct ntb_transport_qp
*qp
;
1117 struct ntb_transport
*nt
;
1118 unsigned int free_queue
;
1121 nt
= ntb_find_transport(pdev
);
1125 free_queue
= ffs(nt
->qp_bitmap
);
1129 /* decrement free_queue to make it zero based */
1132 clear_bit(free_queue
, &nt
->qp_bitmap
);
1134 qp
= &nt
->qps
[free_queue
];
1136 qp
->rx_handler
= handlers
->rx_handler
;
1137 qp
->tx_handler
= handlers
->tx_handler
;
1138 qp
->event_handler
= handlers
->event_handler
;
1140 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1141 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1145 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
,
1149 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1150 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1154 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1158 tasklet_init(&qp
->rx_work
, ntb_transport_rx
, (unsigned long) qp
);
1160 rc
= ntb_register_db_callback(qp
->ndev
, free_queue
, qp
,
1161 ntb_transport_rxc_db
);
1165 dev_info(&pdev
->dev
, "NTB Transport QP %d created\n", qp
->qp_num
);
1170 tasklet_disable(&qp
->rx_work
);
1173 ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1177 ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1179 set_bit(free_queue
, &nt
->qp_bitmap
);
1183 EXPORT_SYMBOL_GPL(ntb_transport_create_queue
);
1186 * ntb_transport_free_queue - Frees NTB transport queue
1187 * @qp: NTB queue to be freed
1189 * Frees NTB transport queue
1191 void ntb_transport_free_queue(struct ntb_transport_qp
*qp
)
1193 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1194 struct ntb_queue_entry
*entry
;
1199 cancel_delayed_work_sync(&qp
->link_work
);
1201 ntb_unregister_db_callback(qp
->ndev
, qp
->qp_num
);
1202 tasklet_disable(&qp
->rx_work
);
1205 ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1209 ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
))) {
1210 dev_warn(&pdev
->dev
, "Freeing item from a non-empty queue\n");
1215 ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1218 set_bit(qp
->qp_num
, &qp
->transport
->qp_bitmap
);
1220 dev_info(&pdev
->dev
, "NTB Transport QP %d freed\n", qp
->qp_num
);
1222 EXPORT_SYMBOL_GPL(ntb_transport_free_queue
);
1225 * ntb_transport_rx_remove - Dequeues enqueued rx packet
1226 * @qp: NTB queue to be freed
1227 * @len: pointer to variable to write enqueued buffers length
1229 * Dequeues unused buffers from receive queue. Should only be used during
1232 * RETURNS: NULL error value on error, or void* for success.
1234 void *ntb_transport_rx_remove(struct ntb_transport_qp
*qp
, unsigned int *len
)
1236 struct ntb_queue_entry
*entry
;
1239 if (!qp
|| qp
->client_ready
== NTB_LINK_UP
)
1242 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
1246 buf
= entry
->cb_data
;
1249 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
,
1254 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove
);
1257 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
1258 * @qp: NTB transport layer queue the entry is to be enqueued on
1259 * @cb: per buffer pointer for callback function to use
1260 * @data: pointer to data buffer that incoming packets will be copied into
1261 * @len: length of the data buffer
1263 * Enqueue a new receive buffer onto the transport queue into which a NTB
1264 * payload can be received into.
1266 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1268 int ntb_transport_rx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1271 struct ntb_queue_entry
*entry
;
1276 entry
= ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
);
1280 entry
->cb_data
= cb
;
1284 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
1289 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue
);
1292 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
1293 * @qp: NTB transport layer queue the entry is to be enqueued on
1294 * @cb: per buffer pointer for callback function to use
1295 * @data: pointer to data buffer that will be sent
1296 * @len: length of the data buffer
1298 * Enqueue a new transmit buffer onto the transport queue from which a NTB
1299 * payload will be transmitted. This assumes that a lock is behing held to
1300 * serialize access to the qp.
1302 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1304 int ntb_transport_tx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1307 struct ntb_queue_entry
*entry
;
1310 if (!qp
|| qp
->qp_link
!= NTB_LINK_UP
|| !len
)
1313 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
);
1317 entry
->cb_data
= cb
;
1322 rc
= ntb_process_tx(qp
, entry
);
1324 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1329 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue
);
1332 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
1333 * @qp: NTB transport layer queue to be enabled
1335 * Notify NTB transport layer of client readiness to use queue
1337 void ntb_transport_link_up(struct ntb_transport_qp
*qp
)
1342 qp
->client_ready
= NTB_LINK_UP
;
1344 if (qp
->transport
->transport_link
== NTB_LINK_UP
)
1345 schedule_delayed_work(&qp
->link_work
, 0);
1347 EXPORT_SYMBOL_GPL(ntb_transport_link_up
);
1350 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1351 * @qp: NTB transport layer queue to be disabled
1353 * Notify NTB transport layer of client's desire to no longer receive data on
1354 * transport queue specified. It is the client's responsibility to ensure all
1355 * entries on queue are purged or otherwise handled appropraitely.
1357 void ntb_transport_link_down(struct ntb_transport_qp
*qp
)
1359 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1365 qp
->client_ready
= NTB_LINK_DOWN
;
1367 rc
= ntb_read_local_spad(qp
->ndev
, QP_LINKS
, &val
);
1369 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
1373 rc
= ntb_write_remote_spad(qp
->ndev
, QP_LINKS
,
1374 val
& ~(1 << qp
->qp_num
));
1376 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
1377 val
& ~(1 << qp
->qp_num
), QP_LINKS
);
1379 if (qp
->qp_link
== NTB_LINK_UP
)
1380 ntb_send_link_down(qp
);
1382 cancel_delayed_work_sync(&qp
->link_work
);
1384 EXPORT_SYMBOL_GPL(ntb_transport_link_down
);
1387 * ntb_transport_link_query - Query transport link state
1388 * @qp: NTB transport layer queue to be queried
1390 * Query connectivity to the remote system of the NTB transport queue
1392 * RETURNS: true for link up or false for link down
1394 bool ntb_transport_link_query(struct ntb_transport_qp
*qp
)
1396 return qp
->qp_link
== NTB_LINK_UP
;
1398 EXPORT_SYMBOL_GPL(ntb_transport_link_query
);
1401 * ntb_transport_qp_num - Query the qp number
1402 * @qp: NTB transport layer queue to be queried
1404 * Query qp number of the NTB transport queue
1406 * RETURNS: a zero based number specifying the qp number
1408 unsigned char ntb_transport_qp_num(struct ntb_transport_qp
*qp
)
1412 EXPORT_SYMBOL_GPL(ntb_transport_qp_num
);
1415 * ntb_transport_max_size - Query the max payload size of a qp
1416 * @qp: NTB transport layer queue to be queried
1418 * Query the maximum payload size permissible on the given qp
1420 * RETURNS: the max payload size of a qp
1423 ntb_transport_max_size(__attribute__((unused
)) struct ntb_transport_qp
*qp
)
1425 return transport_mtu
- sizeof(struct ntb_payload_header
);
1427 EXPORT_SYMBOL_GPL(ntb_transport_max_size
);