2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
15 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * Intel PCIe NTB Linux driver
45 * Contact Information:
46 * Jon Mason <jon.mason@intel.com>
48 #include <linux/debugfs.h>
49 #include <linux/delay.h>
50 #include <linux/dma-mapping.h>
51 #include <linux/errno.h>
52 #include <linux/export.h>
53 #include <linux/interrupt.h>
54 #include <linux/module.h>
55 #include <linux/pci.h>
56 #include <linux/slab.h>
57 #include <linux/types.h>
58 #include <linux/ntb.h>
61 #define NTB_TRANSPORT_VERSION 1
63 static unsigned int transport_mtu
= 0x401E;
64 module_param(transport_mtu
, uint
, 0644);
65 MODULE_PARM_DESC(transport_mtu
, "Maximum size of NTB transport packets");
67 static unsigned char max_num_clients
= 2;
68 module_param(max_num_clients
, byte
, 0644);
69 MODULE_PARM_DESC(max_num_clients
, "Maximum number of NTB transport clients");
71 struct ntb_queue_entry
{
72 /* ntb_queue list reference */
73 struct list_head entry
;
74 /* pointers to data to be transfered */
81 struct ntb_transport_qp
{
82 struct ntb_transport
*transport
;
83 struct ntb_device
*ndev
;
88 u8 qp_num
; /* Only 64 QP's are allowed. 0-63 */
90 void (*tx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
92 struct list_head tx_free_q
;
93 spinlock_t ntb_tx_free_q_lock
;
97 unsigned int tx_max_frame
;
99 void (*rx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
100 void *data
, int len
);
101 struct tasklet_struct rx_work
;
102 struct list_head rx_pend_q
;
103 struct list_head rx_free_q
;
104 spinlock_t ntb_rx_pend_q_lock
;
105 spinlock_t ntb_rx_free_q_lock
;
109 unsigned int rx_max_frame
;
111 void (*event_handler
) (void *data
, int status
);
112 struct delayed_work link_work
;
113 struct work_struct link_cleanup
;
115 struct dentry
*debugfs_dir
;
116 struct dentry
*debugfs_stats
;
130 struct ntb_transport_mw
{
136 struct ntb_transport_client_dev
{
137 struct list_head entry
;
141 struct ntb_transport
{
142 struct list_head entry
;
143 struct list_head client_devs
;
145 struct ntb_device
*ndev
;
146 struct ntb_transport_mw mw
[NTB_NUM_MW
];
147 struct ntb_transport_qp
*qps
;
148 unsigned int max_qps
;
149 unsigned long qp_bitmap
;
151 struct delayed_work link_work
;
152 struct work_struct link_cleanup
;
153 struct dentry
*debugfs_dir
;
157 DESC_DONE_FLAG
= 1 << 0,
158 LINK_DOWN_FLAG
= 1 << 1,
161 struct ntb_payload_header
{
176 #define QP_TO_MW(qp) ((qp) % NTB_NUM_MW)
177 #define NTB_QP_DEF_NUM_ENTRIES 100
178 #define NTB_LINK_DOWN_TIMEOUT 10
180 static int ntb_match_bus(struct device
*dev
, struct device_driver
*drv
)
182 return !strncmp(dev_name(dev
), drv
->name
, strlen(drv
->name
));
185 static int ntb_client_probe(struct device
*dev
)
187 const struct ntb_client
*drv
= container_of(dev
->driver
,
188 struct ntb_client
, driver
);
189 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
193 if (drv
&& drv
->probe
)
194 rc
= drv
->probe(pdev
);
201 static int ntb_client_remove(struct device
*dev
)
203 const struct ntb_client
*drv
= container_of(dev
->driver
,
204 struct ntb_client
, driver
);
205 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
207 if (drv
&& drv
->remove
)
215 struct bus_type ntb_bus_type
= {
217 .match
= ntb_match_bus
,
218 .probe
= ntb_client_probe
,
219 .remove
= ntb_client_remove
,
222 static LIST_HEAD(ntb_transport_list
);
224 static int ntb_bus_init(struct ntb_transport
*nt
)
226 if (list_empty(&ntb_transport_list
)) {
227 int rc
= bus_register(&ntb_bus_type
);
232 list_add(&nt
->entry
, &ntb_transport_list
);
237 static void ntb_bus_remove(struct ntb_transport
*nt
)
239 struct ntb_transport_client_dev
*client_dev
, *cd
;
241 list_for_each_entry_safe(client_dev
, cd
, &nt
->client_devs
, entry
) {
242 dev_err(client_dev
->dev
.parent
, "%s still attached to bus, removing\n",
243 dev_name(&client_dev
->dev
));
244 list_del(&client_dev
->entry
);
245 device_unregister(&client_dev
->dev
);
248 list_del(&nt
->entry
);
250 if (list_empty(&ntb_transport_list
))
251 bus_unregister(&ntb_bus_type
);
254 static void ntb_client_release(struct device
*dev
)
256 struct ntb_transport_client_dev
*client_dev
;
257 client_dev
= container_of(dev
, struct ntb_transport_client_dev
, dev
);
263 * ntb_unregister_client_dev - Unregister NTB client device
264 * @device_name: Name of NTB client device
266 * Unregister an NTB client device with the NTB transport layer
268 void ntb_unregister_client_dev(char *device_name
)
270 struct ntb_transport_client_dev
*client
, *cd
;
271 struct ntb_transport
*nt
;
273 list_for_each_entry(nt
, &ntb_transport_list
, entry
)
274 list_for_each_entry_safe(client
, cd
, &nt
->client_devs
, entry
)
275 if (!strncmp(dev_name(&client
->dev
), device_name
,
276 strlen(device_name
))) {
277 list_del(&client
->entry
);
278 device_unregister(&client
->dev
);
281 EXPORT_SYMBOL_GPL(ntb_unregister_client_dev
);
284 * ntb_register_client_dev - Register NTB client device
285 * @device_name: Name of NTB client device
287 * Register an NTB client device with the NTB transport layer
289 int ntb_register_client_dev(char *device_name
)
291 struct ntb_transport_client_dev
*client_dev
;
292 struct ntb_transport
*nt
;
295 if (list_empty(&ntb_transport_list
))
298 list_for_each_entry(nt
, &ntb_transport_list
, entry
) {
301 client_dev
= kzalloc(sizeof(struct ntb_transport_client_dev
),
308 dev
= &client_dev
->dev
;
310 /* setup and register client devices */
311 dev_set_name(dev
, "%s", device_name
);
312 dev
->bus
= &ntb_bus_type
;
313 dev
->release
= ntb_client_release
;
314 dev
->parent
= &ntb_query_pdev(nt
->ndev
)->dev
;
316 rc
= device_register(dev
);
322 list_add_tail(&client_dev
->entry
, &nt
->client_devs
);
328 ntb_unregister_client_dev(device_name
);
332 EXPORT_SYMBOL_GPL(ntb_register_client_dev
);
335 * ntb_register_client - Register NTB client driver
336 * @drv: NTB client driver to be registered
338 * Register an NTB client driver with the NTB transport layer
340 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
342 int ntb_register_client(struct ntb_client
*drv
)
344 drv
->driver
.bus
= &ntb_bus_type
;
346 if (list_empty(&ntb_transport_list
))
349 return driver_register(&drv
->driver
);
351 EXPORT_SYMBOL_GPL(ntb_register_client
);
354 * ntb_unregister_client - Unregister NTB client driver
355 * @drv: NTB client driver to be unregistered
357 * Unregister an NTB client driver with the NTB transport layer
359 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
361 void ntb_unregister_client(struct ntb_client
*drv
)
363 driver_unregister(&drv
->driver
);
365 EXPORT_SYMBOL_GPL(ntb_unregister_client
);
367 static int debugfs_open(struct inode
*inode
, struct file
*filp
)
369 filp
->private_data
= inode
->i_private
;
373 static ssize_t
debugfs_read(struct file
*filp
, char __user
*ubuf
, size_t count
,
376 struct ntb_transport_qp
*qp
;
378 ssize_t ret
, out_offset
, out_count
;
382 qp
= filp
->private_data
;
384 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
386 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
387 "rx_bytes - \t%llu\n", qp
->rx_bytes
);
388 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
389 "rx_pkts - \t%llu\n", qp
->rx_pkts
);
390 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
391 "rx_ring_empty - %llu\n", qp
->rx_ring_empty
);
392 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
393 "rx_err_no_buf - %llu\n", qp
->rx_err_no_buf
);
394 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
395 "rx_err_oflow - \t%llu\n", qp
->rx_err_oflow
);
396 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
397 "rx_err_ver - \t%llu\n", qp
->rx_err_ver
);
398 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
399 "rx_buff_begin - %p\n", qp
->rx_buff_begin
);
400 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
401 "rx_offset - \t%p\n", qp
->rx_offset
);
402 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
403 "rx_buff_end - \t%p\n", qp
->rx_buff_end
);
405 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
406 "tx_bytes - \t%llu\n", qp
->tx_bytes
);
407 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
408 "tx_pkts - \t%llu\n", qp
->tx_pkts
);
409 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
410 "tx_ring_full - \t%llu\n", qp
->tx_ring_full
);
411 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
412 "tx_mw_begin - \t%p\n", qp
->tx_mw_begin
);
413 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
414 "tx_offset - \t%p\n", qp
->tx_offset
);
415 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
416 "tx_mw_end - \t%p\n", qp
->tx_mw_end
);
418 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
419 "QP Link %s\n", (qp
->qp_link
== NTB_LINK_UP
) ?
422 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, out_offset
);
426 static const struct file_operations ntb_qp_debugfs_stats
= {
427 .owner
= THIS_MODULE
,
428 .open
= debugfs_open
,
429 .read
= debugfs_read
,
432 static void ntb_list_add(spinlock_t
*lock
, struct list_head
*entry
,
433 struct list_head
*list
)
437 spin_lock_irqsave(lock
, flags
);
438 list_add_tail(entry
, list
);
439 spin_unlock_irqrestore(lock
, flags
);
442 static struct ntb_queue_entry
*ntb_list_rm(spinlock_t
*lock
,
443 struct list_head
*list
)
445 struct ntb_queue_entry
*entry
;
448 spin_lock_irqsave(lock
, flags
);
449 if (list_empty(list
)) {
453 entry
= list_first_entry(list
, struct ntb_queue_entry
, entry
);
454 list_del(&entry
->entry
);
456 spin_unlock_irqrestore(lock
, flags
);
461 static void ntb_transport_setup_qp_mw(struct ntb_transport
*nt
,
464 struct ntb_transport_qp
*qp
= &nt
->qps
[qp_num
];
465 unsigned int rx_size
, num_qps_mw
;
466 u8 mw_num
= QP_TO_MW(qp_num
);
469 WARN_ON(nt
->mw
[mw_num
].virt_addr
== 0);
471 if (nt
->max_qps
% NTB_NUM_MW
&& mw_num
< nt
->max_qps
% NTB_NUM_MW
)
472 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
+ 1;
474 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
;
476 rx_size
= nt
->mw
[mw_num
].size
/ num_qps_mw
;
477 qp
->rx_buff_begin
= nt
->mw
[mw_num
].virt_addr
+
478 (qp_num
/ NTB_NUM_MW
* rx_size
);
479 qp
->rx_buff_end
= qp
->rx_buff_begin
+ rx_size
;
480 qp
->rx_offset
= qp
->rx_buff_begin
;
481 qp
->rx_max_frame
= min(transport_mtu
, rx_size
);
483 /* setup the hdr offsets with 0's */
484 for (offset
= qp
->rx_buff_begin
+ qp
->rx_max_frame
-
485 sizeof(struct ntb_payload_header
);
486 offset
< qp
->rx_buff_end
; offset
+= qp
->rx_max_frame
)
487 memset(offset
, 0, sizeof(struct ntb_payload_header
));
493 static int ntb_set_mw(struct ntb_transport
*nt
, int num_mw
, unsigned int size
)
495 struct ntb_transport_mw
*mw
= &nt
->mw
[num_mw
];
496 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
498 /* Alloc memory for receiving data. Must be 4k aligned */
499 mw
->size
= ALIGN(size
, 4096);
501 mw
->virt_addr
= dma_alloc_coherent(&pdev
->dev
, mw
->size
, &mw
->dma_addr
,
503 if (!mw
->virt_addr
) {
504 dev_err(&pdev
->dev
, "Unable to allocate MW buffer of size %d\n",
509 /* Notify HW the memory location of the receive buffer */
510 ntb_set_mw_addr(nt
->ndev
, num_mw
, mw
->dma_addr
);
515 static void ntb_qp_link_cleanup(struct work_struct
*work
)
517 struct ntb_transport_qp
*qp
= container_of(work
,
518 struct ntb_transport_qp
,
520 struct ntb_transport
*nt
= qp
->transport
;
521 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
523 if (qp
->qp_link
== NTB_LINK_DOWN
) {
524 cancel_delayed_work_sync(&qp
->link_work
);
528 if (qp
->event_handler
)
529 qp
->event_handler(qp
->cb_data
, NTB_LINK_DOWN
);
531 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
532 qp
->qp_link
= NTB_LINK_DOWN
;
534 if (nt
->transport_link
== NTB_LINK_UP
)
535 schedule_delayed_work(&qp
->link_work
,
536 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
539 static void ntb_qp_link_down(struct ntb_transport_qp
*qp
)
541 schedule_work(&qp
->link_cleanup
);
544 static void ntb_transport_link_cleanup(struct work_struct
*work
)
546 struct ntb_transport
*nt
= container_of(work
, struct ntb_transport
,
550 if (nt
->transport_link
== NTB_LINK_DOWN
)
551 cancel_delayed_work_sync(&nt
->link_work
);
553 nt
->transport_link
= NTB_LINK_DOWN
;
555 /* Pass along the info to any clients */
556 for (i
= 0; i
< nt
->max_qps
; i
++)
557 if (!test_bit(i
, &nt
->qp_bitmap
))
558 ntb_qp_link_down(&nt
->qps
[i
]);
560 /* The scratchpad registers keep the values if the remote side
561 * goes down, blast them now to give them a sane value the next
562 * time they are accessed
564 for (i
= 0; i
< MAX_SPAD
; i
++)
565 ntb_write_local_spad(nt
->ndev
, i
, 0);
568 static void ntb_transport_event_callback(void *data
, enum ntb_hw_event event
)
570 struct ntb_transport
*nt
= data
;
573 case NTB_EVENT_HW_LINK_UP
:
574 schedule_delayed_work(&nt
->link_work
, 0);
576 case NTB_EVENT_HW_LINK_DOWN
:
577 schedule_work(&nt
->link_cleanup
);
584 static void ntb_transport_link_work(struct work_struct
*work
)
586 struct ntb_transport
*nt
= container_of(work
, struct ntb_transport
,
588 struct ntb_device
*ndev
= nt
->ndev
;
589 struct pci_dev
*pdev
= ntb_query_pdev(ndev
);
593 /* send the local info */
594 rc
= ntb_write_remote_spad(ndev
, VERSION
, NTB_TRANSPORT_VERSION
);
596 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
601 rc
= ntb_write_remote_spad(ndev
, MW0_SZ
, ntb_get_mw_size(ndev
, 0));
603 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
604 (u32
) ntb_get_mw_size(ndev
, 0), MW0_SZ
);
608 rc
= ntb_write_remote_spad(ndev
, MW1_SZ
, ntb_get_mw_size(ndev
, 1));
610 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
611 (u32
) ntb_get_mw_size(ndev
, 1), MW1_SZ
);
615 rc
= ntb_write_remote_spad(ndev
, NUM_QPS
, nt
->max_qps
);
617 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
618 nt
->max_qps
, NUM_QPS
);
622 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
624 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
628 rc
= ntb_write_remote_spad(ndev
, QP_LINKS
, val
);
630 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
635 /* Query the remote side for its info */
636 rc
= ntb_read_remote_spad(ndev
, VERSION
, &val
);
638 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", VERSION
);
642 if (val
!= NTB_TRANSPORT_VERSION
)
644 dev_dbg(&pdev
->dev
, "Remote version = %d\n", val
);
646 rc
= ntb_read_remote_spad(ndev
, NUM_QPS
, &val
);
648 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", NUM_QPS
);
652 if (val
!= nt
->max_qps
)
654 dev_dbg(&pdev
->dev
, "Remote max number of qps = %d\n", val
);
656 rc
= ntb_read_remote_spad(ndev
, MW0_SZ
, &val
);
658 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW0_SZ
);
664 dev_dbg(&pdev
->dev
, "Remote MW0 size = %d\n", val
);
666 rc
= ntb_set_mw(nt
, 0, val
);
670 rc
= ntb_read_remote_spad(ndev
, MW1_SZ
, &val
);
672 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW1_SZ
);
678 dev_dbg(&pdev
->dev
, "Remote MW1 size = %d\n", val
);
680 rc
= ntb_set_mw(nt
, 1, val
);
684 nt
->transport_link
= NTB_LINK_UP
;
686 for (i
= 0; i
< nt
->max_qps
; i
++) {
687 struct ntb_transport_qp
*qp
= &nt
->qps
[i
];
689 ntb_transport_setup_qp_mw(nt
, i
);
691 if (qp
->client_ready
== NTB_LINK_UP
)
692 schedule_delayed_work(&qp
->link_work
, 0);
698 if (ntb_hw_link_status(ndev
))
699 schedule_delayed_work(&nt
->link_work
,
700 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
703 static void ntb_qp_link_work(struct work_struct
*work
)
705 struct ntb_transport_qp
*qp
= container_of(work
,
706 struct ntb_transport_qp
,
708 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
709 struct ntb_transport
*nt
= qp
->transport
;
712 WARN_ON(nt
->transport_link
!= NTB_LINK_UP
);
714 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
716 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
720 rc
= ntb_write_remote_spad(nt
->ndev
, QP_LINKS
, val
| 1 << qp
->qp_num
);
722 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
723 val
| 1 << qp
->qp_num
, QP_LINKS
);
725 /* query remote spad for qp ready bits */
726 rc
= ntb_read_remote_spad(nt
->ndev
, QP_LINKS
, &val
);
728 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", QP_LINKS
);
730 dev_dbg(&pdev
->dev
, "Remote QP link status = %x\n", val
);
732 /* See if the remote side is up */
733 if (1 << qp
->qp_num
& val
) {
734 qp
->qp_link
= NTB_LINK_UP
;
736 dev_info(&pdev
->dev
, "qp %d: Link Up\n", qp
->qp_num
);
737 if (qp
->event_handler
)
738 qp
->event_handler(qp
->cb_data
, NTB_LINK_UP
);
739 } else if (nt
->transport_link
== NTB_LINK_UP
)
740 schedule_delayed_work(&qp
->link_work
,
741 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
744 static void ntb_transport_init_queue(struct ntb_transport
*nt
,
747 struct ntb_transport_qp
*qp
;
748 unsigned int num_qps_mw
, tx_size
;
749 u8 mw_num
= QP_TO_MW(qp_num
);
751 qp
= &nt
->qps
[qp_num
];
755 qp
->qp_link
= NTB_LINK_DOWN
;
756 qp
->client_ready
= NTB_LINK_DOWN
;
757 qp
->event_handler
= NULL
;
759 if (nt
->max_qps
% NTB_NUM_MW
&& mw_num
< nt
->max_qps
% NTB_NUM_MW
)
760 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
+ 1;
762 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
;
764 tx_size
= ntb_get_mw_size(qp
->ndev
, mw_num
) / num_qps_mw
;
765 qp
->tx_mw_begin
= ntb_get_mw_vbase(nt
->ndev
, mw_num
) +
766 (qp_num
/ NTB_NUM_MW
* tx_size
);
767 qp
->tx_mw_end
= qp
->tx_mw_begin
+ tx_size
;
768 qp
->tx_offset
= qp
->tx_mw_begin
;
769 qp
->tx_max_frame
= min(transport_mtu
, tx_size
);
771 if (nt
->debugfs_dir
) {
772 char debugfs_name
[4];
774 snprintf(debugfs_name
, 4, "qp%d", qp_num
);
775 qp
->debugfs_dir
= debugfs_create_dir(debugfs_name
,
778 qp
->debugfs_stats
= debugfs_create_file("stats", S_IRUSR
,
780 &ntb_qp_debugfs_stats
);
783 INIT_DELAYED_WORK(&qp
->link_work
, ntb_qp_link_work
);
784 INIT_WORK(&qp
->link_cleanup
, ntb_qp_link_cleanup
);
786 spin_lock_init(&qp
->ntb_rx_pend_q_lock
);
787 spin_lock_init(&qp
->ntb_rx_free_q_lock
);
788 spin_lock_init(&qp
->ntb_tx_free_q_lock
);
790 INIT_LIST_HEAD(&qp
->rx_pend_q
);
791 INIT_LIST_HEAD(&qp
->rx_free_q
);
792 INIT_LIST_HEAD(&qp
->tx_free_q
);
795 int ntb_transport_init(struct pci_dev
*pdev
)
797 struct ntb_transport
*nt
;
800 nt
= kzalloc(sizeof(struct ntb_transport
), GFP_KERNEL
);
804 if (debugfs_initialized())
805 nt
->debugfs_dir
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
807 nt
->debugfs_dir
= NULL
;
809 nt
->ndev
= ntb_register_transport(pdev
, nt
);
815 nt
->max_qps
= min(nt
->ndev
->max_cbs
, max_num_clients
);
817 nt
->qps
= kcalloc(nt
->max_qps
, sizeof(struct ntb_transport_qp
),
824 nt
->qp_bitmap
= ((u64
) 1 << nt
->max_qps
) - 1;
826 for (i
= 0; i
< nt
->max_qps
; i
++)
827 ntb_transport_init_queue(nt
, i
);
829 INIT_DELAYED_WORK(&nt
->link_work
, ntb_transport_link_work
);
830 INIT_WORK(&nt
->link_cleanup
, ntb_transport_link_cleanup
);
832 rc
= ntb_register_event_callback(nt
->ndev
,
833 ntb_transport_event_callback
);
837 INIT_LIST_HEAD(&nt
->client_devs
);
838 rc
= ntb_bus_init(nt
);
842 if (ntb_hw_link_status(nt
->ndev
))
843 schedule_delayed_work(&nt
->link_work
, 0);
848 ntb_unregister_event_callback(nt
->ndev
);
852 ntb_unregister_transport(nt
->ndev
);
854 debugfs_remove_recursive(nt
->debugfs_dir
);
859 void ntb_transport_free(void *transport
)
861 struct ntb_transport
*nt
= transport
;
862 struct pci_dev
*pdev
;
865 nt
->transport_link
= NTB_LINK_DOWN
;
867 /* verify that all the qp's are freed */
868 for (i
= 0; i
< nt
->max_qps
; i
++)
869 if (!test_bit(i
, &nt
->qp_bitmap
))
870 ntb_transport_free_queue(&nt
->qps
[i
]);
874 cancel_delayed_work_sync(&nt
->link_work
);
876 debugfs_remove_recursive(nt
->debugfs_dir
);
878 ntb_unregister_event_callback(nt
->ndev
);
880 pdev
= ntb_query_pdev(nt
->ndev
);
882 for (i
= 0; i
< NTB_NUM_MW
; i
++)
883 if (nt
->mw
[i
].virt_addr
)
884 dma_free_coherent(&pdev
->dev
, nt
->mw
[i
].size
,
889 ntb_unregister_transport(nt
->ndev
);
893 static void ntb_rx_copy_task(struct ntb_transport_qp
*qp
,
894 struct ntb_queue_entry
*entry
, void *offset
)
897 struct ntb_payload_header
*hdr
;
899 BUG_ON(offset
< qp
->rx_buff_begin
||
900 offset
+ qp
->rx_max_frame
>= qp
->rx_buff_end
);
902 hdr
= offset
+ qp
->rx_max_frame
- sizeof(struct ntb_payload_header
);
903 entry
->len
= hdr
->len
;
905 memcpy(entry
->buf
, offset
, entry
->len
);
907 /* Ensure that the data is fully copied out before clearing the flag */
911 if (qp
->rx_handler
&& qp
->client_ready
== NTB_LINK_UP
)
912 qp
->rx_handler(qp
, qp
->cb_data
, entry
->cb_data
, entry
->len
);
914 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
, &qp
->rx_free_q
);
917 static int ntb_process_rxc(struct ntb_transport_qp
*qp
)
919 struct ntb_payload_header
*hdr
;
920 struct ntb_queue_entry
*entry
;
923 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
925 hdr
= offset
+ qp
->rx_max_frame
-
926 sizeof(struct ntb_payload_header
);
927 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
928 "no buffer - HDR ver %llu, len %d, flags %x\n",
929 hdr
->ver
, hdr
->len
, hdr
->flags
);
934 offset
= qp
->rx_offset
;
935 hdr
= offset
+ qp
->rx_max_frame
- sizeof(struct ntb_payload_header
);
937 if (!(hdr
->flags
& DESC_DONE_FLAG
)) {
938 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
944 if (hdr
->ver
!= qp
->rx_pkts
) {
945 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
946 "qp %d: version mismatch, expected %llu - got %llu\n",
947 qp
->qp_num
, qp
->rx_pkts
, hdr
->ver
);
948 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
954 if (hdr
->flags
& LINK_DOWN_FLAG
) {
955 ntb_qp_link_down(qp
);
957 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
960 /* Ensure that the data is fully copied out before clearing the
968 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
969 "rx offset %p, ver %llu - %d payload received, buf size %d\n",
970 qp
->rx_offset
, hdr
->ver
, hdr
->len
, entry
->len
);
972 if (hdr
->len
<= entry
->len
)
973 ntb_rx_copy_task(qp
, entry
, offset
);
975 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
978 /* Ensure that the data is fully copied out before clearing the
984 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
985 "RX overflow! Wanted %d got %d\n",
986 hdr
->len
, entry
->len
);
989 qp
->rx_bytes
+= hdr
->len
;
993 qp
->rx_offset
+= qp
->rx_max_frame
;
994 if (qp
->rx_offset
+ qp
->rx_max_frame
>= qp
->rx_buff_end
)
995 qp
->rx_offset
= qp
->rx_buff_begin
;
1000 static void ntb_transport_rx(unsigned long data
)
1002 struct ntb_transport_qp
*qp
= (struct ntb_transport_qp
*)data
;
1006 rc
= ntb_process_rxc(qp
);
1010 static void ntb_transport_rxc_db(void *data
, int db_num
)
1012 struct ntb_transport_qp
*qp
= data
;
1014 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%s: doorbell %d received\n",
1017 tasklet_schedule(&qp
->rx_work
);
1020 static void ntb_tx_copy_task(struct ntb_transport_qp
*qp
,
1021 struct ntb_queue_entry
*entry
,
1024 struct ntb_payload_header
*hdr
;
1026 BUG_ON(offset
< qp
->tx_mw_begin
||
1027 offset
+ qp
->tx_max_frame
>= qp
->tx_mw_end
);
1029 memcpy_toio(offset
, entry
->buf
, entry
->len
);
1031 hdr
= offset
+ qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1032 hdr
->len
= entry
->len
;
1033 hdr
->ver
= qp
->tx_pkts
;
1035 /* Ensure that the data is fully copied out before setting the flag */
1037 hdr
->flags
= entry
->flags
| DESC_DONE_FLAG
;
1039 ntb_ring_sdb(qp
->ndev
, qp
->qp_num
);
1041 /* The entry length can only be zero if the packet is intended to be a
1042 * "link down" or similar. Since no payload is being sent in these
1043 * cases, there is nothing to add to the completion queue.
1045 if (entry
->len
> 0) {
1046 qp
->tx_bytes
+= entry
->len
;
1049 qp
->tx_handler(qp
, qp
->cb_data
, entry
->cb_data
,
1053 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
, &qp
->tx_free_q
);
1056 static int ntb_process_tx(struct ntb_transport_qp
*qp
,
1057 struct ntb_queue_entry
*entry
)
1059 struct ntb_payload_header
*hdr
;
1062 offset
= qp
->tx_offset
;
1063 hdr
= offset
+ qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1065 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%lld - offset %p, tx %p, entry len %d flags %x buff %p\n",
1066 qp
->tx_pkts
, offset
, qp
->tx_offset
, entry
->len
, entry
->flags
,
1073 if (entry
->len
> qp
->tx_max_frame
- sizeof(struct ntb_payload_header
)) {
1075 qp
->tx_handler(qp
->cb_data
, qp
, NULL
, -EIO
);
1077 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1082 ntb_tx_copy_task(qp
, entry
, offset
);
1084 qp
->tx_offset
+= qp
->tx_max_frame
;
1085 if (qp
->tx_offset
+ qp
->tx_max_frame
>= qp
->tx_mw_end
)
1086 qp
->tx_offset
= qp
->tx_mw_begin
;
1093 static void ntb_send_link_down(struct ntb_transport_qp
*qp
)
1095 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1096 struct ntb_queue_entry
*entry
;
1099 if (qp
->qp_link
== NTB_LINK_DOWN
)
1102 qp
->qp_link
= NTB_LINK_DOWN
;
1103 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
1105 for (i
= 0; i
< NTB_LINK_DOWN_TIMEOUT
; i
++) {
1106 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
,
1116 entry
->cb_data
= NULL
;
1119 entry
->flags
= LINK_DOWN_FLAG
;
1121 rc
= ntb_process_tx(qp
, entry
);
1123 dev_err(&pdev
->dev
, "ntb: QP%d unable to send linkdown msg\n",
1128 * ntb_transport_create_queue - Create a new NTB transport layer queue
1129 * @rx_handler: receive callback function
1130 * @tx_handler: transmit callback function
1131 * @event_handler: event callback function
1133 * Create a new NTB transport layer queue and provide the queue with a callback
1134 * routine for both transmit and receive. The receive callback routine will be
1135 * used to pass up data when the transport has received it on the queue. The
1136 * transmit callback routine will be called when the transport has completed the
1137 * transmission of the data on the queue and the data is ready to be freed.
1139 * RETURNS: pointer to newly created ntb_queue, NULL on error.
1141 struct ntb_transport_qp
*
1142 ntb_transport_create_queue(void *data
, struct pci_dev
*pdev
,
1143 const struct ntb_queue_handlers
*handlers
)
1145 struct ntb_queue_entry
*entry
;
1146 struct ntb_transport_qp
*qp
;
1147 struct ntb_transport
*nt
;
1148 unsigned int free_queue
;
1151 nt
= ntb_find_transport(pdev
);
1155 free_queue
= ffs(nt
->qp_bitmap
);
1159 /* decrement free_queue to make it zero based */
1162 clear_bit(free_queue
, &nt
->qp_bitmap
);
1164 qp
= &nt
->qps
[free_queue
];
1166 qp
->rx_handler
= handlers
->rx_handler
;
1167 qp
->tx_handler
= handlers
->tx_handler
;
1168 qp
->event_handler
= handlers
->event_handler
;
1170 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1171 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1175 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
,
1179 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1180 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1184 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1188 tasklet_init(&qp
->rx_work
, ntb_transport_rx
, (unsigned long) qp
);
1190 rc
= ntb_register_db_callback(qp
->ndev
, free_queue
, qp
,
1191 ntb_transport_rxc_db
);
1195 dev_info(&pdev
->dev
, "NTB Transport QP %d created\n", qp
->qp_num
);
1200 tasklet_disable(&qp
->rx_work
);
1203 ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1207 ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1209 set_bit(free_queue
, &nt
->qp_bitmap
);
1213 EXPORT_SYMBOL_GPL(ntb_transport_create_queue
);
1216 * ntb_transport_free_queue - Frees NTB transport queue
1217 * @qp: NTB queue to be freed
1219 * Frees NTB transport queue
1221 void ntb_transport_free_queue(struct ntb_transport_qp
*qp
)
1223 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1224 struct ntb_queue_entry
*entry
;
1229 cancel_delayed_work_sync(&qp
->link_work
);
1231 ntb_unregister_db_callback(qp
->ndev
, qp
->qp_num
);
1232 tasklet_disable(&qp
->rx_work
);
1235 ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1239 ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
))) {
1240 dev_warn(&pdev
->dev
, "Freeing item from a non-empty queue\n");
1245 ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1248 set_bit(qp
->qp_num
, &qp
->transport
->qp_bitmap
);
1250 dev_info(&pdev
->dev
, "NTB Transport QP %d freed\n", qp
->qp_num
);
1252 EXPORT_SYMBOL_GPL(ntb_transport_free_queue
);
1255 * ntb_transport_rx_remove - Dequeues enqueued rx packet
1256 * @qp: NTB queue to be freed
1257 * @len: pointer to variable to write enqueued buffers length
1259 * Dequeues unused buffers from receive queue. Should only be used during
1262 * RETURNS: NULL error value on error, or void* for success.
1264 void *ntb_transport_rx_remove(struct ntb_transport_qp
*qp
, unsigned int *len
)
1266 struct ntb_queue_entry
*entry
;
1269 if (!qp
|| qp
->client_ready
== NTB_LINK_UP
)
1272 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
1276 buf
= entry
->cb_data
;
1279 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
,
1284 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove
);
1287 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
1288 * @qp: NTB transport layer queue the entry is to be enqueued on
1289 * @cb: per buffer pointer for callback function to use
1290 * @data: pointer to data buffer that incoming packets will be copied into
1291 * @len: length of the data buffer
1293 * Enqueue a new receive buffer onto the transport queue into which a NTB
1294 * payload can be received into.
1296 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1298 int ntb_transport_rx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1301 struct ntb_queue_entry
*entry
;
1306 entry
= ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
);
1310 entry
->cb_data
= cb
;
1314 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
1319 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue
);
1322 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
1323 * @qp: NTB transport layer queue the entry is to be enqueued on
1324 * @cb: per buffer pointer for callback function to use
1325 * @data: pointer to data buffer that will be sent
1326 * @len: length of the data buffer
1328 * Enqueue a new transmit buffer onto the transport queue from which a NTB
1329 * payload will be transmitted. This assumes that a lock is behing held to
1330 * serialize access to the qp.
1332 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1334 int ntb_transport_tx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1337 struct ntb_queue_entry
*entry
;
1340 if (!qp
|| qp
->qp_link
!= NTB_LINK_UP
|| !len
)
1343 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
);
1347 entry
->cb_data
= cb
;
1352 rc
= ntb_process_tx(qp
, entry
);
1354 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1359 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue
);
1362 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
1363 * @qp: NTB transport layer queue to be enabled
1365 * Notify NTB transport layer of client readiness to use queue
1367 void ntb_transport_link_up(struct ntb_transport_qp
*qp
)
1372 qp
->client_ready
= NTB_LINK_UP
;
1374 if (qp
->transport
->transport_link
== NTB_LINK_UP
)
1375 schedule_delayed_work(&qp
->link_work
, 0);
1377 EXPORT_SYMBOL_GPL(ntb_transport_link_up
);
1380 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1381 * @qp: NTB transport layer queue to be disabled
1383 * Notify NTB transport layer of client's desire to no longer receive data on
1384 * transport queue specified. It is the client's responsibility to ensure all
1385 * entries on queue are purged or otherwise handled appropraitely.
1387 void ntb_transport_link_down(struct ntb_transport_qp
*qp
)
1389 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1395 qp
->client_ready
= NTB_LINK_DOWN
;
1397 rc
= ntb_read_local_spad(qp
->ndev
, QP_LINKS
, &val
);
1399 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
1403 rc
= ntb_write_remote_spad(qp
->ndev
, QP_LINKS
,
1404 val
& ~(1 << qp
->qp_num
));
1406 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
1407 val
& ~(1 << qp
->qp_num
), QP_LINKS
);
1409 if (qp
->qp_link
== NTB_LINK_UP
)
1410 ntb_send_link_down(qp
);
1412 cancel_delayed_work_sync(&qp
->link_work
);
1414 EXPORT_SYMBOL_GPL(ntb_transport_link_down
);
1417 * ntb_transport_link_query - Query transport link state
1418 * @qp: NTB transport layer queue to be queried
1420 * Query connectivity to the remote system of the NTB transport queue
1422 * RETURNS: true for link up or false for link down
1424 bool ntb_transport_link_query(struct ntb_transport_qp
*qp
)
1426 return qp
->qp_link
== NTB_LINK_UP
;
1428 EXPORT_SYMBOL_GPL(ntb_transport_link_query
);
1431 * ntb_transport_qp_num - Query the qp number
1432 * @qp: NTB transport layer queue to be queried
1434 * Query qp number of the NTB transport queue
1436 * RETURNS: a zero based number specifying the qp number
1438 unsigned char ntb_transport_qp_num(struct ntb_transport_qp
*qp
)
1442 EXPORT_SYMBOL_GPL(ntb_transport_qp_num
);
1445 * ntb_transport_max_size - Query the max payload size of a qp
1446 * @qp: NTB transport layer queue to be queried
1448 * Query the maximum payload size permissible on the given qp
1450 * RETURNS: the max payload size of a qp
1452 unsigned int ntb_transport_max_size(struct ntb_transport_qp
*qp
)
1454 return qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1456 EXPORT_SYMBOL_GPL(ntb_transport_max_size
);