1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2015 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/version.h>
23 #include <linux/module.h>
24 #include <linux/crc32.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/pci.h>
27 #include <linux/pci_ids.h>
30 #include <linux/ipv6.h>
31 #include <linux/net_tstamp.h>
32 #include <linux/if_vlan.h>
33 #include <linux/firmware.h>
34 #include <linux/ethtool.h>
35 #include <linux/ptp_clock_kernel.h>
36 #include <linux/types.h>
37 #include <linux/list.h>
38 #include <linux/workqueue.h>
39 #include <linux/interrupt.h>
40 #include "octeon_config.h"
41 #include "liquidio_common.h"
42 #include "octeon_droq.h"
43 #include "octeon_iq.h"
44 #include "response_manager.h"
45 #include "octeon_device.h"
46 #include "octeon_nic.h"
47 #include "octeon_main.h"
48 #include "octeon_network.h"
49 #include "cn66xx_regs.h"
50 #include "cn66xx_device.h"
51 #include "cn68xx_regs.h"
52 #include "cn68xx_device.h"
53 #include "liquidio_image.h"
55 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
56 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
57 MODULE_LICENSE("GPL");
58 MODULE_VERSION(LIQUIDIO_VERSION
);
59 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX
);
60 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX
);
61 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX
);
63 static int ddr_timeout
= 10000;
64 module_param(ddr_timeout
, int, 0644);
65 MODULE_PARM_DESC(ddr_timeout
,
66 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
68 static u32 console_bitmask
;
69 module_param(console_bitmask
, int, 0644);
70 MODULE_PARM_DESC(console_bitmask
,
71 "Bitmask indicating which consoles have debug output redirected to syslog.");
73 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
75 static int debug
= -1;
76 module_param(debug
, int, 0644);
77 MODULE_PARM_DESC(debug
, "NETIF_MSG debug bits");
79 static char fw_type
[LIO_MAX_FW_TYPE_LEN
];
80 module_param_string(fw_type
, fw_type
, sizeof(fw_type
), 0000);
81 MODULE_PARM_DESC(fw_type
, "Type of firmware to be loaded. Default \"nic\"");
84 module_param(conf_type
, int, 0);
85 MODULE_PARM_DESC(conf_type
, "select octeon configuration 0 default 1 ovs");
87 /* Bit mask values for lio->ifstate */
88 #define LIO_IFSTATE_DROQ_OPS 0x01
89 #define LIO_IFSTATE_REGISTERED 0x02
90 #define LIO_IFSTATE_RUNNING 0x04
91 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
93 /* Polling interval for determining when NIC application is alive */
94 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
96 /* runtime link query interval */
97 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000
99 struct liquidio_if_cfg_context
{
102 wait_queue_head_t wc
;
107 struct liquidio_if_cfg_resp
{
109 struct liquidio_if_cfg_info cfg_info
;
113 struct oct_link_status_resp
{
115 struct oct_link_info link_info
;
119 struct oct_timestamp_resp
{
125 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
130 #ifdef __BIG_ENDIAN_BITFIELD
142 /** Octeon device properties to be used by the NIC module.
143 * Each octeon device in the system will be represented
144 * by this structure in the NIC module.
147 #define OCTNIC_MAX_SG (MAX_SKB_FRAGS)
149 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
150 #define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE)
152 /** Structure of a node in list of gather components maintained by
153 * NIC driver for each network device.
155 struct octnic_gather
{
156 /** List manipulation. Next and prev pointers. */
157 struct list_head list
;
159 /** Size of the gather component at sg in bytes. */
162 /** Number of bytes that sg was adjusted to make it 8B-aligned. */
165 /** Gather component that can accommodate max sized fragment list
166 * received from the IP layer.
168 struct octeon_sg_entry
*sg
;
171 /** This structure is used by NIC driver to store information required
172 * to free the sk_buff when the packet has been fetched by Octeon.
173 * Bytes offset below assume worst-case of a 64-bit system.
175 struct octnet_buf_free_info
{
176 /** Bytes 1-8. Pointer to network device private structure. */
179 /** Bytes 9-16. Pointer to sk_buff. */
182 /** Bytes 17-24. Pointer to gather list. */
183 struct octnic_gather
*g
;
185 /** Bytes 25-32. Physical address of skb->data or gather list. */
188 /** Bytes 33-47. Piggybacked soft command, if any */
189 struct octeon_soft_command
*sc
;
193 struct completion init
;
194 struct completion started
;
195 struct pci_dev
*pci_dev
;
200 struct octeon_device_priv
{
201 /** Tasklet structures for this device. */
202 struct tasklet_struct droq_tasklet
;
203 unsigned long napi_mask
;
206 static int octeon_device_init(struct octeon_device
*);
207 static void liquidio_remove(struct pci_dev
*pdev
);
208 static int liquidio_probe(struct pci_dev
*pdev
,
209 const struct pci_device_id
*ent
);
211 static struct handshake handshake
[MAX_OCTEON_DEVICES
];
212 static struct completion first_stage
;
214 static void octeon_droq_bh(unsigned long pdev
)
218 struct octeon_device
*oct
= (struct octeon_device
*)pdev
;
219 struct octeon_device_priv
*oct_priv
=
220 (struct octeon_device_priv
*)oct
->priv
;
222 /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
223 for (q_no
= 0; q_no
< MAX_OCTEON_OUTPUT_QUEUES
; q_no
++) {
224 if (!(oct
->io_qmask
.oq
& (1UL << q_no
)))
226 reschedule
|= octeon_droq_process_packets(oct
, oct
->droq
[q_no
],
231 tasklet_schedule(&oct_priv
->droq_tasklet
);
234 static int lio_wait_for_oq_pkts(struct octeon_device
*oct
)
236 struct octeon_device_priv
*oct_priv
=
237 (struct octeon_device_priv
*)oct
->priv
;
238 int retry
= 100, pkt_cnt
= 0, pending_pkts
= 0;
244 for (i
= 0; i
< MAX_OCTEON_OUTPUT_QUEUES
; i
++) {
245 if (!(oct
->io_qmask
.oq
& (1UL << i
)))
247 pkt_cnt
+= octeon_droq_check_hw_for_pkts(oct
,
251 pending_pkts
+= pkt_cnt
;
252 tasklet_schedule(&oct_priv
->droq_tasklet
);
255 schedule_timeout_uninterruptible(1);
257 } while (retry
-- && pending_pkts
);
262 void octeon_report_tx_completion_to_bql(void *txq
, unsigned int pkts_compl
,
263 unsigned int bytes_compl
)
265 struct netdev_queue
*netdev_queue
= txq
;
267 netdev_tx_completed_queue(netdev_queue
, pkts_compl
, bytes_compl
);
270 void octeon_update_tx_completion_counters(void *buf
, int reqtype
,
271 unsigned int *pkts_compl
,
272 unsigned int *bytes_compl
)
274 struct octnet_buf_free_info
*finfo
;
275 struct sk_buff
*skb
= NULL
;
276 struct octeon_soft_command
*sc
;
279 case REQTYPE_NORESP_NET
:
280 case REQTYPE_NORESP_NET_SG
:
285 case REQTYPE_RESP_NET_SG
:
286 case REQTYPE_RESP_NET
:
288 skb
= sc
->callback_arg
;
296 *bytes_compl
+= skb
->len
;
299 void octeon_report_sent_bytes_to_bql(void *buf
, int reqtype
)
301 struct octnet_buf_free_info
*finfo
;
303 struct octeon_soft_command
*sc
;
304 struct netdev_queue
*txq
;
307 case REQTYPE_NORESP_NET
:
308 case REQTYPE_NORESP_NET_SG
:
313 case REQTYPE_RESP_NET_SG
:
314 case REQTYPE_RESP_NET
:
316 skb
= sc
->callback_arg
;
323 txq
= netdev_get_tx_queue(skb
->dev
, skb_get_queue_mapping(skb
));
324 netdev_tx_sent_queue(txq
, skb
->len
);
327 int octeon_console_debug_enabled(u32 console
)
329 return (console_bitmask
>> (console
)) & 0x1;
333 * \brief Forces all IO queues off on a given device
334 * @param oct Pointer to Octeon device
336 static void force_io_queues_off(struct octeon_device
*oct
)
338 if ((oct
->chip_id
== OCTEON_CN66XX
) ||
339 (oct
->chip_id
== OCTEON_CN68XX
)) {
340 /* Reset the Enable bits for Input Queues. */
341 octeon_write_csr(oct
, CN6XXX_SLI_PKT_INSTR_ENB
, 0);
343 /* Reset the Enable bits for Output Queues. */
344 octeon_write_csr(oct
, CN6XXX_SLI_PKT_OUT_ENB
, 0);
349 * \brief wait for all pending requests to complete
350 * @param oct Pointer to Octeon device
352 * Called during shutdown sequence
354 static int wait_for_pending_requests(struct octeon_device
*oct
)
358 for (i
= 0; i
< 100; i
++) {
360 atomic_read(&oct
->response_list
361 [OCTEON_ORDERED_SC_LIST
].pending_req_count
);
363 schedule_timeout_uninterruptible(HZ
/ 10);
375 * \brief Cause device to go quiet so it can be safely removed/reset/etc
376 * @param oct Pointer to Octeon device
378 static inline void pcierror_quiesce_device(struct octeon_device
*oct
)
382 /* Disable the input and output queues now. No more packets will
383 * arrive from Octeon, but we should wait for all packet processing
386 force_io_queues_off(oct
);
388 /* To allow for in-flight requests */
389 schedule_timeout_uninterruptible(100);
391 if (wait_for_pending_requests(oct
))
392 dev_err(&oct
->pci_dev
->dev
, "There were pending requests\n");
394 /* Force all requests waiting to be fetched by OCTEON to complete. */
395 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES
; i
++) {
396 struct octeon_instr_queue
*iq
;
398 if (!(oct
->io_qmask
.iq
& (1UL << i
)))
400 iq
= oct
->instr_queue
[i
];
402 if (atomic_read(&iq
->instr_pending
)) {
403 spin_lock_bh(&iq
->lock
);
405 iq
->octeon_read_index
= iq
->host_write_index
;
406 iq
->stats
.instr_processed
+=
407 atomic_read(&iq
->instr_pending
);
408 lio_process_iq_request_list(oct
, iq
);
409 spin_unlock_bh(&iq
->lock
);
413 /* Force all pending ordered list requests to time out. */
414 lio_process_ordered_list(oct
, 1);
416 /* We do not need to wait for output queue packets to be processed. */
420 * \brief Cleanup PCI AER uncorrectable error status
421 * @param dev Pointer to PCI device
423 static void cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
428 pr_info("%s :\n", __func__
);
430 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
431 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
432 if (dev
->error_state
== pci_channel_io_normal
)
433 status
&= ~mask
; /* Clear corresponding nonfatal bits */
435 status
&= mask
; /* Clear corresponding fatal bits */
436 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
440 * \brief Stop all PCI IO to a given device
441 * @param dev Pointer to Octeon device
443 static void stop_pci_io(struct octeon_device
*oct
)
445 /* No more instructions will be forwarded. */
446 atomic_set(&oct
->status
, OCT_DEV_IN_RESET
);
448 pci_disable_device(oct
->pci_dev
);
450 /* Disable interrupts */
451 oct
->fn_list
.disable_interrupt(oct
->chip
);
453 pcierror_quiesce_device(oct
);
455 /* Release the interrupt line */
456 free_irq(oct
->pci_dev
->irq
, oct
);
458 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
459 pci_disable_msi(oct
->pci_dev
);
461 dev_dbg(&oct
->pci_dev
->dev
, "Device state is now %s\n",
462 lio_get_state_string(&oct
->status
));
464 /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
465 /* making it a common function for all OCTEON models */
466 cleanup_aer_uncorrect_error_status(oct
->pci_dev
);
470 * \brief called when PCI error is detected
471 * @param pdev Pointer to PCI device
472 * @param state The current pci connection state
474 * This function is called after a PCI bus error affecting
475 * this device has been detected.
477 static pci_ers_result_t
liquidio_pcie_error_detected(struct pci_dev
*pdev
,
478 pci_channel_state_t state
)
480 struct octeon_device
*oct
= pci_get_drvdata(pdev
);
482 /* Non-correctable Non-fatal errors */
483 if (state
== pci_channel_io_normal
) {
484 dev_err(&oct
->pci_dev
->dev
, "Non-correctable non-fatal error reported:\n");
485 cleanup_aer_uncorrect_error_status(oct
->pci_dev
);
486 return PCI_ERS_RESULT_CAN_RECOVER
;
489 /* Non-correctable Fatal errors */
490 dev_err(&oct
->pci_dev
->dev
, "Non-correctable FATAL reported by PCI AER driver\n");
493 /* Always return a DISCONNECT. There is no support for recovery but only
494 * for a clean shutdown.
496 return PCI_ERS_RESULT_DISCONNECT
;
500 * \brief mmio handler
501 * @param pdev Pointer to PCI device
503 static pci_ers_result_t
liquidio_pcie_mmio_enabled(struct pci_dev
*pdev
)
505 /* We should never hit this since we never ask for a reset for a Fatal
506 * Error. We always return DISCONNECT in io_error above.
507 * But play safe and return RECOVERED for now.
509 return PCI_ERS_RESULT_RECOVERED
;
513 * \brief called after the pci bus has been reset.
514 * @param pdev Pointer to PCI device
516 * Restart the card from scratch, as if from a cold-boot. Implementation
517 * resembles the first-half of the octeon_resume routine.
519 static pci_ers_result_t
liquidio_pcie_slot_reset(struct pci_dev
*pdev
)
521 /* We should never hit this since we never ask for a reset for a Fatal
522 * Error. We always return DISCONNECT in io_error above.
523 * But play safe and return RECOVERED for now.
525 return PCI_ERS_RESULT_RECOVERED
;
529 * \brief called when traffic can start flowing again.
530 * @param pdev Pointer to PCI device
532 * This callback is called when the error recovery driver tells us that
533 * its OK to resume normal operation. Implementation resembles the
534 * second-half of the octeon_resume routine.
536 static void liquidio_pcie_resume(struct pci_dev
*pdev
)
538 /* Nothing to be done here. */
543 * \brief called when suspending
544 * @param pdev Pointer to PCI device
545 * @param state state to suspend to
547 static int liquidio_suspend(struct pci_dev
*pdev
, pm_message_t state
)
553 * \brief called when resuming
554 * @param pdev Pointer to PCI device
556 static int liquidio_resume(struct pci_dev
*pdev
)
562 /* For PCI-E Advanced Error Recovery (AER) Interface */
563 static const struct pci_error_handlers liquidio_err_handler
= {
564 .error_detected
= liquidio_pcie_error_detected
,
565 .mmio_enabled
= liquidio_pcie_mmio_enabled
,
566 .slot_reset
= liquidio_pcie_slot_reset
,
567 .resume
= liquidio_pcie_resume
,
570 static const struct pci_device_id liquidio_pci_tbl
[] = {
572 PCI_VENDOR_ID_CAVIUM
, 0x91, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0
575 PCI_VENDOR_ID_CAVIUM
, 0x92, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0
581 MODULE_DEVICE_TABLE(pci
, liquidio_pci_tbl
);
583 static struct pci_driver liquidio_pci_driver
= {
585 .id_table
= liquidio_pci_tbl
,
586 .probe
= liquidio_probe
,
587 .remove
= liquidio_remove
,
588 .err_handler
= &liquidio_err_handler
, /* For AER */
591 .suspend
= liquidio_suspend
,
592 .resume
= liquidio_resume
,
598 * \brief register PCI driver
600 static int liquidio_init_pci(void)
602 return pci_register_driver(&liquidio_pci_driver
);
606 * \brief unregister PCI driver
608 static void liquidio_deinit_pci(void)
610 pci_unregister_driver(&liquidio_pci_driver
);
614 * \brief check interface state
615 * @param lio per-network private data
616 * @param state_flag flag state to check
618 static inline int ifstate_check(struct lio
*lio
, int state_flag
)
620 return atomic_read(&lio
->ifstate
) & state_flag
;
624 * \brief set interface state
625 * @param lio per-network private data
626 * @param state_flag flag state to set
628 static inline void ifstate_set(struct lio
*lio
, int state_flag
)
630 atomic_set(&lio
->ifstate
, (atomic_read(&lio
->ifstate
) | state_flag
));
634 * \brief clear interface state
635 * @param lio per-network private data
636 * @param state_flag flag state to clear
638 static inline void ifstate_reset(struct lio
*lio
, int state_flag
)
640 atomic_set(&lio
->ifstate
, (atomic_read(&lio
->ifstate
) & ~(state_flag
)));
644 * \brief Stop Tx queues
645 * @param netdev network device
647 static inline void txqs_stop(struct net_device
*netdev
)
649 if (netif_is_multiqueue(netdev
)) {
652 for (i
= 0; i
< netdev
->num_tx_queues
; i
++)
653 netif_stop_subqueue(netdev
, i
);
655 netif_stop_queue(netdev
);
660 * \brief Start Tx queues
661 * @param netdev network device
663 static inline void txqs_start(struct net_device
*netdev
)
665 if (netif_is_multiqueue(netdev
)) {
668 for (i
= 0; i
< netdev
->num_tx_queues
; i
++)
669 netif_start_subqueue(netdev
, i
);
671 netif_start_queue(netdev
);
676 * \brief Wake Tx queues
677 * @param netdev network device
679 static inline void txqs_wake(struct net_device
*netdev
)
681 if (netif_is_multiqueue(netdev
)) {
684 for (i
= 0; i
< netdev
->num_tx_queues
; i
++)
685 netif_wake_subqueue(netdev
, i
);
687 netif_wake_queue(netdev
);
692 * \brief Stop Tx queue
693 * @param netdev network device
695 static void stop_txq(struct net_device
*netdev
)
701 * \brief Start Tx queue
702 * @param netdev network device
704 static void start_txq(struct net_device
*netdev
)
706 struct lio
*lio
= GET_LIO(netdev
);
708 if (lio
->linfo
.link
.s
.status
) {
715 * \brief Wake a queue
716 * @param netdev network device
717 * @param q which queue to wake
719 static inline void wake_q(struct net_device
*netdev
, int q
)
721 if (netif_is_multiqueue(netdev
))
722 netif_wake_subqueue(netdev
, q
);
724 netif_wake_queue(netdev
);
728 * \brief Stop a queue
729 * @param netdev network device
730 * @param q which queue to stop
732 static inline void stop_q(struct net_device
*netdev
, int q
)
734 if (netif_is_multiqueue(netdev
))
735 netif_stop_subqueue(netdev
, q
);
737 netif_stop_queue(netdev
);
741 * \brief Check Tx queue status, and take appropriate action
742 * @param lio per-network private data
743 * @returns 0 if full, number of queues woken up otherwise
745 static inline int check_txq_status(struct lio
*lio
)
749 if (netif_is_multiqueue(lio
->netdev
)) {
750 int numqs
= lio
->netdev
->num_tx_queues
;
753 /* check each sub-queue state */
754 for (q
= 0; q
< numqs
; q
++) {
755 iq
= lio
->linfo
.txpciq
[q
& (lio
->linfo
.num_txpciq
- 1)];
756 if (octnet_iq_is_full(lio
->oct_dev
, iq
))
758 wake_q(lio
->netdev
, q
);
762 if (octnet_iq_is_full(lio
->oct_dev
, lio
->txq
))
764 wake_q(lio
->netdev
, lio
->txq
);
771 * Remove the node at the head of the list. The list would be empty at
772 * the end of this call if there are no more nodes in the list.
774 static inline struct list_head
*list_delete_head(struct list_head
*root
)
776 struct list_head
*node
;
778 if ((root
->prev
== root
) && (root
->next
== root
))
790 * \brief Delete gather list
791 * @param lio per-network private data
793 static void delete_glist(struct lio
*lio
)
795 struct octnic_gather
*g
;
798 g
= (struct octnic_gather
*)
799 list_delete_head(&lio
->glist
);
802 kfree((void *)((unsigned long)g
->sg
-
810 * \brief Setup gather list
811 * @param lio per-network private data
813 static int setup_glist(struct lio
*lio
)
816 struct octnic_gather
*g
;
818 INIT_LIST_HEAD(&lio
->glist
);
820 for (i
= 0; i
< lio
->tx_qsize
; i
++) {
821 g
= kzalloc(sizeof(*g
), GFP_KERNEL
);
826 ((ROUNDUP4(OCTNIC_MAX_SG
) >> 2) * OCT_SG_ENTRY_SIZE
);
828 g
->sg
= kmalloc(g
->sg_size
+ 8, GFP_KERNEL
);
834 /* The gather component should be aligned on 64-bit boundary */
835 if (((unsigned long)g
->sg
) & 7) {
836 g
->adjust
= 8 - (((unsigned long)g
->sg
) & 7);
837 g
->sg
= (struct octeon_sg_entry
*)
838 ((unsigned long)g
->sg
+ g
->adjust
);
840 list_add_tail(&g
->list
, &lio
->glist
);
843 if (i
== lio
->tx_qsize
)
851 * \brief Print link information
852 * @param netdev network device
854 static void print_link_info(struct net_device
*netdev
)
856 struct lio
*lio
= GET_LIO(netdev
);
858 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_REGISTERED
) {
859 struct oct_link_info
*linfo
= &lio
->linfo
;
861 if (linfo
->link
.s
.status
) {
862 netif_info(lio
, link
, lio
->netdev
, "%d Mbps %s Duplex UP\n",
864 (linfo
->link
.s
.duplex
) ? "Full" : "Half");
866 netif_info(lio
, link
, lio
->netdev
, "Link Down\n");
872 * \brief Update link status
873 * @param netdev network device
874 * @param ls link status structure
876 * Called on receipt of a link status response from the core application to
877 * update each interface's link status.
879 static inline void update_link_status(struct net_device
*netdev
,
880 union oct_link_status
*ls
)
882 struct lio
*lio
= GET_LIO(netdev
);
884 if ((lio
->intf_open
) && (lio
->linfo
.link
.u64
!= ls
->u64
)) {
885 lio
->linfo
.link
.u64
= ls
->u64
;
887 print_link_info(netdev
);
889 if (lio
->linfo
.link
.s
.status
) {
890 netif_carrier_on(netdev
);
891 /* start_txq(netdev); */
894 netif_carrier_off(netdev
);
901 * \brief Droq packet processor sceduler
902 * @param oct octeon device
905 void liquidio_schedule_droq_pkt_handlers(struct octeon_device
*oct
)
907 struct octeon_device_priv
*oct_priv
=
908 (struct octeon_device_priv
*)oct
->priv
;
910 struct octeon_droq
*droq
;
912 if (oct
->int_status
& OCT_DEV_INTR_PKT_DATA
) {
913 for (oq_no
= 0; oq_no
< MAX_OCTEON_OUTPUT_QUEUES
; oq_no
++) {
914 if (!(oct
->droq_intr
& (1 << oq_no
)))
917 droq
= oct
->droq
[oq_no
];
919 if (droq
->ops
.poll_mode
) {
920 droq
->ops
.napi_fn(droq
);
921 oct_priv
->napi_mask
|= (1 << oq_no
);
923 tasklet_schedule(&oct_priv
->droq_tasklet
);
930 * \brief Interrupt handler for octeon
932 * @param dev octeon device
935 irqreturn_t
liquidio_intr_handler(int irq
__attribute__((unused
)), void *dev
)
937 struct octeon_device
*oct
= (struct octeon_device
*)dev
;
940 /* Disable our interrupts for the duration of ISR */
941 oct
->fn_list
.disable_interrupt(oct
->chip
);
943 ret
= oct
->fn_list
.process_interrupt_regs(oct
);
945 if (ret
== IRQ_HANDLED
)
946 liquidio_schedule_droq_pkt_handlers(oct
);
948 /* Re-enable our interrupts */
949 if (!(atomic_read(&oct
->status
) == OCT_DEV_IN_RESET
))
950 oct
->fn_list
.enable_interrupt(oct
->chip
);
956 * \brief Setup interrupt for octeon device
957 * @param oct octeon device
959 * Enable interrupt in Octeon device as given in the PCI interrupt mask.
961 static int octeon_setup_interrupt(struct octeon_device
*oct
)
965 err
= pci_enable_msi(oct
->pci_dev
);
967 dev_warn(&oct
->pci_dev
->dev
, "Reverting to legacy interrupts. Error: %d\n",
970 oct
->flags
|= LIO_FLAG_MSI_ENABLED
;
972 irqret
= request_irq(oct
->pci_dev
->irq
, liquidio_intr_handler
,
973 IRQF_SHARED
, "octeon", oct
);
975 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
976 pci_disable_msi(oct
->pci_dev
);
977 dev_err(&oct
->pci_dev
->dev
, "Request IRQ failed with code: %d\n",
986 * \brief PCI probe handler
987 * @param pdev PCI device structure
990 static int liquidio_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
992 struct octeon_device
*oct_dev
= NULL
;
993 struct handshake
*hs
;
995 oct_dev
= octeon_allocate_device(pdev
->device
,
996 sizeof(struct octeon_device_priv
));
998 dev_err(&pdev
->dev
, "Unable to allocate device\n");
1002 dev_info(&pdev
->dev
, "Initializing device %x:%x.\n",
1003 (u32
)pdev
->vendor
, (u32
)pdev
->device
);
1005 /* Assign octeon_device for this device to the private data area. */
1006 pci_set_drvdata(pdev
, oct_dev
);
1008 /* set linux specific device pointer */
1009 oct_dev
->pci_dev
= (void *)pdev
;
1011 hs
= &handshake
[oct_dev
->octeon_id
];
1012 init_completion(&hs
->init
);
1013 init_completion(&hs
->started
);
1016 if (oct_dev
->octeon_id
== 0)
1017 /* first LiquidIO NIC is detected */
1018 complete(&first_stage
);
1020 if (octeon_device_init(oct_dev
)) {
1021 liquidio_remove(pdev
);
1025 dev_dbg(&oct_dev
->pci_dev
->dev
, "Device is ready\n");
1031 *\brief Destroy resources associated with octeon device
1032 * @param pdev PCI device structure
1035 static void octeon_destroy_resources(struct octeon_device
*oct
)
1038 struct octeon_device_priv
*oct_priv
=
1039 (struct octeon_device_priv
*)oct
->priv
;
1041 struct handshake
*hs
;
1043 switch (atomic_read(&oct
->status
)) {
1044 case OCT_DEV_RUNNING
:
1045 case OCT_DEV_CORE_OK
:
1047 /* No more instructions will be forwarded. */
1048 atomic_set(&oct
->status
, OCT_DEV_IN_RESET
);
1050 oct
->app_mode
= CVM_DRV_INVALID_APP
;
1051 dev_dbg(&oct
->pci_dev
->dev
, "Device state is now %s\n",
1052 lio_get_state_string(&oct
->status
));
1054 schedule_timeout_uninterruptible(HZ
/ 10);
1057 case OCT_DEV_HOST_OK
:
1060 case OCT_DEV_CONSOLE_INIT_DONE
:
1061 /* Remove any consoles */
1062 octeon_remove_consoles(oct
);
1065 case OCT_DEV_IO_QUEUES_DONE
:
1066 if (wait_for_pending_requests(oct
))
1067 dev_err(&oct
->pci_dev
->dev
, "There were pending requests\n");
1069 if (lio_wait_for_instr_fetch(oct
))
1070 dev_err(&oct
->pci_dev
->dev
, "IQ had pending instructions\n");
1072 /* Disable the input and output queues now. No more packets will
1073 * arrive from Octeon, but we should wait for all packet
1074 * processing to finish.
1076 oct
->fn_list
.disable_io_queues(oct
);
1078 if (lio_wait_for_oq_pkts(oct
))
1079 dev_err(&oct
->pci_dev
->dev
, "OQ had pending packets\n");
1081 /* Disable interrupts */
1082 oct
->fn_list
.disable_interrupt(oct
->chip
);
1084 /* Release the interrupt line */
1085 free_irq(oct
->pci_dev
->irq
, oct
);
1087 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
1088 pci_disable_msi(oct
->pci_dev
);
1090 /* Soft reset the octeon device before exiting */
1091 oct
->fn_list
.soft_reset(oct
);
1093 /* Disable the device, releasing the PCI INT */
1094 pci_disable_device(oct
->pci_dev
);
1097 case OCT_DEV_IN_RESET
:
1098 case OCT_DEV_DROQ_INIT_DONE
:
1099 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1101 for (i
= 0; i
< MAX_OCTEON_OUTPUT_QUEUES
; i
++) {
1102 if (!(oct
->io_qmask
.oq
& (1UL << i
)))
1104 octeon_delete_droq(oct
, i
);
1107 /* Force any pending handshakes to complete */
1108 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
1112 handshake
[oct
->octeon_id
].init_ok
= 0;
1113 complete(&handshake
[oct
->octeon_id
].init
);
1114 handshake
[oct
->octeon_id
].started_ok
= 0;
1115 complete(&handshake
[oct
->octeon_id
].started
);
1120 case OCT_DEV_RESP_LIST_INIT_DONE
:
1121 octeon_delete_response_list(oct
);
1124 case OCT_DEV_SC_BUFF_POOL_INIT_DONE
:
1125 octeon_free_sc_buffer_pool(oct
);
1128 case OCT_DEV_INSTR_QUEUE_INIT_DONE
:
1129 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES
; i
++) {
1130 if (!(oct
->io_qmask
.iq
& (1UL << i
)))
1132 octeon_delete_instr_queue(oct
, i
);
1136 case OCT_DEV_DISPATCH_INIT_DONE
:
1137 octeon_delete_dispatch_list(oct
);
1138 cancel_delayed_work_sync(&oct
->nic_poll_work
.work
);
1141 case OCT_DEV_PCI_MAP_DONE
:
1142 octeon_unmap_pci_barx(oct
, 0);
1143 octeon_unmap_pci_barx(oct
, 1);
1146 case OCT_DEV_BEGIN_STATE
:
1147 /* Nothing to be done here either */
1149 } /* end switch(oct->status) */
1151 tasklet_kill(&oct_priv
->droq_tasklet
);
1155 * \brief Send Rx control command
1156 * @param lio per-network private data
1157 * @param start_stop whether to start or stop
1159 static void send_rx_ctrl_cmd(struct lio
*lio
, int start_stop
)
1161 struct octnic_ctrl_pkt nctrl
;
1162 struct octnic_ctrl_params nparams
;
1164 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
1166 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_RX_CTL
;
1167 nctrl
.ncmd
.s
.param1
= lio
->linfo
.ifidx
;
1168 nctrl
.ncmd
.s
.param2
= start_stop
;
1169 nctrl
.netpndev
= (u64
)lio
->netdev
;
1171 nparams
.resp_order
= OCTEON_RESP_NORESPONSE
;
1173 if (octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
, nparams
) < 0)
1174 netif_info(lio
, rx_err
, lio
->netdev
, "Failed to send RX Control message\n");
1178 * \brief Destroy NIC device interface
1179 * @param oct octeon device
1180 * @param ifidx which interface to destroy
1182 * Cleanup associated with each interface for an Octeon device when NIC
1183 * module is being unloaded or if initialization fails during load.
1185 static void liquidio_destroy_nic_device(struct octeon_device
*oct
, int ifidx
)
1187 struct net_device
*netdev
= oct
->props
[ifidx
].netdev
;
1191 dev_err(&oct
->pci_dev
->dev
, "%s No netdevice ptr for index %d\n",
1196 lio
= GET_LIO(netdev
);
1198 dev_dbg(&oct
->pci_dev
->dev
, "NIC device cleanup\n");
1200 send_rx_ctrl_cmd(lio
, 0);
1202 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_RUNNING
)
1205 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_REGISTERED
)
1206 unregister_netdev(netdev
);
1210 free_netdev(netdev
);
1212 oct
->props
[ifidx
].netdev
= NULL
;
1216 * \brief Stop complete NIC functionality
1217 * @param oct octeon device
1219 static int liquidio_stop_nic_module(struct octeon_device
*oct
)
1224 dev_dbg(&oct
->pci_dev
->dev
, "Stopping network interfaces\n");
1225 if (!oct
->ifcount
) {
1226 dev_err(&oct
->pci_dev
->dev
, "Init for Octeon was not completed\n");
1230 for (i
= 0; i
< oct
->ifcount
; i
++) {
1231 lio
= GET_LIO(oct
->props
[i
].netdev
);
1232 for (j
= 0; j
< lio
->linfo
.num_rxpciq
; j
++)
1233 octeon_unregister_droq_ops(oct
, lio
->linfo
.rxpciq
[j
]);
1236 for (i
= 0; i
< oct
->ifcount
; i
++)
1237 liquidio_destroy_nic_device(oct
, i
);
1239 dev_dbg(&oct
->pci_dev
->dev
, "Network interfaces stopped\n");
1244 * \brief Cleans up resources at unload time
1245 * @param pdev PCI device structure
1247 static void liquidio_remove(struct pci_dev
*pdev
)
1249 struct octeon_device
*oct_dev
= pci_get_drvdata(pdev
);
1251 dev_dbg(&oct_dev
->pci_dev
->dev
, "Stopping device\n");
1253 if (oct_dev
->app_mode
&& (oct_dev
->app_mode
== CVM_DRV_NIC_APP
))
1254 liquidio_stop_nic_module(oct_dev
);
1256 /* Reset the octeon device and cleanup all memory allocated for
1257 * the octeon device by driver.
1259 octeon_destroy_resources(oct_dev
);
1261 dev_info(&oct_dev
->pci_dev
->dev
, "Device removed\n");
1263 /* This octeon device has been removed. Update the global
1264 * data structure to reflect this. Free the device structure.
1266 octeon_free_device_mem(oct_dev
);
1270 * \brief Identify the Octeon device and to map the BAR address space
1271 * @param oct octeon device
1273 static int octeon_chip_specific_setup(struct octeon_device
*oct
)
1278 pci_read_config_dword(oct
->pci_dev
, 0, &dev_id
);
1279 pci_read_config_dword(oct
->pci_dev
, 8, &rev_id
);
1280 oct
->rev_id
= rev_id
& 0xff;
1283 case OCTEON_CN68XX_PCIID
:
1284 oct
->chip_id
= OCTEON_CN68XX
;
1285 ret
= lio_setup_cn68xx_octeon_device(oct
);
1288 case OCTEON_CN66XX_PCIID
:
1289 oct
->chip_id
= OCTEON_CN66XX
;
1290 ret
= lio_setup_cn66xx_octeon_device(oct
);
1293 dev_err(&oct
->pci_dev
->dev
, "Unknown device found (dev_id: %x)\n",
1298 dev_info(&oct
->pci_dev
->dev
, "CN68XX PASS%d.%d %s\n",
1299 OCTEON_MAJOR_REV(oct
),
1300 OCTEON_MINOR_REV(oct
),
1301 octeon_get_conf(oct
)->card_name
);
1307 * \brief PCI initialization for each Octeon device.
1308 * @param oct octeon device
1310 static int octeon_pci_os_setup(struct octeon_device
*oct
)
1312 /* setup PCI stuff first */
1313 if (pci_enable_device(oct
->pci_dev
)) {
1314 dev_err(&oct
->pci_dev
->dev
, "pci_enable_device failed\n");
1318 if (dma_set_mask_and_coherent(&oct
->pci_dev
->dev
, DMA_BIT_MASK(64))) {
1319 dev_err(&oct
->pci_dev
->dev
, "Unexpected DMA device capability\n");
1323 /* Enable PCI DMA Master. */
1324 pci_set_master(oct
->pci_dev
);
1330 * \brief Check Tx queue state for a given network buffer
1331 * @param lio per-network private data
1332 * @param skb network buffer
1334 static inline int check_txq_state(struct lio
*lio
, struct sk_buff
*skb
)
1338 if (netif_is_multiqueue(lio
->netdev
)) {
1339 q
= skb
->queue_mapping
;
1340 iq
= lio
->linfo
.txpciq
[(q
& (lio
->linfo
.num_txpciq
- 1))];
1345 if (octnet_iq_is_full(lio
->oct_dev
, iq
))
1347 wake_q(lio
->netdev
, q
);
1352 * \brief Unmap and free network buffer
1355 static void free_netbuf(void *buf
)
1357 struct sk_buff
*skb
;
1358 struct octnet_buf_free_info
*finfo
;
1361 finfo
= (struct octnet_buf_free_info
*)buf
;
1365 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
, finfo
->dptr
, skb
->len
,
1368 check_txq_state(lio
, skb
);
1370 recv_buffer_free((struct sk_buff
*)skb
);
1374 * \brief Unmap and free gather buffer
1377 static void free_netsgbuf(void *buf
)
1379 struct octnet_buf_free_info
*finfo
;
1380 struct sk_buff
*skb
;
1382 struct octnic_gather
*g
;
1385 finfo
= (struct octnet_buf_free_info
*)buf
;
1389 frags
= skb_shinfo(skb
)->nr_frags
;
1391 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1392 g
->sg
[0].ptr
[0], (skb
->len
- skb
->data_len
),
1397 struct skb_frag_struct
*frag
= &skb_shinfo(skb
)->frags
[i
- 1];
1399 pci_unmap_page((lio
->oct_dev
)->pci_dev
,
1400 g
->sg
[(i
>> 2)].ptr
[(i
& 3)],
1401 frag
->size
, DMA_TO_DEVICE
);
1405 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1406 finfo
->dptr
, g
->sg_size
,
1409 spin_lock(&lio
->lock
);
1410 list_add_tail(&g
->list
, &lio
->glist
);
1411 spin_unlock(&lio
->lock
);
1413 check_txq_state(lio
, skb
); /* mq support: sub-queue state check */
1415 recv_buffer_free((struct sk_buff
*)skb
);
1419 * \brief Unmap and free gather buffer with response
1422 static void free_netsgbuf_with_resp(void *buf
)
1424 struct octeon_soft_command
*sc
;
1425 struct octnet_buf_free_info
*finfo
;
1426 struct sk_buff
*skb
;
1428 struct octnic_gather
*g
;
1431 sc
= (struct octeon_soft_command
*)buf
;
1432 skb
= (struct sk_buff
*)sc
->callback_arg
;
1433 finfo
= (struct octnet_buf_free_info
*)&skb
->cb
;
1437 frags
= skb_shinfo(skb
)->nr_frags
;
1439 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1440 g
->sg
[0].ptr
[0], (skb
->len
- skb
->data_len
),
1445 struct skb_frag_struct
*frag
= &skb_shinfo(skb
)->frags
[i
- 1];
1447 pci_unmap_page((lio
->oct_dev
)->pci_dev
,
1448 g
->sg
[(i
>> 2)].ptr
[(i
& 3)],
1449 frag
->size
, DMA_TO_DEVICE
);
1453 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1454 finfo
->dptr
, g
->sg_size
,
1457 spin_lock(&lio
->lock
);
1458 list_add_tail(&g
->list
, &lio
->glist
);
1459 spin_unlock(&lio
->lock
);
1461 /* Don't free the skb yet */
1463 check_txq_state(lio
, skb
);
1467 * \brief Adjust ptp frequency
1468 * @param ptp PTP clock info
1469 * @param ppb how much to adjust by, in parts-per-billion
1471 static int liquidio_ptp_adjfreq(struct ptp_clock_info
*ptp
, s32 ppb
)
1473 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1474 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1476 unsigned long flags
;
1477 bool neg_adj
= false;
1484 /* The hardware adds the clock compensation value to the
1485 * PTP clock on every coprocessor clock cycle, so we
1486 * compute the delta in terms of coprocessor clocks.
1488 delta
= (u64
)ppb
<< 32;
1489 do_div(delta
, oct
->coproc_clock_rate
);
1491 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1492 comp
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1497 lio_pci_writeq(oct
, comp
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1498 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1504 * \brief Adjust ptp time
1505 * @param ptp PTP clock info
1506 * @param delta how much to adjust by, in nanosecs
1508 static int liquidio_ptp_adjtime(struct ptp_clock_info
*ptp
, s64 delta
)
1510 unsigned long flags
;
1511 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1513 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1514 lio
->ptp_adjust
+= delta
;
1515 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1521 * \brief Get hardware clock time, including any adjustment
1522 * @param ptp PTP clock info
1523 * @param ts timespec
1525 static int liquidio_ptp_gettime(struct ptp_clock_info
*ptp
,
1526 struct timespec64
*ts
)
1530 unsigned long flags
;
1531 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1532 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1534 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1535 ns
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_HI
);
1536 ns
+= lio
->ptp_adjust
;
1537 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1539 ts
->tv_sec
= div_u64_rem(ns
, 1000000000ULL, &remainder
);
1540 ts
->tv_nsec
= remainder
;
1546 * \brief Set hardware clock time. Reset adjustment
1547 * @param ptp PTP clock info
1548 * @param ts timespec
1550 static int liquidio_ptp_settime(struct ptp_clock_info
*ptp
,
1551 const struct timespec64
*ts
)
1554 unsigned long flags
;
1555 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1556 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1558 ns
= timespec_to_ns(ts
);
1560 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1561 lio_pci_writeq(oct
, ns
, CN6XXX_MIO_PTP_CLOCK_HI
);
1562 lio
->ptp_adjust
= 0;
1563 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1569 * \brief Check if PTP is enabled
1570 * @param ptp PTP clock info
1572 * @param on is it on
1574 static int liquidio_ptp_enable(struct ptp_clock_info
*ptp
,
1575 struct ptp_clock_request
*rq
, int on
)
1581 * \brief Open PTP clock source
1582 * @param netdev network device
1584 static void oct_ptp_open(struct net_device
*netdev
)
1586 struct lio
*lio
= GET_LIO(netdev
);
1587 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1589 spin_lock_init(&lio
->ptp_lock
);
1591 snprintf(lio
->ptp_info
.name
, 16, "%s", netdev
->name
);
1592 lio
->ptp_info
.owner
= THIS_MODULE
;
1593 lio
->ptp_info
.max_adj
= 250000000;
1594 lio
->ptp_info
.n_alarm
= 0;
1595 lio
->ptp_info
.n_ext_ts
= 0;
1596 lio
->ptp_info
.n_per_out
= 0;
1597 lio
->ptp_info
.pps
= 0;
1598 lio
->ptp_info
.adjfreq
= liquidio_ptp_adjfreq
;
1599 lio
->ptp_info
.adjtime
= liquidio_ptp_adjtime
;
1600 lio
->ptp_info
.gettime64
= liquidio_ptp_gettime
;
1601 lio
->ptp_info
.settime64
= liquidio_ptp_settime
;
1602 lio
->ptp_info
.enable
= liquidio_ptp_enable
;
1604 lio
->ptp_adjust
= 0;
1606 lio
->ptp_clock
= ptp_clock_register(&lio
->ptp_info
,
1607 &oct
->pci_dev
->dev
);
1609 if (IS_ERR(lio
->ptp_clock
))
1610 lio
->ptp_clock
= NULL
;
1614 * \brief Init PTP clock
1615 * @param oct octeon device
1617 static void liquidio_ptp_init(struct octeon_device
*oct
)
1619 u64 clock_comp
, cfg
;
1621 clock_comp
= (u64
)NSEC_PER_SEC
<< 32;
1622 do_div(clock_comp
, oct
->coproc_clock_rate
);
1623 lio_pci_writeq(oct
, clock_comp
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1626 cfg
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_CFG
);
1627 lio_pci_writeq(oct
, cfg
| 0x01, CN6XXX_MIO_PTP_CLOCK_CFG
);
1631 * \brief Load firmware to device
1632 * @param oct octeon device
1634 * Maps device to firmware filename, requests firmware, and downloads it
1636 static int load_firmware(struct octeon_device
*oct
)
1639 const struct firmware
*fw
;
1640 char fw_name
[LIO_MAX_FW_FILENAME_LEN
];
1643 if (strncmp(fw_type
, LIO_FW_NAME_TYPE_NONE
,
1644 sizeof(LIO_FW_NAME_TYPE_NONE
)) == 0) {
1645 dev_info(&oct
->pci_dev
->dev
, "Skipping firmware load\n");
1649 if (fw_type
[0] == '\0')
1650 tmp_fw_type
= LIO_FW_NAME_TYPE_NIC
;
1652 tmp_fw_type
= fw_type
;
1654 sprintf(fw_name
, "%s%s%s_%s%s", LIO_FW_DIR
, LIO_FW_BASE_NAME
,
1655 octeon_get_conf(oct
)->card_name
, tmp_fw_type
,
1656 LIO_FW_NAME_SUFFIX
);
1658 ret
= request_firmware(&fw
, fw_name
, &oct
->pci_dev
->dev
);
1660 dev_err(&oct
->pci_dev
->dev
, "Request firmware failed. Could not find file %s.\n.",
1665 ret
= octeon_download_firmware(oct
, fw
->data
, fw
->size
);
1667 release_firmware(fw
);
1673 * \brief Setup output queue
1674 * @param oct octeon device
1675 * @param q_no which queue
1676 * @param num_descs how many descriptors
1677 * @param desc_size size of each descriptor
1678 * @param app_ctx application context
1680 static int octeon_setup_droq(struct octeon_device
*oct
, int q_no
, int num_descs
,
1681 int desc_size
, void *app_ctx
)
1685 dev_dbg(&oct
->pci_dev
->dev
, "Creating Droq: %d\n", q_no
);
1686 /* droq creation and local register settings. */
1687 ret_val
= octeon_create_droq(oct
, q_no
, num_descs
, desc_size
, app_ctx
);
1692 dev_dbg(&oct
->pci_dev
->dev
, "Using default droq %d\n", q_no
);
1695 /* tasklet creation for the droq */
1697 /* Enable the droq queues */
1698 octeon_set_droq_pkt_op(oct
, q_no
, 1);
1700 /* Send Credit for Octeon Output queues. Credits are always
1701 * sent after the output queue is enabled.
1703 writel(oct
->droq
[q_no
]->max_count
,
1704 oct
->droq
[q_no
]->pkts_credit_reg
);
1710 * \brief Callback for getting interface configuration
1711 * @param status status of request
1712 * @param buf pointer to resp structure
1714 static void if_cfg_callback(struct octeon_device
*oct
,
1718 struct octeon_soft_command
*sc
= (struct octeon_soft_command
*)buf
;
1719 struct liquidio_if_cfg_resp
*resp
;
1720 struct liquidio_if_cfg_context
*ctx
;
1722 resp
= (struct liquidio_if_cfg_resp
*)sc
->virtrptr
;
1723 ctx
= (struct liquidio_if_cfg_context
*)sc
->ctxptr
;
1725 oct
= lio_get_device(ctx
->octeon_id
);
1727 dev_err(&oct
->pci_dev
->dev
, "nic if cfg instruction failed. Status: %llx\n",
1728 CVM_CAST64(resp
->status
));
1729 ACCESS_ONCE(ctx
->cond
) = 1;
1731 /* This barrier is required to be sure that the response has been
1732 * written fully before waking up the handler
1736 wake_up_interruptible(&ctx
->wc
);
1740 * \brief Select queue based on hash
1741 * @param dev Net device
1742 * @param skb sk_buff structure
1743 * @returns selected queue number
1745 static u16
select_q(struct net_device
*dev
, struct sk_buff
*skb
,
1746 void *accel_priv
, select_queue_fallback_t fallback
)
1752 /* select queue on chosen queue_mapping or core */
1753 qindex
= skb_rx_queue_recorded(skb
) ?
1754 skb_get_rx_queue(skb
) : smp_processor_id();
1755 return (u16
)(qindex
& (lio
->linfo
.num_txpciq
- 1));
1758 /** Routine to push packets arriving on Octeon interface upto network layer.
1759 * @param oct_id - octeon device id.
1760 * @param skbuff - skbuff struct to be passed to network layer.
1761 * @param len - size of total data received.
1762 * @param rh - Control header associated with the packet
1763 * @param param - additional control data with the packet
1766 liquidio_push_packet(u32 octeon_id
,
1769 union octeon_rh
*rh
,
1772 struct napi_struct
*napi
= param
;
1773 struct octeon_device
*oct
= lio_get_device(octeon_id
);
1774 struct sk_buff
*skb
= (struct sk_buff
*)skbuff
;
1775 struct skb_shared_hwtstamps
*shhwtstamps
;
1777 struct net_device
*netdev
=
1778 (struct net_device
*)oct
->props
[rh
->r_dh
.link
].netdev
;
1779 struct octeon_droq
*droq
= container_of(param
, struct octeon_droq
,
1782 int packet_was_received
;
1783 struct lio
*lio
= GET_LIO(netdev
);
1785 /* Do not proceed if the interface is not in RUNNING state. */
1786 if (!ifstate_check(lio
, LIO_IFSTATE_RUNNING
)) {
1787 recv_buffer_free(skb
);
1788 droq
->stats
.rx_dropped
++;
1794 if (rh
->r_dh
.has_hwtstamp
) {
1795 /* timestamp is included from the hardware at the
1796 * beginning of the packet.
1798 if (ifstate_check(lio
,
1799 LIO_IFSTATE_RX_TIMESTAMP_ENABLED
)) {
1800 /* Nanoseconds are in the first 64-bits
1803 memcpy(&ns
, (skb
->data
), sizeof(ns
));
1804 shhwtstamps
= skb_hwtstamps(skb
);
1805 shhwtstamps
->hwtstamp
=
1806 ns_to_ktime(ns
+ lio
->ptp_adjust
);
1808 skb_pull(skb
, sizeof(ns
));
1811 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1813 if ((netdev
->features
& NETIF_F_RXCSUM
) &&
1814 (rh
->r_dh
.csum_verified
== CNNIC_CSUM_VERIFIED
))
1815 /* checksum has already been verified */
1816 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1818 skb
->ip_summed
= CHECKSUM_NONE
;
1820 packet_was_received
= napi_gro_receive(napi
, skb
) != GRO_DROP
;
1822 if (packet_was_received
) {
1823 droq
->stats
.rx_bytes_received
+= len
;
1824 droq
->stats
.rx_pkts_received
++;
1825 netdev
->last_rx
= jiffies
;
1827 droq
->stats
.rx_dropped
++;
1828 netif_info(lio
, rx_err
, lio
->netdev
,
1829 "droq:%d error rx_dropped:%llu\n",
1830 droq
->q_no
, droq
->stats
.rx_dropped
);
1834 recv_buffer_free(skb
);
1839 * \brief wrapper for calling napi_schedule
1840 * @param param parameters to pass to napi_schedule
1842 * Used when scheduling on different CPUs
1844 static void napi_schedule_wrapper(void *param
)
1846 struct napi_struct
*napi
= param
;
1848 napi_schedule(napi
);
1852 * \brief callback when receive interrupt occurs and we are in NAPI mode
1853 * @param arg pointer to octeon output queue
1855 static void liquidio_napi_drv_callback(void *arg
)
1857 struct octeon_droq
*droq
= arg
;
1858 int this_cpu
= smp_processor_id();
1860 if (droq
->cpu_id
== this_cpu
) {
1861 napi_schedule(&droq
->napi
);
1863 struct call_single_data
*csd
= &droq
->csd
;
1865 csd
->func
= napi_schedule_wrapper
;
1866 csd
->info
= &droq
->napi
;
1869 smp_call_function_single_async(droq
->cpu_id
, csd
);
1874 * \brief Main NAPI poll function
1875 * @param droq octeon output queue
1876 * @param budget maximum number of items to process
1878 static int liquidio_napi_do_rx(struct octeon_droq
*droq
, int budget
)
1881 struct lio
*lio
= GET_LIO(droq
->napi
.dev
);
1882 struct octeon_device
*oct
= lio
->oct_dev
;
1884 work_done
= octeon_process_droq_poll_cmd(oct
, droq
->q_no
,
1885 POLL_EVENT_PROCESS_PKTS
,
1887 if (work_done
< 0) {
1888 netif_info(lio
, rx_err
, lio
->netdev
,
1889 "Receive work_done < 0, rxq:%d\n", droq
->q_no
);
1890 goto octnet_napi_finish
;
1893 if (work_done
> budget
)
1894 dev_err(&oct
->pci_dev
->dev
, ">>>> %s work_done: %d budget: %d\n",
1895 __func__
, work_done
, budget
);
1900 napi_complete(&droq
->napi
);
1901 octeon_process_droq_poll_cmd(oct
, droq
->q_no
, POLL_EVENT_ENABLE_INTR
,
1907 * \brief Entry point for NAPI polling
1908 * @param napi NAPI structure
1909 * @param budget maximum number of items to process
1911 static int liquidio_napi_poll(struct napi_struct
*napi
, int budget
)
1913 struct octeon_droq
*droq
;
1916 droq
= container_of(napi
, struct octeon_droq
, napi
);
1918 work_done
= liquidio_napi_do_rx(droq
, budget
);
1920 if (work_done
< budget
) {
1921 napi_complete(napi
);
1922 octeon_process_droq_poll_cmd(droq
->oct_dev
, droq
->q_no
,
1923 POLL_EVENT_ENABLE_INTR
, 0);
1931 * \brief Setup input and output queues
1932 * @param octeon_dev octeon device
1933 * @param net_device Net device
1935 * Note: Queues are with respect to the octeon device. Thus
1936 * an input queue is for egress packets, and output queues
1937 * are for ingress packets.
1939 static inline int setup_io_queues(struct octeon_device
*octeon_dev
,
1940 struct net_device
*net_device
)
1942 static int first_time
= 1;
1943 static struct octeon_droq_ops droq_ops
;
1945 static int cpu_id_modulus
;
1946 struct octeon_droq
*droq
;
1947 struct napi_struct
*napi
;
1948 int q
, q_no
, retval
= 0;
1952 lio
= GET_LIO(net_device
);
1955 memset(&droq_ops
, 0, sizeof(struct octeon_droq_ops
));
1957 droq_ops
.fptr
= liquidio_push_packet
;
1959 droq_ops
.poll_mode
= 1;
1960 droq_ops
.napi_fn
= liquidio_napi_drv_callback
;
1962 cpu_id_modulus
= num_present_cpus();
1966 for (q
= 0; q
< lio
->linfo
.num_rxpciq
; q
++) {
1967 q_no
= lio
->linfo
.rxpciq
[q
];
1969 retval
= octeon_setup_droq(octeon_dev
, q_no
,
1970 CFG_GET_NUM_RX_DESCS_NIC_IF
1971 (octeon_get_conf(octeon_dev
),
1973 CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
1974 (octeon_get_conf(octeon_dev
),
1977 dev_err(&octeon_dev
->pci_dev
->dev
,
1978 " %s : Runtime DROQ(RxQ) creation failed.\n",
1983 droq
= octeon_dev
->droq
[q_no
];
1985 netif_napi_add(net_device
, napi
, liquidio_napi_poll
, 64);
1987 /* designate a CPU for this droq */
1988 droq
->cpu_id
= cpu_id
;
1990 if (cpu_id
>= cpu_id_modulus
)
1993 octeon_register_droq_ops(octeon_dev
, q_no
, &droq_ops
);
1997 for (q
= 0; q
< lio
->linfo
.num_txpciq
; q
++) {
1998 num_tx_descs
= CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2001 retval
= octeon_setup_iq(octeon_dev
, lio
->linfo
.txpciq
[q
],
2003 netdev_get_tx_queue(net_device
, q
));
2005 dev_err(&octeon_dev
->pci_dev
->dev
,
2006 " %s : Runtime IQ(TxQ) creation failed.\n",
2016 * \brief Poll routine for checking transmit queue status
2017 * @param work work_struct data structure
2019 static void octnet_poll_check_txq_status(struct work_struct
*work
)
2021 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
2022 struct lio
*lio
= (struct lio
*)wk
->ctxptr
;
2024 if (!ifstate_check(lio
, LIO_IFSTATE_RUNNING
))
2027 check_txq_status(lio
);
2028 queue_delayed_work(lio
->txq_status_wq
.wq
,
2029 &lio
->txq_status_wq
.wk
.work
, msecs_to_jiffies(1));
2033 * \brief Sets up the txq poll check
2034 * @param netdev network device
2036 static inline void setup_tx_poll_fn(struct net_device
*netdev
)
2038 struct lio
*lio
= GET_LIO(netdev
);
2039 struct octeon_device
*oct
= lio
->oct_dev
;
2041 lio
->txq_status_wq
.wq
= create_workqueue("txq-status");
2042 if (!lio
->txq_status_wq
.wq
) {
2043 dev_err(&oct
->pci_dev
->dev
, "unable to create cavium txq status wq\n");
2046 INIT_DELAYED_WORK(&lio
->txq_status_wq
.wk
.work
,
2047 octnet_poll_check_txq_status
);
2048 lio
->txq_status_wq
.wk
.ctxptr
= lio
;
2049 queue_delayed_work(lio
->txq_status_wq
.wq
,
2050 &lio
->txq_status_wq
.wk
.work
, msecs_to_jiffies(1));
2054 * \brief Net device open for LiquidIO
2055 * @param netdev network device
2057 static int liquidio_open(struct net_device
*netdev
)
2059 struct lio
*lio
= GET_LIO(netdev
);
2060 struct octeon_device
*oct
= lio
->oct_dev
;
2061 struct napi_struct
*napi
, *n
;
2063 list_for_each_entry_safe(napi
, n
, &netdev
->napi_list
, dev_list
)
2066 oct_ptp_open(netdev
);
2068 ifstate_set(lio
, LIO_IFSTATE_RUNNING
);
2069 setup_tx_poll_fn(netdev
);
2072 netif_info(lio
, ifup
, lio
->netdev
, "Interface Open, ready for traffic\n");
2073 try_module_get(THIS_MODULE
);
2075 /* tell Octeon to start forwarding packets to host */
2076 send_rx_ctrl_cmd(lio
, 1);
2078 /* Ready for link status updates */
2081 dev_info(&oct
->pci_dev
->dev
, "%s interface is opened\n",
2088 * \brief Net device stop for LiquidIO
2089 * @param netdev network device
2091 static int liquidio_stop(struct net_device
*netdev
)
2093 struct napi_struct
*napi
, *n
;
2094 struct lio
*lio
= GET_LIO(netdev
);
2095 struct octeon_device
*oct
= lio
->oct_dev
;
2097 netif_info(lio
, ifdown
, lio
->netdev
, "Stopping interface!\n");
2098 /* Inform that netif carrier is down */
2100 lio
->linfo
.link
.s
.status
= 0;
2102 netif_carrier_off(netdev
);
2104 /* tell Octeon to stop forwarding packets to host */
2105 send_rx_ctrl_cmd(lio
, 0);
2107 cancel_delayed_work_sync(&lio
->txq_status_wq
.wk
.work
);
2108 flush_workqueue(lio
->txq_status_wq
.wq
);
2109 destroy_workqueue(lio
->txq_status_wq
.wq
);
2111 if (lio
->ptp_clock
) {
2112 ptp_clock_unregister(lio
->ptp_clock
);
2113 lio
->ptp_clock
= NULL
;
2116 ifstate_reset(lio
, LIO_IFSTATE_RUNNING
);
2118 /* This is a hack that allows DHCP to continue working. */
2119 set_bit(__LINK_STATE_START
, &lio
->netdev
->state
);
2121 list_for_each_entry_safe(napi
, n
, &netdev
->napi_list
, dev_list
)
2126 dev_info(&oct
->pci_dev
->dev
, "%s interface is stopped\n", netdev
->name
);
2127 module_put(THIS_MODULE
);
2132 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr
)
2134 struct octnic_ctrl_pkt
*nctrl
= (struct octnic_ctrl_pkt
*)nctrl_ptr
;
2135 struct net_device
*netdev
= (struct net_device
*)nctrl
->netpndev
;
2136 struct lio
*lio
= GET_LIO(netdev
);
2137 struct octeon_device
*oct
= lio
->oct_dev
;
2139 switch (nctrl
->ncmd
.s
.cmd
) {
2140 case OCTNET_CMD_CHANGE_DEVFLAGS
:
2141 case OCTNET_CMD_SET_MULTI_LIST
:
2144 case OCTNET_CMD_CHANGE_MACADDR
:
2145 /* If command is successful, change the MACADDR. */
2146 netif_info(lio
, probe
, lio
->netdev
, " MACAddr changed to 0x%llx\n",
2147 CVM_CAST64(nctrl
->udd
[0]));
2148 dev_info(&oct
->pci_dev
->dev
, "%s MACAddr changed to 0x%llx\n",
2149 netdev
->name
, CVM_CAST64(nctrl
->udd
[0]));
2150 memcpy(netdev
->dev_addr
, ((u8
*)&nctrl
->udd
[0]) + 2, ETH_ALEN
);
2153 case OCTNET_CMD_CHANGE_MTU
:
2154 /* If command is successful, change the MTU. */
2155 netif_info(lio
, probe
, lio
->netdev
, " MTU Changed from %d to %d\n",
2156 netdev
->mtu
, nctrl
->ncmd
.s
.param2
);
2157 dev_info(&oct
->pci_dev
->dev
, "%s MTU Changed from %d to %d\n",
2158 netdev
->name
, netdev
->mtu
,
2159 nctrl
->ncmd
.s
.param2
);
2160 netdev
->mtu
= nctrl
->ncmd
.s
.param2
;
2163 case OCTNET_CMD_GPIO_ACCESS
:
2164 netif_info(lio
, probe
, lio
->netdev
, "LED Flashing visual identification\n");
2168 case OCTNET_CMD_LRO_ENABLE
:
2169 dev_info(&oct
->pci_dev
->dev
, "%s LRO Enabled\n", netdev
->name
);
2172 case OCTNET_CMD_LRO_DISABLE
:
2173 dev_info(&oct
->pci_dev
->dev
, "%s LRO Disabled\n",
2177 case OCTNET_CMD_VERBOSE_ENABLE
:
2178 dev_info(&oct
->pci_dev
->dev
, "%s LRO Enabled\n", netdev
->name
);
2181 case OCTNET_CMD_VERBOSE_DISABLE
:
2182 dev_info(&oct
->pci_dev
->dev
, "%s LRO Disabled\n",
2186 case OCTNET_CMD_SET_SETTINGS
:
2187 dev_info(&oct
->pci_dev
->dev
, "%s settings changed\n",
2193 dev_err(&oct
->pci_dev
->dev
, "%s Unknown cmd %d\n", __func__
,
2199 * \brief Converts a mask based on net device flags
2200 * @param netdev network device
2202 * This routine generates a octnet_ifflags mask from the net device flags
2203 * received from the OS.
2205 static inline enum octnet_ifflags
get_new_flags(struct net_device
*netdev
)
2207 enum octnet_ifflags f
= OCTNET_IFFLAG_UNICAST
;
2209 if (netdev
->flags
& IFF_PROMISC
)
2210 f
|= OCTNET_IFFLAG_PROMISC
;
2212 if (netdev
->flags
& IFF_ALLMULTI
)
2213 f
|= OCTNET_IFFLAG_ALLMULTI
;
2215 if (netdev
->flags
& IFF_MULTICAST
) {
2216 f
|= OCTNET_IFFLAG_MULTICAST
;
2218 /* Accept all multicast addresses if there are more than we
2221 if (netdev_mc_count(netdev
) > MAX_OCTEON_MULTICAST_ADDR
)
2222 f
|= OCTNET_IFFLAG_ALLMULTI
;
2225 if (netdev
->flags
& IFF_BROADCAST
)
2226 f
|= OCTNET_IFFLAG_BROADCAST
;
2232 * \brief Net device set_multicast_list
2233 * @param netdev network device
2235 static void liquidio_set_mcast_list(struct net_device
*netdev
)
2237 struct lio
*lio
= GET_LIO(netdev
);
2238 struct octeon_device
*oct
= lio
->oct_dev
;
2239 struct octnic_ctrl_pkt nctrl
;
2240 struct octnic_ctrl_params nparams
;
2241 struct netdev_hw_addr
*ha
;
2244 int mc_count
= min(netdev_mc_count(netdev
), MAX_OCTEON_MULTICAST_ADDR
);
2246 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2248 /* Create a ctrl pkt command to be sent to core app. */
2250 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_SET_MULTI_LIST
;
2251 nctrl
.ncmd
.s
.param1
= lio
->linfo
.ifidx
;
2252 nctrl
.ncmd
.s
.param2
= get_new_flags(netdev
);
2253 nctrl
.ncmd
.s
.param3
= mc_count
;
2254 nctrl
.ncmd
.s
.more
= mc_count
;
2255 nctrl
.netpndev
= (u64
)netdev
;
2256 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2258 /* copy all the addresses into the udd */
2261 netdev_for_each_mc_addr(ha
, netdev
) {
2263 memcpy(((u8
*)mc
) + 2, ha
->addr
, ETH_ALEN
);
2264 /* no need to swap bytes */
2266 if (++mc
> &nctrl
.udd
[mc_count
])
2270 /* Apparently, any activity in this call from the kernel has to
2271 * be atomic. So we won't wait for response.
2273 nctrl
.wait_time
= 0;
2275 nparams
.resp_order
= OCTEON_RESP_NORESPONSE
;
2277 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
, nparams
);
2279 dev_err(&oct
->pci_dev
->dev
, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2285 * \brief Net device set_mac_address
2286 * @param netdev network device
2288 static int liquidio_set_mac(struct net_device
*netdev
, void *p
)
2291 struct lio
*lio
= GET_LIO(netdev
);
2292 struct octeon_device
*oct
= lio
->oct_dev
;
2293 struct sockaddr
*addr
= (struct sockaddr
*)p
;
2294 struct octnic_ctrl_pkt nctrl
;
2295 struct octnic_ctrl_params nparams
;
2297 if ((!is_valid_ether_addr(addr
->sa_data
)) ||
2298 (ifstate_check(lio
, LIO_IFSTATE_RUNNING
)))
2299 return -EADDRNOTAVAIL
;
2301 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2304 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_CHANGE_MACADDR
;
2305 nctrl
.ncmd
.s
.param1
= lio
->linfo
.ifidx
;
2306 nctrl
.ncmd
.s
.param2
= 0;
2307 nctrl
.ncmd
.s
.more
= 1;
2308 nctrl
.netpndev
= (u64
)netdev
;
2309 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2310 nctrl
.wait_time
= 100;
2313 /* The MAC Address is presented in network byte order. */
2314 memcpy((u8
*)&nctrl
.udd
[0] + 2, addr
->sa_data
, ETH_ALEN
);
2316 nparams
.resp_order
= OCTEON_RESP_ORDERED
;
2318 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
, nparams
);
2320 dev_err(&oct
->pci_dev
->dev
, "MAC Address change failed\n");
2323 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
2324 memcpy(((u8
*)&lio
->linfo
.hw_addr
) + 2, addr
->sa_data
, ETH_ALEN
);
2330 * \brief Net device get_stats
2331 * @param netdev network device
2333 static struct net_device_stats
*liquidio_get_stats(struct net_device
*netdev
)
2335 struct lio
*lio
= GET_LIO(netdev
);
2336 struct net_device_stats
*stats
= &netdev
->stats
;
2337 struct octeon_device
*oct
;
2338 u64 pkts
= 0, drop
= 0, bytes
= 0;
2339 struct oct_droq_stats
*oq_stats
;
2340 struct oct_iq_stats
*iq_stats
;
2341 int i
, iq_no
, oq_no
;
2345 for (i
= 0; i
< lio
->linfo
.num_txpciq
; i
++) {
2346 iq_no
= lio
->linfo
.txpciq
[i
];
2347 iq_stats
= &oct
->instr_queue
[iq_no
]->stats
;
2348 pkts
+= iq_stats
->tx_done
;
2349 drop
+= iq_stats
->tx_dropped
;
2350 bytes
+= iq_stats
->tx_tot_bytes
;
2353 stats
->tx_packets
= pkts
;
2354 stats
->tx_bytes
= bytes
;
2355 stats
->tx_dropped
= drop
;
2361 for (i
= 0; i
< lio
->linfo
.num_rxpciq
; i
++) {
2362 oq_no
= lio
->linfo
.rxpciq
[i
];
2363 oq_stats
= &oct
->droq
[oq_no
]->stats
;
2364 pkts
+= oq_stats
->rx_pkts_received
;
2365 drop
+= (oq_stats
->rx_dropped
+
2366 oq_stats
->dropped_nodispatch
+
2367 oq_stats
->dropped_toomany
+
2368 oq_stats
->dropped_nomem
);
2369 bytes
+= oq_stats
->rx_bytes_received
;
2372 stats
->rx_bytes
= bytes
;
2373 stats
->rx_packets
= pkts
;
2374 stats
->rx_dropped
= drop
;
2380 * \brief Net device change_mtu
2381 * @param netdev network device
2383 static int liquidio_change_mtu(struct net_device
*netdev
, int new_mtu
)
2385 struct lio
*lio
= GET_LIO(netdev
);
2386 struct octeon_device
*oct
= lio
->oct_dev
;
2387 struct octnic_ctrl_pkt nctrl
;
2388 struct octnic_ctrl_params nparams
;
2389 int max_frm_size
= new_mtu
+ OCTNET_FRM_HEADER_SIZE
;
2392 /* Limit the MTU to make sure the ethernet packets are between 64 bytes
2395 if ((max_frm_size
< OCTNET_MIN_FRM_SIZE
) ||
2396 (max_frm_size
> OCTNET_MAX_FRM_SIZE
)) {
2397 dev_err(&oct
->pci_dev
->dev
, "Invalid MTU: %d\n", new_mtu
);
2398 dev_err(&oct
->pci_dev
->dev
, "Valid range %d and %d\n",
2399 (OCTNET_MIN_FRM_SIZE
- OCTNET_FRM_HEADER_SIZE
),
2400 (OCTNET_MAX_FRM_SIZE
- OCTNET_FRM_HEADER_SIZE
));
2404 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2407 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_CHANGE_MTU
;
2408 nctrl
.ncmd
.s
.param1
= lio
->linfo
.ifidx
;
2409 nctrl
.ncmd
.s
.param2
= new_mtu
;
2410 nctrl
.wait_time
= 100;
2411 nctrl
.netpndev
= (u64
)netdev
;
2412 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2414 nparams
.resp_order
= OCTEON_RESP_ORDERED
;
2416 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
, nparams
);
2418 dev_err(&oct
->pci_dev
->dev
, "Failed to set MTU\n");
2428 * \brief Handler for SIOCSHWTSTAMP ioctl
2429 * @param netdev network device
2430 * @param ifr interface request
2431 * @param cmd command
2433 static int hwtstamp_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
2435 struct hwtstamp_config conf
;
2436 struct lio
*lio
= GET_LIO(netdev
);
2438 if (copy_from_user(&conf
, ifr
->ifr_data
, sizeof(conf
)))
2444 switch (conf
.tx_type
) {
2445 case HWTSTAMP_TX_ON
:
2446 case HWTSTAMP_TX_OFF
:
2452 switch (conf
.rx_filter
) {
2453 case HWTSTAMP_FILTER_NONE
:
2455 case HWTSTAMP_FILTER_ALL
:
2456 case HWTSTAMP_FILTER_SOME
:
2457 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT
:
2458 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC
:
2459 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ
:
2460 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT
:
2461 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC
:
2462 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ
:
2463 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT
:
2464 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC
:
2465 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ
:
2466 case HWTSTAMP_FILTER_PTP_V2_EVENT
:
2467 case HWTSTAMP_FILTER_PTP_V2_SYNC
:
2468 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ
:
2469 conf
.rx_filter
= HWTSTAMP_FILTER_ALL
;
2475 if (conf
.rx_filter
== HWTSTAMP_FILTER_ALL
)
2476 ifstate_set(lio
, LIO_IFSTATE_RX_TIMESTAMP_ENABLED
);
2479 ifstate_reset(lio
, LIO_IFSTATE_RX_TIMESTAMP_ENABLED
);
2481 return copy_to_user(ifr
->ifr_data
, &conf
, sizeof(conf
)) ? -EFAULT
: 0;
2485 * \brief ioctl handler
2486 * @param netdev network device
2487 * @param ifr interface request
2488 * @param cmd command
2490 static int liquidio_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
2494 return hwtstamp_ioctl(netdev
, ifr
, cmd
);
2501 * \brief handle a Tx timestamp response
2502 * @param status response status
2503 * @param buf pointer to skb
2505 static void handle_timestamp(struct octeon_device
*oct
,
2509 struct octnet_buf_free_info
*finfo
;
2510 struct octeon_soft_command
*sc
;
2511 struct oct_timestamp_resp
*resp
;
2513 struct sk_buff
*skb
= (struct sk_buff
*)buf
;
2515 finfo
= (struct octnet_buf_free_info
*)skb
->cb
;
2519 resp
= (struct oct_timestamp_resp
*)sc
->virtrptr
;
2521 if (status
!= OCTEON_REQUEST_DONE
) {
2522 dev_err(&oct
->pci_dev
->dev
, "Tx timestamp instruction failed. Status: %llx\n",
2523 CVM_CAST64(status
));
2524 resp
->timestamp
= 0;
2527 octeon_swap_8B_data(&resp
->timestamp
, 1);
2529 if (unlikely((skb_shinfo(skb
)->tx_flags
| SKBTX_IN_PROGRESS
) != 0)) {
2530 struct skb_shared_hwtstamps ts
;
2531 u64 ns
= resp
->timestamp
;
2533 netif_info(lio
, tx_done
, lio
->netdev
,
2534 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2535 skb
, (unsigned long long)ns
);
2536 ts
.hwtstamp
= ns_to_ktime(ns
+ lio
->ptp_adjust
);
2537 skb_tstamp_tx(skb
, &ts
);
2540 octeon_free_soft_command(oct
, sc
);
2541 recv_buffer_free(skb
);
2544 /* \brief Send a data packet that will be timestamped
2545 * @param oct octeon device
2546 * @param ndata pointer to network data
2547 * @param finfo pointer to private network data
2549 static inline int send_nic_timestamp_pkt(struct octeon_device
*oct
,
2550 struct octnic_data_pkt
*ndata
,
2551 struct octnet_buf_free_info
*finfo
,
2555 struct octeon_soft_command
*sc
;
2556 struct octeon_instr_ih
*ih
;
2557 struct octeon_instr_rdp
*rdp
;
2563 sc
= octeon_alloc_soft_command_resp(oct
, &ndata
->cmd
,
2564 sizeof(struct oct_timestamp_resp
));
2568 dev_err(&oct
->pci_dev
->dev
, "No memory for timestamped data packet\n");
2569 return IQ_SEND_FAILED
;
2572 if (ndata
->reqtype
== REQTYPE_NORESP_NET
)
2573 ndata
->reqtype
= REQTYPE_RESP_NET
;
2574 else if (ndata
->reqtype
== REQTYPE_NORESP_NET_SG
)
2575 ndata
->reqtype
= REQTYPE_RESP_NET_SG
;
2577 sc
->callback
= handle_timestamp
;
2578 sc
->callback_arg
= finfo
->skb
;
2579 sc
->iq_no
= ndata
->q_no
;
2581 ih
= (struct octeon_instr_ih
*)&sc
->cmd
.ih
;
2582 rdp
= (struct octeon_instr_rdp
*)&sc
->cmd
.rdp
;
2584 ring_doorbell
= !xmit_more
;
2585 retval
= octeon_send_command(oct
, sc
->iq_no
, ring_doorbell
, &sc
->cmd
,
2586 sc
, ih
->dlengsz
, ndata
->reqtype
);
2589 dev_err(&oct
->pci_dev
->dev
, "timestamp data packet failed status: %x\n",
2591 octeon_free_soft_command(oct
, sc
);
2593 netif_info(lio
, tx_queued
, lio
->netdev
, "Queued timestamp packet\n");
2599 static inline int is_ipv4(struct sk_buff
*skb
)
2601 return (skb
->protocol
== htons(ETH_P_IP
)) &&
2602 (ip_hdr(skb
)->version
== 4);
2605 static inline int is_vlan(struct sk_buff
*skb
)
2607 return skb
->protocol
== htons(ETH_P_8021Q
);
2610 static inline int is_ip_fragmented(struct sk_buff
*skb
)
2612 /* The Don't fragment and Reserved flag fields are ignored.
2613 * IP is fragmented if
2614 * - the More fragments bit is set (indicating this IP is a fragment
2615 * with more to follow; the current offset could be 0 ).
2616 * - ths offset field is non-zero.
2618 return (ip_hdr(skb
)->frag_off
& htons(IP_MF
| IP_OFFSET
)) ? 1 : 0;
2621 static inline int is_ipv6(struct sk_buff
*skb
)
2623 return (skb
->protocol
== htons(ETH_P_IPV6
)) &&
2624 (ipv6_hdr(skb
)->version
== 6);
2627 static inline int is_with_extn_hdr(struct sk_buff
*skb
)
2629 return (ipv6_hdr(skb
)->nexthdr
!= IPPROTO_TCP
) &&
2630 (ipv6_hdr(skb
)->nexthdr
!= IPPROTO_UDP
);
2633 static inline int is_tcpudp(struct sk_buff
*skb
)
2635 return (ip_hdr(skb
)->protocol
== IPPROTO_TCP
) ||
2636 (ip_hdr(skb
)->protocol
== IPPROTO_UDP
);
2639 static inline u32
get_ipv4_5tuple_tag(struct sk_buff
*skb
)
2642 struct iphdr
*iphdr
= ip_hdr(skb
);
2644 tag
= crc32(0, &iphdr
->protocol
, 1);
2645 tag
= crc32(tag
, (u8
*)&iphdr
->saddr
, 8);
2646 tag
= crc32(tag
, skb_transport_header(skb
), 4);
2650 static inline u32
get_ipv6_5tuple_tag(struct sk_buff
*skb
)
2653 struct ipv6hdr
*ipv6hdr
= ipv6_hdr(skb
);
2655 tag
= crc32(0, &ipv6hdr
->nexthdr
, 1);
2656 tag
= crc32(tag
, (u8
*)&ipv6hdr
->saddr
, 32);
2657 tag
= crc32(tag
, skb_transport_header(skb
), 4);
2661 /** \brief Transmit networks packets to the Octeon interface
2662 * @param skbuff skbuff struct to be passed to network layer.
2663 * @param netdev pointer to network device
2664 * @returns whether the packet was transmitted to the device okay or not
2665 * (NETDEV_TX_OK or NETDEV_TX_BUSY)
2667 static int liquidio_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
2670 struct octnet_buf_free_info
*finfo
;
2671 union octnic_cmd_setup cmdsetup
;
2672 struct octnic_data_pkt ndata
;
2673 struct octeon_device
*oct
;
2674 struct oct_iq_stats
*stats
;
2675 int cpu
= 0, status
= 0;
2676 int q_idx
= 0, iq_no
= 0;
2680 lio
= GET_LIO(netdev
);
2683 if (netif_is_multiqueue(netdev
)) {
2684 cpu
= skb
->queue_mapping
;
2685 q_idx
= (cpu
& (lio
->linfo
.num_txpciq
- 1));
2686 iq_no
= lio
->linfo
.txpciq
[q_idx
];
2691 stats
= &oct
->instr_queue
[iq_no
]->stats
;
2693 /* Check for all conditions in which the current packet cannot be
2696 if (!(atomic_read(&lio
->ifstate
) & LIO_IFSTATE_RUNNING
) ||
2697 (!lio
->linfo
.link
.s
.status
) ||
2699 netif_info(lio
, tx_err
, lio
->netdev
,
2700 "Transmit failed link_status : %d\n",
2701 lio
->linfo
.link
.s
.status
);
2702 goto lio_xmit_failed
;
2705 /* Use space in skb->cb to store info used to unmap and
2708 finfo
= (struct octnet_buf_free_info
*)skb
->cb
;
2713 /* Prepare the attributes for the data to be passed to OSI. */
2714 memset(&ndata
, 0, sizeof(struct octnic_data_pkt
));
2716 ndata
.buf
= (void *)finfo
;
2720 if (netif_is_multiqueue(netdev
)) {
2721 if (octnet_iq_is_full(oct
, ndata
.q_no
)) {
2722 /* defer sending if queue is full */
2723 netif_info(lio
, tx_err
, lio
->netdev
, "Transmit failed iq:%d full\n",
2725 stats
->tx_iq_busy
++;
2726 return NETDEV_TX_BUSY
;
2729 if (octnet_iq_is_full(oct
, lio
->txq
)) {
2730 /* defer sending if queue is full */
2731 stats
->tx_iq_busy
++;
2732 netif_info(lio
, tx_err
, lio
->netdev
, "Transmit failed iq:%d full\n",
2734 return NETDEV_TX_BUSY
;
2737 /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n",
2738 * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no );
2741 ndata
.datasize
= skb
->len
;
2744 cmdsetup
.s
.ifidx
= lio
->linfo
.ifidx
;
2746 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2747 if (is_ipv4(skb
) && !is_ip_fragmented(skb
) && is_tcpudp(skb
)) {
2748 tag
= get_ipv4_5tuple_tag(skb
);
2750 cmdsetup
.s
.cksum_offset
= sizeof(struct ethhdr
) + 1;
2752 if (ip_hdr(skb
)->ihl
> 5)
2753 cmdsetup
.s
.ipv4opts_ipv6exthdr
=
2754 OCT_PKT_PARAM_IPV4OPTS
;
2756 } else if (is_ipv6(skb
)) {
2757 tag
= get_ipv6_5tuple_tag(skb
);
2759 cmdsetup
.s
.cksum_offset
= sizeof(struct ethhdr
) + 1;
2761 if (is_with_extn_hdr(skb
))
2762 cmdsetup
.s
.ipv4opts_ipv6exthdr
=
2763 OCT_PKT_PARAM_IPV6EXTHDR
;
2765 } else if (is_vlan(skb
)) {
2766 if (vlan_eth_hdr(skb
)->h_vlan_encapsulated_proto
2767 == htons(ETH_P_IP
) &&
2768 !is_ip_fragmented(skb
) && is_tcpudp(skb
)) {
2769 tag
= get_ipv4_5tuple_tag(skb
);
2771 cmdsetup
.s
.cksum_offset
=
2772 sizeof(struct vlan_ethhdr
) + 1;
2774 if (ip_hdr(skb
)->ihl
> 5)
2775 cmdsetup
.s
.ipv4opts_ipv6exthdr
=
2776 OCT_PKT_PARAM_IPV4OPTS
;
2778 } else if (vlan_eth_hdr(skb
)->h_vlan_encapsulated_proto
2779 == htons(ETH_P_IPV6
)) {
2780 tag
= get_ipv6_5tuple_tag(skb
);
2782 cmdsetup
.s
.cksum_offset
=
2783 sizeof(struct vlan_ethhdr
) + 1;
2785 if (is_with_extn_hdr(skb
))
2786 cmdsetup
.s
.ipv4opts_ipv6exthdr
=
2787 OCT_PKT_PARAM_IPV6EXTHDR
;
2791 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
2792 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
2793 cmdsetup
.s
.timestamp
= 1;
2796 if (skb_shinfo(skb
)->nr_frags
== 0) {
2797 cmdsetup
.s
.u
.datasize
= skb
->len
;
2798 octnet_prepare_pci_cmd(&ndata
.cmd
, &cmdsetup
, tag
);
2799 /* Offload checksum calculation for TCP/UDP packets */
2800 ndata
.cmd
.dptr
= dma_map_single(&oct
->pci_dev
->dev
,
2804 if (dma_mapping_error(&oct
->pci_dev
->dev
, ndata
.cmd
.dptr
)) {
2805 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 1\n",
2807 return NETDEV_TX_BUSY
;
2810 finfo
->dptr
= ndata
.cmd
.dptr
;
2812 ndata
.reqtype
= REQTYPE_NORESP_NET
;
2816 struct skb_frag_struct
*frag
;
2817 struct octnic_gather
*g
;
2819 spin_lock(&lio
->lock
);
2820 g
= (struct octnic_gather
*)list_delete_head(&lio
->glist
);
2821 spin_unlock(&lio
->lock
);
2824 netif_info(lio
, tx_err
, lio
->netdev
,
2825 "Transmit scatter gather: glist null!\n");
2826 goto lio_xmit_failed
;
2829 cmdsetup
.s
.gather
= 1;
2830 cmdsetup
.s
.u
.gatherptrs
= (skb_shinfo(skb
)->nr_frags
+ 1);
2831 octnet_prepare_pci_cmd(&ndata
.cmd
, &cmdsetup
, tag
);
2833 memset(g
->sg
, 0, g
->sg_size
);
2835 g
->sg
[0].ptr
[0] = dma_map_single(&oct
->pci_dev
->dev
,
2837 (skb
->len
- skb
->data_len
),
2839 if (dma_mapping_error(&oct
->pci_dev
->dev
, g
->sg
[0].ptr
[0])) {
2840 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 2\n",
2842 return NETDEV_TX_BUSY
;
2844 add_sg_size(&g
->sg
[0], (skb
->len
- skb
->data_len
), 0);
2846 frags
= skb_shinfo(skb
)->nr_frags
;
2849 frag
= &skb_shinfo(skb
)->frags
[i
- 1];
2851 g
->sg
[(i
>> 2)].ptr
[(i
& 3)] =
2852 dma_map_page(&oct
->pci_dev
->dev
,
2858 add_sg_size(&g
->sg
[(i
>> 2)], frag
->size
, (i
& 3));
2862 ndata
.cmd
.dptr
= dma_map_single(&oct
->pci_dev
->dev
,
2865 if (dma_mapping_error(&oct
->pci_dev
->dev
, ndata
.cmd
.dptr
)) {
2866 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 3\n",
2868 dma_unmap_single(&oct
->pci_dev
->dev
, g
->sg
[0].ptr
[0],
2869 skb
->len
- skb
->data_len
,
2871 return NETDEV_TX_BUSY
;
2874 finfo
->dptr
= ndata
.cmd
.dptr
;
2877 ndata
.reqtype
= REQTYPE_NORESP_NET_SG
;
2880 if (skb_shinfo(skb
)->gso_size
) {
2881 struct octeon_instr_irh
*irh
=
2882 (struct octeon_instr_irh
*)&ndata
.cmd
.irh
;
2883 union tx_info
*tx_info
= (union tx_info
*)&ndata
.cmd
.ossp
[0];
2885 irh
->len
= 1; /* to indicate that ossp[0] contains tx_info */
2886 tx_info
->s
.gso_size
= skb_shinfo(skb
)->gso_size
;
2887 tx_info
->s
.gso_segs
= skb_shinfo(skb
)->gso_segs
;
2890 xmit_more
= skb
->xmit_more
;
2892 if (unlikely(cmdsetup
.s
.timestamp
))
2893 status
= send_nic_timestamp_pkt(oct
, &ndata
, finfo
, xmit_more
);
2895 status
= octnet_send_nic_data_pkt(oct
, &ndata
, xmit_more
);
2896 if (status
== IQ_SEND_FAILED
)
2897 goto lio_xmit_failed
;
2899 netif_info(lio
, tx_queued
, lio
->netdev
, "Transmit queued successfully\n");
2901 if (status
== IQ_SEND_STOP
)
2902 stop_q(lio
->netdev
, q_idx
);
2904 netdev
->trans_start
= jiffies
;
2907 stats
->tx_tot_bytes
+= skb
->len
;
2909 return NETDEV_TX_OK
;
2912 stats
->tx_dropped
++;
2913 netif_info(lio
, tx_err
, lio
->netdev
, "IQ%d Transmit dropped:%llu\n",
2914 iq_no
, stats
->tx_dropped
);
2915 dma_unmap_single(&oct
->pci_dev
->dev
, ndata
.cmd
.dptr
,
2916 ndata
.datasize
, DMA_TO_DEVICE
);
2917 recv_buffer_free(skb
);
2918 return NETDEV_TX_OK
;
2921 /** \brief Network device Tx timeout
2922 * @param netdev pointer to network device
2924 static void liquidio_tx_timeout(struct net_device
*netdev
)
2928 lio
= GET_LIO(netdev
);
2930 netif_info(lio
, tx_err
, lio
->netdev
,
2931 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2932 netdev
->stats
.tx_dropped
);
2933 netdev
->trans_start
= jiffies
;
2937 int liquidio_set_feature(struct net_device
*netdev
, int cmd
)
2939 struct lio
*lio
= GET_LIO(netdev
);
2940 struct octeon_device
*oct
= lio
->oct_dev
;
2941 struct octnic_ctrl_pkt nctrl
;
2942 struct octnic_ctrl_params nparams
;
2945 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2948 nctrl
.ncmd
.s
.cmd
= cmd
;
2949 nctrl
.ncmd
.s
.param1
= lio
->linfo
.ifidx
;
2950 nctrl
.ncmd
.s
.param2
= OCTNIC_LROIPV4
| OCTNIC_LROIPV6
;
2951 nctrl
.wait_time
= 100;
2952 nctrl
.netpndev
= (u64
)netdev
;
2953 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2955 nparams
.resp_order
= OCTEON_RESP_NORESPONSE
;
2957 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
, nparams
);
2959 dev_err(&oct
->pci_dev
->dev
, "Feature change failed in core (ret: 0x%x)\n",
2965 /** \brief Net device fix features
2966 * @param netdev pointer to network device
2967 * @param request features requested
2968 * @returns updated features list
2970 static netdev_features_t
liquidio_fix_features(struct net_device
*netdev
,
2971 netdev_features_t request
)
2973 struct lio
*lio
= netdev_priv(netdev
);
2975 if ((request
& NETIF_F_RXCSUM
) &&
2976 !(lio
->dev_capability
& NETIF_F_RXCSUM
))
2977 request
&= ~NETIF_F_RXCSUM
;
2979 if ((request
& NETIF_F_HW_CSUM
) &&
2980 !(lio
->dev_capability
& NETIF_F_HW_CSUM
))
2981 request
&= ~NETIF_F_HW_CSUM
;
2983 if ((request
& NETIF_F_TSO
) && !(lio
->dev_capability
& NETIF_F_TSO
))
2984 request
&= ~NETIF_F_TSO
;
2986 if ((request
& NETIF_F_TSO6
) && !(lio
->dev_capability
& NETIF_F_TSO6
))
2987 request
&= ~NETIF_F_TSO6
;
2989 if ((request
& NETIF_F_LRO
) && !(lio
->dev_capability
& NETIF_F_LRO
))
2990 request
&= ~NETIF_F_LRO
;
2992 /*Disable LRO if RXCSUM is off */
2993 if (!(request
& NETIF_F_RXCSUM
) && (netdev
->features
& NETIF_F_LRO
) &&
2994 (lio
->dev_capability
& NETIF_F_LRO
))
2995 request
&= ~NETIF_F_LRO
;
3000 /** \brief Net device set features
3001 * @param netdev pointer to network device
3002 * @param features features to enable/disable
3004 static int liquidio_set_features(struct net_device
*netdev
,
3005 netdev_features_t features
)
3007 struct lio
*lio
= netdev_priv(netdev
);
3009 if (!((netdev
->features
^ features
) & NETIF_F_LRO
))
3012 if ((features
& NETIF_F_LRO
) && (lio
->dev_capability
& NETIF_F_LRO
))
3013 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_ENABLE
);
3014 else if (!(features
& NETIF_F_LRO
) &&
3015 (lio
->dev_capability
& NETIF_F_LRO
))
3016 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_DISABLE
);
3021 static struct net_device_ops lionetdevops
= {
3022 .ndo_open
= liquidio_open
,
3023 .ndo_stop
= liquidio_stop
,
3024 .ndo_start_xmit
= liquidio_xmit
,
3025 .ndo_get_stats
= liquidio_get_stats
,
3026 .ndo_set_mac_address
= liquidio_set_mac
,
3027 .ndo_set_rx_mode
= liquidio_set_mcast_list
,
3028 .ndo_tx_timeout
= liquidio_tx_timeout
,
3029 .ndo_change_mtu
= liquidio_change_mtu
,
3030 .ndo_do_ioctl
= liquidio_ioctl
,
3031 .ndo_fix_features
= liquidio_fix_features
,
3032 .ndo_set_features
= liquidio_set_features
,
3035 /** \brief Entry point for the liquidio module
3037 static int __init
liquidio_init(void)
3040 struct handshake
*hs
;
3042 init_completion(&first_stage
);
3044 octeon_init_device_list(conf_type
);
3046 if (liquidio_init_pci())
3049 wait_for_completion_timeout(&first_stage
, msecs_to_jiffies(1000));
3051 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
3054 wait_for_completion(&hs
->init
);
3056 /* init handshake failed */
3057 dev_err(&hs
->pci_dev
->dev
,
3058 "Failed to init device\n");
3059 liquidio_deinit_pci();
3065 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
3068 wait_for_completion_timeout(&hs
->started
,
3069 msecs_to_jiffies(30000));
3070 if (!hs
->started_ok
) {
3071 /* starter handshake failed */
3072 dev_err(&hs
->pci_dev
->dev
,
3073 "Firmware failed to start\n");
3074 liquidio_deinit_pci();
3083 static int lio_nic_info(struct octeon_recv_info
*recv_info
, void *buf
)
3085 struct octeon_device
*oct
= (struct octeon_device
*)buf
;
3086 struct octeon_recv_pkt
*recv_pkt
= recv_info
->recv_pkt
;
3088 union oct_link_status
*ls
;
3091 if ((recv_pkt
->buffer_size
[0] != sizeof(*ls
)) ||
3092 (recv_pkt
->rh
.r_nic_info
.ifidx
> oct
->ifcount
)) {
3093 dev_err(&oct
->pci_dev
->dev
, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3094 recv_pkt
->buffer_size
[0],
3095 recv_pkt
->rh
.r_nic_info
.ifidx
);
3099 ifidx
= recv_pkt
->rh
.r_nic_info
.ifidx
;
3100 ls
= (union oct_link_status
*)get_rbd(recv_pkt
->buffer_ptr
[0]);
3102 octeon_swap_8B_data((u64
*)ls
, (sizeof(union oct_link_status
)) >> 3);
3104 update_link_status(oct
->props
[ifidx
].netdev
, ls
);
3107 for (i
= 0; i
< recv_pkt
->buffer_count
; i
++)
3108 recv_buffer_free(recv_pkt
->buffer_ptr
[i
]);
3109 octeon_free_recv_info(recv_info
);
3114 * \brief Setup network interfaces
3115 * @param octeon_dev octeon device
3117 * Called during init time for each device. It assumes the NIC
3118 * is already up and running. The link information for each
3119 * interface is passed in link_info.
3121 static int setup_nic_devices(struct octeon_device
*octeon_dev
)
3123 struct lio
*lio
= NULL
;
3124 struct net_device
*netdev
;
3126 struct octeon_soft_command
*sc
;
3127 struct liquidio_if_cfg_context
*ctx
;
3128 struct liquidio_if_cfg_resp
*resp
;
3129 struct octdev_props
*props
;
3130 int retval
, num_iqueues
, num_oqueues
, q_no
;
3132 int num_cpus
= num_online_cpus();
3133 union oct_nic_if_cfg if_cfg
;
3134 unsigned int base_queue
;
3135 unsigned int gmx_port_id
;
3136 u32 resp_size
, ctx_size
;
3138 /* This is to handle link status changes */
3139 octeon_register_dispatch_fn(octeon_dev
, OPCODE_NIC
,
3141 lio_nic_info
, octeon_dev
);
3143 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3144 * They are handled directly.
3146 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_NORESP_NET
,
3149 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_NORESP_NET_SG
,
3152 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_RESP_NET_SG
,
3153 free_netsgbuf_with_resp
);
3155 for (i
= 0; i
< octeon_dev
->ifcount
; i
++) {
3156 resp_size
= sizeof(struct liquidio_if_cfg_resp
);
3157 ctx_size
= sizeof(struct liquidio_if_cfg_context
);
3158 sc
= (struct octeon_soft_command
*)
3159 octeon_alloc_soft_command(octeon_dev
, 0,
3160 resp_size
, ctx_size
);
3161 resp
= (struct liquidio_if_cfg_resp
*)sc
->virtrptr
;
3162 ctx
= (struct liquidio_if_cfg_context
*)sc
->ctxptr
;
3165 CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev
), i
);
3167 CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev
), i
);
3169 CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev
), i
);
3171 CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev
), i
);
3172 if (num_iqueues
> num_cpus
)
3173 num_iqueues
= num_cpus
;
3174 if (num_oqueues
> num_cpus
)
3175 num_oqueues
= num_cpus
;
3176 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3177 "requesting config for interface %d, iqs %d, oqs %d\n",
3178 i
, num_iqueues
, num_oqueues
);
3179 ACCESS_ONCE(ctx
->cond
) = 0;
3180 ctx
->octeon_id
= lio_get_device_id(octeon_dev
);
3181 init_waitqueue_head(&ctx
->wc
);
3184 if_cfg
.s
.num_iqueues
= num_iqueues
;
3185 if_cfg
.s
.num_oqueues
= num_oqueues
;
3186 if_cfg
.s
.base_queue
= base_queue
;
3187 if_cfg
.s
.gmx_port_id
= gmx_port_id
;
3188 octeon_prepare_soft_command(octeon_dev
, sc
, OPCODE_NIC
,
3189 OPCODE_NIC_IF_CFG
, i
,
3192 sc
->callback
= if_cfg_callback
;
3193 sc
->callback_arg
= sc
;
3194 sc
->wait_time
= 1000;
3196 retval
= octeon_send_soft_command(octeon_dev
, sc
);
3198 dev_err(&octeon_dev
->pci_dev
->dev
,
3199 "iq/oq config failed status: %x\n",
3201 /* Soft instr is freed by driver in case of failure. */
3202 goto setup_nic_dev_fail
;
3205 /* Sleep on a wait queue till the cond flag indicates that the
3206 * response arrived or timed-out.
3208 sleep_cond(&ctx
->wc
, &ctx
->cond
);
3209 retval
= resp
->status
;
3211 dev_err(&octeon_dev
->pci_dev
->dev
, "iq/oq config failed\n");
3212 goto setup_nic_dev_fail
;
3215 octeon_swap_8B_data((u64
*)(&resp
->cfg_info
),
3216 (sizeof(struct liquidio_if_cfg_info
)) >> 3);
3218 num_iqueues
= hweight64(resp
->cfg_info
.iqmask
);
3219 num_oqueues
= hweight64(resp
->cfg_info
.oqmask
);
3221 if (!(num_iqueues
) || !(num_oqueues
)) {
3222 dev_err(&octeon_dev
->pci_dev
->dev
,
3223 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3224 resp
->cfg_info
.iqmask
,
3225 resp
->cfg_info
.oqmask
);
3226 goto setup_nic_dev_fail
;
3228 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3229 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3230 i
, resp
->cfg_info
.iqmask
, resp
->cfg_info
.oqmask
,
3231 num_iqueues
, num_oqueues
);
3232 netdev
= alloc_etherdev_mq(LIO_SIZE
, num_iqueues
);
3235 dev_err(&octeon_dev
->pci_dev
->dev
, "Device allocation failed\n");
3236 goto setup_nic_dev_fail
;
3239 props
= &octeon_dev
->props
[i
];
3240 props
->netdev
= netdev
;
3242 if (num_iqueues
> 1)
3243 lionetdevops
.ndo_select_queue
= select_q
;
3245 /* Associate the routines that will handle different
3248 netdev
->netdev_ops
= &lionetdevops
;
3250 lio
= GET_LIO(netdev
);
3252 memset(lio
, 0, sizeof(struct lio
));
3254 lio
->linfo
.ifidx
= resp
->cfg_info
.ifidx
;
3255 lio
->ifidx
= resp
->cfg_info
.ifidx
;
3257 lio
->linfo
.num_rxpciq
= num_oqueues
;
3258 lio
->linfo
.num_txpciq
= num_iqueues
;
3259 q_mask
= resp
->cfg_info
.oqmask
;
3260 /* q_mask is 0-based and already verified mask is nonzero */
3261 for (j
= 0; j
< num_oqueues
; j
++) {
3262 q_no
= __ffs64(q_mask
);
3263 q_mask
&= (~(1UL << q_no
));
3264 lio
->linfo
.rxpciq
[j
] = q_no
;
3266 q_mask
= resp
->cfg_info
.iqmask
;
3267 for (j
= 0; j
< num_iqueues
; j
++) {
3268 q_no
= __ffs64(q_mask
);
3269 q_mask
&= (~(1UL << q_no
));
3270 lio
->linfo
.txpciq
[j
] = q_no
;
3272 lio
->linfo
.hw_addr
= resp
->cfg_info
.linfo
.hw_addr
;
3273 lio
->linfo
.gmxport
= resp
->cfg_info
.linfo
.gmxport
;
3274 lio
->linfo
.link
.u64
= resp
->cfg_info
.linfo
.link
.u64
;
3276 lio
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
3278 lio
->dev_capability
= NETIF_F_HIGHDMA
3279 | NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
3280 | NETIF_F_SG
| NETIF_F_RXCSUM
3281 | NETIF_F_TSO
| NETIF_F_TSO6
3283 netif_set_gso_max_size(netdev
, OCTNIC_GSO_MAX_SIZE
);
3285 netdev
->features
= lio
->dev_capability
;
3286 netdev
->vlan_features
= lio
->dev_capability
;
3288 netdev
->hw_features
= lio
->dev_capability
;
3290 /* Point to the properties for octeon device to which this
3291 * interface belongs.
3293 lio
->oct_dev
= octeon_dev
;
3294 lio
->octprops
= props
;
3295 lio
->netdev
= netdev
;
3296 spin_lock_init(&lio
->lock
);
3298 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3299 "if%d gmx: %d hw_addr: 0x%llx\n", i
,
3300 lio
->linfo
.gmxport
, CVM_CAST64(lio
->linfo
.hw_addr
));
3302 /* 64-bit swap required on LE machines */
3303 octeon_swap_8B_data(&lio
->linfo
.hw_addr
, 1);
3304 for (j
= 0; j
< 6; j
++)
3305 mac
[j
] = *((u8
*)(((u8
*)&lio
->linfo
.hw_addr
) + 2 + j
));
3307 /* Copy MAC Address to OS network device structure */
3309 ether_addr_copy(netdev
->dev_addr
, mac
);
3311 if (setup_io_queues(octeon_dev
, netdev
)) {
3312 dev_err(&octeon_dev
->pci_dev
->dev
, "I/O queues creation failed\n");
3313 goto setup_nic_dev_fail
;
3316 ifstate_set(lio
, LIO_IFSTATE_DROQ_OPS
);
3318 /* By default all interfaces on a single Octeon uses the same
3321 lio
->txq
= lio
->linfo
.txpciq
[0];
3322 lio
->rxq
= lio
->linfo
.rxpciq
[0];
3324 lio
->tx_qsize
= octeon_get_tx_qsize(octeon_dev
, lio
->txq
);
3325 lio
->rx_qsize
= octeon_get_rx_qsize(octeon_dev
, lio
->rxq
);
3327 if (setup_glist(lio
)) {
3328 dev_err(&octeon_dev
->pci_dev
->dev
,
3329 "Gather list allocation failed\n");
3330 goto setup_nic_dev_fail
;
3333 /* Register ethtool support */
3334 liquidio_set_ethtool_ops(netdev
);
3336 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_ENABLE
);
3338 if ((debug
!= -1) && (debug
& NETIF_MSG_HW
))
3339 liquidio_set_feature(netdev
, OCTNET_CMD_VERBOSE_ENABLE
);
3341 /* Register the network device with the OS */
3342 if (register_netdev(netdev
)) {
3343 dev_err(&octeon_dev
->pci_dev
->dev
, "Device registration failed\n");
3344 goto setup_nic_dev_fail
;
3347 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3348 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3349 i
, mac
[0], mac
[1], mac
[2], mac
[3], mac
[4], mac
[5]);
3350 netif_carrier_off(netdev
);
3352 if (lio
->linfo
.link
.s
.status
) {
3353 netif_carrier_on(netdev
);
3356 netif_carrier_off(netdev
);
3359 ifstate_set(lio
, LIO_IFSTATE_REGISTERED
);
3361 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3362 "NIC ifidx:%d Setup successful\n", i
);
3364 octeon_free_soft_command(octeon_dev
, sc
);
3371 octeon_free_soft_command(octeon_dev
, sc
);
3374 dev_err(&octeon_dev
->pci_dev
->dev
,
3375 "NIC ifidx:%d Setup failed\n", i
);
3376 liquidio_destroy_nic_device(octeon_dev
, i
);
3382 * \brief initialize the NIC
3383 * @param oct octeon device
3385 * This initialization routine is called once the Octeon device application is
3388 static int liquidio_init_nic_module(struct octeon_device
*oct
)
3390 struct oct_intrmod_cfg
*intrmod_cfg
;
3392 int num_nic_ports
= CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct
));
3394 dev_dbg(&oct
->pci_dev
->dev
, "Initializing network interfaces\n");
3396 /* only default iq and oq were initialized
3397 * initialize the rest as well
3399 /* run port_config command for each port */
3400 oct
->ifcount
= num_nic_ports
;
3402 memset(oct
->props
, 0,
3403 sizeof(struct octdev_props
) * num_nic_ports
);
3405 retval
= setup_nic_devices(oct
);
3407 dev_err(&oct
->pci_dev
->dev
, "Setup NIC devices failed\n");
3408 goto octnet_init_failure
;
3411 liquidio_ptp_init(oct
);
3413 /* Initialize interrupt moderation params */
3414 intrmod_cfg
= &((struct octeon_device
*)oct
)->intrmod
;
3415 intrmod_cfg
->intrmod_enable
= 1;
3416 intrmod_cfg
->intrmod_check_intrvl
= LIO_INTRMOD_CHECK_INTERVAL
;
3417 intrmod_cfg
->intrmod_maxpkt_ratethr
= LIO_INTRMOD_MAXPKT_RATETHR
;
3418 intrmod_cfg
->intrmod_minpkt_ratethr
= LIO_INTRMOD_MINPKT_RATETHR
;
3419 intrmod_cfg
->intrmod_maxcnt_trigger
= LIO_INTRMOD_MAXCNT_TRIGGER
;
3420 intrmod_cfg
->intrmod_maxtmr_trigger
= LIO_INTRMOD_MAXTMR_TRIGGER
;
3421 intrmod_cfg
->intrmod_mintmr_trigger
= LIO_INTRMOD_MINTMR_TRIGGER
;
3422 intrmod_cfg
->intrmod_mincnt_trigger
= LIO_INTRMOD_MINCNT_TRIGGER
;
3424 dev_dbg(&oct
->pci_dev
->dev
, "Network interfaces ready\n");
3428 octnet_init_failure
:
3436 * \brief starter callback that invokes the remaining initialization work after
3437 * the NIC is up and running.
3438 * @param octptr work struct work_struct
3440 static void nic_starter(struct work_struct
*work
)
3442 struct octeon_device
*oct
;
3443 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
3445 oct
= (struct octeon_device
*)wk
->ctxptr
;
3447 if (atomic_read(&oct
->status
) == OCT_DEV_RUNNING
)
3450 /* If the status of the device is CORE_OK, the core
3451 * application has reported its application type. Call
3452 * any registered handlers now and move to the RUNNING
3455 if (atomic_read(&oct
->status
) != OCT_DEV_CORE_OK
) {
3456 schedule_delayed_work(&oct
->nic_poll_work
.work
,
3457 LIQUIDIO_STARTER_POLL_INTERVAL_MS
);
3461 atomic_set(&oct
->status
, OCT_DEV_RUNNING
);
3463 if (oct
->app_mode
&& oct
->app_mode
== CVM_DRV_NIC_APP
) {
3464 dev_dbg(&oct
->pci_dev
->dev
, "Starting NIC module\n");
3466 if (liquidio_init_nic_module(oct
))
3467 dev_err(&oct
->pci_dev
->dev
, "NIC initialization failed\n");
3469 handshake
[oct
->octeon_id
].started_ok
= 1;
3471 dev_err(&oct
->pci_dev
->dev
,
3472 "Unexpected application running on NIC (%d). Check firmware.\n",
3476 complete(&handshake
[oct
->octeon_id
].started
);
3480 * \brief Device initialization for each Octeon device that is probed
3481 * @param octeon_dev octeon device
3483 static int octeon_device_init(struct octeon_device
*octeon_dev
)
3486 struct octeon_device_priv
*oct_priv
=
3487 (struct octeon_device_priv
*)octeon_dev
->priv
;
3488 atomic_set(&octeon_dev
->status
, OCT_DEV_BEGIN_STATE
);
3490 /* Enable access to the octeon device and make its DMA capability
3493 if (octeon_pci_os_setup(octeon_dev
))
3496 /* Identify the Octeon type and map the BAR address space. */
3497 if (octeon_chip_specific_setup(octeon_dev
)) {
3498 dev_err(&octeon_dev
->pci_dev
->dev
, "Chip specific setup failed\n");
3502 atomic_set(&octeon_dev
->status
, OCT_DEV_PCI_MAP_DONE
);
3504 octeon_dev
->app_mode
= CVM_DRV_INVALID_APP
;
3506 /* Do a soft reset of the Octeon device. */
3507 if (octeon_dev
->fn_list
.soft_reset(octeon_dev
))
3510 /* Initialize the dispatch mechanism used to push packets arriving on
3511 * Octeon Output queues.
3513 if (octeon_init_dispatch_list(octeon_dev
))
3516 octeon_register_dispatch_fn(octeon_dev
, OPCODE_NIC
,
3517 OPCODE_NIC_CORE_DRV_ACTIVE
,
3518 octeon_core_drv_init
,
3521 INIT_DELAYED_WORK(&octeon_dev
->nic_poll_work
.work
, nic_starter
);
3522 octeon_dev
->nic_poll_work
.ctxptr
= (void *)octeon_dev
;
3523 schedule_delayed_work(&octeon_dev
->nic_poll_work
.work
,
3524 LIQUIDIO_STARTER_POLL_INTERVAL_MS
);
3526 atomic_set(&octeon_dev
->status
, OCT_DEV_DISPATCH_INIT_DONE
);
3528 octeon_set_io_queues_off(octeon_dev
);
3530 /* Setup the data structures that manage this Octeon's Input queues. */
3531 if (octeon_setup_instr_queues(octeon_dev
)) {
3532 dev_err(&octeon_dev
->pci_dev
->dev
,
3533 "instruction queue initialization failed\n");
3534 /* On error, release any previously allocated queues */
3535 for (j
= 0; j
< octeon_dev
->num_iqs
; j
++)
3536 octeon_delete_instr_queue(octeon_dev
, j
);
3539 atomic_set(&octeon_dev
->status
, OCT_DEV_INSTR_QUEUE_INIT_DONE
);
3541 /* Initialize soft command buffer pool
3543 if (octeon_setup_sc_buffer_pool(octeon_dev
)) {
3544 dev_err(&octeon_dev
->pci_dev
->dev
, "sc buffer pool allocation failed\n");
3547 atomic_set(&octeon_dev
->status
, OCT_DEV_SC_BUFF_POOL_INIT_DONE
);
3549 /* Initialize lists to manage the requests of different types that
3550 * arrive from user & kernel applications for this octeon device.
3552 if (octeon_setup_response_list(octeon_dev
)) {
3553 dev_err(&octeon_dev
->pci_dev
->dev
, "Response list allocation failed\n");
3556 atomic_set(&octeon_dev
->status
, OCT_DEV_RESP_LIST_INIT_DONE
);
3558 if (octeon_setup_output_queues(octeon_dev
)) {
3559 dev_err(&octeon_dev
->pci_dev
->dev
, "Output queue initialization failed\n");
3560 /* Release any previously allocated queues */
3561 for (j
= 0; j
< octeon_dev
->num_oqs
; j
++)
3562 octeon_delete_droq(octeon_dev
, j
);
3565 atomic_set(&octeon_dev
->status
, OCT_DEV_DROQ_INIT_DONE
);
3567 /* The input and output queue registers were setup earlier (the queues
3568 * were not enabled). Any additional registers that need to be
3569 * programmed should be done now.
3571 ret
= octeon_dev
->fn_list
.setup_device_regs(octeon_dev
);
3573 dev_err(&octeon_dev
->pci_dev
->dev
,
3574 "Failed to configure device registers\n");
3578 /* Initialize the tasklet that handles output queue packet processing.*/
3579 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Initializing droq tasklet\n");
3580 tasklet_init(&oct_priv
->droq_tasklet
, octeon_droq_bh
,
3581 (unsigned long)octeon_dev
);
3583 /* Setup the interrupt handler and record the INT SUM register address
3585 octeon_setup_interrupt(octeon_dev
);
3587 /* Enable Octeon device interrupts */
3588 octeon_dev
->fn_list
.enable_interrupt(octeon_dev
->chip
);
3590 /* Enable the input and output queues for this Octeon device */
3591 octeon_dev
->fn_list
.enable_io_queues(octeon_dev
);
3593 atomic_set(&octeon_dev
->status
, OCT_DEV_IO_QUEUES_DONE
);
3595 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Waiting for DDR initialization...\n");
3597 if (ddr_timeout
== 0) {
3598 dev_info(&octeon_dev
->pci_dev
->dev
,
3599 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3602 schedule_timeout_uninterruptible(HZ
* LIO_RESET_SECS
);
3604 /* Wait for the octeon to initialize DDR after the soft-reset. */
3605 ret
= octeon_wait_for_ddr_init(octeon_dev
, &ddr_timeout
);
3607 dev_err(&octeon_dev
->pci_dev
->dev
,
3608 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3613 if (octeon_wait_for_bootloader(octeon_dev
, 1000) != 0) {
3614 dev_err(&octeon_dev
->pci_dev
->dev
, "Board not responding\n");
3618 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Initializing consoles\n");
3619 ret
= octeon_init_consoles(octeon_dev
);
3621 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not access board consoles\n");
3624 ret
= octeon_add_console(octeon_dev
, 0);
3626 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not access board console\n");
3630 atomic_set(&octeon_dev
->status
, OCT_DEV_CONSOLE_INIT_DONE
);
3632 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Loading firmware\n");
3633 ret
= load_firmware(octeon_dev
);
3635 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not load firmware to board\n");
3639 handshake
[octeon_dev
->octeon_id
].init_ok
= 1;
3640 complete(&handshake
[octeon_dev
->octeon_id
].init
);
3642 atomic_set(&octeon_dev
->status
, OCT_DEV_HOST_OK
);
3644 /* Send Credit for Octeon Output queues. Credits are always sent after
3645 * the output queue is enabled.
3647 for (j
= 0; j
< octeon_dev
->num_oqs
; j
++)
3648 writel(octeon_dev
->droq
[j
]->max_count
,
3649 octeon_dev
->droq
[j
]->pkts_credit_reg
);
3651 /* Packets can start arriving on the output queues from this point. */
3657 * \brief Exits the module
3659 static void __exit
liquidio_exit(void)
3661 liquidio_deinit_pci();
3663 pr_info("LiquidIO network module is now unloaded\n");
3666 module_init(liquidio_init
);
3667 module_exit(liquidio_exit
);