[deliverable/linux.git] / drivers / crypto / qat / qat_dh895xcc / adf_drv.c

/*
  This file is provided under a dual BSD/GPLv2 license.  When using or
  redistributing this file, you may do so under either license.

  GPL LICENSE SUMMARY
  Copyright(c) 2014 Intel Corporation.
  This program is free software; you can redistribute it and/or modify
  it under the terms of version 2 of the GNU General Public License as
  published by the Free Software Foundation.

  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

  Contact Information:
  qat-linux@intel.com

  BSD LICENSE
  Copyright(c) 2014 Intel Corporation.
  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions
  are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the
      distribution.
    * Neither the name of Intel Corporation nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
#include <linux/io.h>
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
#include <adf_cfg.h>
#include <adf_transport_access_macros.h>
#include "adf_dh895xcc_hw_data.h"
#include "adf_drv.h"

static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;

#define ADF_SYSTEM_DEVICE(device_id) \
	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}

static const struct pci_device_id adf_pci_tbl[] = {
	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
	{0,}
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);

static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
static void adf_remove(struct pci_dev *dev);

static struct pci_driver adf_driver = {
	.id_table = adf_pci_tbl,
	.name = adf_driver_name,
	.probe = adf_probe,
	.remove = adf_remove
};

static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
{
	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
	int i;

	adf_dev_shutdown(accel_dev);

	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i];

		if (bar->virt_addr)
			pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	}

	if (accel_dev->hw_device) {
		switch (accel_pci_dev->pci_dev->device) {
		case ADF_DH895XCC_PCI_DEVICE_ID:
			adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
			break;
		default:
			break;
		}
		kfree(accel_dev->hw_device);
	}
	adf_cfg_dev_remove(accel_dev);
	debugfs_remove(accel_dev->debugfs_dir);
	adf_devmgr_rm_dev(accel_dev);
	pci_release_regions(accel_pci_dev->pci_dev);
	pci_disable_device(accel_pci_dev->pci_dev);
	kfree(accel_dev);
}

static int adf_dev_configure(struct adf_accel_dev *accel_dev)
{
	int cpus = num_online_cpus();
	int banks = GET_MAX_BANKS(accel_dev);
	int instances = min(cpus, banks);
	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
	int i;
	unsigned long val;

	if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
		goto err;
	if (adf_cfg_section_add(accel_dev, "Accelerator0"))
		goto err;
	for (i = 0; i < instances; i++) {
		val = i;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
			 i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
		val = 128;
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 512;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 0;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 2;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 8;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 10;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = ADF_COALESCING_DEF_TIME;
		snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
		if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
						key, (void *)&val, ADF_DEC))
			goto err;
	}

	val = i;
	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
					ADF_NUM_CY, (void *)&val, ADF_DEC))
		goto err;

	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
	return 0;
err:
	dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
	return -EINVAL;
}

static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
	struct adf_accel_dev *accel_dev;
	struct adf_accel_pci *accel_pci_dev;
	struct adf_hw_device_data *hw_data;
	char name[ADF_DEVICE_NAME_LENGTH];
	unsigned int i, bar_nr;
	int ret;

	switch (ent->device) {
	case ADF_DH895XCC_PCI_DEVICE_ID:
		break;
	default:
		dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
		return -ENODEV;
	}

	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
		/* If the accelerator is connected to a node with no memory
		 * there is no point in using the accelerator since the remote
		 * memory transaction will be very slow. */
		dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
		return -EINVAL;
	}

	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
				 dev_to_node(&pdev->dev));
	if (!accel_dev)
		return -ENOMEM;

	INIT_LIST_HEAD(&accel_dev->crypto_list);

	/* Add accel device to accel table.
	 * This should be called before adf_cleanup_accel is called */
	if (adf_devmgr_add_dev(accel_dev)) {
		dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
		kfree(accel_dev);
		return -EFAULT;
	}

	accel_dev->owner = THIS_MODULE;
	/* Allocate and configure device configuration structure */
	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
			       dev_to_node(&pdev->dev));
	if (!hw_data) {
		ret = -ENOMEM;
		goto out_err;
	}

	accel_dev->hw_device = hw_data;
	switch (ent->device) {
	case ADF_DH895XCC_PCI_DEVICE_ID:
		adf_init_hw_data_dh895xcc(accel_dev->hw_device);
		break;
	default:
		return -ENODEV;
	}
	accel_pci_dev = &accel_dev->accel_pci_dev;
	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
	pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
			      &hw_data->fuses);

	/* Get Accelerators and Accelerators Engines masks */
	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	accel_pci_dev->sku = hw_data->get_sku(hw_data);
	accel_pci_dev->pci_dev = pdev;
	/* If the device has no acceleration engines then ignore it. */
	if (!hw_data->accel_mask || !hw_data->ae_mask ||
	    ((~hw_data->ae_mask) & 0x01)) {
		dev_err(&pdev->dev, "No acceleration units found");
		ret = -EFAULT;
		goto out_err;
	}

	/* Create dev top level debugfs entry */
	snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
		 hw_data->dev_class->name, hw_data->instance_id);
	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
	if (!accel_dev->debugfs_dir) {
		dev_err(&pdev->dev, "Could not create debugfs dir\n");
		ret = -EINVAL;
		goto out_err;
	}

	/* Create device configuration table */
	ret = adf_cfg_dev_add(accel_dev);
	if (ret)
		goto out_err;

	pcie_set_readrq(pdev, 1024);

	/* enable PCI device */
	if (pci_enable_device(pdev)) {
		ret = -EFAULT;
		goto out_err;
	}

	/* set dma identifier */
	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
		if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
			dev_err(&pdev->dev, "No usable DMA configuration\n");
			ret = -EFAULT;
			goto out_err;
		} else {
			pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
		}

	} else {
		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	}

	if (pci_request_regions(pdev, adf_driver_name)) {
		ret = -EFAULT;
		goto out_err;
	}

	/* Read accelerator capabilities mask */
	pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
			      &hw_data->accel_capabilities_mask);

	/* Find and map all the device's BARS */
	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i];

		bar_nr = i * 2;
		bar->base_addr = pci_resource_start(pdev, bar_nr);
		if (!bar->base_addr)
			break;
		bar->size = pci_resource_len(pdev, bar_nr);
		bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
		if (!bar->virt_addr) {
			dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
			ret = -EFAULT;
			goto out_err;
		}
	}
	pci_set_master(pdev);

	if (adf_enable_aer(accel_dev, &adf_driver)) {
		dev_err(&pdev->dev, "Failed to enable aer\n");
		ret = -EFAULT;
		goto out_err;
	}

	if (pci_save_state(pdev)) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		ret = -ENOMEM;
		goto out_err;
	}

	ret = adf_dev_configure(accel_dev);
	if (ret)
		goto out_err;

	ret = adf_dev_init(accel_dev);
	if (ret)
		goto out_err;

	ret = adf_dev_start(accel_dev);
	if (ret) {
		adf_dev_stop(accel_dev);
		goto out_err;
	}

	return 0;
out_err:
	adf_cleanup_accel(accel_dev);
	return ret;
}

static void adf_remove(struct pci_dev *pdev)
{
	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);

	if (!accel_dev) {
		pr_err("QAT: Driver removal failed\n");
		return;
	}
	if (adf_dev_stop(accel_dev))
		dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
	adf_disable_aer(accel_dev);
	adf_cleanup_accel(accel_dev);
}

static int __init adfdrv_init(void)
{
	request_module("intel_qat");

	if (pci_register_driver(&adf_driver)) {
		pr_err("QAT: Driver initialization failed\n");
		return -EFAULT;
	}
	return 0;
}

static void __exit adfdrv_release(void)
{
	pci_unregister_driver(&adf_driver);
}

module_init(adfdrv_init);
module_exit(adfdrv_release);

MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel");
MODULE_FIRMWARE(ADF_DH895XCC_FW);
MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
MODULE_VERSION(ADF_DRV_VERSION);
Commit	Line	Data
7afa232e TS	1	/*
	2	This file is provided under a dual BSD/GPLv2 license. When using or
	3	redistributing this file, you may do so under either license.
	4
	5	GPL LICENSE SUMMARY
	6	Copyright(c) 2014 Intel Corporation.
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of version 2 of the GNU General Public License as
	9	published by the Free Software Foundation.
	10
	11	This program is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	General Public License for more details.
	15
	16	Contact Information:
	17	qat-linux@intel.com
	18
	19	BSD LICENSE
	20	Copyright(c) 2014 Intel Corporation.
	21	Redistribution and use in source and binary forms, with or without
	22	modification, are permitted provided that the following conditions
	23	are met:
	24
	25	* Redistributions of source code must retain the above copyright
	26	notice, this list of conditions and the following disclaimer.
	27	* Redistributions in binary form must reproduce the above copyright
	28	notice, this list of conditions and the following disclaimer in
	29	the documentation and/or other materials provided with the
	30	distribution.
	31	* Neither the name of Intel Corporation nor the names of its
	32	contributors may be used to endorse or promote products derived
	33	from this software without specific prior written permission.
	34
	35	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	36	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	37	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	38	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	39	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	40	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	41	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	42	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	43	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	44	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	45	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	46	*/
	47	#include <linux/kernel.h>
	48	#include <linux/module.h>
	49	#include <linux/pci.h>
	50	#include <linux/init.h>
	51	#include <linux/types.h>
	52	#include <linux/fs.h>
	53	#include <linux/slab.h>
	54	#include <linux/errno.h>
	55	#include <linux/device.h>
	56	#include <linux/dma-mapping.h>
	57	#include <linux/platform_device.h>
	58	#include <linux/workqueue.h>
	59	#include <linux/io.h>
	60	#include <adf_accel_devices.h>
	61	#include <adf_common_drv.h>
	62	#include <adf_cfg.h>
	63	#include <adf_transport_access_macros.h>
	64	#include "adf_dh895xcc_hw_data.h"
65	#include "adf_drv.h"
66
67	static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
68
69	#define ADF_SYSTEM_DEVICE(device_id) \
70	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
71
72	static const struct pci_device_id adf_pci_tbl[] = {
73	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
74	{0,}
75	};
76	MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
77
78	static int adf_probe(struct pci_dev dev, const struct pci_device_id ent);
79	static void adf_remove(struct pci_dev *dev);
80
81	static struct pci_driver adf_driver = {
82	.id_table = adf_pci_tbl,
83	.name = adf_driver_name,
84	.probe = adf_probe,
85	.remove = adf_remove
86	};
87
88	static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
89	{
90	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
91	int i;
92
22e4dda0	93	adf_dev_shutdown(accel_dev);
7afa232e TS	94
	95	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
	96	struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
	97
	98	if (bar->virt_addr)
	99	pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	100	}
	101
	102	if (accel_dev->hw_device) {
5995752e	103	switch (accel_pci_dev->pci_dev->device) {
7afa232e TS	104	case ADF_DH895XCC_PCI_DEVICE_ID:
	105	adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
	106	break;
	107	default:
	108	break;
	109	}
	110	kfree(accel_dev->hw_device);
	111	}
	112	adf_cfg_dev_remove(accel_dev);
	113	debugfs_remove(accel_dev->debugfs_dir);
	114	adf_devmgr_rm_dev(accel_dev);
	115	pci_release_regions(accel_pci_dev->pci_dev);
	116	pci_disable_device(accel_pci_dev->pci_dev);
	117	kfree(accel_dev);
	118	}
	119
22e4dda0	120	static int adf_dev_configure(struct adf_accel_dev *accel_dev)
7afa232e TS	121	{
	122	int cpus = num_online_cpus();
	123	int banks = GET_MAX_BANKS(accel_dev);
	124	int instances = min(cpus, banks);
	125	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
	126	int i;
	127	unsigned long val;
	128
	129	if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
	130	goto err;
	131	if (adf_cfg_section_add(accel_dev, "Accelerator0"))
	132	goto err;
	133	for (i = 0; i < instances; i++) {
	134	val = i;
	135	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
	136	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	137	key, (void *)&val, ADF_DEC))
	138	goto err;
	139
	140	snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
	141	i);
	142	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	143	key, (void *)&val, ADF_DEC))
	144	goto err;
	145
	146	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
	147	val = 128;
	148	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	149	key, (void *)&val, ADF_DEC))
	150	goto err;
	151
	152	val = 512;
	153	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
	154	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	155	key, (void *)&val, ADF_DEC))
	156	goto err;
	157
	158	val = 0;
	159	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
	160	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	161	key, (void *)&val, ADF_DEC))
	162	goto err;
	163
	164	val = 2;
	165	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
	166	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	167	key, (void *)&val, ADF_DEC))
	168	goto err;
	169
7afa232e TS	170	val = 8;
	171	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
	172	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	173	key, (void *)&val, ADF_DEC))
	174	goto err;
	175
	176	val = 10;
	177	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
	178	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	179	key, (void *)&val, ADF_DEC))
	180	goto err;
	181
7afa232e TS	182	val = ADF_COALESCING_DEF_TIME;
	183	snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
	184	if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
	185	key, (void *)&val, ADF_DEC))
	186	goto err;
	187	}
	188
	189	val = i;
	190	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	191	ADF_NUM_CY, (void *)&val, ADF_DEC))
	192	goto err;
	193
	194	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
22e4dda0	195	return 0;
7afa232e TS	196	err:
	197	dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
	198	return -EINVAL;
	199	}
	200
	201	static int adf_probe(struct pci_dev pdev, const struct pci_device_id ent)
	202	{
	203	struct adf_accel_dev *accel_dev;
	204	struct adf_accel_pci *accel_pci_dev;
	205	struct adf_hw_device_data *hw_data;
7afa232e TS	206	char name[ADF_DEVICE_NAME_LENGTH];
7afa232e TS	207	unsigned int i, bar_nr;
7afa232e TS	208	int ret;
	209
	210	switch (ent->device) {
	211	case ADF_DH895XCC_PCI_DEVICE_ID:
	212	break;
	213	default:
	214	dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
	215	return -ENODEV;
	216	}
	217
09adc878 TS	218	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
	219	/* If the accelerator is connected to a node with no memory
	220	* there is no point in using the accelerator since the remote
	221	* memory transaction will be very slow. */
	222	dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
	223	return -EINVAL;
	224	}
	225
	226	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
64a31be3	227	dev_to_node(&pdev->dev));
7afa232e TS	228	if (!accel_dev)
	229	return -ENOMEM;
	230
7afa232e TS	231	INIT_LIST_HEAD(&accel_dev->crypto_list);
	232
	233	/* Add accel device to accel table.
	234	* This should be called before adf_cleanup_accel is called */
	235	if (adf_devmgr_add_dev(accel_dev)) {
	236	dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
	237	kfree(accel_dev);
	238	return -EFAULT;
	239	}
	240
	241	accel_dev->owner = THIS_MODULE;
	242	/* Allocate and configure device configuration structure */
09adc878 TS	243	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
09adc878 TS	244	dev_to_node(&pdev->dev));
7afa232e TS	245	if (!hw_data) {
	246	ret = -ENOMEM;
	247	goto out_err;
	248	}
	249
	250	accel_dev->hw_device = hw_data;
	251	switch (ent->device) {
	252	case ADF_DH895XCC_PCI_DEVICE_ID:
	253	adf_init_hw_data_dh895xcc(accel_dev->hw_device);
	254	break;
	255	default:
	256	return -ENODEV;
	257	}
	258	accel_pci_dev = &accel_dev->accel_pci_dev;
	259	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
	260	pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
	261	&hw_data->fuses);
	262
	263	/* Get Accelerators and Accelerators Engines masks */
	264	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	265	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	266	accel_pci_dev->sku = hw_data->get_sku(hw_data);
	267	accel_pci_dev->pci_dev = pdev;
	268	/* If the device has no acceleration engines then ignore it. */
	269	if (!hw_data->accel_mask \|\| !hw_data->ae_mask \|\|
	270	((~hw_data->ae_mask) & 0x01)) {
	271	dev_err(&pdev->dev, "No acceleration units found");
	272	ret = -EFAULT;
	273	goto out_err;
	274	}
	275
	276	/* Create dev top level debugfs entry */
	277	snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
	278	hw_data->dev_class->name, hw_data->instance_id);
	279	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
	280	if (!accel_dev->debugfs_dir) {
	281	dev_err(&pdev->dev, "Could not create debugfs dir\n");
	282	ret = -EINVAL;
	283	goto out_err;
	284	}
	285
	286	/* Create device configuration table */
	287	ret = adf_cfg_dev_add(accel_dev);
	288	if (ret)
	289	goto out_err;
	290
e4fa1460 TS	291	pcie_set_readrq(pdev, 1024);
e4fa1460 TS	292
7afa232e TS	293	/* enable PCI device */
	294	if (pci_enable_device(pdev)) {
	295	ret = -EFAULT;
	296	goto out_err;
	297	}
	298
	299	/* set dma identifier */
	300	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
	301	if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
	302	dev_err(&pdev->dev, "No usable DMA configuration\n");
	303	ret = -EFAULT;
	304	goto out_err;
	305	} else {
	306	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
	307	}
	308
	309	} else {
	310	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	311	}
	312
	313	if (pci_request_regions(pdev, adf_driver_name)) {
	314	ret = -EFAULT;
	315	goto out_err;
	316	}
	317
	318	/* Read accelerator capabilities mask */
	319	pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
	320	&hw_data->accel_capabilities_mask);
	321
	322	/* Find and map all the device's BARS */
	323	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
	324	struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
	325
	326	bar_nr = i * 2;
	327	bar->base_addr = pci_resource_start(pdev, bar_nr);
	328	if (!bar->base_addr)
	329	break;
	330	bar->size = pci_resource_len(pdev, bar_nr);
	331	bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
	332	if (!bar->virt_addr) {
	333	dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
	334	ret = -EFAULT;
	335	goto out_err;
	336	}
7afa232e TS	337	}
	338	pci_set_master(pdev);
	339
	340	if (adf_enable_aer(accel_dev, &adf_driver)) {
	341	dev_err(&pdev->dev, "Failed to enable aer\n");
	342	ret = -EFAULT;
	343	goto out_err;
	344	}
	345
7afa232e TS	346	if (pci_save_state(pdev)) {
	347	dev_err(&pdev->dev, "Failed to save pci state\n");
	348	ret = -ENOMEM;
	349	goto out_err;
	350	}
	351
22e4dda0 AB	352	ret = adf_dev_configure(accel_dev);
	353	if (ret)
	354	goto out_err;
	355
	356	ret = adf_dev_init(accel_dev);
	357	if (ret)
	358	goto out_err;
7afa232e	359
22e4dda0	360	ret = adf_dev_start(accel_dev);
7afa232e TS	361	if (ret) {
	362	adf_dev_stop(accel_dev);
	363	goto out_err;
	364	}
	365
	366	return 0;
	367	out_err:
	368	adf_cleanup_accel(accel_dev);
	369	return ret;
	370	}
	371
83ce01d2	372	static void adf_remove(struct pci_dev *pdev)
7afa232e TS	373	{
	374	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
	375
	376	if (!accel_dev) {
	377	pr_err("QAT: Driver removal failed\n");
	378	return;
	379	}
	380	if (adf_dev_stop(accel_dev))
	381	dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
	382	adf_disable_aer(accel_dev);
	383	adf_cleanup_accel(accel_dev);
	384	}
	385
	386	static int __init adfdrv_init(void)
	387	{
	388	request_module("intel_qat");
7afa232e TS	389
	390	if (pci_register_driver(&adf_driver)) {
	391	pr_err("QAT: Driver initialization failed\n");
	392	return -EFAULT;
	393	}
	394	return 0;
	395	}
	396
	397	static void __exit adfdrv_release(void)
	398	{
	399	pci_unregister_driver(&adf_driver);
7afa232e TS	400	}
	401
	402	module_init(adfdrv_init);
	403	module_exit(adfdrv_release);
	404
	405	MODULE_LICENSE("Dual BSD/GPL");
	406	MODULE_AUTHOR("Intel");
dcd93e83	407	MODULE_FIRMWARE(ADF_DH895XCC_FW);
7afa232e	408	MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
551d7ed2	409	MODULE_VERSION(ADF_DRV_VERSION);