[deliverable/linux.git] / drivers / crypto / qat / qat_dh895xcc / adf_drv.c

/*
  This file is provided under a dual BSD/GPLv2 license.  When using or
  redistributing this file, you may do so under either license.

  GPL LICENSE SUMMARY
  Copyright(c) 2014 Intel Corporation.
  This program is free software; you can redistribute it and/or modify
  it under the terms of version 2 of the GNU General Public License as
  published by the Free Software Foundation.

  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

  Contact Information:
  qat-linux@intel.com

  BSD LICENSE
  Copyright(c) 2014 Intel Corporation.
  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions
  are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the
      distribution.
    * Neither the name of Intel Corporation nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
#include <linux/io.h>
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
#include <adf_cfg.h>
#include "adf_dh895xcc_hw_data.h"

#define ADF_SYSTEM_DEVICE(device_id) \
	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}

static const struct pci_device_id adf_pci_tbl[] = {
	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
	{0,}
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);

static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
static void adf_remove(struct pci_dev *dev);

static struct pci_driver adf_driver = {
	.id_table = adf_pci_tbl,
	.name = ADF_DH895XCC_DEVICE_NAME,
	.probe = adf_probe,
	.remove = adf_remove,
	.sriov_configure = adf_sriov_configure,
};

static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
{
	pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
	pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
}

static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
{
	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
	int i;

	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i];

		if (bar->virt_addr)
			pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	}

	if (accel_dev->hw_device) {
		switch (accel_pci_dev->pci_dev->device) {
		case ADF_DH895XCC_PCI_DEVICE_ID:
			adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
			break;
		default:
			break;
		}
		kfree(accel_dev->hw_device);
		accel_dev->hw_device = NULL;
	}
	adf_cfg_dev_remove(accel_dev);
	debugfs_remove(accel_dev->debugfs_dir);
	adf_devmgr_rm_dev(accel_dev, NULL);
}

static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
	struct adf_accel_dev *accel_dev;
	struct adf_accel_pci *accel_pci_dev;
	struct adf_hw_device_data *hw_data;
	char name[ADF_DEVICE_NAME_LENGTH];
	unsigned int i, bar_nr;
	int ret, bar_mask;

	switch (ent->device) {
	case ADF_DH895XCC_PCI_DEVICE_ID:
		break;
	default:
		dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
		return -ENODEV;
	}

	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
		/* If the accelerator is connected to a node with no memory
		 * there is no point in using the accelerator since the remote
		 * memory transaction will be very slow. */
		dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
		return -EINVAL;
	}

	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
				 dev_to_node(&pdev->dev));
	if (!accel_dev)
		return -ENOMEM;

	INIT_LIST_HEAD(&accel_dev->crypto_list);
	accel_pci_dev = &accel_dev->accel_pci_dev;
	accel_pci_dev->pci_dev = pdev;

	/* Add accel device to accel table.
	 * This should be called before adf_cleanup_accel is called */
	if (adf_devmgr_add_dev(accel_dev, NULL)) {
		dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
		kfree(accel_dev);
		return -EFAULT;
	}

	accel_dev->owner = THIS_MODULE;
	/* Allocate and configure device configuration structure */
	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
			       dev_to_node(&pdev->dev));
	if (!hw_data) {
		ret = -ENOMEM;
		goto out_err;
	}

	accel_dev->hw_device = hw_data;
	adf_init_hw_data_dh895xcc(accel_dev->hw_device);
	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
	pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET,
			      &hw_data->fuses);

	/* Get Accelerators and Accelerators Engines masks */
	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	accel_pci_dev->sku = hw_data->get_sku(hw_data);
	/* If the device has no acceleration engines then ignore it. */
	if (!hw_data->accel_mask || !hw_data->ae_mask ||
	    ((~hw_data->ae_mask) & 0x01)) {
		dev_err(&pdev->dev, "No acceleration units found");
		ret = -EFAULT;
		goto out_err;
	}

	/* Create dev top level debugfs entry */
	snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d",
		 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
		 pdev->bus->number, PCI_SLOT(pdev->devfn),
		 PCI_FUNC(pdev->devfn));

	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
	if (!accel_dev->debugfs_dir) {
		dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
		ret = -EINVAL;
		goto out_err;
	}

	/* Create device configuration table */
	ret = adf_cfg_dev_add(accel_dev);
	if (ret)
		goto out_err;

	pcie_set_readrq(pdev, 1024);

	/* enable PCI device */
	if (pci_enable_device(pdev)) {
		ret = -EFAULT;
		goto out_err;
	}

	/* set dma identifier */
	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
		if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
			dev_err(&pdev->dev, "No usable DMA configuration\n");
			ret = -EFAULT;
			goto out_err_disable;
		} else {
			pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
		}

	} else {
		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	}

	if (pci_request_regions(pdev, ADF_DH895XCC_DEVICE_NAME)) {
		ret = -EFAULT;
		goto out_err_disable;
	}

	/* Read accelerator capabilities mask */
	pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET,
			      &hw_data->accel_capabilities_mask);

	/* Find and map all the device's BARS */
	i = 0;
	bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
	for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask,
			 ADF_PCI_MAX_BARS * 2) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];

		bar->base_addr = pci_resource_start(pdev, bar_nr);
		if (!bar->base_addr)
			break;
		bar->size = pci_resource_len(pdev, bar_nr);
		bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
		if (!bar->virt_addr) {
			dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
			ret = -EFAULT;
			goto out_err_free_reg;
		}
	}
	pci_set_master(pdev);

	if (adf_enable_aer(accel_dev, &adf_driver)) {
		dev_err(&pdev->dev, "Failed to enable aer\n");
		ret = -EFAULT;
		goto out_err_free_reg;
	}

	if (pci_save_state(pdev)) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		ret = -ENOMEM;
		goto out_err_free_reg;
	}

	ret = qat_crypto_dev_config(accel_dev);
	if (ret)
		goto out_err_free_reg;

	ret = adf_dev_init(accel_dev);
	if (ret)
		goto out_err_dev_shutdown;

	ret = adf_dev_start(accel_dev);
	if (ret)
		goto out_err_dev_stop;

	return ret;

out_err_dev_stop:
	adf_dev_stop(accel_dev);
out_err_dev_shutdown:
	adf_dev_shutdown(accel_dev);
out_err_free_reg:
	pci_release_regions(accel_pci_dev->pci_dev);
out_err_disable:
	pci_disable_device(accel_pci_dev->pci_dev);
out_err:
	adf_cleanup_accel(accel_dev);
	kfree(accel_dev);
	return ret;
}

static void adf_remove(struct pci_dev *pdev)
{
	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);

	if (!accel_dev) {
		pr_err("QAT: Driver removal failed\n");
		return;
	}
	if (adf_dev_stop(accel_dev))
		dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");

	adf_dev_shutdown(accel_dev);
	adf_disable_aer(accel_dev);
	adf_cleanup_accel(accel_dev);
	adf_cleanup_pci_dev(accel_dev);
	kfree(accel_dev);
}

static int __init adfdrv_init(void)
{
	request_module("intel_qat");

	if (pci_register_driver(&adf_driver)) {
		pr_err("QAT: Driver initialization failed\n");
		return -EFAULT;
	}
	return 0;
}

static void __exit adfdrv_release(void)
{
	pci_unregister_driver(&adf_driver);
}

module_init(adfdrv_init);
module_exit(adfdrv_release);

MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel");
MODULE_FIRMWARE(ADF_DH895XCC_FW);
MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
MODULE_VERSION(ADF_DRV_VERSION);
Commit	Line	Data
7afa232e TS	1	/*
	2	This file is provided under a dual BSD/GPLv2 license. When using or
	3	redistributing this file, you may do so under either license.
	4
	5	GPL LICENSE SUMMARY
	6	Copyright(c) 2014 Intel Corporation.
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of version 2 of the GNU General Public License as
	9	published by the Free Software Foundation.
	10
	11	This program is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	General Public License for more details.
	15
	16	Contact Information:
	17	qat-linux@intel.com
	18
	19	BSD LICENSE
	20	Copyright(c) 2014 Intel Corporation.
	21	Redistribution and use in source and binary forms, with or without
	22	modification, are permitted provided that the following conditions
	23	are met:
	24
	25	* Redistributions of source code must retain the above copyright
	26	notice, this list of conditions and the following disclaimer.
	27	* Redistributions in binary form must reproduce the above copyright
	28	notice, this list of conditions and the following disclaimer in
	29	the documentation and/or other materials provided with the
	30	distribution.
	31	* Neither the name of Intel Corporation nor the names of its
	32	contributors may be used to endorse or promote products derived
	33	from this software without specific prior written permission.
	34
	35	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	36	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	37	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	38	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	39	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	40	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	41	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	42	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	43	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	44	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	45	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	46	*/
	47	#include <linux/kernel.h>
	48	#include <linux/module.h>
	49	#include <linux/pci.h>
	50	#include <linux/init.h>
	51	#include <linux/types.h>
	52	#include <linux/fs.h>
	53	#include <linux/slab.h>
	54	#include <linux/errno.h>
	55	#include <linux/device.h>
	56	#include <linux/dma-mapping.h>
	57	#include <linux/platform_device.h>
	58	#include <linux/workqueue.h>
	59	#include <linux/io.h>
	60	#include <adf_accel_devices.h>
	61	#include <adf_common_drv.h>
	62	#include <adf_cfg.h>
7afa232e	63	#include "adf_dh895xcc_hw_data.h"
7afa232e TS	64
	65	#define ADF_SYSTEM_DEVICE(device_id) \
	66	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
	67
	68	static const struct pci_device_id adf_pci_tbl[] = {
	69	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
	70	{0,}
	71	};
	72	MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
	73
	74	static int adf_probe(struct pci_dev dev, const struct pci_device_id ent);
	75	static void adf_remove(struct pci_dev *dev);
	76
	77	static struct pci_driver adf_driver = {
	78	.id_table = adf_pci_tbl,
1a72d3a6	79	.name = ADF_DH895XCC_DEVICE_NAME,
7afa232e	80	.probe = adf_probe,
ed8ccaef TS	81	.remove = adf_remove,
ed8ccaef TS	82	.sriov_configure = adf_sriov_configure,
7afa232e TS	83	};
7afa232e TS	84
ed8ccaef TS	85	static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
	86	{
	87	pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
	88	pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
	89	}
	90
7afa232e TS	91	static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
	92	{
	93	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
	94	int i;
	95
7afa232e TS	96	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
	97	struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
	98
	99	if (bar->virt_addr)
	100	pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	101	}
	102
	103	if (accel_dev->hw_device) {
5995752e	104	switch (accel_pci_dev->pci_dev->device) {
7afa232e TS	105	case ADF_DH895XCC_PCI_DEVICE_ID:
	106	adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
	107	break;
	108	default:
	109	break;
	110	}
	111	kfree(accel_dev->hw_device);
ed8ccaef	112	accel_dev->hw_device = NULL;
7afa232e TS	113	}
	114	adf_cfg_dev_remove(accel_dev);
	115	debugfs_remove(accel_dev->debugfs_dir);
ed8ccaef	116	adf_devmgr_rm_dev(accel_dev, NULL);
7afa232e TS	117	}
7afa232e TS	118
7afa232e TS	119	static int adf_probe(struct pci_dev pdev, const struct pci_device_id ent)
	120	{
	121	struct adf_accel_dev *accel_dev;
	122	struct adf_accel_pci *accel_pci_dev;
	123	struct adf_hw_device_data *hw_data;
7afa232e TS	124	char name[ADF_DEVICE_NAME_LENGTH];
7afa232e TS	125	unsigned int i, bar_nr;
ed8ccaef	126	int ret, bar_mask;
7afa232e TS	127
	128	switch (ent->device) {
	129	case ADF_DH895XCC_PCI_DEVICE_ID:
	130	break;
	131	default:
	132	dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
	133	return -ENODEV;
	134	}
	135
09adc878 TS	136	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
	137	/* If the accelerator is connected to a node with no memory
	138	* there is no point in using the accelerator since the remote
	139	* memory transaction will be very slow. */
	140	dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
	141	return -EINVAL;
	142	}
	143
	144	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
64a31be3	145	dev_to_node(&pdev->dev));
7afa232e TS	146	if (!accel_dev)
	147	return -ENOMEM;
	148
7afa232e	149	INIT_LIST_HEAD(&accel_dev->crypto_list);
ed8ccaef TS	150	accel_pci_dev = &accel_dev->accel_pci_dev;
ed8ccaef TS	151	accel_pci_dev->pci_dev = pdev;
7afa232e TS	152
	153	/* Add accel device to accel table.
	154	* This should be called before adf_cleanup_accel is called */
ed8ccaef	155	if (adf_devmgr_add_dev(accel_dev, NULL)) {
7afa232e TS	156	dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
	157	kfree(accel_dev);
	158	return -EFAULT;
	159	}
	160
	161	accel_dev->owner = THIS_MODULE;
	162	/* Allocate and configure device configuration structure */
09adc878 TS	163	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
09adc878 TS	164	dev_to_node(&pdev->dev));
7afa232e TS	165	if (!hw_data) {
	166	ret = -ENOMEM;
	167	goto out_err;
	168	}
	169
	170	accel_dev->hw_device = hw_data;
c52b6733	171	adf_init_hw_data_dh895xcc(accel_dev->hw_device);
7afa232e	172	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
1a72d3a6	173	pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET,
7afa232e TS	174	&hw_data->fuses);
	175
	176	/* Get Accelerators and Accelerators Engines masks */
	177	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	178	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	179	accel_pci_dev->sku = hw_data->get_sku(hw_data);
7afa232e TS	180	/* If the device has no acceleration engines then ignore it. */
	181	if (!hw_data->accel_mask \|\| !hw_data->ae_mask \|\|
	182	((~hw_data->ae_mask) & 0x01)) {
	183	dev_err(&pdev->dev, "No acceleration units found");
	184	ret = -EFAULT;
	185	goto out_err;
	186	}
	187
	188	/* Create dev top level debugfs entry */
ed8ccaef TS	189	snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d",
	190	ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
	191	pdev->bus->number, PCI_SLOT(pdev->devfn),
	192	PCI_FUNC(pdev->devfn));
	193
7afa232e TS	194	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
7afa232e TS	195	if (!accel_dev->debugfs_dir) {
ed8ccaef	196	dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
7afa232e TS	197	ret = -EINVAL;
	198	goto out_err;
	199	}
	200
	201	/* Create device configuration table */
	202	ret = adf_cfg_dev_add(accel_dev);
	203	if (ret)
	204	goto out_err;
	205
e4fa1460 TS	206	pcie_set_readrq(pdev, 1024);
e4fa1460 TS	207
7afa232e TS	208	/* enable PCI device */
	209	if (pci_enable_device(pdev)) {
	210	ret = -EFAULT;
	211	goto out_err;
	212	}
	213
	214	/* set dma identifier */
	215	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
	216	if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
	217	dev_err(&pdev->dev, "No usable DMA configuration\n");
	218	ret = -EFAULT;
ed8ccaef	219	goto out_err_disable;
7afa232e TS	220	} else {
	221	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
	222	}
	223
	224	} else {
	225	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	226	}
	227
1a72d3a6	228	if (pci_request_regions(pdev, ADF_DH895XCC_DEVICE_NAME)) {
7afa232e	229	ret = -EFAULT;
ed8ccaef	230	goto out_err_disable;
7afa232e TS	231	}
	232
	233	/* Read accelerator capabilities mask */
1a72d3a6	234	pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET,
7afa232e TS	235	&hw_data->accel_capabilities_mask);
	236
	237	/* Find and map all the device's BARS */
ed8ccaef TS	238	i = 0;
	239	bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
	240	for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask,
	241	ADF_PCI_MAX_BARS * 2) {
	242	struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];
7afa232e	243
7afa232e TS	244	bar->base_addr = pci_resource_start(pdev, bar_nr);
	245	if (!bar->base_addr)
	246	break;
	247	bar->size = pci_resource_len(pdev, bar_nr);
	248	bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
	249	if (!bar->virt_addr) {
ed8ccaef	250	dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
7afa232e	251	ret = -EFAULT;
ed8ccaef	252	goto out_err_free_reg;
7afa232e	253	}
7afa232e TS	254	}
	255	pci_set_master(pdev);
	256
	257	if (adf_enable_aer(accel_dev, &adf_driver)) {
	258	dev_err(&pdev->dev, "Failed to enable aer\n");
	259	ret = -EFAULT;
ed8ccaef	260	goto out_err_free_reg;
7afa232e TS	261	}
7afa232e TS	262
7afa232e TS	263	if (pci_save_state(pdev)) {
	264	dev_err(&pdev->dev, "Failed to save pci state\n");
	265	ret = -ENOMEM;
ed8ccaef	266	goto out_err_free_reg;
7afa232e TS	267	}
7afa232e TS	268
1a72d3a6	269	ret = qat_crypto_dev_config(accel_dev);
22e4dda0	270	if (ret)
ed8ccaef	271	goto out_err_free_reg;
22e4dda0 AB	272
	273	ret = adf_dev_init(accel_dev);
	274	if (ret)
ed8ccaef	275	goto out_err_dev_shutdown;
7afa232e	276
22e4dda0	277	ret = adf_dev_start(accel_dev);
ed8ccaef TS	278	if (ret)
ed8ccaef TS	279	goto out_err_dev_stop;
7afa232e	280
ed8ccaef TS	281	return ret;
	282
	283	out_err_dev_stop:
	284	adf_dev_stop(accel_dev);
	285	out_err_dev_shutdown:
	286	adf_dev_shutdown(accel_dev);
	287	out_err_free_reg:
	288	pci_release_regions(accel_pci_dev->pci_dev);
	289	out_err_disable:
	290	pci_disable_device(accel_pci_dev->pci_dev);
7afa232e TS	291	out_err:
7afa232e TS	292	adf_cleanup_accel(accel_dev);
ed8ccaef	293	kfree(accel_dev);
7afa232e TS	294	return ret;
	295	}
	296
83ce01d2	297	static void adf_remove(struct pci_dev *pdev)
7afa232e TS	298	{
	299	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
	300
	301	if (!accel_dev) {
	302	pr_err("QAT: Driver removal failed\n");
	303	return;
	304	}
	305	if (adf_dev_stop(accel_dev))
	306	dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
ed8ccaef TS	307
ed8ccaef TS	308	adf_dev_shutdown(accel_dev);
7afa232e TS	309	adf_disable_aer(accel_dev);
7afa232e TS	310	adf_cleanup_accel(accel_dev);
ed8ccaef TS	311	adf_cleanup_pci_dev(accel_dev);
ed8ccaef TS	312	kfree(accel_dev);
7afa232e TS	313	}
	314
	315	static int __init adfdrv_init(void)
	316	{
	317	request_module("intel_qat");
7afa232e TS	318
	319	if (pci_register_driver(&adf_driver)) {
	320	pr_err("QAT: Driver initialization failed\n");
	321	return -EFAULT;
	322	}
	323	return 0;
	324	}
	325
	326	static void __exit adfdrv_release(void)
	327	{
	328	pci_unregister_driver(&adf_driver);
7afa232e TS	329	}
	330
	331	module_init(adfdrv_init);
	332	module_exit(adfdrv_release);
	333
	334	MODULE_LICENSE("Dual BSD/GPL");
	335	MODULE_AUTHOR("Intel");
dcd93e83	336	MODULE_FIRMWARE(ADF_DH895XCC_FW);
7afa232e	337	MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
551d7ed2	338	MODULE_VERSION(ADF_DRV_VERSION);