drivers/crypto/qat/qat_dh895xcc/adf_drv.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/kernel.h>
  48 #include <linux/module.h>
  49 #include <linux/pci.h>
  50 #include <linux/init.h>
  51 #include <linux/types.h>
  52 #include <linux/fs.h>
  53 #include <linux/slab.h>
  54 #include <linux/errno.h>
  55 #include <linux/device.h>
  56 #include <linux/dma-mapping.h>
  57 #include <linux/platform_device.h>
  58 #include <linux/workqueue.h>
  59 #include <linux/io.h>
  60 #include <adf_accel_devices.h>
  61 #include <adf_common_drv.h>
  62 #include <adf_cfg.h>
  63 #include <adf_transport_access_macros.h>
  64 #include "adf_dh895xcc_hw_data.h"
  65 #include "adf_drv.h"
  66
  67 static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
  68
  69 #define ADF_SYSTEM_DEVICE(device_id) \
  70         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  71
  72 static const struct pci_device_id adf_pci_tbl[] = {
  73         ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
  74         {0,}
  75 };
  76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
  77
  78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
  79 static void adf_remove(struct pci_dev *dev);
  80
  81 static struct pci_driver adf_driver = {
  82         .id_table = adf_pci_tbl,
  83         .name = adf_driver_name,
  84         .probe = adf_probe,
  85         .remove = adf_remove
  86 };
  87
  88 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
  89 {
  90         struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
  91         int i;
  92
  93         adf_exit_admin_comms(accel_dev);
  94         adf_exit_arb(accel_dev);
  95         adf_cleanup_etr_data(accel_dev);
  96
  97         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
  98                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
  99
 100                 if (bar->virt_addr)
 101                         pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
 102         }
 103
 104         if (accel_dev->hw_device) {
 105                 switch (accel_dev->hw_device->pci_dev_id) {
 106                 case ADF_DH895XCC_PCI_DEVICE_ID:
 107                         adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
 108                         break;
 109                 default:
 110                         break;
 111                 }
 112                 kfree(accel_dev->hw_device);
 113         }
 114         adf_cfg_dev_remove(accel_dev);
 115         debugfs_remove(accel_dev->debugfs_dir);
 116         adf_devmgr_rm_dev(accel_dev);
 117         pci_release_regions(accel_pci_dev->pci_dev);
 118         pci_disable_device(accel_pci_dev->pci_dev);
 119         kfree(accel_dev);
 120 }
 121
 122 static uint8_t adf_get_dev_node_id(struct pci_dev *pdev)
 123 {
 124         unsigned int bus_per_cpu = 0;
 125         struct cpuinfo_x86 *c = &cpu_data(num_online_cpus() - 1);
 126
 127         if (!c->phys_proc_id)
 128                 return 0;
 129
 130         bus_per_cpu = 256 / (c->phys_proc_id + 1);
 131
 132         if (bus_per_cpu != 0)
 133                 return pdev->bus->number / bus_per_cpu;
 134         return 0;
 135 }
 136
 137 static int qat_dev_start(struct adf_accel_dev *accel_dev)
 138 {
 139         int cpus = num_online_cpus();
 140         int banks = GET_MAX_BANKS(accel_dev);
 141         int instances = min(cpus, banks);
 142         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 143         int i;
 144         unsigned long val;
 145
 146         if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
 147                 goto err;
 148         if (adf_cfg_section_add(accel_dev, "Accelerator0"))
 149                 goto err;
 150         for (i = 0; i < instances; i++) {
 151                 val = i;
 152                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
 153                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 154                                                 key, (void *)&val, ADF_DEC))
 155                         goto err;
 156
 157                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
 158                          i);
 159                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 160                                                 key, (void *)&val, ADF_DEC))
 161                         goto err;
 162
 163                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
 164                 val = 128;
 165                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 166                                                 key, (void *)&val, ADF_DEC))
 167                         goto err;
 168
 169                 val = 512;
 170                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
 171                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 172                                                 key, (void *)&val, ADF_DEC))
 173                         goto err;
 174
 175                 val = 0;
 176                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
 177                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 178                                                 key, (void *)&val, ADF_DEC))
 179                         goto err;
 180
 181                 val = 2;
 182                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
 183                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 184                                                 key, (void *)&val, ADF_DEC))
 185                         goto err;
 186
 187                 val = 4;
 188                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i);
 189                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 190                                                 key, (void *)&val, ADF_DEC))
 191                         goto err;
 192
 193                 val = 8;
 194                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
 195                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 196                                                 key, (void *)&val, ADF_DEC))
 197                         goto err;
 198
 199                 val = 10;
 200                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
 201                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 202                                                 key, (void *)&val, ADF_DEC))
 203                         goto err;
 204
 205                 val = 12;
 206                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i);
 207                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 208                                                 key, (void *)&val, ADF_DEC))
 209                         goto err;
 210
 211                 val = ADF_COALESCING_DEF_TIME;
 212                 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
 213                 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
 214                                                 key, (void *)&val, ADF_DEC))
 215                         goto err;
 216         }
 217
 218         val = i;
 219         if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 220                                         ADF_NUM_CY, (void *)&val, ADF_DEC))
 221                 goto err;
 222
 223         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 224         return adf_dev_start(accel_dev);
 225 err:
 226         dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
 227         return -EINVAL;
 228 }
 229
 230 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 231 {
 232         struct adf_accel_dev *accel_dev;
 233         struct adf_accel_pci *accel_pci_dev;
 234         struct adf_hw_device_data *hw_data;
 235         void __iomem *pmisc_bar_addr = NULL;
 236         char name[ADF_DEVICE_NAME_LENGTH];
 237         unsigned int i, bar_nr;
 238         uint8_t node;
 239         int ret;
 240
 241         switch (ent->device) {
 242         case ADF_DH895XCC_PCI_DEVICE_ID:
 243                 break;
 244         default:
 245                 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
 246                 return -ENODEV;
 247         }
 248
 249         node = adf_get_dev_node_id(pdev);
 250         accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, node);
 251         if (!accel_dev)
 252                 return -ENOMEM;
 253
 254         accel_dev->numa_node = node;
 255         INIT_LIST_HEAD(&accel_dev->crypto_list);
 256
 257         /* Add accel device to accel table.
 258          * This should be called before adf_cleanup_accel is called */
 259         if (adf_devmgr_add_dev(accel_dev)) {
 260                 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
 261                 kfree(accel_dev);
 262                 return -EFAULT;
 263         }
 264
 265         accel_dev->owner = THIS_MODULE;
 266         /* Allocate and configure device configuration structure */
 267         hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, node);
 268         if (!hw_data) {
 269                 ret = -ENOMEM;
 270                 goto out_err;
 271         }
 272
 273         accel_dev->hw_device = hw_data;
 274         switch (ent->device) {
 275         case ADF_DH895XCC_PCI_DEVICE_ID:
 276                 adf_init_hw_data_dh895xcc(accel_dev->hw_device);
 277                 break;
 278         default:
 279                 return -ENODEV;
 280         }
 281         accel_pci_dev = &accel_dev->accel_pci_dev;
 282         pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
 283         pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
 284                               &hw_data->fuses);
 285
 286         /* Get Accelerators and Accelerators Engines masks */
 287         hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
 288         hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
 289         accel_pci_dev->sku = hw_data->get_sku(hw_data);
 290         accel_pci_dev->pci_dev = pdev;
 291         /* If the device has no acceleration engines then ignore it. */
 292         if (!hw_data->accel_mask || !hw_data->ae_mask ||
 293             ((~hw_data->ae_mask) & 0x01)) {
 294                 dev_err(&pdev->dev, "No acceleration units found");
 295                 ret = -EFAULT;
 296                 goto out_err;
 297         }
 298
 299         /* Create dev top level debugfs entry */
 300         snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
 301                  hw_data->dev_class->name, hw_data->instance_id);
 302         accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
 303         if (!accel_dev->debugfs_dir) {
 304                 dev_err(&pdev->dev, "Could not create debugfs dir\n");
 305                 ret = -EINVAL;
 306                 goto out_err;
 307         }
 308
 309         /* Create device configuration table */
 310         ret = adf_cfg_dev_add(accel_dev);
 311         if (ret)
 312                 goto out_err;
 313
 314         /* enable PCI device */
 315         if (pci_enable_device(pdev)) {
 316                 ret = -EFAULT;
 317                 goto out_err;
 318         }
 319
 320         /* set dma identifier */
 321         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
 322                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
 323                         dev_err(&pdev->dev, "No usable DMA configuration\n");
 324                         ret = -EFAULT;
 325                         goto out_err;
 326                 } else {
 327                         pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 328                 }
 329
 330         } else {
 331                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 332         }
 333
 334         if (pci_request_regions(pdev, adf_driver_name)) {
 335                 ret = -EFAULT;
 336                 goto out_err;
 337         }
 338
 339         /* Read accelerator capabilities mask */
 340         pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
 341                               &hw_data->accel_capabilities_mask);
 342
 343         /* Find and map all the device's BARS */
 344         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
 345                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
 346
 347                 bar_nr = i * 2;
 348                 bar->base_addr = pci_resource_start(pdev, bar_nr);
 349                 if (!bar->base_addr)
 350                         break;
 351                 bar->size = pci_resource_len(pdev, bar_nr);
 352                 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
 353                 if (!bar->virt_addr) {
 354                         dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
 355                         ret = -EFAULT;
 356                         goto out_err;
 357                 }
 358                 if (i == ADF_DH895XCC_PMISC_BAR)
 359                         pmisc_bar_addr = bar->virt_addr;
 360         }
 361         pci_set_master(pdev);
 362
 363         if (adf_enable_aer(accel_dev, &adf_driver)) {
 364                 dev_err(&pdev->dev, "Failed to enable aer\n");
 365                 ret = -EFAULT;
 366                 goto out_err;
 367         }
 368
 369         if (adf_init_etr_data(accel_dev)) {
 370                 dev_err(&pdev->dev, "Failed initialize etr\n");
 371                 ret = -EFAULT;
 372                 goto out_err;
 373         }
 374
 375         if (adf_init_admin_comms(accel_dev)) {
 376                 dev_err(&pdev->dev, "Failed initialize admin comms\n");
 377                 ret = -EFAULT;
 378                 goto out_err;
 379         }
 380
 381         if (adf_init_arb(accel_dev)) {
 382                 dev_err(&pdev->dev, "Failed initialize hw arbiter\n");
 383                 ret = -EFAULT;
 384                 goto out_err;
 385         }
 386         if (pci_save_state(pdev)) {
 387                 dev_err(&pdev->dev, "Failed to save pci state\n");
 388                 ret = -ENOMEM;
 389                 goto out_err;
 390         }
 391
 392         /* Enable bundle and misc interrupts */
 393         ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
 394                    ADF_DH895XCC_SMIA0_MASK);
 395         ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
 396                    ADF_DH895XCC_SMIA1_MASK);
 397
 398         ret = qat_dev_start(accel_dev);
 399         if (ret) {
 400                 adf_dev_stop(accel_dev);
 401                 goto out_err;
 402         }
 403
 404         return 0;
 405 out_err:
 406         adf_cleanup_accel(accel_dev);
 407         return ret;
 408 }
 409
 410 static void __exit adf_remove(struct pci_dev *pdev)
 411 {
 412         struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
 413
 414         if (!accel_dev) {
 415                 pr_err("QAT: Driver removal failed\n");
 416                 return;
 417         }
 418         if (adf_dev_stop(accel_dev))
 419                 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
 420         adf_disable_aer(accel_dev);
 421         adf_cleanup_accel(accel_dev);
 422 }
 423
 424 static int __init adfdrv_init(void)
 425 {
 426         request_module("intel_qat");
 427         if (qat_admin_register())
 428                 return -EFAULT;
 429
 430         if (pci_register_driver(&adf_driver)) {
 431                 pr_err("QAT: Driver initialization failed\n");
 432                 return -EFAULT;
 433         }
 434         return 0;
 435 }
 436
 437 static void __exit adfdrv_release(void)
 438 {
 439         pci_unregister_driver(&adf_driver);
 440         qat_admin_unregister();
 441 }
 442
 443 module_init(adfdrv_init);
 444 module_exit(adfdrv_release);
 445
 446 MODULE_LICENSE("Dual BSD/GPL");
 447 MODULE_AUTHOR("Intel");
 448 MODULE_FIRMWARE("qat_895xcc.bin");
 449 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");