[deliverable/linux.git] / drivers / scsi / virtio_scsi.c

/*
 * Virtio SCSI HBA driver
 *
 * Copyright IBM Corp. 2010
 * Copyright Red Hat, Inc. 2011
 *
 * Authors:
 *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
 *  Paolo Bonzini   <pbonzini@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mempool.h>
#include <linux/virtio.h>
#include <linux/virtio_ids.h>
#include <linux/virtio_config.h>
#include <linux/virtio_scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_cmnd.h>

#define VIRTIO_SCSI_MEMPOOL_SZ 64

/* Command queue element */
struct virtio_scsi_cmd {
	struct scsi_cmnd *sc;
	struct completion *comp;
	union {
		struct virtio_scsi_cmd_req       cmd;
		struct virtio_scsi_ctrl_tmf_req  tmf;
		struct virtio_scsi_ctrl_an_req   an;
	} req;
	union {
		struct virtio_scsi_cmd_resp      cmd;
		struct virtio_scsi_ctrl_tmf_resp tmf;
		struct virtio_scsi_ctrl_an_resp  an;
		struct virtio_scsi_event         evt;
	} resp;
} ____cacheline_aligned_in_smp;

struct virtio_scsi_vq {
	/* Protects vq */
	spinlock_t vq_lock;

	struct virtqueue *vq;
};

/* Per-target queue state */
struct virtio_scsi_target_state {
	/* Protects sg.  Lock hierarchy is tgt_lock -> vq_lock.  */
	spinlock_t tgt_lock;

	/* For sglist construction when adding commands to the virtqueue.  */
	struct scatterlist sg[];
};

/* Driver instance state */
struct virtio_scsi {
	struct virtio_device *vdev;

	struct virtio_scsi_vq ctrl_vq;
	struct virtio_scsi_vq event_vq;
	struct virtio_scsi_vq req_vq;

	struct virtio_scsi_target_state *tgt[];
};

static struct kmem_cache *virtscsi_cmd_cache;
static mempool_t *virtscsi_cmd_pool;

static inline struct Scsi_Host *virtio_scsi_host(struct virtio_device *vdev)
{
	return vdev->priv;
}

static void virtscsi_compute_resid(struct scsi_cmnd *sc, u32 resid)
{
	if (!resid)
		return;

	if (!scsi_bidi_cmnd(sc)) {
		scsi_set_resid(sc, resid);
		return;
	}

	scsi_in(sc)->resid = min(resid, scsi_in(sc)->length);
	scsi_out(sc)->resid = resid - scsi_in(sc)->resid;
}

/**
 * virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
 *
 * Called with vq_lock held.
 */
static void virtscsi_complete_cmd(void *buf)
{
	struct virtio_scsi_cmd *cmd = buf;
	struct scsi_cmnd *sc = cmd->sc;
	struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;

	dev_dbg(&sc->device->sdev_gendev,
		"cmd %p response %u status %#02x sense_len %u\n",
		sc, resp->response, resp->status, resp->sense_len);

	sc->result = resp->status;
	virtscsi_compute_resid(sc, resp->resid);
	switch (resp->response) {
	case VIRTIO_SCSI_S_OK:
		set_host_byte(sc, DID_OK);
		break;
	case VIRTIO_SCSI_S_OVERRUN:
		set_host_byte(sc, DID_ERROR);
		break;
	case VIRTIO_SCSI_S_ABORTED:
		set_host_byte(sc, DID_ABORT);
		break;
	case VIRTIO_SCSI_S_BAD_TARGET:
		set_host_byte(sc, DID_BAD_TARGET);
		break;
	case VIRTIO_SCSI_S_RESET:
		set_host_byte(sc, DID_RESET);
		break;
	case VIRTIO_SCSI_S_BUSY:
		set_host_byte(sc, DID_BUS_BUSY);
		break;
	case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
		set_host_byte(sc, DID_TRANSPORT_DISRUPTED);
		break;
	case VIRTIO_SCSI_S_TARGET_FAILURE:
		set_host_byte(sc, DID_TARGET_FAILURE);
		break;
	case VIRTIO_SCSI_S_NEXUS_FAILURE:
		set_host_byte(sc, DID_NEXUS_FAILURE);
		break;
	default:
		scmd_printk(KERN_WARNING, sc, "Unknown response %d",
			    resp->response);
		/* fall through */
	case VIRTIO_SCSI_S_FAILURE:
		set_host_byte(sc, DID_ERROR);
		break;
	}

	WARN_ON(resp->sense_len > VIRTIO_SCSI_SENSE_SIZE);
	if (sc->sense_buffer) {
		memcpy(sc->sense_buffer, resp->sense,
		       min_t(u32, resp->sense_len, VIRTIO_SCSI_SENSE_SIZE));
		if (resp->sense_len)
			set_driver_byte(sc, DRIVER_SENSE);
	}

	mempool_free(cmd, virtscsi_cmd_pool);
	sc->scsi_done(sc);
}

static void virtscsi_vq_done(struct virtqueue *vq, void (*fn)(void *buf))
{
	void *buf;
	unsigned int len;

	do {
		virtqueue_disable_cb(vq);
		while ((buf = virtqueue_get_buf(vq, &len)) != NULL)
			fn(buf);
	} while (!virtqueue_enable_cb(vq));
}

static void virtscsi_req_done(struct virtqueue *vq)
{
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->req_vq.vq_lock, flags);
	virtscsi_vq_done(vq, virtscsi_complete_cmd);
	spin_unlock_irqrestore(&vscsi->req_vq.vq_lock, flags);
};

static void virtscsi_complete_free(void *buf)
{
	struct virtio_scsi_cmd *cmd = buf;

	if (cmd->comp)
		complete_all(cmd->comp);
	else
		mempool_free(cmd, virtscsi_cmd_pool);
}

static void virtscsi_ctrl_done(struct virtqueue *vq)
{
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->ctrl_vq.vq_lock, flags);
	virtscsi_vq_done(vq, virtscsi_complete_free);
	spin_unlock_irqrestore(&vscsi->ctrl_vq.vq_lock, flags);
};

static void virtscsi_event_done(struct virtqueue *vq)
{
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);
	virtscsi_vq_done(vq, virtscsi_complete_free);
	spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);
};

static void virtscsi_map_sgl(struct scatterlist *sg, unsigned int *p_idx,
			     struct scsi_data_buffer *sdb)
{
	struct sg_table *table = &sdb->table;
	struct scatterlist *sg_elem;
	unsigned int idx = *p_idx;
	int i;

	for_each_sg(table->sgl, sg_elem, table->nents, i)
		sg_set_buf(&sg[idx++], sg_virt(sg_elem), sg_elem->length);

	*p_idx = idx;
}

/**
 * virtscsi_map_cmd - map a scsi_cmd to a virtqueue scatterlist
 * @vscsi	: virtio_scsi state
 * @cmd		: command structure
 * @out_num	: number of read-only elements
 * @in_num	: number of write-only elements
 * @req_size	: size of the request buffer
 * @resp_size	: size of the response buffer
 *
 * Called with tgt_lock held.
 */
static void virtscsi_map_cmd(struct virtio_scsi_target_state *tgt,
			     struct virtio_scsi_cmd *cmd,
			     unsigned *out_num, unsigned *in_num,
			     size_t req_size, size_t resp_size)
{
	struct scsi_cmnd *sc = cmd->sc;
	struct scatterlist *sg = tgt->sg;
	unsigned int idx = 0;

	/* Request header.  */
	sg_set_buf(&sg[idx++], &cmd->req, req_size);

	/* Data-out buffer.  */
	if (sc && sc->sc_data_direction != DMA_FROM_DEVICE)
		virtscsi_map_sgl(sg, &idx, scsi_out(sc));

	*out_num = idx;

	/* Response header.  */
	sg_set_buf(&sg[idx++], &cmd->resp, resp_size);

	/* Data-in buffer */
	if (sc && sc->sc_data_direction != DMA_TO_DEVICE)
		virtscsi_map_sgl(sg, &idx, scsi_in(sc));

	*in_num = idx - *out_num;
}

static int virtscsi_kick_cmd(struct virtio_scsi_target_state *tgt,
			     struct virtio_scsi_vq *vq,
			     struct virtio_scsi_cmd *cmd,
			     size_t req_size, size_t resp_size, gfp_t gfp)
{
	unsigned int out_num, in_num;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&tgt->tgt_lock, flags);
	virtscsi_map_cmd(tgt, cmd, &out_num, &in_num, req_size, resp_size);

	spin_lock(&vq->vq_lock);
	ret = virtqueue_add_buf(vq->vq, tgt->sg, out_num, in_num, cmd, gfp);
	spin_unlock(&tgt->tgt_lock);
	if (ret >= 0)
		ret = virtqueue_kick_prepare(vq->vq);

	spin_unlock_irqrestore(&vq->vq_lock, flags);

	if (ret > 0)
		virtqueue_notify(vq->vq);
	return ret;
}

static int virtscsi_queuecommand(struct Scsi_Host *sh, struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sh);
	struct virtio_scsi_target_state *tgt = vscsi->tgt[sc->device->id];
	struct virtio_scsi_cmd *cmd;
	int ret;

	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
	BUG_ON(scsi_sg_count(sc) > shost->sg_tablesize);

	/* TODO: check feature bit and fail if unsupported?  */
	BUG_ON(sc->sc_data_direction == DMA_BIDIRECTIONAL);

	dev_dbg(&sc->device->sdev_gendev,
		"cmd %p CDB: %#02x\n", sc, sc->cmnd[0]);

	ret = SCSI_MLQUEUE_HOST_BUSY;
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_ATOMIC);
	if (!cmd)
		goto out;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.cmd = (struct virtio_scsi_cmd_req){
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
		.tag = (unsigned long)sc,
		.task_attr = VIRTIO_SCSI_S_SIMPLE,
		.prio = 0,
		.crn = 0,
	};

	BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
	memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);

	if (virtscsi_kick_cmd(tgt, &vscsi->req_vq, cmd,
			      sizeof cmd->req.cmd, sizeof cmd->resp.cmd,
			      GFP_ATOMIC) >= 0)
		ret = 0;

out:
	return ret;
}

static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)
{
	DECLARE_COMPLETION_ONSTACK(comp);
	struct virtio_scsi_target_state *tgt = vscsi->tgt[cmd->sc->device->id];
	int ret = FAILED;

	cmd->comp = &comp;
	if (virtscsi_kick_cmd(tgt, &vscsi->ctrl_vq, cmd,
			      sizeof cmd->req.tmf, sizeof cmd->resp.tmf,
			      GFP_NOIO) < 0)
		goto out;

	wait_for_completion(&comp);
	if (cmd->resp.tmf.response == VIRTIO_SCSI_S_OK ||
	    cmd->resp.tmf.response == VIRTIO_SCSI_S_FUNCTION_SUCCEEDED)
		ret = SUCCESS;

out:
	mempool_free(cmd, virtscsi_cmd_pool);
	return ret;
}

static int virtscsi_device_reset(struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	struct virtio_scsi_cmd *cmd;

	sdev_printk(KERN_INFO, sc->device, "device reset\n");
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	if (!cmd)
		return FAILED;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
		.type = VIRTIO_SCSI_T_TMF,
		.subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET,
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
	};
	return virtscsi_tmf(vscsi, cmd);
}

static int virtscsi_abort(struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	struct virtio_scsi_cmd *cmd;

	scmd_printk(KERN_INFO, sc, "abort\n");
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	if (!cmd)
		return FAILED;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
		.type = VIRTIO_SCSI_T_TMF,
		.subtype = VIRTIO_SCSI_T_TMF_ABORT_TASK,
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
		.tag = (unsigned long)sc,
	};
	return virtscsi_tmf(vscsi, cmd);
}

static struct scsi_host_template virtscsi_host_template = {
	.module = THIS_MODULE,
	.name = "Virtio SCSI HBA",
	.proc_name = "virtio_scsi",
	.queuecommand = virtscsi_queuecommand,
	.this_id = -1,
	.eh_abort_handler = virtscsi_abort,
	.eh_device_reset_handler = virtscsi_device_reset,

	.can_queue = 1024,
	.dma_boundary = UINT_MAX,
	.use_clustering = ENABLE_CLUSTERING,
};

#define virtscsi_config_get(vdev, fld) \
	({ \
		typeof(((struct virtio_scsi_config *)0)->fld) __val; \
		vdev->config->get(vdev, \
				  offsetof(struct virtio_scsi_config, fld), \
				  &__val, sizeof(__val)); \
		__val; \
	})

#define virtscsi_config_set(vdev, fld, val) \
	(void)({ \
		typeof(((struct virtio_scsi_config *)0)->fld) __val = (val); \
		vdev->config->set(vdev, \
				  offsetof(struct virtio_scsi_config, fld), \
				  &__val, sizeof(__val)); \
	})

static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
			     struct virtqueue *vq)
{
	spin_lock_init(&virtscsi_vq->vq_lock);
	virtscsi_vq->vq = vq;
}

static struct virtio_scsi_target_state *virtscsi_alloc_tgt(
	struct virtio_device *vdev, int sg_elems)
{
	struct virtio_scsi_target_state *tgt;
	gfp_t gfp_mask = GFP_KERNEL;

	/* We need extra sg elements at head and tail.  */
	tgt = kmalloc(sizeof(*tgt) + sizeof(tgt->sg[0]) * (sg_elems + 2),
		      gfp_mask);

	if (!tgt)
		return NULL;

	spin_lock_init(&tgt->tgt_lock);
	sg_init_table(tgt->sg, sg_elems + 2);
	return tgt;
}

static void virtscsi_remove_vqs(struct virtio_device *vdev)
{
	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	u32 i, num_targets;

	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

	num_targets = sh->max_id;
	for (i = 0; i < num_targets; i++) {
		kfree(vscsi->tgt[i]);
		vscsi->tgt[i] = NULL;
	}

	vdev->config->del_vqs(vdev);
}

static int virtscsi_init(struct virtio_device *vdev,
			 struct virtio_scsi *vscsi, int num_targets)
{
	int err;
	struct virtqueue *vqs[3];
	u32 i, sg_elems;

	vq_callback_t *callbacks[] = {
		virtscsi_ctrl_done,
		virtscsi_event_done,
		virtscsi_req_done
	};
	const char *names[] = {
		"control",
		"event",
		"request"
	};

	/* Discover virtqueues and write information to configuration.  */
	err = vdev->config->find_vqs(vdev, 3, vqs, callbacks, names);
	if (err)
		return err;

	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);
	virtscsi_init_vq(&vscsi->event_vq, vqs[1]);
	virtscsi_init_vq(&vscsi->req_vq, vqs[2]);

	virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
	virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);

	/* We need to know how many segments before we allocate.  */
	sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;

	for (i = 0; i < num_targets; i++) {
		vscsi->tgt[i] = virtscsi_alloc_tgt(vdev, sg_elems);
		if (!vscsi->tgt[i]) {
			err = -ENOMEM;
			goto out;
		}
	}
	err = 0;

out:
	if (err)
		virtscsi_remove_vqs(vdev);
	return err;
}

static int __devinit virtscsi_probe(struct virtio_device *vdev)
{
	struct Scsi_Host *shost;
	struct virtio_scsi *vscsi;
	int err;
	u32 sg_elems, num_targets;
	u32 cmd_per_lun;

	/* Allocate memory and link the structs together.  */
	num_targets = virtscsi_config_get(vdev, max_target) + 1;
	shost = scsi_host_alloc(&virtscsi_host_template,
		sizeof(*vscsi)
		+ num_targets * sizeof(struct virtio_scsi_target_state));

	if (!shost)
		return -ENOMEM;

	sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;
	shost->sg_tablesize = sg_elems;
	vscsi = shost_priv(shost);
	vscsi->vdev = vdev;
	vdev->priv = shost;

	err = virtscsi_init(vdev, vscsi, num_targets);
	if (err)
		goto virtscsi_init_failed;

	cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
	shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
	shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
	shost->max_lun = virtscsi_config_get(vdev, max_lun) + 1;
	shost->max_id = num_targets;
	shost->max_channel = 0;
	shost->max_cmd_len = VIRTIO_SCSI_CDB_SIZE;
	err = scsi_add_host(shost, &vdev->dev);
	if (err)
		goto scsi_add_host_failed;

	scsi_scan_host(shost);

	return 0;

scsi_add_host_failed:
	vdev->config->del_vqs(vdev);
virtscsi_init_failed:
	scsi_host_put(shost);
	return err;
}

static void __devexit virtscsi_remove(struct virtio_device *vdev)
{
	struct Scsi_Host *shost = virtio_scsi_host(vdev);

	scsi_remove_host(shost);

	virtscsi_remove_vqs(vdev);
	scsi_host_put(shost);
}

#ifdef CONFIG_PM
static int virtscsi_freeze(struct virtio_device *vdev)
{
	virtscsi_remove_vqs(vdev);
	return 0;
}

static int virtscsi_restore(struct virtio_device *vdev)
{
	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);

	return virtscsi_init(vdev, vscsi, sh->max_id);
}
#endif

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_SCSI, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

static struct virtio_driver virtio_scsi_driver = {
	.driver.name = KBUILD_MODNAME,
	.driver.owner = THIS_MODULE,
	.id_table = id_table,
	.probe = virtscsi_probe,
#ifdef CONFIG_PM
	.freeze = virtscsi_freeze,
	.restore = virtscsi_restore,
#endif
	.remove = __devexit_p(virtscsi_remove),
};

static int __init init(void)
{
	int ret = -ENOMEM;

	virtscsi_cmd_cache = KMEM_CACHE(virtio_scsi_cmd, 0);
	if (!virtscsi_cmd_cache) {
		printk(KERN_ERR "kmem_cache_create() for "
				"virtscsi_cmd_cache failed\n");
		goto error;
	}


	virtscsi_cmd_pool =
		mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ,
					 virtscsi_cmd_cache);
	if (!virtscsi_cmd_pool) {
		printk(KERN_ERR "mempool_create() for"
				"virtscsi_cmd_pool failed\n");
		goto error;
	}
	ret = register_virtio_driver(&virtio_scsi_driver);
	if (ret < 0)
		goto error;

	return 0;

error:
	if (virtscsi_cmd_pool) {
		mempool_destroy(virtscsi_cmd_pool);
		virtscsi_cmd_pool = NULL;
	}
	if (virtscsi_cmd_cache) {
		kmem_cache_destroy(virtscsi_cmd_cache);
		virtscsi_cmd_cache = NULL;
	}
	return ret;
}

static void __exit fini(void)
{
	unregister_virtio_driver(&virtio_scsi_driver);
	mempool_destroy(virtscsi_cmd_pool);
	kmem_cache_destroy(virtscsi_cmd_cache);
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio SCSI HBA driver");
MODULE_LICENSE("GPL");
Commit	Line	Data
4fe74b1c PB	1	/*
	2	* Virtio SCSI HBA driver
	3	*
	4	* Copyright IBM Corp. 2010
	5	* Copyright Red Hat, Inc. 2011
	6	*
	7	* Authors:
	8	* Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
	9	* Paolo Bonzini <pbonzini@redhat.com>
	10	*
	11	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	12	* See the COPYING file in the top-level directory.
	13	*
	14	*/
	15
	16	#include <linux/module.h>
	17	#include <linux/slab.h>
	18	#include <linux/mempool.h>
	19	#include <linux/virtio.h>
	20	#include <linux/virtio_ids.h>
	21	#include <linux/virtio_config.h>
	22	#include <linux/virtio_scsi.h>
	23	#include <scsi/scsi_host.h>
	24	#include <scsi/scsi_device.h>
	25	#include <scsi/scsi_cmnd.h>
	26
	27	#define VIRTIO_SCSI_MEMPOOL_SZ 64
	28
	29	/* Command queue element */
	30	struct virtio_scsi_cmd {
	31	struct scsi_cmnd *sc;
	32	struct completion *comp;
	33	union {
	34	struct virtio_scsi_cmd_req cmd;
	35	struct virtio_scsi_ctrl_tmf_req tmf;
	36	struct virtio_scsi_ctrl_an_req an;
	37	} req;
	38	union {
	39	struct virtio_scsi_cmd_resp cmd;
	40	struct virtio_scsi_ctrl_tmf_resp tmf;
	41	struct virtio_scsi_ctrl_an_resp an;
	42	struct virtio_scsi_event evt;
	43	} resp;
	44	} ____cacheline_aligned_in_smp;
	45
139fe45a PB	46	struct virtio_scsi_vq {
	47	/* Protects vq */
	48	spinlock_t vq_lock;
	49
	50	struct virtqueue *vq;
	51	};
	52
2bd37f0f PB	53	/* Per-target queue state */
	54	struct virtio_scsi_target_state {
	55	/* Protects sg. Lock hierarchy is tgt_lock -> vq_lock. */
	56	spinlock_t tgt_lock;
	57
	58	/* For sglist construction when adding commands to the virtqueue. */
	59	struct scatterlist sg[];
	60	};
	61
4fe74b1c PB	62	/* Driver instance state */
4fe74b1c PB	63	struct virtio_scsi {
4fe74b1c	64	struct virtio_device *vdev;
2bd37f0f	65
139fe45a PB	66	struct virtio_scsi_vq ctrl_vq;
	67	struct virtio_scsi_vq event_vq;
	68	struct virtio_scsi_vq req_vq;
4fe74b1c	69
2bd37f0f	70	struct virtio_scsi_target_state *tgt[];
4fe74b1c PB	71	};
	72
	73	static struct kmem_cache *virtscsi_cmd_cache;
	74	static mempool_t *virtscsi_cmd_pool;
	75
	76	static inline struct Scsi_Host virtio_scsi_host(struct virtio_device vdev)
	77	{
	78	return vdev->priv;
	79	}
	80
	81	static void virtscsi_compute_resid(struct scsi_cmnd *sc, u32 resid)
	82	{
	83	if (!resid)
	84	return;
	85
	86	if (!scsi_bidi_cmnd(sc)) {
	87	scsi_set_resid(sc, resid);
	88	return;
	89	}
	90
	91	scsi_in(sc)->resid = min(resid, scsi_in(sc)->length);
	92	scsi_out(sc)->resid = resid - scsi_in(sc)->resid;
	93	}
	94
	95	/**
	96	* virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
	97	*
	98	* Called with vq_lock held.
	99	*/
	100	static void virtscsi_complete_cmd(void *buf)
	101	{
	102	struct virtio_scsi_cmd *cmd = buf;
	103	struct scsi_cmnd *sc = cmd->sc;
	104	struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
	105
	106	dev_dbg(&sc->device->sdev_gendev,
	107	"cmd %p response %u status %#02x sense_len %u\n",
	108	sc, resp->response, resp->status, resp->sense_len);
	109
	110	sc->result = resp->status;
	111	virtscsi_compute_resid(sc, resp->resid);
	112	switch (resp->response) {
	113	case VIRTIO_SCSI_S_OK:
	114	set_host_byte(sc, DID_OK);
	115	break;
	116	case VIRTIO_SCSI_S_OVERRUN:
	117	set_host_byte(sc, DID_ERROR);
	118	break;
	119	case VIRTIO_SCSI_S_ABORTED:
	120	set_host_byte(sc, DID_ABORT);
	121	break;
	122	case VIRTIO_SCSI_S_BAD_TARGET:
	123	set_host_byte(sc, DID_BAD_TARGET);
	124	break;
	125	case VIRTIO_SCSI_S_RESET:
	126	set_host_byte(sc, DID_RESET);
	127	break;
	128	case VIRTIO_SCSI_S_BUSY:
	129	set_host_byte(sc, DID_BUS_BUSY);
	130	break;
	131	case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
	132	set_host_byte(sc, DID_TRANSPORT_DISRUPTED);
	133	break;
	134	case VIRTIO_SCSI_S_TARGET_FAILURE:
135	set_host_byte(sc, DID_TARGET_FAILURE);
136	break;
137	case VIRTIO_SCSI_S_NEXUS_FAILURE:
138	set_host_byte(sc, DID_NEXUS_FAILURE);
139	break;
140	default:
141	scmd_printk(KERN_WARNING, sc, "Unknown response %d",
142	resp->response);
143	/* fall through */
144	case VIRTIO_SCSI_S_FAILURE:
145	set_host_byte(sc, DID_ERROR);
146	break;
147	}
148
149	WARN_ON(resp->sense_len > VIRTIO_SCSI_SENSE_SIZE);
150	if (sc->sense_buffer) {
151	memcpy(sc->sense_buffer, resp->sense,
152	min_t(u32, resp->sense_len, VIRTIO_SCSI_SENSE_SIZE));
153	if (resp->sense_len)
154	set_driver_byte(sc, DRIVER_SENSE);
155	}
156
157	mempool_free(cmd, virtscsi_cmd_pool);
158	sc->scsi_done(sc);
159	}
160
161	static void virtscsi_vq_done(struct virtqueue vq, void (fn)(void *buf))
162	{
4fe74b1c	163	void *buf;
4fe74b1c PB	164	unsigned int len;
4fe74b1c PB	165
4fe74b1c PB	166	do {
	167	virtqueue_disable_cb(vq);
	168	while ((buf = virtqueue_get_buf(vq, &len)) != NULL)
	169	fn(buf);
	170	} while (!virtqueue_enable_cb(vq));
4fe74b1c PB	171	}
	172
	173	static void virtscsi_req_done(struct virtqueue *vq)
	174	{
139fe45a PB	175	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	176	struct virtio_scsi *vscsi = shost_priv(sh);
	177	unsigned long flags;
	178
	179	spin_lock_irqsave(&vscsi->req_vq.vq_lock, flags);
4fe74b1c	180	virtscsi_vq_done(vq, virtscsi_complete_cmd);
139fe45a	181	spin_unlock_irqrestore(&vscsi->req_vq.vq_lock, flags);
4fe74b1c PB	182	};
	183
	184	static void virtscsi_complete_free(void *buf)
	185	{
	186	struct virtio_scsi_cmd *cmd = buf;
	187
	188	if (cmd->comp)
	189	complete_all(cmd->comp);
e4594bb5 PB	190	else
e4594bb5 PB	191	mempool_free(cmd, virtscsi_cmd_pool);
4fe74b1c PB	192	}
	193
	194	static void virtscsi_ctrl_done(struct virtqueue *vq)
	195	{
139fe45a PB	196	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	197	struct virtio_scsi *vscsi = shost_priv(sh);
	198	unsigned long flags;
	199
	200	spin_lock_irqsave(&vscsi->ctrl_vq.vq_lock, flags);
4fe74b1c	201	virtscsi_vq_done(vq, virtscsi_complete_free);
139fe45a	202	spin_unlock_irqrestore(&vscsi->ctrl_vq.vq_lock, flags);
4fe74b1c PB	203	};
	204
	205	static void virtscsi_event_done(struct virtqueue *vq)
	206	{
139fe45a PB	207	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	208	struct virtio_scsi *vscsi = shost_priv(sh);
	209	unsigned long flags;
	210
	211	spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);
4fe74b1c	212	virtscsi_vq_done(vq, virtscsi_complete_free);
139fe45a	213	spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);
4fe74b1c PB	214	};
	215
	216	static void virtscsi_map_sgl(struct scatterlist sg, unsigned int p_idx,
	217	struct scsi_data_buffer *sdb)
	218	{
	219	struct sg_table *table = &sdb->table;
	220	struct scatterlist *sg_elem;
	221	unsigned int idx = *p_idx;
	222	int i;
	223
	224	for_each_sg(table->sgl, sg_elem, table->nents, i)
	225	sg_set_buf(&sg[idx++], sg_virt(sg_elem), sg_elem->length);
	226
	227	*p_idx = idx;
	228	}
	229
	230	/**
	231	* virtscsi_map_cmd - map a scsi_cmd to a virtqueue scatterlist
	232	* @vscsi : virtio_scsi state
	233	* @cmd : command structure
	234	* @out_num : number of read-only elements
	235	* @in_num : number of write-only elements
	236	* @req_size : size of the request buffer
	237	* @resp_size : size of the response buffer
	238	*
2bd37f0f	239	* Called with tgt_lock held.
4fe74b1c	240	*/
2bd37f0f	241	static void virtscsi_map_cmd(struct virtio_scsi_target_state *tgt,
4fe74b1c PB	242	struct virtio_scsi_cmd *cmd,
	243	unsigned out_num, unsigned in_num,
	244	size_t req_size, size_t resp_size)
	245	{
	246	struct scsi_cmnd *sc = cmd->sc;
2bd37f0f	247	struct scatterlist *sg = tgt->sg;
4fe74b1c PB	248	unsigned int idx = 0;
4fe74b1c PB	249
4fe74b1c PB	250	/* Request header. */
	251	sg_set_buf(&sg[idx++], &cmd->req, req_size);
	252
	253	/* Data-out buffer. */
	254	if (sc && sc->sc_data_direction != DMA_FROM_DEVICE)
	255	virtscsi_map_sgl(sg, &idx, scsi_out(sc));
	256
	257	*out_num = idx;
	258
	259	/* Response header. */
	260	sg_set_buf(&sg[idx++], &cmd->resp, resp_size);
	261
	262	/* Data-in buffer */
	263	if (sc && sc->sc_data_direction != DMA_TO_DEVICE)
	264	virtscsi_map_sgl(sg, &idx, scsi_in(sc));
	265
	266	in_num = idx - out_num;
	267	}
	268
2bd37f0f PB	269	static int virtscsi_kick_cmd(struct virtio_scsi_target_state *tgt,
2bd37f0f PB	270	struct virtio_scsi_vq *vq,
4fe74b1c PB	271	struct virtio_scsi_cmd *cmd,
	272	size_t req_size, size_t resp_size, gfp_t gfp)
	273	{
	274	unsigned int out_num, in_num;
	275	unsigned long flags;
	276	int ret;
	277
2bd37f0f PB	278	spin_lock_irqsave(&tgt->tgt_lock, flags);
2bd37f0f PB	279	virtscsi_map_cmd(tgt, cmd, &out_num, &in_num, req_size, resp_size);
4fe74b1c	280
139fe45a	281	spin_lock(&vq->vq_lock);
2bd37f0f PB	282	ret = virtqueue_add_buf(vq->vq, tgt->sg, out_num, in_num, cmd, gfp);
2bd37f0f PB	283	spin_unlock(&tgt->tgt_lock);
4fe74b1c	284	if (ret >= 0)
139fe45a PB	285	ret = virtqueue_kick_prepare(vq->vq);
139fe45a PB	286
bce750b1	287	spin_unlock_irqrestore(&vq->vq_lock, flags);
4fe74b1c	288
b5ee8f28	289	if (ret > 0)
139fe45a	290	virtqueue_notify(vq->vq);
4fe74b1c PB	291	return ret;
	292	}
	293
	294	static int virtscsi_queuecommand(struct Scsi_Host sh, struct scsi_cmnd sc)
	295	{
	296	struct virtio_scsi *vscsi = shost_priv(sh);
2bd37f0f	297	struct virtio_scsi_target_state *tgt = vscsi->tgt[sc->device->id];
4fe74b1c PB	298	struct virtio_scsi_cmd *cmd;
	299	int ret;
	300
2bd37f0f PB	301	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
	302	BUG_ON(scsi_sg_count(sc) > shost->sg_tablesize);
	303
	304	/* TODO: check feature bit and fail if unsupported? */
	305	BUG_ON(sc->sc_data_direction == DMA_BIDIRECTIONAL);
	306
4fe74b1c PB	307	dev_dbg(&sc->device->sdev_gendev,
	308	"cmd %p CDB: %#02x\n", sc, sc->cmnd[0]);
	309
	310	ret = SCSI_MLQUEUE_HOST_BUSY;
	311	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_ATOMIC);
	312	if (!cmd)
	313	goto out;
	314
	315	memset(cmd, 0, sizeof(*cmd));
	316	cmd->sc = sc;
	317	cmd->req.cmd = (struct virtio_scsi_cmd_req){
	318	.lun[0] = 1,
	319	.lun[1] = sc->device->id,
	320	.lun[2] = (sc->device->lun >> 8) \| 0x40,
	321	.lun[3] = sc->device->lun & 0xff,
	322	.tag = (unsigned long)sc,
	323	.task_attr = VIRTIO_SCSI_S_SIMPLE,
	324	.prio = 0,
	325	.crn = 0,
	326	};
	327
	328	BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
	329	memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);
	330
2bd37f0f	331	if (virtscsi_kick_cmd(tgt, &vscsi->req_vq, cmd,
4fe74b1c PB	332	sizeof cmd->req.cmd, sizeof cmd->resp.cmd,
	333	GFP_ATOMIC) >= 0)
	334	ret = 0;
	335
	336	out:
	337	return ret;
	338	}
	339
	340	static int virtscsi_tmf(struct virtio_scsi vscsi, struct virtio_scsi_cmd cmd)
	341	{
	342	DECLARE_COMPLETION_ONSTACK(comp);
2bd37f0f	343	struct virtio_scsi_target_state *tgt = vscsi->tgt[cmd->sc->device->id];
e4594bb5	344	int ret = FAILED;
4fe74b1c PB	345
4fe74b1c PB	346	cmd->comp = &comp;
2bd37f0f	347	if (virtscsi_kick_cmd(tgt, &vscsi->ctrl_vq, cmd,
e4594bb5 PB	348	sizeof cmd->req.tmf, sizeof cmd->resp.tmf,
	349	GFP_NOIO) < 0)
	350	goto out;
4fe74b1c PB	351
4fe74b1c PB	352	wait_for_completion(&comp);
e4594bb5 PB	353	if (cmd->resp.tmf.response == VIRTIO_SCSI_S_OK \|\|
	354	cmd->resp.tmf.response == VIRTIO_SCSI_S_FUNCTION_SUCCEEDED)
	355	ret = SUCCESS;
4fe74b1c	356
e4594bb5 PB	357	out:
	358	mempool_free(cmd, virtscsi_cmd_pool);
	359	return ret;
4fe74b1c PB	360	}
	361
	362	static int virtscsi_device_reset(struct scsi_cmnd *sc)
	363	{
	364	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	365	struct virtio_scsi_cmd *cmd;
	366
	367	sdev_printk(KERN_INFO, sc->device, "device reset\n");
	368	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	369	if (!cmd)
	370	return FAILED;
	371
	372	memset(cmd, 0, sizeof(*cmd));
	373	cmd->sc = sc;
	374	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
	375	.type = VIRTIO_SCSI_T_TMF,
	376	.subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET,
	377	.lun[0] = 1,
	378	.lun[1] = sc->device->id,
	379	.lun[2] = (sc->device->lun >> 8) \| 0x40,
	380	.lun[3] = sc->device->lun & 0xff,
	381	};
	382	return virtscsi_tmf(vscsi, cmd);
	383	}
	384
	385	static int virtscsi_abort(struct scsi_cmnd *sc)
	386	{
	387	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	388	struct virtio_scsi_cmd *cmd;
	389
	390	scmd_printk(KERN_INFO, sc, "abort\n");
	391	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	392	if (!cmd)
	393	return FAILED;
	394
	395	memset(cmd, 0, sizeof(*cmd));
	396	cmd->sc = sc;
	397	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
	398	.type = VIRTIO_SCSI_T_TMF,
	399	.subtype = VIRTIO_SCSI_T_TMF_ABORT_TASK,
	400	.lun[0] = 1,
	401	.lun[1] = sc->device->id,
	402	.lun[2] = (sc->device->lun >> 8) \| 0x40,
	403	.lun[3] = sc->device->lun & 0xff,
	404	.tag = (unsigned long)sc,
	405	};
	406	return virtscsi_tmf(vscsi, cmd);
	407	}
	408
	409	static struct scsi_host_template virtscsi_host_template = {
	410	.module = THIS_MODULE,
	411	.name = "Virtio SCSI HBA",
	412	.proc_name = "virtio_scsi",
	413	.queuecommand = virtscsi_queuecommand,
	414	.this_id = -1,
	415	.eh_abort_handler = virtscsi_abort,
	416	.eh_device_reset_handler = virtscsi_device_reset,
	417
	418	.can_queue = 1024,
	419	.dma_boundary = UINT_MAX,
	420	.use_clustering = ENABLE_CLUSTERING,
	421	};
	422
	423	#define virtscsi_config_get(vdev, fld) \
424	({ \
425	typeof(((struct virtio_scsi_config *)0)->fld) __val; \
426	vdev->config->get(vdev, \
427	offsetof(struct virtio_scsi_config, fld), \
428	&__val, sizeof(__val)); \
429	__val; \
430	})
431
432	#define virtscsi_config_set(vdev, fld, val) \
433	(void)({ \
434	typeof(((struct virtio_scsi_config *)0)->fld) __val = (val); \
435	vdev->config->set(vdev, \
436	offsetof(struct virtio_scsi_config, fld), \
437	&__val, sizeof(__val)); \
438	})
439
139fe45a PB	440	static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
	441	struct virtqueue *vq)
	442	{
	443	spin_lock_init(&virtscsi_vq->vq_lock);
	444	virtscsi_vq->vq = vq;
	445	}
	446
2bd37f0f PB	447	static struct virtio_scsi_target_state *virtscsi_alloc_tgt(
	448	struct virtio_device *vdev, int sg_elems)
	449	{
	450	struct virtio_scsi_target_state *tgt;
	451	gfp_t gfp_mask = GFP_KERNEL;
	452
	453	/* We need extra sg elements at head and tail. */
	454	tgt = kmalloc(sizeof(tgt) + sizeof(tgt->sg[0]) (sg_elems + 2),
	455	gfp_mask);
	456
	457	if (!tgt)
	458	return NULL;
	459
	460	spin_lock_init(&tgt->tgt_lock);
	461	sg_init_table(tgt->sg, sg_elems + 2);
	462	return tgt;
	463	}
	464
	465	static void virtscsi_remove_vqs(struct virtio_device *vdev)
	466	{
	467	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	468	struct virtio_scsi *vscsi = shost_priv(sh);
	469	u32 i, num_targets;
	470
	471	/* Stop all the virtqueues. */
	472	vdev->config->reset(vdev);
	473
	474	num_targets = sh->max_id;
	475	for (i = 0; i < num_targets; i++) {
	476	kfree(vscsi->tgt[i]);
	477	vscsi->tgt[i] = NULL;
	478	}
	479
	480	vdev->config->del_vqs(vdev);
	481	}
	482
4fe74b1c	483	static int virtscsi_init(struct virtio_device *vdev,
2bd37f0f	484	struct virtio_scsi *vscsi, int num_targets)
4fe74b1c PB	485	{
	486	int err;
	487	struct virtqueue *vqs[3];
2bd37f0f PB	488	u32 i, sg_elems;
2bd37f0f PB	489
4fe74b1c PB	490	vq_callback_t *callbacks[] = {
	491	virtscsi_ctrl_done,
	492	virtscsi_event_done,
	493	virtscsi_req_done
	494	};
	495	const char *names[] = {
	496	"control",
	497	"event",
	498	"request"
	499	};
	500
	501	/* Discover virtqueues and write information to configuration. */
	502	err = vdev->config->find_vqs(vdev, 3, vqs, callbacks, names);
	503	if (err)
	504	return err;
	505
139fe45a PB	506	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);
	507	virtscsi_init_vq(&vscsi->event_vq, vqs[1]);
	508	virtscsi_init_vq(&vscsi->req_vq, vqs[2]);
4fe74b1c PB	509
	510	virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
	511	virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
2bd37f0f PB	512
	513	/* We need to know how many segments before we allocate. */
	514	sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;
	515
	516	for (i = 0; i < num_targets; i++) {
	517	vscsi->tgt[i] = virtscsi_alloc_tgt(vdev, sg_elems);
	518	if (!vscsi->tgt[i]) {
	519	err = -ENOMEM;
	520	goto out;
	521	}
	522	}
	523	err = 0;
	524
	525	out:
	526	if (err)
	527	virtscsi_remove_vqs(vdev);
	528	return err;
4fe74b1c PB	529	}
	530
	531	static int __devinit virtscsi_probe(struct virtio_device *vdev)
	532	{
	533	struct Scsi_Host *shost;
	534	struct virtio_scsi *vscsi;
	535	int err;
2bd37f0f	536	u32 sg_elems, num_targets;
4fe74b1c PB	537	u32 cmd_per_lun;
4fe74b1c PB	538
4fe74b1c	539	/* Allocate memory and link the structs together. */
2bd37f0f	540	num_targets = virtscsi_config_get(vdev, max_target) + 1;
4fe74b1c	541	shost = scsi_host_alloc(&virtscsi_host_template,
2bd37f0f PB	542	sizeof(*vscsi)
2bd37f0f PB	543	+ num_targets * sizeof(struct virtio_scsi_target_state));
4fe74b1c PB	544
	545	if (!shost)
	546	return -ENOMEM;
	547
2bd37f0f	548	sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;
4fe74b1c PB	549	shost->sg_tablesize = sg_elems;
	550	vscsi = shost_priv(shost);
	551	vscsi->vdev = vdev;
	552	vdev->priv = shost;
	553
2bd37f0f	554	err = virtscsi_init(vdev, vscsi, num_targets);
4fe74b1c PB	555	if (err)
	556	goto virtscsi_init_failed;
	557
	558	cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
	559	shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
	560	shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
	561	shost->max_lun = virtscsi_config_get(vdev, max_lun) + 1;
2bd37f0f	562	shost->max_id = num_targets;
4fe74b1c PB	563	shost->max_channel = 0;
	564	shost->max_cmd_len = VIRTIO_SCSI_CDB_SIZE;
	565	err = scsi_add_host(shost, &vdev->dev);
	566	if (err)
	567	goto scsi_add_host_failed;
	568
	569	scsi_scan_host(shost);
	570
	571	return 0;
	572
	573	scsi_add_host_failed:
	574	vdev->config->del_vqs(vdev);
	575	virtscsi_init_failed:
	576	scsi_host_put(shost);
	577	return err;
	578	}
	579
4fe74b1c PB	580	static void __devexit virtscsi_remove(struct virtio_device *vdev)
	581	{
	582	struct Scsi_Host *shost = virtio_scsi_host(vdev);
	583
	584	scsi_remove_host(shost);
	585
	586	virtscsi_remove_vqs(vdev);
	587	scsi_host_put(shost);
	588	}
	589
	590	#ifdef CONFIG_PM
	591	static int virtscsi_freeze(struct virtio_device *vdev)
	592	{
	593	virtscsi_remove_vqs(vdev);
	594	return 0;
	595	}
	596
	597	static int virtscsi_restore(struct virtio_device *vdev)
	598	{
	599	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	600	struct virtio_scsi *vscsi = shost_priv(sh);
	601
2bd37f0f	602	return virtscsi_init(vdev, vscsi, sh->max_id);
4fe74b1c PB	603	}
	604	#endif
	605
	606	static struct virtio_device_id id_table[] = {
	607	{ VIRTIO_ID_SCSI, VIRTIO_DEV_ANY_ID },
	608	{ 0 },
	609	};
	610
	611	static struct virtio_driver virtio_scsi_driver = {
	612	.driver.name = KBUILD_MODNAME,
	613	.driver.owner = THIS_MODULE,
	614	.id_table = id_table,
	615	.probe = virtscsi_probe,
	616	#ifdef CONFIG_PM
	617	.freeze = virtscsi_freeze,
	618	.restore = virtscsi_restore,
	619	#endif
	620	.remove = __devexit_p(virtscsi_remove),
	621	};
	622
	623	static int __init init(void)
	624	{
	625	int ret = -ENOMEM;
	626
	627	virtscsi_cmd_cache = KMEM_CACHE(virtio_scsi_cmd, 0);
	628	if (!virtscsi_cmd_cache) {
	629	printk(KERN_ERR "kmem_cache_create() for "
	630	"virtscsi_cmd_cache failed\n");
	631	goto error;
	632	}
	633
	634
	635	virtscsi_cmd_pool =
	636	mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ,
	637	virtscsi_cmd_cache);
	638	if (!virtscsi_cmd_pool) {
	639	printk(KERN_ERR "mempool_create() for"
	640	"virtscsi_cmd_pool failed\n");
	641	goto error;
	642	}
	643	ret = register_virtio_driver(&virtio_scsi_driver);
	644	if (ret < 0)
	645	goto error;
	646
	647	return 0;
	648
	649	error:
	650	if (virtscsi_cmd_pool) {
	651	mempool_destroy(virtscsi_cmd_pool);
	652	virtscsi_cmd_pool = NULL;
	653	}
	654	if (virtscsi_cmd_cache) {
	655	kmem_cache_destroy(virtscsi_cmd_cache);
	656	virtscsi_cmd_cache = NULL;
	657	}
	658	return ret;
	659	}
	660
	661	static void __exit fini(void)
	662	{
	663	unregister_virtio_driver(&virtio_scsi_driver);
	664	mempool_destroy(virtscsi_cmd_pool);
	665	kmem_cache_destroy(virtscsi_cmd_cache);
	666	}
667	module_init(init);
668	module_exit(fini);
669
670	MODULE_DEVICE_TABLE(virtio, id_table);
671	MODULE_DESCRIPTION("Virtio SCSI HBA driver");
672	MODULE_LICENSE("GPL");