#include <linux/vmalloc.h>
#include <linux/uio_driver.h>
#include <linux/stringify.h>
+#include <linux/bitops.h>
#include <net/genetlink.h>
#include <scsi/scsi_common.h>
#include <scsi/scsi_proto.h>
#define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
+#define DATA_BLOCK_BITS 256
+#define DATA_BLOCK_SIZE 4096
+
#define CMDR_SIZE (16 * 4096)
-#define DATA_SIZE (257 * 4096)
+#define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE)
#define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE)
u32 cmdr_size;
u32 cmdr_last_cleaned;
/* Offset of data ring from start of mb */
+ /* Must add data_off and mb_addr to get the address */
size_t data_off;
size_t data_size;
- /* Ring head + tail values. */
- /* Must add data_off and mb_addr to get the address */
- size_t data_head;
- size_t data_tail;
+
+ DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
wait_queue_head_t wait_cmdr;
/* TODO should this be a mutex? */
uint16_t cmd_id;
- /* Can't use se_cmd->data_length when cleaning up expired cmds, because if
+ /* Can't use se_cmd when cleaning up expired cmds, because if
cmd has been completed then accessing se_cmd is off limits */
- size_t data_length;
+ DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
unsigned long deadline;
tcmu_cmd->se_cmd = se_cmd;
tcmu_cmd->tcmu_dev = udev;
- tcmu_cmd->data_length = se_cmd->data_length;
-
- if (se_cmd->se_cmd_flags & SCF_BIDI) {
- BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
- tcmu_cmd->data_length += se_cmd->t_bidi_data_sg->length;
- }
-
tcmu_cmd->deadline = jiffies + msecs_to_jiffies(TCMU_TIME_OUT);
idr_preload(GFP_KERNEL);
return size - head;
}
+static inline void new_iov(struct iovec **iov, int *iov_cnt,
+ struct tcmu_dev *udev)
+{
+ struct iovec *iovec;
+
+ if (*iov_cnt != 0)
+ (*iov)++;
+ (*iov_cnt)++;
+
+ iovec = *iov;
+ memset(iovec, 0, sizeof(struct iovec));
+}
+
#define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
+/* offset is relative to mb_addr */
+static inline size_t get_block_offset(struct tcmu_dev *dev,
+ int block, int remaining)
+{
+ return dev->data_off + block * DATA_BLOCK_SIZE +
+ DATA_BLOCK_SIZE - remaining;
+}
+
+static inline size_t iov_tail(struct tcmu_dev *udev, struct iovec *iov)
+{
+ return (size_t)iov->iov_base + iov->iov_len;
+}
+
static void alloc_and_scatter_data_area(struct tcmu_dev *udev,
struct scatterlist *data_sg, unsigned int data_nents,
struct iovec **iov, int *iov_cnt, bool copy_data)
{
- int i;
+ int i, block;
+ int block_remaining = 0;
void *from, *to;
- size_t copy_bytes;
+ size_t copy_bytes, to_offset;
struct scatterlist *sg;
for_each_sg(data_sg, sg, data_nents, i) {
- copy_bytes = min_t(size_t, sg->length,
- head_to_end(udev->data_head, udev->data_size));
+ int sg_remaining = sg->length;
from = kmap_atomic(sg_page(sg)) + sg->offset;
- to = (void *) udev->mb_addr + udev->data_off + udev->data_head;
-
- if (copy_data) {
- memcpy(to, from, copy_bytes);
- tcmu_flush_dcache_range(to, copy_bytes);
- }
-
- /* Even iov_base is relative to mb_addr */
- (*iov)->iov_len = copy_bytes;
- (*iov)->iov_base = (void __user *) udev->data_off +
- udev->data_head;
- (*iov_cnt)++;
- (*iov)++;
-
- UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size);
-
- /* Uh oh, we wrapped the buffer. Must split sg across 2 iovs. */
- if (sg->length != copy_bytes) {
- void *from_skip = from + copy_bytes;
-
- copy_bytes = sg->length - copy_bytes;
-
- (*iov)->iov_len = copy_bytes;
- (*iov)->iov_base = (void __user *) udev->data_off +
- udev->data_head;
-
+ while (sg_remaining > 0) {
+ if (block_remaining == 0) {
+ block = find_first_zero_bit(udev->data_bitmap,
+ DATA_BLOCK_BITS);
+ block_remaining = DATA_BLOCK_SIZE;
+ set_bit(block, udev->data_bitmap);
+ }
+ copy_bytes = min_t(size_t, sg_remaining,
+ block_remaining);
+ to_offset = get_block_offset(udev, block,
+ block_remaining);
+ to = (void *)udev->mb_addr + to_offset;
+ if (*iov_cnt != 0 &&
+ to_offset == iov_tail(udev, *iov)) {
+ (*iov)->iov_len += copy_bytes;
+ } else {
+ new_iov(iov, iov_cnt, udev);
+ (*iov)->iov_base = (void __user *) to_offset;
+ (*iov)->iov_len = copy_bytes;
+ }
if (copy_data) {
- to = (void *) udev->mb_addr +
- udev->data_off + udev->data_head;
- memcpy(to, from_skip, copy_bytes);
+ memcpy(to, from + sg->length - sg_remaining,
+ copy_bytes);
tcmu_flush_dcache_range(to, copy_bytes);
}
-
- (*iov_cnt)++;
- (*iov)++;
-
- UPDATE_HEAD(udev->data_head,
- copy_bytes, udev->data_size);
+ sg_remaining -= copy_bytes;
+ block_remaining -= copy_bytes;
}
-
kunmap_atomic(from - sg->offset);
}
}
-static void gather_and_free_data_area(struct tcmu_dev *udev,
- struct scatterlist *data_sg, unsigned int data_nents)
+static void free_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd)
{
- int i;
+ bitmap_xor(udev->data_bitmap, udev->data_bitmap, cmd->data_bitmap,
+ DATA_BLOCK_BITS);
+}
+
+static void gather_data_area(struct tcmu_dev *udev, unsigned long *cmd_bitmap,
+ struct scatterlist *data_sg, unsigned int data_nents)
+{
+ int i, block;
+ int block_remaining = 0;
void *from, *to;
- size_t copy_bytes;
+ size_t copy_bytes, from_offset;
struct scatterlist *sg;
- /* It'd be easier to look at entry's iovec again, but UAM */
for_each_sg(data_sg, sg, data_nents, i) {
- copy_bytes = min_t(size_t, sg->length,
- head_to_end(udev->data_tail, udev->data_size));
-
+ int sg_remaining = sg->length;
to = kmap_atomic(sg_page(sg)) + sg->offset;
- WARN_ON(sg->length + sg->offset > PAGE_SIZE);
- from = (void *) udev->mb_addr +
- udev->data_off + udev->data_tail;
- tcmu_flush_dcache_range(from, copy_bytes);
- memcpy(to, from, copy_bytes);
-
- UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size);
-
- /* Uh oh, wrapped the data buffer for this sg's data */
- if (sg->length != copy_bytes) {
- void *to_skip = to + copy_bytes;
-
- from = (void *) udev->mb_addr +
- udev->data_off + udev->data_tail;
- WARN_ON(udev->data_tail);
- copy_bytes = sg->length - copy_bytes;
+ while (sg_remaining > 0) {
+ if (block_remaining == 0) {
+ block = find_first_bit(cmd_bitmap,
+ DATA_BLOCK_BITS);
+ block_remaining = DATA_BLOCK_SIZE;
+ clear_bit(block, cmd_bitmap);
+ }
+ copy_bytes = min_t(size_t, sg_remaining,
+ block_remaining);
+ from_offset = get_block_offset(udev, block,
+ block_remaining);
+ from = (void *) udev->mb_addr + from_offset;
tcmu_flush_dcache_range(from, copy_bytes);
- memcpy(to_skip, from, copy_bytes);
+ memcpy(to + sg->length - sg_remaining, from,
+ copy_bytes);
- UPDATE_HEAD(udev->data_tail,
- copy_bytes, udev->data_size);
+ sg_remaining -= copy_bytes;
+ block_remaining -= copy_bytes;
}
kunmap_atomic(to - sg->offset);
}
}
+static inline size_t spc_bitmap_free(unsigned long *bitmap)
+{
+ return DATA_BLOCK_SIZE * (DATA_BLOCK_BITS -
+ bitmap_weight(bitmap, DATA_BLOCK_BITS));
+}
+
/*
* We can't queue a command until we have space available on the cmd ring *and*
* space available on the data ring.
static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed)
{
struct tcmu_mailbox *mb = udev->mb_addr;
- size_t space;
+ size_t space, cmd_needed;
u32 cmd_head;
- size_t cmd_needed;
tcmu_flush_dcache_range(mb, sizeof(*mb));
return false;
}
- space = spc_free(udev->data_head, udev->data_tail, udev->data_size);
+ space = spc_bitmap_free(udev->data_bitmap);
if (space < data_needed) {
- pr_debug("no data space: %zu %zu %zu\n", udev->data_head,
- udev->data_tail, udev->data_size);
+ pr_debug("no data space: only %zu available, but ask for %zu\n",
+ space, data_needed);
return false;
}
uint32_t cmd_head;
uint64_t cdb_off;
bool copy_to_data_area;
+ size_t data_length;
+ DECLARE_BITMAP(old_bitmap, DATA_BLOCK_BITS);
if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
return -EINVAL;
* Must be a certain minimum size for response sense info, but
* also may be larger if the iov array is large.
*
- * iovs = sgl_nents+1, for end-of-ring case, plus another 1
- * b/c size == offsetof one-past-element.
+ * We prepare way too many iovs for potential uses here, because it's
+ * expensive to tell how many regions are freed in the bitmap
*/
base_command_size = max(offsetof(struct tcmu_cmd_entry,
- req.iov[se_cmd->t_bidi_data_nents +
- se_cmd->t_data_nents + 2]),
+ req.iov[se_cmd->t_bidi_data_nents +
+ se_cmd->t_data_nents]),
sizeof(struct tcmu_cmd_entry));
command_size = base_command_size
+ round_up(scsi_command_size(se_cmd->t_task_cdb), TCMU_OP_ALIGN_SIZE);
mb = udev->mb_addr;
cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
+ data_length = se_cmd->data_length;
+ if (se_cmd->se_cmd_flags & SCF_BIDI) {
+ BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
+ data_length += se_cmd->t_bidi_data_sg->length;
+ }
if ((command_size > (udev->cmdr_size / 2))
- || tcmu_cmd->data_length > (udev->data_size - 1))
+ || data_length > udev->data_size)
pr_warn("TCMU: Request of size %zu/%zu may be too big for %u/%zu "
- "cmd/data ring buffers\n", command_size, tcmu_cmd->data_length,
+ "cmd/data ring buffers\n", command_size, data_length,
udev->cmdr_size, udev->data_size);
- while (!is_ring_space_avail(udev, command_size, tcmu_cmd->data_length)) {
+ while (!is_ring_space_avail(udev, command_size, data_length)) {
int ret;
DEFINE_WAIT(__wait);
entry->hdr.kflags = 0;
entry->hdr.uflags = 0;
+ bitmap_copy(old_bitmap, udev->data_bitmap, DATA_BLOCK_BITS);
+
/*
* Fix up iovecs, and handle if allocation in data ring wrapped.
*/
se_cmd->t_bidi_data_nents, &iov, &iov_cnt, false);
entry->req.iov_bidi_cnt = iov_cnt;
+ /* cmd's data_bitmap is what changed in process */
+ bitmap_xor(tcmu_cmd->data_bitmap, old_bitmap, udev->data_bitmap,
+ DATA_BLOCK_BITS);
+
/* All offsets relative to mb_addr, not start of entry! */
cdb_off = CMDR_OFF + cmd_head + base_command_size;
memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
struct tcmu_dev *udev = cmd->tcmu_dev;
if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
- /* cmd has been completed already from timeout, just reclaim data
- ring space */
- UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);
+ /*
+ * cmd has been completed already from timeout, just reclaim
+ * data ring space and free cmd
+ */
+ free_data_area(udev, cmd);
+
+ kmem_cache_free(tcmu_cmd_cache, cmd);
return;
}
if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
- UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);
+ free_data_area(udev, cmd);
pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
cmd->se_cmd);
entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
} else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer,
se_cmd->scsi_sense_length);
-
- UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);
+ free_data_area(udev, cmd);
} else if (se_cmd->se_cmd_flags & SCF_BIDI) {
- /* Discard data_out buffer */
- UPDATE_HEAD(udev->data_tail,
- (size_t)se_cmd->t_data_sg->length, udev->data_size);
+ DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS);
- /* Get Data-In buffer */
- gather_and_free_data_area(udev,
+ /* Get Data-In buffer before clean up */
+ bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS);
+ gather_data_area(udev, bitmap,
se_cmd->t_bidi_data_sg, se_cmd->t_bidi_data_nents);
+ free_data_area(udev, cmd);
} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
- gather_and_free_data_area(udev,
+ DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS);
+
+ bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS);
+ gather_data_area(udev, bitmap,
se_cmd->t_data_sg, se_cmd->t_data_nents);
+ free_data_area(udev, cmd);
} else if (se_cmd->data_direction == DMA_TO_DEVICE) {
- UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);
+ free_data_area(udev, cmd);
} else if (se_cmd->data_direction != DMA_NONE) {
pr_warn("TCMU: data direction was %d!\n",
se_cmd->data_direction);
mb = udev->mb_addr;
mb->version = TCMU_MAILBOX_VERSION;
+ mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC;
mb->cmdr_off = CMDR_OFF;
mb->cmdr_size = udev->cmdr_size;
WARN_ON(!PAGE_ALIGNED(udev->data_off));
WARN_ON(udev->data_size % PAGE_SIZE);
+ WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
info->version = __stringify(TCMU_MAILBOX_VERSION);
return ret;
}
-static int tcmu_check_pending_cmd(int id, void *p, void *data)
+static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
{
- struct tcmu_cmd *cmd = p;
-
- if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
+ if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
+ kmem_cache_free(tcmu_cmd_cache, cmd);
return 0;
+ }
return -EINVAL;
}
static void tcmu_free_device(struct se_device *dev)
{
struct tcmu_dev *udev = TCMU_DEV(dev);
+ struct tcmu_cmd *cmd;
+ bool all_expired = true;
int i;
del_timer_sync(&udev->timeout);
/* Upper layer should drain all requests before calling this */
spin_lock_irq(&udev->commands_lock);
- i = idr_for_each(&udev->commands, tcmu_check_pending_cmd, NULL);
+ idr_for_each_entry(&udev->commands, cmd, i) {
+ if (tcmu_check_and_free_pending_cmd(cmd) != 0)
+ all_expired = false;
+ }
idr_destroy(&udev->commands);
spin_unlock_irq(&udev->commands_lock);
- WARN_ON(i);
+ WARN_ON(!all_expired);
/* Device was configured */
if (udev->uio_info.uio_dev) {
projects / deliverable / linux.git / blobdiff