* @buf: the buffer struct
* @size: total size of the buffer
*
- * Returns a pointer to the resulting buffer, NULL if unsuccessful
+ * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
+ * passed in size will get page aligned, if it isn't already.
*/
-static void *relay_alloc_buf(struct rchan_buf *buf, unsigned long size)
+static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)
{
void *mem;
unsigned int i, j, n_pages;
- size = PAGE_ALIGN(size);
- n_pages = size >> PAGE_SHIFT;
+ *size = PAGE_ALIGN(*size);
+ n_pages = *size >> PAGE_SHIFT;
buf->page_array = kcalloc(n_pages, sizeof(struct page *), GFP_KERNEL);
if (!buf->page_array)
if (!mem)
goto depopulate;
- memset(mem, 0, size);
+ memset(mem, 0, *size);
buf->page_count = n_pages;
return mem;
/**
* relay_create_buf - allocate and initialize a channel buffer
- * @alloc_size: size of the buffer to allocate
- * @n_subbufs: number of sub-buffers in the channel
+ * @chan: the relay channel
*
- * Returns channel buffer if successful, NULL otherwise
+ * Returns channel buffer if successful, %NULL otherwise.
*/
struct rchan_buf *relay_create_buf(struct rchan *chan)
{
if (!buf->padding)
goto free_buf;
- buf->start = relay_alloc_buf(buf, chan->alloc_size);
+ buf->start = relay_alloc_buf(buf, &chan->alloc_size);
if (!buf->start)
goto free_buf;
/**
* relay_destroy_channel - free the channel struct
+ * @kref: target kernel reference that contains the relay channel
*
* Should only be called from kref_put().
*/
/**
* relay_remove_buf - remove a channel buffer
+ * @kref: target kernel reference that contains the relay buffer
*
* Removes the file from the fileystem, which also frees the
* rchan_buf_struct and the channel buffer. Should only be called from
* reason waking is deferred is that calling directly from write
* causes problems if you're writing from say the scheduler.
*/
-static void wakeup_readers(void *private)
+static void wakeup_readers(struct work_struct *work)
{
- struct rchan_buf *buf = private;
+ struct rchan_buf *buf =
+ container_of(work, struct rchan_buf, wake_readers.work);
wake_up_interruptible(&buf->read_wait);
}
if (init) {
init_waitqueue_head(&buf->read_wait);
kref_init(&buf->kref);
- INIT_WORK(&buf->wake_readers, NULL, NULL);
+ INIT_DELAYED_WORK(&buf->wake_readers, NULL);
} else {
cancel_delayed_work(&buf->wake_readers);
flush_scheduled_work();
}
EXPORT_SYMBOL_GPL(relay_reset);
-/**
+/*
* relay_open_buf - create a new relay channel buffer
*
* Internal - used by relay_open().
/**
* relay_open - create a new relay channel
* @base_filename: base name of files to create
- * @parent: dentry of parent directory, NULL for root directory
+ * @parent: dentry of parent directory, %NULL for root directory
* @subbuf_size: size of sub-buffers
* @n_subbufs: number of sub-buffers
* @cb: client callback functions
*
- * Returns channel pointer if successful, NULL otherwise.
+ * Returns channel pointer if successful, %NULL otherwise.
*
* Creates a channel buffer for each cpu using the sizes and
* attributes specified. The created channel buffer files
old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
buf->padding[old_subbuf] = buf->prev_padding;
buf->subbufs_produced++;
+ buf->dentry->d_inode->i_size += buf->chan->subbuf_size -
+ buf->padding[old_subbuf];
+ smp_mb();
if (waitqueue_active(&buf->read_wait)) {
- PREPARE_WORK(&buf->wake_readers, wakeup_readers, buf);
+ PREPARE_DELAYED_WORK(&buf->wake_readers,
+ wakeup_readers);
schedule_delayed_work(&buf->wake_readers, 1);
}
}
* subbufs_consumed should be the number of sub-buffers newly consumed,
* not the total consumed.
*
- * NOTE: kernel clients don't need to call this function if the channel
+ * NOTE: Kernel clients don't need to call this function if the channel
* mode is 'overwrite'.
*/
void relay_subbufs_consumed(struct rchan *chan,
* relay_flush - close the channel
* @chan: the channel
*
- * Flushes all channel buffers i.e. forces buffer switch.
+ * Flushes all channel buffers, i.e. forces buffer switch.
*/
void relay_flush(struct rchan *chan)
{
*/
static int relay_file_open(struct inode *inode, struct file *filp)
{
- struct rchan_buf *buf = inode->u.generic_ip;
+ struct rchan_buf *buf = inode->i_private;
kref_get(&buf->kref);
filp->private_data = buf;
return 0;
}
-/**
+/*
* relay_file_read_consume - update the consumed count for the buffer
*/
static void relay_file_read_consume(struct rchan_buf *buf,
}
}
-/**
+/*
* relay_file_read_avail - boolean, are there unconsumed bytes available?
*/
static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos)
{
- size_t bytes_produced, bytes_consumed, write_offset;
size_t subbuf_size = buf->chan->subbuf_size;
size_t n_subbufs = buf->chan->n_subbufs;
- size_t produced = buf->subbufs_produced % n_subbufs;
- size_t consumed = buf->subbufs_consumed % n_subbufs;
+ size_t produced = buf->subbufs_produced;
+ size_t consumed = buf->subbufs_consumed;
- write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;
+ relay_file_read_consume(buf, read_pos, 0);
- if (consumed > produced) {
- if ((produced > n_subbufs) &&
- (produced + n_subbufs - consumed <= n_subbufs))
- produced += n_subbufs;
- } else if (consumed == produced) {
- if (buf->offset > subbuf_size) {
- produced += n_subbufs;
- if (buf->subbufs_produced == buf->subbufs_consumed)
- consumed += n_subbufs;
- }
+ if (unlikely(buf->offset > subbuf_size)) {
+ if (produced == consumed)
+ return 0;
+ return 1;
}
- if (buf->offset > subbuf_size)
- bytes_produced = (produced - 1) * subbuf_size + write_offset;
- else
- bytes_produced = produced * subbuf_size + write_offset;
- bytes_consumed = consumed * subbuf_size + buf->bytes_consumed;
-
- if (bytes_produced == bytes_consumed)
+ if (unlikely(produced - consumed >= n_subbufs)) {
+ consumed = (produced / n_subbufs) * n_subbufs;
+ buf->subbufs_consumed = consumed;
+ }
+
+ produced = (produced % n_subbufs) * subbuf_size + buf->offset;
+ consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;
+
+ if (consumed > produced)
+ produced += n_subbufs * subbuf_size;
+
+ if (consumed == produced)
return 0;
- relay_file_read_consume(buf, read_pos, 0);
-
return 1;
}
/**
* relay_file_read_subbuf_avail - return bytes available in sub-buffer
+ * @read_pos: file read position
+ * @buf: relay channel buffer
*/
static size_t relay_file_read_subbuf_avail(size_t read_pos,
struct rchan_buf *buf)
/**
* relay_file_read_start_pos - find the first available byte to read
+ * @read_pos: file read position
+ * @buf: relay channel buffer
*
* If the read_pos is in the middle of padding, return the
* position of the first actually available byte, otherwise
/**
* relay_file_read_end_pos - return the new read position
+ * @read_pos: file read position
+ * @buf: relay channel buffer
+ * @count: number of bytes to be read
*/
static size_t relay_file_read_end_pos(struct rchan_buf *buf,
size_t read_pos,
return end_pos;
}
-/**
- * relay_file_read - read file op for relay files
- * @filp: the file
- * @buffer: the userspace buffer
- * @count: number of bytes to read
- * @ppos: position to read from
- *
- * Reads count bytes or the number of bytes available in the
- * current sub-buffer being read, whichever is smaller.
+/*
+ * subbuf_read_actor - read up to one subbuf's worth of data
*/
-static ssize_t relay_file_read(struct file *filp,
- char __user *buffer,
- size_t count,
- loff_t *ppos)
+static int subbuf_read_actor(size_t read_start,
+ struct rchan_buf *buf,
+ size_t avail,
+ read_descriptor_t *desc,
+ read_actor_t actor)
{
- struct rchan_buf *buf = filp->private_data;
- struct inode *inode = filp->f_dentry->d_inode;
- size_t read_start, avail;
- ssize_t ret = 0;
void *from;
+ int ret = 0;
- mutex_lock(&inode->i_mutex);
- if(!relay_file_read_avail(buf, *ppos))
- goto out;
+ from = buf->start + read_start;
+ ret = avail;
+ if (copy_to_user(desc->arg.buf, from, avail)) {
+ desc->error = -EFAULT;
+ ret = 0;
+ }
+ desc->arg.data += ret;
+ desc->written += ret;
+ desc->count -= ret;
- read_start = relay_file_read_start_pos(*ppos, buf);
- avail = relay_file_read_subbuf_avail(read_start, buf);
- if (!avail)
- goto out;
+ return ret;
+}
- from = buf->start + read_start;
- ret = count = min(count, avail);
- if (copy_to_user(buffer, from, count)) {
- ret = -EFAULT;
- goto out;
+/*
+ * subbuf_send_actor - send up to one subbuf's worth of data
+ */
+static int subbuf_send_actor(size_t read_start,
+ struct rchan_buf *buf,
+ size_t avail,
+ read_descriptor_t *desc,
+ read_actor_t actor)
+{
+ unsigned long pidx, poff;
+ unsigned int subbuf_pages;
+ int ret = 0;
+
+ subbuf_pages = buf->chan->alloc_size >> PAGE_SHIFT;
+ pidx = (read_start / PAGE_SIZE) % subbuf_pages;
+ poff = read_start & ~PAGE_MASK;
+ while (avail) {
+ struct page *p = buf->page_array[pidx];
+ unsigned int len;
+
+ len = PAGE_SIZE - poff;
+ if (len > avail)
+ len = avail;
+
+ len = actor(desc, p, poff, len);
+ if (desc->error)
+ break;
+
+ avail -= len;
+ ret += len;
+ poff = 0;
+ pidx = (pidx + 1) % subbuf_pages;
}
- relay_file_read_consume(buf, read_start, count);
- *ppos = relay_file_read_end_pos(buf, read_start, count);
-out:
- mutex_unlock(&inode->i_mutex);
+
return ret;
}
+typedef int (*subbuf_actor_t) (size_t read_start,
+ struct rchan_buf *buf,
+ size_t avail,
+ read_descriptor_t *desc,
+ read_actor_t actor);
+
+/*
+ * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries
+ */
+static inline ssize_t relay_file_read_subbufs(struct file *filp,
+ loff_t *ppos,
+ subbuf_actor_t subbuf_actor,
+ read_actor_t actor,
+ read_descriptor_t *desc)
+{
+ struct rchan_buf *buf = filp->private_data;
+ size_t read_start, avail;
+ int ret;
+
+ if (!desc->count)
+ return 0;
+
+ mutex_lock(&filp->f_dentry->d_inode->i_mutex);
+ do {
+ if (!relay_file_read_avail(buf, *ppos))
+ break;
+
+ read_start = relay_file_read_start_pos(*ppos, buf);
+ avail = relay_file_read_subbuf_avail(read_start, buf);
+ if (!avail)
+ break;
+
+ avail = min(desc->count, avail);
+ ret = subbuf_actor(read_start, buf, avail, desc, actor);
+ if (desc->error < 0)
+ break;
+
+ if (ret) {
+ relay_file_read_consume(buf, read_start, ret);
+ *ppos = relay_file_read_end_pos(buf, read_start, ret);
+ }
+ } while (desc->count && ret);
+ mutex_unlock(&filp->f_dentry->d_inode->i_mutex);
+
+ return desc->written;
+}
+
+static ssize_t relay_file_read(struct file *filp,
+ char __user *buffer,
+ size_t count,
+ loff_t *ppos)
+{
+ read_descriptor_t desc;
+ desc.written = 0;
+ desc.count = count;
+ desc.arg.buf = buffer;
+ desc.error = 0;
+ return relay_file_read_subbufs(filp, ppos, subbuf_read_actor,
+ NULL, &desc);
+}
+
+static ssize_t relay_file_sendfile(struct file *filp,
+ loff_t *ppos,
+ size_t count,
+ read_actor_t actor,
+ void *target)
+{
+ read_descriptor_t desc;
+ desc.written = 0;
+ desc.count = count;
+ desc.arg.data = target;
+ desc.error = 0;
+ return relay_file_read_subbufs(filp, ppos, subbuf_send_actor,
+ actor, &desc);
+}
+
struct file_operations relay_file_operations = {
.open = relay_file_open,
.poll = relay_file_poll,
.read = relay_file_read,
.llseek = no_llseek,
.release = relay_file_release,
+ .sendfile = relay_file_sendfile,
};
EXPORT_SYMBOL_GPL(relay_file_operations);