[deliverable/linux.git] / drivers / media / v4l2-core / videobuf2-vmalloc.c

/*
 * videobuf2-vmalloc.c - vmalloc memory allocator for videobuf2
 *
 * Copyright (C) 2010 Samsung Electronics
 *
 * Author: Pawel Osciak <pawel@osciak.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 */

#include <linux/io.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

#include <media/videobuf2-core.h>
#include <media/videobuf2-vmalloc.h>
#include <media/videobuf2-memops.h>

struct vb2_vmalloc_buf {
	void				*vaddr;
	struct page			**pages;
	struct vm_area_struct		*vma;
	enum dma_data_direction		dma_dir;
	unsigned long			size;
	unsigned int			n_pages;
	atomic_t			refcount;
	struct vb2_vmarea_handler	handler;
	struct dma_buf			*dbuf;
};

static void vb2_vmalloc_put(void *buf_priv);

static void *vb2_vmalloc_alloc(void *alloc_ctx, unsigned long size,
			       enum dma_data_direction dma_dir, gfp_t gfp_flags)
{
	struct vb2_vmalloc_buf *buf;

	buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags);
	if (!buf)
		return NULL;

	buf->size = size;
	buf->vaddr = vmalloc_user(buf->size);
	buf->dma_dir = dma_dir;
	buf->handler.refcount = &buf->refcount;
	buf->handler.put = vb2_vmalloc_put;
	buf->handler.arg = buf;

	if (!buf->vaddr) {
		pr_debug("vmalloc of size %ld failed\n", buf->size);
		kfree(buf);
		return NULL;
	}

	atomic_inc(&buf->refcount);
	return buf;
}

static void vb2_vmalloc_put(void *buf_priv)
{
	struct vb2_vmalloc_buf *buf = buf_priv;

	if (atomic_dec_and_test(&buf->refcount)) {
		vfree(buf->vaddr);
		kfree(buf);
	}
}

static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr,
				     unsigned long size,
				     enum dma_data_direction dma_dir)
{
	struct vb2_vmalloc_buf *buf;
	unsigned long first, last;
	int n_pages, offset;
	struct vm_area_struct *vma;
	dma_addr_t physp;

	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	if (!buf)
		return NULL;

	buf->dma_dir = dma_dir;
	offset = vaddr & ~PAGE_MASK;
	buf->size = size;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, vaddr);
	if (vma && (vma->vm_flags & VM_PFNMAP) && (vma->vm_pgoff)) {
		if (vb2_get_contig_userptr(vaddr, size, &vma, &physp))
			goto fail_pages_array_alloc;
		buf->vma = vma;
		buf->vaddr = (__force void *)ioremap_nocache(physp, size);
		if (!buf->vaddr)
			goto fail_pages_array_alloc;
	} else {
		first = vaddr >> PAGE_SHIFT;
		last  = (vaddr + size - 1) >> PAGE_SHIFT;
		buf->n_pages = last - first + 1;
		buf->pages = kzalloc(buf->n_pages * sizeof(struct page *),
				     GFP_KERNEL);
		if (!buf->pages)
			goto fail_pages_array_alloc;

		/* current->mm->mmap_sem is taken by videobuf2 core */
		n_pages = get_user_pages(current, current->mm,
					 vaddr & PAGE_MASK, buf->n_pages,
					 dma_dir == DMA_FROM_DEVICE,
					 1, /* force */
					 buf->pages, NULL);
		if (n_pages != buf->n_pages)
			goto fail_get_user_pages;

		buf->vaddr = vm_map_ram(buf->pages, buf->n_pages, -1,
					PAGE_KERNEL);
		if (!buf->vaddr)
			goto fail_get_user_pages;
	}
	up_read(&current->mm->mmap_sem);

	buf->vaddr += offset;
	return buf;

fail_get_user_pages:
	pr_debug("get_user_pages requested/got: %d/%d]\n", n_pages,
		 buf->n_pages);
	while (--n_pages >= 0)
		put_page(buf->pages[n_pages]);
	kfree(buf->pages);

fail_pages_array_alloc:
	up_read(&current->mm->mmap_sem);
	kfree(buf);

	return NULL;
}

static void vb2_vmalloc_put_userptr(void *buf_priv)
{
	struct vb2_vmalloc_buf *buf = buf_priv;
	unsigned long vaddr = (unsigned long)buf->vaddr & PAGE_MASK;
	unsigned int i;

	down_read(&current->mm->mmap_sem);
	if (buf->pages) {
		if (vaddr)
			vm_unmap_ram((void *)vaddr, buf->n_pages);
		for (i = 0; i < buf->n_pages; ++i) {
			if (buf->dma_dir == DMA_FROM_DEVICE)
				set_page_dirty_lock(buf->pages[i]);
			put_page(buf->pages[i]);
		}
		kfree(buf->pages);
	} else {
		vb2_put_vma(buf->vma);
		iounmap((__force void __iomem *)buf->vaddr);
	}
	up_read(&current->mm->mmap_sem);
	kfree(buf);
}

static void *vb2_vmalloc_vaddr(void *buf_priv)
{
	struct vb2_vmalloc_buf *buf = buf_priv;

	if (!buf->vaddr) {
		pr_err("Address of an unallocated plane requested "
		       "or cannot map user pointer\n");
		return NULL;
	}

	return buf->vaddr;
}

static unsigned int vb2_vmalloc_num_users(void *buf_priv)
{
	struct vb2_vmalloc_buf *buf = buf_priv;
	return atomic_read(&buf->refcount);
}

static int vb2_vmalloc_mmap(void *buf_priv, struct vm_area_struct *vma)
{
	struct vb2_vmalloc_buf *buf = buf_priv;
	int ret;

	if (!buf) {
		pr_err("No memory to map\n");
		return -EINVAL;
	}

	ret = remap_vmalloc_range(vma, buf->vaddr, 0);
	if (ret) {
		pr_err("Remapping vmalloc memory, error: %d\n", ret);
		return ret;
	}

	/*
	 * Make sure that vm_areas for 2 buffers won't be merged together
	 */
	vma->vm_flags		|= VM_DONTEXPAND;

	/*
	 * Use common vm_area operations to track buffer refcount.
	 */
	vma->vm_private_data	= &buf->handler;
	vma->vm_ops		= &vb2_common_vm_ops;

	vma->vm_ops->open(vma);

	return 0;
}

#ifdef CONFIG_HAS_DMA
/*********************************************/
/*         DMABUF ops for exporters          */
/*********************************************/

struct vb2_vmalloc_attachment {
	struct sg_table sgt;
	enum dma_data_direction dma_dir;
};

static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *dev,
	struct dma_buf_attachment *dbuf_attach)
{
	struct vb2_vmalloc_attachment *attach;
	struct vb2_vmalloc_buf *buf = dbuf->priv;
	int num_pages = PAGE_ALIGN(buf->size) / PAGE_SIZE;
	struct sg_table *sgt;
	struct scatterlist *sg;
	void *vaddr = buf->vaddr;
	int ret;
	int i;

	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
	if (!attach)
		return -ENOMEM;

	sgt = &attach->sgt;
	ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
	if (ret) {
		kfree(attach);
		return ret;
	}
	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
		struct page *page = vmalloc_to_page(vaddr);

		if (!page) {
			sg_free_table(sgt);
			kfree(attach);
			return -ENOMEM;
		}
		sg_set_page(sg, page, PAGE_SIZE, 0);
		vaddr += PAGE_SIZE;
	}

	attach->dma_dir = DMA_NONE;
	dbuf_attach->priv = attach;
	return 0;
}

static void vb2_vmalloc_dmabuf_ops_detach(struct dma_buf *dbuf,
	struct dma_buf_attachment *db_attach)
{
	struct vb2_vmalloc_attachment *attach = db_attach->priv;
	struct sg_table *sgt;

	if (!attach)
		return;

	sgt = &attach->sgt;

	/* release the scatterlist cache */
	if (attach->dma_dir != DMA_NONE)
		dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
			attach->dma_dir);
	sg_free_table(sgt);
	kfree(attach);
	db_attach->priv = NULL;
}

static struct sg_table *vb2_vmalloc_dmabuf_ops_map(
	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
{
	struct vb2_vmalloc_attachment *attach = db_attach->priv;
	/* stealing dmabuf mutex to serialize map/unmap operations */
	struct mutex *lock = &db_attach->dmabuf->lock;
	struct sg_table *sgt;

	mutex_lock(lock);

	sgt = &attach->sgt;
	/* return previously mapped sg table */
	if (attach->dma_dir == dma_dir) {
		mutex_unlock(lock);
		return sgt;
	}

	/* release any previous cache */
	if (attach->dma_dir != DMA_NONE) {
		dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
			attach->dma_dir);
		attach->dma_dir = DMA_NONE;
	}

	/* mapping to the client with new direction */
	sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
				dma_dir);
	if (!sgt->nents) {
		pr_err("failed to map scatterlist\n");
		mutex_unlock(lock);
		return ERR_PTR(-EIO);
	}

	attach->dma_dir = dma_dir;

	mutex_unlock(lock);

	return sgt;
}

static void vb2_vmalloc_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
	struct sg_table *sgt, enum dma_data_direction dma_dir)
{
	/* nothing to be done here */
}

static void vb2_vmalloc_dmabuf_ops_release(struct dma_buf *dbuf)
{
	/* drop reference obtained in vb2_vmalloc_get_dmabuf */
	vb2_vmalloc_put(dbuf->priv);
}

static void *vb2_vmalloc_dmabuf_ops_kmap(struct dma_buf *dbuf, unsigned long pgnum)
{
	struct vb2_vmalloc_buf *buf = dbuf->priv;

	return buf->vaddr + pgnum * PAGE_SIZE;
}

static void *vb2_vmalloc_dmabuf_ops_vmap(struct dma_buf *dbuf)
{
	struct vb2_vmalloc_buf *buf = dbuf->priv;

	return buf->vaddr;
}

static int vb2_vmalloc_dmabuf_ops_mmap(struct dma_buf *dbuf,
	struct vm_area_struct *vma)
{
	return vb2_vmalloc_mmap(dbuf->priv, vma);
}

static struct dma_buf_ops vb2_vmalloc_dmabuf_ops = {
	.attach = vb2_vmalloc_dmabuf_ops_attach,
	.detach = vb2_vmalloc_dmabuf_ops_detach,
	.map_dma_buf = vb2_vmalloc_dmabuf_ops_map,
	.unmap_dma_buf = vb2_vmalloc_dmabuf_ops_unmap,
	.kmap = vb2_vmalloc_dmabuf_ops_kmap,
	.kmap_atomic = vb2_vmalloc_dmabuf_ops_kmap,
	.vmap = vb2_vmalloc_dmabuf_ops_vmap,
	.mmap = vb2_vmalloc_dmabuf_ops_mmap,
	.release = vb2_vmalloc_dmabuf_ops_release,
};

static struct dma_buf *vb2_vmalloc_get_dmabuf(void *buf_priv, unsigned long flags)
{
	struct vb2_vmalloc_buf *buf = buf_priv;
	struct dma_buf *dbuf;
	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

	exp_info.ops = &vb2_vmalloc_dmabuf_ops;
	exp_info.size = buf->size;
	exp_info.flags = flags;
	exp_info.priv = buf;

	if (WARN_ON(!buf->vaddr))
		return NULL;

	dbuf = dma_buf_export(&exp_info);
	if (IS_ERR(dbuf))
		return NULL;

	/* dmabuf keeps reference to vb2 buffer */
	atomic_inc(&buf->refcount);

	return dbuf;
}
#endif /* CONFIG_HAS_DMA */


/*********************************************/
/*       callbacks for DMABUF buffers        */
/*********************************************/

static int vb2_vmalloc_map_dmabuf(void *mem_priv)
{
	struct vb2_vmalloc_buf *buf = mem_priv;

	buf->vaddr = dma_buf_vmap(buf->dbuf);

	return buf->vaddr ? 0 : -EFAULT;
}

static void vb2_vmalloc_unmap_dmabuf(void *mem_priv)
{
	struct vb2_vmalloc_buf *buf = mem_priv;

	dma_buf_vunmap(buf->dbuf, buf->vaddr);
	buf->vaddr = NULL;
}

static void vb2_vmalloc_detach_dmabuf(void *mem_priv)
{
	struct vb2_vmalloc_buf *buf = mem_priv;

	if (buf->vaddr)
		dma_buf_vunmap(buf->dbuf, buf->vaddr);

	kfree(buf);
}

static void *vb2_vmalloc_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf,
	unsigned long size, enum dma_data_direction dma_dir)
{
	struct vb2_vmalloc_buf *buf;

	if (dbuf->size < size)
		return ERR_PTR(-EFAULT);

	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	if (!buf)
		return ERR_PTR(-ENOMEM);

	buf->dbuf = dbuf;
	buf->dma_dir = dma_dir;
	buf->size = size;

	return buf;
}


const struct vb2_mem_ops vb2_vmalloc_memops = {
	.alloc		= vb2_vmalloc_alloc,
	.put		= vb2_vmalloc_put,
	.get_userptr	= vb2_vmalloc_get_userptr,
	.put_userptr	= vb2_vmalloc_put_userptr,
#ifdef CONFIG_HAS_DMA
	.get_dmabuf	= vb2_vmalloc_get_dmabuf,
#endif
	.map_dmabuf	= vb2_vmalloc_map_dmabuf,
	.unmap_dmabuf	= vb2_vmalloc_unmap_dmabuf,
	.attach_dmabuf	= vb2_vmalloc_attach_dmabuf,
	.detach_dmabuf	= vb2_vmalloc_detach_dmabuf,
	.vaddr		= vb2_vmalloc_vaddr,
	.mmap		= vb2_vmalloc_mmap,
	.num_users	= vb2_vmalloc_num_users,
};
EXPORT_SYMBOL_GPL(vb2_vmalloc_memops);

MODULE_DESCRIPTION("vmalloc memory handling routines for videobuf2");
MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
3c18ff06 PO	1	/*
	2	* videobuf2-vmalloc.c - vmalloc memory allocator for videobuf2
	3	*
	4	* Copyright (C) 2010 Samsung Electronics
	5	*
95072084	6	* Author: Pawel Osciak <pawel@osciak.com>
3c18ff06 PO	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License as published by
	10	* the Free Software Foundation.
	11	*/
	12
570d2a48	13	#include <linux/io.h>
3c18ff06 PO	14	#include <linux/module.h>
3c18ff06 PO	15	#include <linux/mm.h>
4419b8ac	16	#include <linux/sched.h>
3c18ff06 PO	17	#include <linux/slab.h>
	18	#include <linux/vmalloc.h>
	19
	20	#include <media/videobuf2-core.h>
ddb9fa25	21	#include <media/videobuf2-vmalloc.h>
3c18ff06 PO	22	#include <media/videobuf2-memops.h>
	23
	24	struct vb2_vmalloc_buf {
	25	void *vaddr;
4419b8ac	26	struct page **pages;
570d2a48	27	struct vm_area_struct *vma;
cd474037	28	enum dma_data_direction dma_dir;
3c18ff06	29	unsigned long size;
4419b8ac	30	unsigned int n_pages;
3c18ff06 PO	31	atomic_t refcount;
3c18ff06 PO	32	struct vb2_vmarea_handler handler;
89d2ee08	33	struct dma_buf *dbuf;
3c18ff06 PO	34	};
	35
	36	static void vb2_vmalloc_put(void *buf_priv);
	37
d935c57e HV	38	static void vb2_vmalloc_alloc(void alloc_ctx, unsigned long size,
d935c57e HV	39	enum dma_data_direction dma_dir, gfp_t gfp_flags)
3c18ff06 PO	40	{
	41	struct vb2_vmalloc_buf *buf;
	42
b6ba2057	43	buf = kzalloc(sizeof(*buf), GFP_KERNEL \| gfp_flags);
3c18ff06 PO	44	if (!buf)
	45	return NULL;
	46
	47	buf->size = size;
	48	buf->vaddr = vmalloc_user(buf->size);
d935c57e	49	buf->dma_dir = dma_dir;
3c18ff06 PO	50	buf->handler.refcount = &buf->refcount;
	51	buf->handler.put = vb2_vmalloc_put;
	52	buf->handler.arg = buf;
	53
	54	if (!buf->vaddr) {
4419b8ac	55	pr_debug("vmalloc of size %ld failed\n", buf->size);
3c18ff06 PO	56	kfree(buf);
	57	return NULL;
	58	}
	59
	60	atomic_inc(&buf->refcount);
3c18ff06 PO	61	return buf;
	62	}
	63
	64	static void vb2_vmalloc_put(void *buf_priv)
	65	{
	66	struct vb2_vmalloc_buf *buf = buf_priv;
	67
	68	if (atomic_dec_and_test(&buf->refcount)) {
3c18ff06 PO	69	vfree(buf->vaddr);
	70	kfree(buf);
	71	}
	72	}
	73
4419b8ac	74	static void vb2_vmalloc_get_userptr(void alloc_ctx, unsigned long vaddr,
cd474037 HV	75	unsigned long size,
cd474037 HV	76	enum dma_data_direction dma_dir)
4419b8ac AP	77	{
	78	struct vb2_vmalloc_buf *buf;
	79	unsigned long first, last;
	80	int n_pages, offset;
570d2a48 JM	81	struct vm_area_struct *vma;
570d2a48 JM	82	dma_addr_t physp;
4419b8ac AP	83
	84	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	85	if (!buf)
	86	return NULL;
	87
cd474037	88	buf->dma_dir = dma_dir;
4419b8ac AP	89	offset = vaddr & ~PAGE_MASK;
	90	buf->size = size;
	91
48b25a3a	92	down_read(&current->mm->mmap_sem);
570d2a48 JM	93	vma = find_vma(current->mm, vaddr);
	94	if (vma && (vma->vm_flags & VM_PFNMAP) && (vma->vm_pgoff)) {
	95	if (vb2_get_contig_userptr(vaddr, size, &vma, &physp))
	96	goto fail_pages_array_alloc;
	97	buf->vma = vma;
7c424dd1	98	buf->vaddr = (__force void *)ioremap_nocache(physp, size);
570d2a48 JM	99	if (!buf->vaddr)
	100	goto fail_pages_array_alloc;
	101	} else {
	102	first = vaddr >> PAGE_SHIFT;
	103	last = (vaddr + size - 1) >> PAGE_SHIFT;
	104	buf->n_pages = last - first + 1;
	105	buf->pages = kzalloc(buf->n_pages * sizeof(struct page *),
	106	GFP_KERNEL);
	107	if (!buf->pages)
	108	goto fail_pages_array_alloc;
	109
	110	/* current->mm->mmap_sem is taken by videobuf2 core */
	111	n_pages = get_user_pages(current, current->mm,
	112	vaddr & PAGE_MASK, buf->n_pages,
cd474037 HV	113	dma_dir == DMA_FROM_DEVICE,
cd474037 HV	114	1, /* force */
570d2a48 JM	115	buf->pages, NULL);
	116	if (n_pages != buf->n_pages)
	117	goto fail_get_user_pages;
	118
	119	buf->vaddr = vm_map_ram(buf->pages, buf->n_pages, -1,
	120	PAGE_KERNEL);
	121	if (!buf->vaddr)
	122	goto fail_get_user_pages;
	123	}
48b25a3a	124	up_read(&current->mm->mmap_sem);
4419b8ac AP	125
	126	buf->vaddr += offset;
	127	return buf;
	128
	129	fail_get_user_pages:
	130	pr_debug("get_user_pages requested/got: %d/%d]\n", n_pages,
	131	buf->n_pages);
	132	while (--n_pages >= 0)
	133	put_page(buf->pages[n_pages]);
	134	kfree(buf->pages);
	135
	136	fail_pages_array_alloc:
48b25a3a	137	up_read(&current->mm->mmap_sem);
4419b8ac AP	138	kfree(buf);
	139
	140	return NULL;
	141	}
	142
	143	static void vb2_vmalloc_put_userptr(void *buf_priv)
3c18ff06 PO	144	{
3c18ff06 PO	145	struct vb2_vmalloc_buf *buf = buf_priv;
4419b8ac AP	146	unsigned long vaddr = (unsigned long)buf->vaddr & PAGE_MASK;
	147	unsigned int i;
	148
48b25a3a	149	down_read(&current->mm->mmap_sem);
570d2a48 JM	150	if (buf->pages) {
	151	if (vaddr)
	152	vm_unmap_ram((void *)vaddr, buf->n_pages);
	153	for (i = 0; i < buf->n_pages; ++i) {
cd474037	154	if (buf->dma_dir == DMA_FROM_DEVICE)
570d2a48 JM	155	set_page_dirty_lock(buf->pages[i]);
	156	put_page(buf->pages[i]);
	157	}
	158	kfree(buf->pages);
	159	} else {
7c57529b	160	vb2_put_vma(buf->vma);
7c424dd1	161	iounmap((__force void __iomem *)buf->vaddr);
4419b8ac	162	}
48b25a3a	163	up_read(&current->mm->mmap_sem);
4419b8ac AP	164	kfree(buf);
4419b8ac AP	165	}
3c18ff06	166
4419b8ac AP	167	static void vb2_vmalloc_vaddr(void buf_priv)
	168	{
	169	struct vb2_vmalloc_buf *buf = buf_priv;
3c18ff06 PO	170
3c18ff06 PO	171	if (!buf->vaddr) {
4419b8ac AP	172	pr_err("Address of an unallocated plane requested "
4419b8ac AP	173	"or cannot map user pointer\n");
3c18ff06 PO	174	return NULL;
	175	}
	176
	177	return buf->vaddr;
	178	}
	179
	180	static unsigned int vb2_vmalloc_num_users(void *buf_priv)
	181	{
	182	struct vb2_vmalloc_buf *buf = buf_priv;
	183	return atomic_read(&buf->refcount);
	184	}
	185
	186	static int vb2_vmalloc_mmap(void buf_priv, struct vm_area_struct vma)
	187	{
	188	struct vb2_vmalloc_buf *buf = buf_priv;
	189	int ret;
	190
	191	if (!buf) {
4419b8ac	192	pr_err("No memory to map\n");
3c18ff06 PO	193	return -EINVAL;
	194	}
	195
	196	ret = remap_vmalloc_range(vma, buf->vaddr, 0);
	197	if (ret) {
4419b8ac	198	pr_err("Remapping vmalloc memory, error: %d\n", ret);
3c18ff06 PO	199	return ret;
	200	}
	201
	202	/*
	203	* Make sure that vm_areas for 2 buffers won't be merged together
	204	*/
	205	vma->vm_flags \|= VM_DONTEXPAND;
	206
	207	/*
	208	* Use common vm_area operations to track buffer refcount.
	209	*/
	210	vma->vm_private_data = &buf->handler;
	211	vma->vm_ops = &vb2_common_vm_ops;
	212
	213	vma->vm_ops->open(vma);
	214
	215	return 0;
	216	}
	217
99f3cd52	218	#ifdef CONFIG_HAS_DMA
46506350 HV	219	/*********************************************/
	220	/* DMABUF ops for exporters */
	221	/*********************************************/
	222
	223	struct vb2_vmalloc_attachment {
	224	struct sg_table sgt;
	225	enum dma_data_direction dma_dir;
	226	};
	227
	228	static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf dbuf, struct device dev,
	229	struct dma_buf_attachment *dbuf_attach)
	230	{
	231	struct vb2_vmalloc_attachment *attach;
	232	struct vb2_vmalloc_buf *buf = dbuf->priv;
	233	int num_pages = PAGE_ALIGN(buf->size) / PAGE_SIZE;
	234	struct sg_table *sgt;
	235	struct scatterlist *sg;
	236	void *vaddr = buf->vaddr;
	237	int ret;
	238	int i;
	239
	240	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
	241	if (!attach)
	242	return -ENOMEM;
	243
	244	sgt = &attach->sgt;
	245	ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
	246	if (ret) {
	247	kfree(attach);
	248	return ret;
	249	}
	250	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
	251	struct page *page = vmalloc_to_page(vaddr);
	252
	253	if (!page) {
	254	sg_free_table(sgt);
	255	kfree(attach);
	256	return -ENOMEM;
	257	}
	258	sg_set_page(sg, page, PAGE_SIZE, 0);
	259	vaddr += PAGE_SIZE;
	260	}
	261
	262	attach->dma_dir = DMA_NONE;
	263	dbuf_attach->priv = attach;
	264	return 0;
	265	}
	266
	267	static void vb2_vmalloc_dmabuf_ops_detach(struct dma_buf *dbuf,
	268	struct dma_buf_attachment *db_attach)
	269	{
	270	struct vb2_vmalloc_attachment *attach = db_attach->priv;
	271	struct sg_table *sgt;
	272
	273	if (!attach)
	274	return;
	275
	276	sgt = &attach->sgt;
	277
	278	/* release the scatterlist cache */
	279	if (attach->dma_dir != DMA_NONE)
	280	dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	281	attach->dma_dir);
	282	sg_free_table(sgt);
283	kfree(attach);
284	db_attach->priv = NULL;
285	}
286
287	static struct sg_table *vb2_vmalloc_dmabuf_ops_map(
288	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
289	{
290	struct vb2_vmalloc_attachment *attach = db_attach->priv;
291	/* stealing dmabuf mutex to serialize map/unmap operations */
292	struct mutex *lock = &db_attach->dmabuf->lock;
293	struct sg_table *sgt;
46506350 HV	294
	295	mutex_lock(lock);
	296
	297	sgt = &attach->sgt;
	298	/* return previously mapped sg table */
	299	if (attach->dma_dir == dma_dir) {
	300	mutex_unlock(lock);
	301	return sgt;
	302	}
	303
	304	/* release any previous cache */
	305	if (attach->dma_dir != DMA_NONE) {
	306	dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	307	attach->dma_dir);
	308	attach->dma_dir = DMA_NONE;
	309	}
	310
	311	/* mapping to the client with new direction */
ba81c6ed RR	312	sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	313	dma_dir);
	314	if (!sgt->nents) {
46506350 HV	315	pr_err("failed to map scatterlist\n");
	316	mutex_unlock(lock);
	317	return ERR_PTR(-EIO);
	318	}
	319
	320	attach->dma_dir = dma_dir;
	321
	322	mutex_unlock(lock);
	323
	324	return sgt;
	325	}
	326
	327	static void vb2_vmalloc_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
	328	struct sg_table *sgt, enum dma_data_direction dma_dir)
	329	{
	330	/* nothing to be done here */
	331	}
	332
	333	static void vb2_vmalloc_dmabuf_ops_release(struct dma_buf *dbuf)
	334	{
	335	/* drop reference obtained in vb2_vmalloc_get_dmabuf */
	336	vb2_vmalloc_put(dbuf->priv);
	337	}
	338
	339	static void vb2_vmalloc_dmabuf_ops_kmap(struct dma_buf dbuf, unsigned long pgnum)
	340	{
	341	struct vb2_vmalloc_buf *buf = dbuf->priv;
	342
	343	return buf->vaddr + pgnum * PAGE_SIZE;
	344	}
	345
	346	static void vb2_vmalloc_dmabuf_ops_vmap(struct dma_buf dbuf)
	347	{
	348	struct vb2_vmalloc_buf *buf = dbuf->priv;
	349
	350	return buf->vaddr;
	351	}
	352
	353	static int vb2_vmalloc_dmabuf_ops_mmap(struct dma_buf *dbuf,
	354	struct vm_area_struct *vma)
	355	{
	356	return vb2_vmalloc_mmap(dbuf->priv, vma);
	357	}
	358
	359	static struct dma_buf_ops vb2_vmalloc_dmabuf_ops = {
	360	.attach = vb2_vmalloc_dmabuf_ops_attach,
	361	.detach = vb2_vmalloc_dmabuf_ops_detach,
	362	.map_dma_buf = vb2_vmalloc_dmabuf_ops_map,
	363	.unmap_dma_buf = vb2_vmalloc_dmabuf_ops_unmap,
	364	.kmap = vb2_vmalloc_dmabuf_ops_kmap,
	365	.kmap_atomic = vb2_vmalloc_dmabuf_ops_kmap,
	366	.vmap = vb2_vmalloc_dmabuf_ops_vmap,
	367	.mmap = vb2_vmalloc_dmabuf_ops_mmap,
	368	.release = vb2_vmalloc_dmabuf_ops_release,
	369	};
	370
	371	static struct dma_buf vb2_vmalloc_get_dmabuf(void buf_priv, unsigned long flags)
	372	{
	373	struct vb2_vmalloc_buf *buf = buf_priv;
	374	struct dma_buf *dbuf;
d8fbe341 SS	375	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	376
	377	exp_info.ops = &vb2_vmalloc_dmabuf_ops;
	378	exp_info.size = buf->size;
	379	exp_info.flags = flags;
	380	exp_info.priv = buf;
46506350 HV	381
	382	if (WARN_ON(!buf->vaddr))
	383	return NULL;
	384
d8fbe341	385	dbuf = dma_buf_export(&exp_info);
46506350 HV	386	if (IS_ERR(dbuf))
	387	return NULL;
	388
	389	/* dmabuf keeps reference to vb2 buffer */
	390	atomic_inc(&buf->refcount);
	391
	392	return dbuf;
	393	}
99f3cd52 GU	394	#endif /* CONFIG_HAS_DMA */
99f3cd52 GU	395
46506350	396
89d2ee08 TS	397	/*********************************************/
	398	/* callbacks for DMABUF buffers */
	399	/*********************************************/
	400
	401	static int vb2_vmalloc_map_dmabuf(void *mem_priv)
	402	{
	403	struct vb2_vmalloc_buf *buf = mem_priv;
	404
	405	buf->vaddr = dma_buf_vmap(buf->dbuf);
	406
	407	return buf->vaddr ? 0 : -EFAULT;
	408	}
	409
	410	static void vb2_vmalloc_unmap_dmabuf(void *mem_priv)
	411	{
	412	struct vb2_vmalloc_buf *buf = mem_priv;
	413
	414	dma_buf_vunmap(buf->dbuf, buf->vaddr);
	415	buf->vaddr = NULL;
	416	}
	417
	418	static void vb2_vmalloc_detach_dmabuf(void *mem_priv)
	419	{
	420	struct vb2_vmalloc_buf *buf = mem_priv;
	421
	422	if (buf->vaddr)
	423	dma_buf_vunmap(buf->dbuf, buf->vaddr);
	424
	425	kfree(buf);
	426	}
	427
	428	static void vb2_vmalloc_attach_dmabuf(void alloc_ctx, struct dma_buf *dbuf,
cd474037	429	unsigned long size, enum dma_data_direction dma_dir)
89d2ee08 TS	430	{
	431	struct vb2_vmalloc_buf *buf;
	432
	433	if (dbuf->size < size)
	434	return ERR_PTR(-EFAULT);
	435
	436	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	437	if (!buf)
	438	return ERR_PTR(-ENOMEM);
	439
	440	buf->dbuf = dbuf;
cd474037	441	buf->dma_dir = dma_dir;
89d2ee08 TS	442	buf->size = size;
	443
	444	return buf;
	445	}
	446
	447
3c18ff06 PO	448	const struct vb2_mem_ops vb2_vmalloc_memops = {
	449	.alloc = vb2_vmalloc_alloc,
	450	.put = vb2_vmalloc_put,
4419b8ac AP	451	.get_userptr = vb2_vmalloc_get_userptr,
4419b8ac AP	452	.put_userptr = vb2_vmalloc_put_userptr,
99f3cd52	453	#ifdef CONFIG_HAS_DMA
46506350	454	.get_dmabuf = vb2_vmalloc_get_dmabuf,
99f3cd52	455	#endif
89d2ee08 TS	456	.map_dmabuf = vb2_vmalloc_map_dmabuf,
	457	.unmap_dmabuf = vb2_vmalloc_unmap_dmabuf,
	458	.attach_dmabuf = vb2_vmalloc_attach_dmabuf,
	459	.detach_dmabuf = vb2_vmalloc_detach_dmabuf,
3c18ff06 PO	460	.vaddr = vb2_vmalloc_vaddr,
	461	.mmap = vb2_vmalloc_mmap,
	462	.num_users = vb2_vmalloc_num_users,
	463	};
	464	EXPORT_SYMBOL_GPL(vb2_vmalloc_memops);
	465
	466	MODULE_DESCRIPTION("vmalloc memory handling routines for videobuf2");
95072084	467	MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
3c18ff06	468	MODULE_LICENSE("GPL");