[deliverable/linux.git] / drivers / gpu / drm / msm / msm_gem.c

/*
 * Copyright (C) 2013 Red Hat
 * Author: Rob Clark <robdclark@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/spinlock.h>
#include <linux/shmem_fs.h>
#include <linux/dma-buf.h>

#include "msm_drv.h"
#include "msm_gem.h"
#include "msm_gpu.h"
#include "msm_mmu.h"

static dma_addr_t physaddr(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	struct msm_drm_private *priv = obj->dev->dev_private;
	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
			priv->vram.paddr;
}

/* allocate pages from VRAM carveout, used when no IOMMU: */
static struct page **get_pages_vram(struct drm_gem_object *obj,
		int npages)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	struct msm_drm_private *priv = obj->dev->dev_private;
	dma_addr_t paddr;
	struct page **p;
	int ret, i;

	p = drm_malloc_ab(npages, sizeof(struct page *));
	if (!p)
		return ERR_PTR(-ENOMEM);

	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node,
			npages, 0, DRM_MM_SEARCH_DEFAULT);
	if (ret) {
		drm_free_large(p);
		return ERR_PTR(ret);
	}

	paddr = physaddr(obj);
	for (i = 0; i < npages; i++) {
		p[i] = phys_to_page(paddr);
		paddr += PAGE_SIZE;
	}

	return p;
}

/* called with dev->struct_mutex held */
static struct page **get_pages(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	if (!msm_obj->pages) {
		struct drm_device *dev = obj->dev;
		struct page **p;
		int npages = obj->size >> PAGE_SHIFT;

		if (iommu_present(&platform_bus_type))
			p = drm_gem_get_pages(obj, 0);
		else
			p = get_pages_vram(obj, npages);

		if (IS_ERR(p)) {
			dev_err(dev->dev, "could not get pages: %ld\n",
					PTR_ERR(p));
			return p;
		}

		msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
		if (IS_ERR(msm_obj->sgt)) {
			dev_err(dev->dev, "failed to allocate sgt\n");
			return ERR_CAST(msm_obj->sgt);
		}

		msm_obj->pages = p;

		/* For non-cached buffers, ensure the new pages are clean
		 * because display controller, GPU, etc. are not coherent:
		 */
		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
			dma_map_sg(dev->dev, msm_obj->sgt->sgl,
					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	}

	return msm_obj->pages;
}

static void put_pages(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	if (msm_obj->pages) {
		/* For non-cached buffers, ensure the new pages are clean
		 * because display controller, GPU, etc. are not coherent:
		 */
		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
			dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
		sg_free_table(msm_obj->sgt);
		kfree(msm_obj->sgt);

		if (iommu_present(&platform_bus_type))
			drm_gem_put_pages(obj, msm_obj->pages, true, false);
		else
			drm_mm_remove_node(msm_obj->vram_node);

		msm_obj->pages = NULL;
	}
}

struct page **msm_gem_get_pages(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct page **p;
	mutex_lock(&dev->struct_mutex);
	p = get_pages(obj);
	mutex_unlock(&dev->struct_mutex);
	return p;
}

void msm_gem_put_pages(struct drm_gem_object *obj)
{
	/* when we start tracking the pin count, then do something here */
}

int msm_gem_mmap_obj(struct drm_gem_object *obj,
		struct vm_area_struct *vma)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	vma->vm_flags &= ~VM_PFNMAP;
	vma->vm_flags |= VM_MIXEDMAP;

	if (msm_obj->flags & MSM_BO_WC) {
		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	} else if (msm_obj->flags & MSM_BO_UNCACHED) {
		vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
	} else {
		/*
		 * Shunt off cached objs to shmem file so they have their own
		 * address_space (so unmap_mapping_range does what we want,
		 * in particular in the case of mmap'd dmabufs)
		 */
		fput(vma->vm_file);
		get_file(obj->filp);
		vma->vm_pgoff = 0;
		vma->vm_file  = obj->filp;

		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
	}

	return 0;
}

int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
{
	int ret;

	ret = drm_gem_mmap(filp, vma);
	if (ret) {
		DBG("mmap failed: %d", ret);
		return ret;
	}

	return msm_gem_mmap_obj(vma->vm_private_data, vma);
}

int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct drm_gem_object *obj = vma->vm_private_data;
	struct drm_device *dev = obj->dev;
	struct page **pages;
	unsigned long pfn;
	pgoff_t pgoff;
	int ret;

	/* Make sure we don't parallel update on a fault, nor move or remove
	 * something from beneath our feet
	 */
	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		goto out;

	/* make sure we have pages attached now */
	pages = get_pages(obj);
	if (IS_ERR(pages)) {
		ret = PTR_ERR(pages);
		goto out_unlock;
	}

	/* We don't use vmf->pgoff since that has the fake offset: */
	pgoff = ((unsigned long)vmf->virtual_address -
			vma->vm_start) >> PAGE_SHIFT;

	pfn = page_to_pfn(pages[pgoff]);

	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
			pfn, pfn << PAGE_SHIFT);

	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);

out_unlock:
	mutex_unlock(&dev->struct_mutex);
out:
	switch (ret) {
	case -EAGAIN:
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
		return VM_FAULT_NOPAGE;
	case -ENOMEM:
		return VM_FAULT_OOM;
	default:
		return VM_FAULT_SIGBUS;
	}
}

/** get mmap offset */
static uint64_t mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	int ret;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	/* Make it mmapable */
	ret = drm_gem_create_mmap_offset(obj);

	if (ret) {
		dev_err(dev->dev, "could not allocate mmap offset\n");
		return 0;
	}

	return drm_vma_node_offset_addr(&obj->vma_node);
}

uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
{
	uint64_t offset;
	mutex_lock(&obj->dev->struct_mutex);
	offset = mmap_offset(obj);
	mutex_unlock(&obj->dev->struct_mutex);
	return offset;
}

/* should be called under struct_mutex.. although it can be called
 * from atomic context without struct_mutex to acquire an extra
 * iova ref if you know one is already held.
 *
 * That means when I do eventually need to add support for unpinning
 * the refcnt counter needs to be atomic_t.
 */
int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
		uint32_t *iova)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret = 0;

	if (!msm_obj->domain[id].iova) {
		struct msm_drm_private *priv = obj->dev->dev_private;
		struct msm_mmu *mmu = priv->mmus[id];
		struct page **pages = get_pages(obj);

		if (IS_ERR(pages))
			return PTR_ERR(pages);

		if (iommu_present(&platform_bus_type)) {
			uint32_t offset = (uint32_t)mmap_offset(obj);
			ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
					obj->size, IOMMU_READ | IOMMU_WRITE);
			msm_obj->domain[id].iova = offset;
		} else {
			msm_obj->domain[id].iova = physaddr(obj);
		}
	}

	if (!ret)
		*iova = msm_obj->domain[id].iova;

	return ret;
}

int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret;

	/* this is safe right now because we don't unmap until the
	 * bo is deleted:
	 */
	if (msm_obj->domain[id].iova) {
		*iova = msm_obj->domain[id].iova;
		return 0;
	}

	mutex_lock(&obj->dev->struct_mutex);
	ret = msm_gem_get_iova_locked(obj, id, iova);
	mutex_unlock(&obj->dev->struct_mutex);
	return ret;
}

void msm_gem_put_iova(struct drm_gem_object *obj, int id)
{
	// XXX TODO ..
	// NOTE: probably don't need a _locked() version.. we wouldn't
	// normally unmap here, but instead just mark that it could be
	// unmapped (if the iova refcnt drops to zero), but then later
	// if another _get_iova_locked() fails we can start unmapping
	// things that are no longer needed..
}

int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
		struct drm_mode_create_dumb *args)
{
	args->pitch = align_pitch(args->width, args->bpp);
	args->size  = PAGE_ALIGN(args->pitch * args->height);
	return msm_gem_new_handle(dev, file, args->size,
			MSM_BO_SCANOUT | MSM_BO_WC, &args->handle);
}

int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
		uint32_t handle, uint64_t *offset)
{
	struct drm_gem_object *obj;
	int ret = 0;

	/* GEM does all our handle to object mapping */
	obj = drm_gem_object_lookup(dev, file, handle);
	if (obj == NULL) {
		ret = -ENOENT;
		goto fail;
	}

	*offset = msm_gem_mmap_offset(obj);

	drm_gem_object_unreference_unlocked(obj);

fail:
	return ret;
}

void *msm_gem_vaddr_locked(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
	if (!msm_obj->vaddr) {
		struct page **pages = get_pages(obj);
		if (IS_ERR(pages))
			return ERR_CAST(pages);
		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
				VM_MAP, pgprot_writecombine(PAGE_KERNEL));
	}
	return msm_obj->vaddr;
}

void *msm_gem_vaddr(struct drm_gem_object *obj)
{
	void *ret;
	mutex_lock(&obj->dev->struct_mutex);
	ret = msm_gem_vaddr_locked(obj);
	mutex_unlock(&obj->dev->struct_mutex);
	return ret;
}

/* setup callback for when bo is no longer busy..
 * TODO probably want to differentiate read vs write..
 */
int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
		struct msm_fence_cb *cb)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret = 0;

	mutex_lock(&dev->struct_mutex);
	if (!list_empty(&cb->work.entry)) {
		ret = -EINVAL;
	} else if (is_active(msm_obj)) {
		cb->fence = max(msm_obj->read_fence, msm_obj->write_fence);
		list_add_tail(&cb->work.entry, &priv->fence_cbs);
	} else {
		queue_work(priv->wq, &cb->work);
	}
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

void msm_gem_move_to_active(struct drm_gem_object *obj,
		struct msm_gpu *gpu, bool write, uint32_t fence)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	msm_obj->gpu = gpu;
	if (write)
		msm_obj->write_fence = fence;
	else
		msm_obj->read_fence = fence;
	list_del_init(&msm_obj->mm_list);
	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
}

void msm_gem_move_to_inactive(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	msm_obj->gpu = NULL;
	msm_obj->read_fence = 0;
	msm_obj->write_fence = 0;
	list_del_init(&msm_obj->mm_list);
	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
}

int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op,
		struct timespec *timeout)
{
	struct drm_device *dev = obj->dev;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret = 0;

	if (is_active(msm_obj)) {
		uint32_t fence = 0;

		if (op & MSM_PREP_READ)
			fence = msm_obj->write_fence;
		if (op & MSM_PREP_WRITE)
			fence = max(fence, msm_obj->read_fence);
		if (op & MSM_PREP_NOSYNC)
			timeout = NULL;

		ret = msm_wait_fence_interruptable(dev, fence, timeout);
	}

	/* TODO cache maintenance */

	return ret;
}

int msm_gem_cpu_fini(struct drm_gem_object *obj)
{
	/* TODO cache maintenance */
	return 0;
}

#ifdef CONFIG_DEBUG_FS
void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
{
	struct drm_device *dev = obj->dev;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	uint64_t off = drm_vma_node_start(&obj->vma_node);

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	seq_printf(m, "%08x: %c(r=%u,w=%u) %2d (%2d) %08llx %p %d\n",
			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
			msm_obj->read_fence, msm_obj->write_fence,
			obj->name, obj->refcount.refcount.counter,
			off, msm_obj->vaddr, obj->size);
}

void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
{
	struct msm_gem_object *msm_obj;
	int count = 0;
	size_t size = 0;

	list_for_each_entry(msm_obj, list, mm_list) {
		struct drm_gem_object *obj = &msm_obj->base;
		seq_printf(m, "   ");
		msm_gem_describe(obj, m);
		count++;
		size += obj->size;
	}

	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
}
#endif

void msm_gem_free_object(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = obj->dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int id;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	/* object should not be on active list: */
	WARN_ON(is_active(msm_obj));

	list_del(&msm_obj->mm_list);

	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
		struct msm_mmu *mmu = priv->mmus[id];
		if (mmu && msm_obj->domain[id].iova) {
			uint32_t offset = (uint32_t)mmap_offset(obj);
			mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
		}
	}

	drm_gem_free_mmap_offset(obj);

	if (obj->import_attach) {
		if (msm_obj->vaddr)
			dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);

		/* Don't drop the pages for imported dmabuf, as they are not
		 * ours, just free the array we allocated:
		 */
		if (msm_obj->pages)
			drm_free_large(msm_obj->pages);

	} else {
		if (msm_obj->vaddr)
			vunmap(msm_obj->vaddr);
		put_pages(obj);
	}

	if (msm_obj->resv == &msm_obj->_resv)
		reservation_object_fini(msm_obj->resv);

	drm_gem_object_release(obj);

	kfree(msm_obj);
}

/* convenience method to construct a GEM buffer object, and userspace handle */
int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
		uint32_t size, uint32_t flags, uint32_t *handle)
{
	struct drm_gem_object *obj;
	int ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	obj = msm_gem_new(dev, size, flags);

	mutex_unlock(&dev->struct_mutex);

	if (IS_ERR(obj))
		return PTR_ERR(obj);

	ret = drm_gem_handle_create(file, obj, handle);

	/* drop reference from allocate - handle holds it now */
	drm_gem_object_unreference_unlocked(obj);

	return ret;
}

static int msm_gem_new_impl(struct drm_device *dev,
		uint32_t size, uint32_t flags,
		struct drm_gem_object **obj)
{
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj;
	unsigned sz;

	switch (flags & MSM_BO_CACHE_MASK) {
	case MSM_BO_UNCACHED:
	case MSM_BO_CACHED:
	case MSM_BO_WC:
		break;
	default:
		dev_err(dev->dev, "invalid cache flag: %x\n",
				(flags & MSM_BO_CACHE_MASK));
		return -EINVAL;
	}

	sz = sizeof(*msm_obj);
	if (!iommu_present(&platform_bus_type))
		sz += sizeof(struct drm_mm_node);

	msm_obj = kzalloc(sz, GFP_KERNEL);
	if (!msm_obj)
		return -ENOMEM;

	if (!iommu_present(&platform_bus_type))
		msm_obj->vram_node = (void *)&msm_obj[1];

	msm_obj->flags = flags;

	msm_obj->resv = &msm_obj->_resv;
	reservation_object_init(msm_obj->resv);

	INIT_LIST_HEAD(&msm_obj->submit_entry);
	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);

	*obj = &msm_obj->base;

	return 0;
}

struct drm_gem_object *msm_gem_new(struct drm_device *dev,
		uint32_t size, uint32_t flags)
{
	struct drm_gem_object *obj = NULL;
	int ret;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	size = PAGE_ALIGN(size);

	ret = msm_gem_new_impl(dev, size, flags, &obj);
	if (ret)
		goto fail;

	if (iommu_present(&platform_bus_type)) {
		ret = drm_gem_object_init(dev, obj, size);
		if (ret)
			goto fail;
	} else {
		drm_gem_private_object_init(dev, obj, size);
	}

	return obj;

fail:
	if (obj)
		drm_gem_object_unreference(obj);

	return ERR_PTR(ret);
}

struct drm_gem_object *msm_gem_import(struct drm_device *dev,
		uint32_t size, struct sg_table *sgt)
{
	struct msm_gem_object *msm_obj;
	struct drm_gem_object *obj;
	int ret, npages;

	/* if we don't have IOMMU, don't bother pretending we can import: */
	if (!iommu_present(&platform_bus_type)) {
		dev_err(dev->dev, "cannot import without IOMMU\n");
		return ERR_PTR(-EINVAL);
	}

	size = PAGE_ALIGN(size);

	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
	if (ret)
		goto fail;

	drm_gem_private_object_init(dev, obj, size);

	npages = size / PAGE_SIZE;

	msm_obj = to_msm_bo(obj);
	msm_obj->sgt = sgt;
	msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
	if (!msm_obj->pages) {
		ret = -ENOMEM;
		goto fail;
	}

	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
	if (ret)
		goto fail;

	return obj;

fail:
	if (obj)
		drm_gem_object_unreference_unlocked(obj);

	return ERR_PTR(ret);
}
Commit	Line	Data
c8afe684 RC	1	/*
	2	* Copyright (C) 2013 Red Hat
	3	* Author: Rob Clark <robdclark@gmail.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published by
	7	* the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/spinlock.h>
	19	#include <linux/shmem_fs.h>
05b84911	20	#include <linux/dma-buf.h>
c8afe684 RC	21
	22	#include "msm_drv.h"
	23	#include "msm_gem.h"
7198e6b0	24	#include "msm_gpu.h"
871d812a	25	#include "msm_mmu.h"
c8afe684	26
871d812a RC	27	static dma_addr_t physaddr(struct drm_gem_object *obj)
	28	{
	29	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	30	struct msm_drm_private *priv = obj->dev->dev_private;
	31	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
	32	priv->vram.paddr;
	33	}
	34
	35	/* allocate pages from VRAM carveout, used when no IOMMU: */
	36	static struct page *get_pages_vram(struct drm_gem_object obj,
	37	int npages)
	38	{
	39	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	40	struct msm_drm_private *priv = obj->dev->dev_private;
	41	dma_addr_t paddr;
	42	struct page **p;
	43	int ret, i;
	44
	45	p = drm_malloc_ab(npages, sizeof(struct page *));
	46	if (!p)
	47	return ERR_PTR(-ENOMEM);
	48
	49	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node,
	50	npages, 0, DRM_MM_SEARCH_DEFAULT);
	51	if (ret) {
	52	drm_free_large(p);
	53	return ERR_PTR(ret);
	54	}
	55
	56	paddr = physaddr(obj);
	57	for (i = 0; i < npages; i++) {
	58	p[i] = phys_to_page(paddr);
	59	paddr += PAGE_SIZE;
	60	}
	61
	62	return p;
	63	}
c8afe684 RC	64
	65	/* called with dev->struct_mutex held */
	66	static struct page *get_pages(struct drm_gem_object obj)
	67	{
	68	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	69
	70	if (!msm_obj->pages) {
	71	struct drm_device *dev = obj->dev;
871d812a	72	struct page **p;
c8afe684 RC	73	int npages = obj->size >> PAGE_SHIFT;
c8afe684 RC	74
871d812a RC	75	if (iommu_present(&platform_bus_type))
	76	p = drm_gem_get_pages(obj, 0);
	77	else
	78	p = get_pages_vram(obj, npages);
	79
c8afe684 RC	80	if (IS_ERR(p)) {
	81	dev_err(dev->dev, "could not get pages: %ld\n",
	82	PTR_ERR(p));
	83	return p;
	84	}
	85
	86	msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
1f70e079	87	if (IS_ERR(msm_obj->sgt)) {
c8afe684	88	dev_err(dev->dev, "failed to allocate sgt\n");
1f70e079	89	return ERR_CAST(msm_obj->sgt);
c8afe684 RC	90	}
	91
	92	msm_obj->pages = p;
	93
	94	/* For non-cached buffers, ensure the new pages are clean
	95	* because display controller, GPU, etc. are not coherent:
	96	*/
	97	if (msm_obj->flags & (MSM_BO_WC\|MSM_BO_UNCACHED))
	98	dma_map_sg(dev->dev, msm_obj->sgt->sgl,
	99	msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	100	}
	101
	102	return msm_obj->pages;
	103	}
	104
	105	static void put_pages(struct drm_gem_object *obj)
	106	{
	107	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	108
	109	if (msm_obj->pages) {
	110	/* For non-cached buffers, ensure the new pages are clean
	111	* because display controller, GPU, etc. are not coherent:
	112	*/
	113	if (msm_obj->flags & (MSM_BO_WC\|MSM_BO_UNCACHED))
	114	dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
	115	msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	116	sg_free_table(msm_obj->sgt);
	117	kfree(msm_obj->sgt);
	118
871d812a RC	119	if (iommu_present(&platform_bus_type))
	120	drm_gem_put_pages(obj, msm_obj->pages, true, false);
	121	else
	122	drm_mm_remove_node(msm_obj->vram_node);
	123
c8afe684 RC	124	msm_obj->pages = NULL;
	125	}
	126	}
	127
05b84911 RC	128	struct page *msm_gem_get_pages(struct drm_gem_object obj)
	129	{
	130	struct drm_device *dev = obj->dev;
	131	struct page **p;
	132	mutex_lock(&dev->struct_mutex);
	133	p = get_pages(obj);
	134	mutex_unlock(&dev->struct_mutex);
	135	return p;
	136	}
	137
	138	void msm_gem_put_pages(struct drm_gem_object *obj)
	139	{
	140	/* when we start tracking the pin count, then do something here */
	141	}
	142
c8afe684 RC	143	int msm_gem_mmap_obj(struct drm_gem_object *obj,
	144	struct vm_area_struct *vma)
	145	{
	146	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	147
	148	vma->vm_flags &= ~VM_PFNMAP;
	149	vma->vm_flags \|= VM_MIXEDMAP;
	150
	151	if (msm_obj->flags & MSM_BO_WC) {
	152	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	153	} else if (msm_obj->flags & MSM_BO_UNCACHED) {
	154	vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
	155	} else {
	156	/*
	157	* Shunt off cached objs to shmem file so they have their own
	158	* address_space (so unmap_mapping_range does what we want,
	159	* in particular in the case of mmap'd dmabufs)
	160	*/
	161	fput(vma->vm_file);
	162	get_file(obj->filp);
	163	vma->vm_pgoff = 0;
	164	vma->vm_file = obj->filp;
	165
	166	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
	167	}
	168
	169	return 0;
	170	}
	171
	172	int msm_gem_mmap(struct file filp, struct vm_area_struct vma)
	173	{
	174	int ret;
	175
	176	ret = drm_gem_mmap(filp, vma);
	177	if (ret) {
	178	DBG("mmap failed: %d", ret);
	179	return ret;
	180	}
	181
	182	return msm_gem_mmap_obj(vma->vm_private_data, vma);
	183	}
	184
	185	int msm_gem_fault(struct vm_area_struct vma, struct vm_fault vmf)
	186	{
	187	struct drm_gem_object *obj = vma->vm_private_data;
c8afe684 RC	188	struct drm_device *dev = obj->dev;
	189	struct page **pages;
	190	unsigned long pfn;
	191	pgoff_t pgoff;
	192	int ret;
	193
	194	/* Make sure we don't parallel update on a fault, nor move or remove
	195	* something from beneath our feet
	196	*/
	197	ret = mutex_lock_interruptible(&dev->struct_mutex);
	198	if (ret)
	199	goto out;
	200
	201	/* make sure we have pages attached now */
	202	pages = get_pages(obj);
	203	if (IS_ERR(pages)) {
	204	ret = PTR_ERR(pages);
	205	goto out_unlock;
	206	}
	207
	208	/* We don't use vmf->pgoff since that has the fake offset: */
	209	pgoff = ((unsigned long)vmf->virtual_address -
	210	vma->vm_start) >> PAGE_SHIFT;
	211
871d812a	212	pfn = page_to_pfn(pages[pgoff]);
c8afe684 RC	213
	214	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
	215	pfn, pfn << PAGE_SHIFT);
	216
	217	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
	218
	219	out_unlock:
	220	mutex_unlock(&dev->struct_mutex);
	221	out:
	222	switch (ret) {
	223	case -EAGAIN:
c8afe684 RC	224	case 0:
	225	case -ERESTARTSYS:
	226	case -EINTR:
505886d5 RC	227	case -EBUSY:
	228	/*
	229	* EBUSY is ok: this just means that another thread
	230	* already did the job.
	231	*/
c8afe684 RC	232	return VM_FAULT_NOPAGE;
	233	case -ENOMEM:
	234	return VM_FAULT_OOM;
	235	default:
	236	return VM_FAULT_SIGBUS;
	237	}
	238	}
	239
	240	/** get mmap offset */
	241	static uint64_t mmap_offset(struct drm_gem_object *obj)
	242	{
	243	struct drm_device *dev = obj->dev;
	244	int ret;
	245
	246	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	247
	248	/* Make it mmapable */
	249	ret = drm_gem_create_mmap_offset(obj);
	250
	251	if (ret) {
	252	dev_err(dev->dev, "could not allocate mmap offset\n");
	253	return 0;
	254	}
	255
	256	return drm_vma_node_offset_addr(&obj->vma_node);
	257	}
	258
	259	uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
	260	{
	261	uint64_t offset;
	262	mutex_lock(&obj->dev->struct_mutex);
	263	offset = mmap_offset(obj);
	264	mutex_unlock(&obj->dev->struct_mutex);
	265	return offset;
	266	}
	267
c8afe684 RC	268	/* should be called under struct_mutex.. although it can be called
	269	* from atomic context without struct_mutex to acquire an extra
	270	* iova ref if you know one is already held.
	271	*
	272	* That means when I do eventually need to add support for unpinning
	273	* the refcnt counter needs to be atomic_t.
	274	*/
	275	int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
	276	uint32_t *iova)
	277	{
	278	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	279	int ret = 0;
	280
	281	if (!msm_obj->domain[id].iova) {
	282	struct msm_drm_private *priv = obj->dev->dev_private;
871d812a RC	283	struct msm_mmu *mmu = priv->mmus[id];
	284	struct page **pages = get_pages(obj);
	285
c8afe684 RC	286	if (IS_ERR(pages))
c8afe684 RC	287	return PTR_ERR(pages);
871d812a RC	288
	289	if (iommu_present(&platform_bus_type)) {
	290	uint32_t offset = (uint32_t)mmap_offset(obj);
	291	ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
	292	obj->size, IOMMU_READ \| IOMMU_WRITE);
	293	msm_obj->domain[id].iova = offset;
	294	} else {
	295	msm_obj->domain[id].iova = physaddr(obj);
	296	}
c8afe684 RC	297	}
	298
	299	if (!ret)
	300	*iova = msm_obj->domain[id].iova;
	301
	302	return ret;
	303	}
	304
	305	int msm_gem_get_iova(struct drm_gem_object obj, int id, uint32_t iova)
	306	{
edd4fc63	307	struct msm_gem_object *msm_obj = to_msm_bo(obj);
c8afe684	308	int ret;
edd4fc63 RC	309
	310	/* this is safe right now because we don't unmap until the
	311	* bo is deleted:
	312	*/
	313	if (msm_obj->domain[id].iova) {
	314	*iova = msm_obj->domain[id].iova;
	315	return 0;
	316	}
	317
c8afe684 RC	318	mutex_lock(&obj->dev->struct_mutex);
	319	ret = msm_gem_get_iova_locked(obj, id, iova);
	320	mutex_unlock(&obj->dev->struct_mutex);
	321	return ret;
	322	}
	323
	324	void msm_gem_put_iova(struct drm_gem_object *obj, int id)
	325	{
	326	// XXX TODO ..
	327	// NOTE: probably don't need a _locked() version.. we wouldn't
	328	// normally unmap here, but instead just mark that it could be
	329	// unmapped (if the iova refcnt drops to zero), but then later
	330	// if another _get_iova_locked() fails we can start unmapping
	331	// things that are no longer needed..
	332	}
	333
	334	int msm_gem_dumb_create(struct drm_file file, struct drm_device dev,
	335	struct drm_mode_create_dumb *args)
	336	{
	337	args->pitch = align_pitch(args->width, args->bpp);
	338	args->size = PAGE_ALIGN(args->pitch * args->height);
	339	return msm_gem_new_handle(dev, file, args->size,
	340	MSM_BO_SCANOUT \| MSM_BO_WC, &args->handle);
	341	}
	342
c8afe684 RC	343	int msm_gem_dumb_map_offset(struct drm_file file, struct drm_device dev,
	344	uint32_t handle, uint64_t *offset)
	345	{
	346	struct drm_gem_object *obj;
	347	int ret = 0;
	348
	349	/* GEM does all our handle to object mapping */
	350	obj = drm_gem_object_lookup(dev, file, handle);
	351	if (obj == NULL) {
	352	ret = -ENOENT;
	353	goto fail;
	354	}
	355
	356	*offset = msm_gem_mmap_offset(obj);
	357
	358	drm_gem_object_unreference_unlocked(obj);
	359
	360	fail:
	361	return ret;
	362	}
	363
	364	void msm_gem_vaddr_locked(struct drm_gem_object obj)
	365	{
	366	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	367	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
	368	if (!msm_obj->vaddr) {
	369	struct page **pages = get_pages(obj);
	370	if (IS_ERR(pages))
	371	return ERR_CAST(pages);
	372	msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
	373	VM_MAP, pgprot_writecombine(PAGE_KERNEL));
	374	}
	375	return msm_obj->vaddr;
	376	}
	377
	378	void msm_gem_vaddr(struct drm_gem_object obj)
	379	{
	380	void *ret;
	381	mutex_lock(&obj->dev->struct_mutex);
	382	ret = msm_gem_vaddr_locked(obj);
	383	mutex_unlock(&obj->dev->struct_mutex);
	384	return ret;
	385	}
	386
edd4fc63 RC	387	/* setup callback for when bo is no longer busy..
	388	* TODO probably want to differentiate read vs write..
	389	*/
	390	int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
	391	struct msm_fence_cb *cb)
c8afe684 RC	392	{
	393	struct drm_device *dev = obj->dev;
	394	struct msm_drm_private *priv = dev->dev_private;
7198e6b0 RC	395	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	396	int ret = 0;
	397
	398	mutex_lock(&dev->struct_mutex);
edd4fc63	399	if (!list_empty(&cb->work.entry)) {
7198e6b0 RC	400	ret = -EINVAL;
7198e6b0 RC	401	} else if (is_active(msm_obj)) {
edd4fc63 RC	402	cb->fence = max(msm_obj->read_fence, msm_obj->write_fence);
edd4fc63 RC	403	list_add_tail(&cb->work.entry, &priv->fence_cbs);
7198e6b0	404	} else {
edd4fc63	405	queue_work(priv->wq, &cb->work);
7198e6b0 RC	406	}
	407	mutex_unlock(&dev->struct_mutex);
	408
	409	return ret;
	410	}
	411
	412	void msm_gem_move_to_active(struct drm_gem_object *obj,
bf6811f3	413	struct msm_gpu *gpu, bool write, uint32_t fence)
7198e6b0 RC	414	{
	415	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	416	msm_obj->gpu = gpu;
bf6811f3 RC	417	if (write)
	418	msm_obj->write_fence = fence;
	419	else
	420	msm_obj->read_fence = fence;
7198e6b0 RC	421	list_del_init(&msm_obj->mm_list);
	422	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
	423	}
	424
	425	void msm_gem_move_to_inactive(struct drm_gem_object *obj)
	426	{
	427	struct drm_device *dev = obj->dev;
	428	struct msm_drm_private *priv = dev->dev_private;
	429	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	430
	431	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	432
	433	msm_obj->gpu = NULL;
bf6811f3 RC	434	msm_obj->read_fence = 0;
bf6811f3 RC	435	msm_obj->write_fence = 0;
7198e6b0 RC	436	list_del_init(&msm_obj->mm_list);
7198e6b0 RC	437	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
7198e6b0 RC	438	}
	439
	440	int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op,
	441	struct timespec *timeout)
	442	{
	443	struct drm_device *dev = obj->dev;
	444	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	445	int ret = 0;
	446
f816f272	447	if (is_active(msm_obj)) {
bf6811f3	448	uint32_t fence = 0;
f816f272	449
bf6811f3 RC	450	if (op & MSM_PREP_READ)
	451	fence = msm_obj->write_fence;
	452	if (op & MSM_PREP_WRITE)
	453	fence = max(fence, msm_obj->read_fence);
f816f272 RC	454	if (op & MSM_PREP_NOSYNC)
	455	timeout = NULL;
	456
bf6811f3 RC	457	ret = msm_wait_fence_interruptable(dev, fence, timeout);
bf6811f3 RC	458	}
7198e6b0 RC	459
7198e6b0 RC	460	/* TODO cache maintenance */
c8afe684	461
7198e6b0 RC	462	return ret;
7198e6b0 RC	463	}
c8afe684	464
7198e6b0 RC	465	int msm_gem_cpu_fini(struct drm_gem_object *obj)
	466	{
	467	/* TODO cache maintenance */
c8afe684 RC	468	return 0;
	469	}
	470
	471	#ifdef CONFIG_DEBUG_FS
	472	void msm_gem_describe(struct drm_gem_object obj, struct seq_file m)
	473	{
	474	struct drm_device *dev = obj->dev;
	475	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	476	uint64_t off = drm_vma_node_start(&obj->vma_node);
	477
	478	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
bf6811f3	479	seq_printf(m, "%08x: %c(r=%u,w=%u) %2d (%2d) %08llx %p %d\n",
7198e6b0	480	msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
bf6811f3 RC	481	msm_obj->read_fence, msm_obj->write_fence,
bf6811f3 RC	482	obj->name, obj->refcount.refcount.counter,
c8afe684 RC	483	off, msm_obj->vaddr, obj->size);
	484	}
	485
	486	void msm_gem_describe_objects(struct list_head list, struct seq_file m)
	487	{
	488	struct msm_gem_object *msm_obj;
	489	int count = 0;
	490	size_t size = 0;
	491
	492	list_for_each_entry(msm_obj, list, mm_list) {
	493	struct drm_gem_object *obj = &msm_obj->base;
	494	seq_printf(m, " ");
	495	msm_gem_describe(obj, m);
	496	count++;
	497	size += obj->size;
	498	}
	499
	500	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
	501	}
	502	#endif
	503
	504	void msm_gem_free_object(struct drm_gem_object *obj)
	505	{
	506	struct drm_device *dev = obj->dev;
871d812a	507	struct msm_drm_private *priv = obj->dev->dev_private;
c8afe684 RC	508	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	509	int id;
	510
	511	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	512
7198e6b0 RC	513	/* object should not be on active list: */
	514	WARN_ON(is_active(msm_obj));
	515
c8afe684 RC	516	list_del(&msm_obj->mm_list);
	517
	518	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
871d812a RC	519	struct msm_mmu *mmu = priv->mmus[id];
871d812a RC	520	if (mmu && msm_obj->domain[id].iova) {
c8afe684	521	uint32_t offset = (uint32_t)mmap_offset(obj);
871d812a	522	mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
c8afe684 RC	523	}
	524	}
	525
	526	drm_gem_free_mmap_offset(obj);
	527
05b84911 RC	528	if (obj->import_attach) {
	529	if (msm_obj->vaddr)
	530	dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
	531
	532	/* Don't drop the pages for imported dmabuf, as they are not
	533	* ours, just free the array we allocated:
	534	*/
	535	if (msm_obj->pages)
	536	drm_free_large(msm_obj->pages);
c8afe684	537
05b84911 RC	538	} else {
	539	if (msm_obj->vaddr)
	540	vunmap(msm_obj->vaddr);
	541	put_pages(obj);
	542	}
c8afe684	543
7198e6b0 RC	544	if (msm_obj->resv == &msm_obj->_resv)
	545	reservation_object_fini(msm_obj->resv);
	546
c8afe684 RC	547	drm_gem_object_release(obj);
	548
	549	kfree(msm_obj);
	550	}
	551
	552	/* convenience method to construct a GEM buffer object, and userspace handle */
	553	int msm_gem_new_handle(struct drm_device dev, struct drm_file file,
	554	uint32_t size, uint32_t flags, uint32_t *handle)
	555	{
	556	struct drm_gem_object *obj;
	557	int ret;
	558
	559	ret = mutex_lock_interruptible(&dev->struct_mutex);
	560	if (ret)
	561	return ret;
	562
	563	obj = msm_gem_new(dev, size, flags);
	564
	565	mutex_unlock(&dev->struct_mutex);
	566
	567	if (IS_ERR(obj))
	568	return PTR_ERR(obj);
	569
	570	ret = drm_gem_handle_create(file, obj, handle);
	571
	572	/* drop reference from allocate - handle holds it now */
	573	drm_gem_object_unreference_unlocked(obj);
	574
	575	return ret;
	576	}
	577
05b84911 RC	578	static int msm_gem_new_impl(struct drm_device *dev,
	579	uint32_t size, uint32_t flags,
	580	struct drm_gem_object **obj)
c8afe684 RC	581	{
	582	struct msm_drm_private *priv = dev->dev_private;
	583	struct msm_gem_object *msm_obj;
871d812a	584	unsigned sz;
c8afe684 RC	585
	586	switch (flags & MSM_BO_CACHE_MASK) {
	587	case MSM_BO_UNCACHED:
	588	case MSM_BO_CACHED:
	589	case MSM_BO_WC:
	590	break;
	591	default:
	592	dev_err(dev->dev, "invalid cache flag: %x\n",
	593	(flags & MSM_BO_CACHE_MASK));
05b84911	594	return -EINVAL;
c8afe684 RC	595	}
c8afe684 RC	596
871d812a RC	597	sz = sizeof(*msm_obj);
	598	if (!iommu_present(&platform_bus_type))
	599	sz += sizeof(struct drm_mm_node);
	600
	601	msm_obj = kzalloc(sz, GFP_KERNEL);
05b84911 RC	602	if (!msm_obj)
05b84911 RC	603	return -ENOMEM;
c8afe684	604
871d812a RC	605	if (!iommu_present(&platform_bus_type))
	606	msm_obj->vram_node = (void *)&msm_obj[1];
	607
c8afe684 RC	608	msm_obj->flags = flags;
c8afe684 RC	609
7198e6b0 RC	610	msm_obj->resv = &msm_obj->_resv;
7198e6b0 RC	611	reservation_object_init(msm_obj->resv);
c8afe684	612
7198e6b0	613	INIT_LIST_HEAD(&msm_obj->submit_entry);
c8afe684 RC	614	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
c8afe684 RC	615
05b84911 RC	616	*obj = &msm_obj->base;
	617
	618	return 0;
	619	}
	620
	621	struct drm_gem_object msm_gem_new(struct drm_device dev,
	622	uint32_t size, uint32_t flags)
	623	{
871d812a	624	struct drm_gem_object *obj = NULL;
05b84911 RC	625	int ret;
	626
	627	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	628
	629	size = PAGE_ALIGN(size);
	630
	631	ret = msm_gem_new_impl(dev, size, flags, &obj);
	632	if (ret)
	633	goto fail;
	634
871d812a RC	635	if (iommu_present(&platform_bus_type)) {
	636	ret = drm_gem_object_init(dev, obj, size);
	637	if (ret)
	638	goto fail;
	639	} else {
	640	drm_gem_private_object_init(dev, obj, size);
	641	}
05b84911 RC	642
	643	return obj;
	644
	645	fail:
	646	if (obj)
9999f105	647	drm_gem_object_unreference(obj);
05b84911 RC	648
	649	return ERR_PTR(ret);
	650	}
	651
	652	struct drm_gem_object msm_gem_import(struct drm_device dev,
	653	uint32_t size, struct sg_table *sgt)
	654	{
	655	struct msm_gem_object *msm_obj;
	656	struct drm_gem_object *obj;
	657	int ret, npages;
	658
871d812a RC	659	/* if we don't have IOMMU, don't bother pretending we can import: */
	660	if (!iommu_present(&platform_bus_type)) {
	661	dev_err(dev->dev, "cannot import without IOMMU\n");
	662	return ERR_PTR(-EINVAL);
	663	}
	664
05b84911 RC	665	size = PAGE_ALIGN(size);
	666
	667	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
	668	if (ret)
	669	goto fail;
	670
	671	drm_gem_private_object_init(dev, obj, size);
	672
	673	npages = size / PAGE_SIZE;
	674
	675	msm_obj = to_msm_bo(obj);
	676	msm_obj->sgt = sgt;
	677	msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
	678	if (!msm_obj->pages) {
	679	ret = -ENOMEM;
	680	goto fail;
	681	}
	682
	683	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
	684	if (ret)
	685	goto fail;
	686
c8afe684 RC	687	return obj;
	688
	689	fail:
	690	if (obj)
	691	drm_gem_object_unreference_unlocked(obj);
	692
	693	return ERR_PTR(ret);
	694	}