[deliverable/linux.git] / drivers / gpu / drm / msm / msm_gem.c

/*
 * Copyright (C) 2013 Red Hat
 * Author: Rob Clark <robdclark@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/spinlock.h>
#include <linux/shmem_fs.h>
#include <linux/dma-buf.h>

#include "msm_drv.h"
#include "msm_gem.h"
#include "msm_gpu.h"
#include "msm_mmu.h"

static dma_addr_t physaddr(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	struct msm_drm_private *priv = obj->dev->dev_private;
	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
			priv->vram.paddr;
}

/* allocate pages from VRAM carveout, used when no IOMMU: */
static struct page **get_pages_vram(struct drm_gem_object *obj,
		int npages)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	struct msm_drm_private *priv = obj->dev->dev_private;
	dma_addr_t paddr;
	struct page **p;
	int ret, i;

	p = drm_malloc_ab(npages, sizeof(struct page *));
	if (!p)
		return ERR_PTR(-ENOMEM);

	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node,
			npages, 0, DRM_MM_SEARCH_DEFAULT);
	if (ret) {
		drm_free_large(p);
		return ERR_PTR(ret);
	}

	paddr = physaddr(obj);
	for (i = 0; i < npages; i++) {
		p[i] = phys_to_page(paddr);
		paddr += PAGE_SIZE;
	}

	return p;
}

/* called with dev->struct_mutex held */
static struct page **get_pages(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	if (!msm_obj->pages) {
		struct drm_device *dev = obj->dev;
		struct page **p;
		int npages = obj->size >> PAGE_SHIFT;

		if (iommu_present(&platform_bus_type))
			p = drm_gem_get_pages(obj, 0);
		else
			p = get_pages_vram(obj, npages);

		if (IS_ERR(p)) {
			dev_err(dev->dev, "could not get pages: %ld\n",
					PTR_ERR(p));
			return p;
		}

		msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
		if (IS_ERR(msm_obj->sgt)) {
			dev_err(dev->dev, "failed to allocate sgt\n");
			return ERR_CAST(msm_obj->sgt);
		}

		msm_obj->pages = p;

		/* For non-cached buffers, ensure the new pages are clean
		 * because display controller, GPU, etc. are not coherent:
		 */
		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
			dma_map_sg(dev->dev, msm_obj->sgt->sgl,
					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	}

	return msm_obj->pages;
}

static void put_pages(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	if (msm_obj->pages) {
		/* For non-cached buffers, ensure the new pages are clean
		 * because display controller, GPU, etc. are not coherent:
		 */
		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
			dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
		sg_free_table(msm_obj->sgt);
		kfree(msm_obj->sgt);

		if (iommu_present(&platform_bus_type))
			drm_gem_put_pages(obj, msm_obj->pages, true, false);
		else {
			drm_mm_remove_node(msm_obj->vram_node);
			drm_free_large(msm_obj->pages);
		}

		msm_obj->pages = NULL;
	}
}

struct page **msm_gem_get_pages(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct page **p;
	mutex_lock(&dev->struct_mutex);
	p = get_pages(obj);
	mutex_unlock(&dev->struct_mutex);
	return p;
}

void msm_gem_put_pages(struct drm_gem_object *obj)
{
	/* when we start tracking the pin count, then do something here */
}

int msm_gem_mmap_obj(struct drm_gem_object *obj,
		struct vm_area_struct *vma)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	vma->vm_flags &= ~VM_PFNMAP;
	vma->vm_flags |= VM_MIXEDMAP;

	if (msm_obj->flags & MSM_BO_WC) {
		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	} else if (msm_obj->flags & MSM_BO_UNCACHED) {
		vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
	} else {
		/*
		 * Shunt off cached objs to shmem file so they have their own
		 * address_space (so unmap_mapping_range does what we want,
		 * in particular in the case of mmap'd dmabufs)
		 */
		fput(vma->vm_file);
		get_file(obj->filp);
		vma->vm_pgoff = 0;
		vma->vm_file  = obj->filp;

		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
	}

	return 0;
}

int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
{
	int ret;

	ret = drm_gem_mmap(filp, vma);
	if (ret) {
		DBG("mmap failed: %d", ret);
		return ret;
	}

	return msm_gem_mmap_obj(vma->vm_private_data, vma);
}

int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct drm_gem_object *obj = vma->vm_private_data;
	struct drm_device *dev = obj->dev;
	struct page **pages;
	unsigned long pfn;
	pgoff_t pgoff;
	int ret;

	/* Make sure we don't parallel update on a fault, nor move or remove
	 * something from beneath our feet
	 */
	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		goto out;

	/* make sure we have pages attached now */
	pages = get_pages(obj);
	if (IS_ERR(pages)) {
		ret = PTR_ERR(pages);
		goto out_unlock;
	}

	/* We don't use vmf->pgoff since that has the fake offset: */
	pgoff = ((unsigned long)vmf->virtual_address -
			vma->vm_start) >> PAGE_SHIFT;

	pfn = page_to_pfn(pages[pgoff]);

	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
			pfn, pfn << PAGE_SHIFT);

	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);

out_unlock:
	mutex_unlock(&dev->struct_mutex);
out:
	switch (ret) {
	case -EAGAIN:
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
		return VM_FAULT_NOPAGE;
	case -ENOMEM:
		return VM_FAULT_OOM;
	default:
		return VM_FAULT_SIGBUS;
	}
}

/** get mmap offset */
static uint64_t mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	int ret;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	/* Make it mmapable */
	ret = drm_gem_create_mmap_offset(obj);

	if (ret) {
		dev_err(dev->dev, "could not allocate mmap offset\n");
		return 0;
	}

	return drm_vma_node_offset_addr(&obj->vma_node);
}

uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
{
	uint64_t offset;
	mutex_lock(&obj->dev->struct_mutex);
	offset = mmap_offset(obj);
	mutex_unlock(&obj->dev->struct_mutex);
	return offset;
}

/* should be called under struct_mutex.. although it can be called
 * from atomic context without struct_mutex to acquire an extra
 * iova ref if you know one is already held.
 *
 * That means when I do eventually need to add support for unpinning
 * the refcnt counter needs to be atomic_t.
 */
int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
		uint32_t *iova)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	struct drm_device *dev = obj->dev;
	int ret = 0;

	if (!msm_obj->domain[id].iova) {
		struct msm_drm_private *priv = obj->dev->dev_private;
		struct msm_mmu *mmu = priv->mmus[id];
		struct page **pages = get_pages(obj);

		if (!mmu) {
			dev_err(dev->dev, "null MMU pointer\n");
			return -EINVAL;
		}

		if (IS_ERR(pages))
			return PTR_ERR(pages);

		if (iommu_present(&platform_bus_type)) {
			uint32_t offset = (uint32_t)mmap_offset(obj);
			ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
					obj->size, IOMMU_READ | IOMMU_WRITE);
			msm_obj->domain[id].iova = offset;
		} else {
			msm_obj->domain[id].iova = physaddr(obj);
		}
	}

	if (!ret)
		*iova = msm_obj->domain[id].iova;

	return ret;
}

int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret;

	/* this is safe right now because we don't unmap until the
	 * bo is deleted:
	 */
	if (msm_obj->domain[id].iova) {
		*iova = msm_obj->domain[id].iova;
		return 0;
	}

	mutex_lock(&obj->dev->struct_mutex);
	ret = msm_gem_get_iova_locked(obj, id, iova);
	mutex_unlock(&obj->dev->struct_mutex);
	return ret;
}

void msm_gem_put_iova(struct drm_gem_object *obj, int id)
{
	// XXX TODO ..
	// NOTE: probably don't need a _locked() version.. we wouldn't
	// normally unmap here, but instead just mark that it could be
	// unmapped (if the iova refcnt drops to zero), but then later
	// if another _get_iova_locked() fails we can start unmapping
	// things that are no longer needed..
}

int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
		struct drm_mode_create_dumb *args)
{
	args->pitch = align_pitch(args->width, args->bpp);
	args->size  = PAGE_ALIGN(args->pitch * args->height);
	return msm_gem_new_handle(dev, file, args->size,
			MSM_BO_SCANOUT | MSM_BO_WC, &args->handle);
}

int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
		uint32_t handle, uint64_t *offset)
{
	struct drm_gem_object *obj;
	int ret = 0;

	/* GEM does all our handle to object mapping */
	obj = drm_gem_object_lookup(dev, file, handle);
	if (obj == NULL) {
		ret = -ENOENT;
		goto fail;
	}

	*offset = msm_gem_mmap_offset(obj);

	drm_gem_object_unreference_unlocked(obj);

fail:
	return ret;
}

void *msm_gem_vaddr_locked(struct drm_gem_object *obj)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
	if (!msm_obj->vaddr) {
		struct page **pages = get_pages(obj);
		if (IS_ERR(pages))
			return ERR_CAST(pages);
		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
				VM_MAP, pgprot_writecombine(PAGE_KERNEL));
	}
	return msm_obj->vaddr;
}

void *msm_gem_vaddr(struct drm_gem_object *obj)
{
	void *ret;
	mutex_lock(&obj->dev->struct_mutex);
	ret = msm_gem_vaddr_locked(obj);
	mutex_unlock(&obj->dev->struct_mutex);
	return ret;
}

/* setup callback for when bo is no longer busy..
 * TODO probably want to differentiate read vs write..
 */
int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
		struct msm_fence_cb *cb)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret = 0;

	mutex_lock(&dev->struct_mutex);
	if (!list_empty(&cb->work.entry)) {
		ret = -EINVAL;
	} else if (is_active(msm_obj)) {
		cb->fence = max(msm_obj->read_fence, msm_obj->write_fence);
		list_add_tail(&cb->work.entry, &priv->fence_cbs);
	} else {
		queue_work(priv->wq, &cb->work);
	}
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

void msm_gem_move_to_active(struct drm_gem_object *obj,
		struct msm_gpu *gpu, bool write, uint32_t fence)
{
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	msm_obj->gpu = gpu;
	if (write)
		msm_obj->write_fence = fence;
	else
		msm_obj->read_fence = fence;
	list_del_init(&msm_obj->mm_list);
	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
}

void msm_gem_move_to_inactive(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	msm_obj->gpu = NULL;
	msm_obj->read_fence = 0;
	msm_obj->write_fence = 0;
	list_del_init(&msm_obj->mm_list);
	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
}

int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op,
		struct timespec *timeout)
{
	struct drm_device *dev = obj->dev;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int ret = 0;

	if (is_active(msm_obj)) {
		uint32_t fence = 0;

		if (op & MSM_PREP_READ)
			fence = msm_obj->write_fence;
		if (op & MSM_PREP_WRITE)
			fence = max(fence, msm_obj->read_fence);
		if (op & MSM_PREP_NOSYNC)
			timeout = NULL;

		ret = msm_wait_fence_interruptable(dev, fence, timeout);
	}

	/* TODO cache maintenance */

	return ret;
}

int msm_gem_cpu_fini(struct drm_gem_object *obj)
{
	/* TODO cache maintenance */
	return 0;
}

#ifdef CONFIG_DEBUG_FS
void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
{
	struct drm_device *dev = obj->dev;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	uint64_t off = drm_vma_node_start(&obj->vma_node);

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	seq_printf(m, "%08x: %c(r=%u,w=%u) %2d (%2d) %08llx %p %d\n",
			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
			msm_obj->read_fence, msm_obj->write_fence,
			obj->name, obj->refcount.refcount.counter,
			off, msm_obj->vaddr, obj->size);
}

void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
{
	struct msm_gem_object *msm_obj;
	int count = 0;
	size_t size = 0;

	list_for_each_entry(msm_obj, list, mm_list) {
		struct drm_gem_object *obj = &msm_obj->base;
		seq_printf(m, "   ");
		msm_gem_describe(obj, m);
		count++;
		size += obj->size;
	}

	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
}
#endif

void msm_gem_free_object(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct msm_drm_private *priv = obj->dev->dev_private;
	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	int id;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	/* object should not be on active list: */
	WARN_ON(is_active(msm_obj));

	list_del(&msm_obj->mm_list);

	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
		struct msm_mmu *mmu = priv->mmus[id];
		if (mmu && msm_obj->domain[id].iova) {
			uint32_t offset = (uint32_t)mmap_offset(obj);
			mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
		}
	}

	drm_gem_free_mmap_offset(obj);

	if (obj->import_attach) {
		if (msm_obj->vaddr)
			dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);

		/* Don't drop the pages for imported dmabuf, as they are not
		 * ours, just free the array we allocated:
		 */
		if (msm_obj->pages)
			drm_free_large(msm_obj->pages);

	} else {
		if (msm_obj->vaddr)
			vunmap(msm_obj->vaddr);
		put_pages(obj);
	}

	if (msm_obj->resv == &msm_obj->_resv)
		reservation_object_fini(msm_obj->resv);

	drm_gem_object_release(obj);

	kfree(msm_obj);
}

/* convenience method to construct a GEM buffer object, and userspace handle */
int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
		uint32_t size, uint32_t flags, uint32_t *handle)
{
	struct drm_gem_object *obj;
	int ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	obj = msm_gem_new(dev, size, flags);

	mutex_unlock(&dev->struct_mutex);

	if (IS_ERR(obj))
		return PTR_ERR(obj);

	ret = drm_gem_handle_create(file, obj, handle);

	/* drop reference from allocate - handle holds it now */
	drm_gem_object_unreference_unlocked(obj);

	return ret;
}

static int msm_gem_new_impl(struct drm_device *dev,
		uint32_t size, uint32_t flags,
		struct drm_gem_object **obj)
{
	struct msm_drm_private *priv = dev->dev_private;
	struct msm_gem_object *msm_obj;
	unsigned sz;

	switch (flags & MSM_BO_CACHE_MASK) {
	case MSM_BO_UNCACHED:
	case MSM_BO_CACHED:
	case MSM_BO_WC:
		break;
	default:
		dev_err(dev->dev, "invalid cache flag: %x\n",
				(flags & MSM_BO_CACHE_MASK));
		return -EINVAL;
	}

	sz = sizeof(*msm_obj);
	if (!iommu_present(&platform_bus_type))
		sz += sizeof(struct drm_mm_node);

	msm_obj = kzalloc(sz, GFP_KERNEL);
	if (!msm_obj)
		return -ENOMEM;

	if (!iommu_present(&platform_bus_type))
		msm_obj->vram_node = (void *)&msm_obj[1];

	msm_obj->flags = flags;

	msm_obj->resv = &msm_obj->_resv;
	reservation_object_init(msm_obj->resv);

	INIT_LIST_HEAD(&msm_obj->submit_entry);
	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);

	*obj = &msm_obj->base;

	return 0;
}

struct drm_gem_object *msm_gem_new(struct drm_device *dev,
		uint32_t size, uint32_t flags)
{
	struct drm_gem_object *obj = NULL;
	int ret;

	WARN_ON(!mutex_is_locked(&dev->struct_mutex));

	size = PAGE_ALIGN(size);

	ret = msm_gem_new_impl(dev, size, flags, &obj);
	if (ret)
		goto fail;

	if (iommu_present(&platform_bus_type)) {
		ret = drm_gem_object_init(dev, obj, size);
		if (ret)
			goto fail;
	} else {
		drm_gem_private_object_init(dev, obj, size);
	}

	return obj;

fail:
	if (obj)
		drm_gem_object_unreference(obj);

	return ERR_PTR(ret);
}

struct drm_gem_object *msm_gem_import(struct drm_device *dev,
		uint32_t size, struct sg_table *sgt)
{
	struct msm_gem_object *msm_obj;
	struct drm_gem_object *obj;
	int ret, npages;

	/* if we don't have IOMMU, don't bother pretending we can import: */
	if (!iommu_present(&platform_bus_type)) {
		dev_err(dev->dev, "cannot import without IOMMU\n");
		return ERR_PTR(-EINVAL);
	}

	size = PAGE_ALIGN(size);

	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
	if (ret)
		goto fail;

	drm_gem_private_object_init(dev, obj, size);

	npages = size / PAGE_SIZE;

	msm_obj = to_msm_bo(obj);
	msm_obj->sgt = sgt;
	msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
	if (!msm_obj->pages) {
		ret = -ENOMEM;
		goto fail;
	}

	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
	if (ret)
		goto fail;

	return obj;

fail:
	if (obj)
		drm_gem_object_unreference_unlocked(obj);

	return ERR_PTR(ret);
}
Commit	Line	Data
c8afe684 RC	1	/*
	2	* Copyright (C) 2013 Red Hat
	3	* Author: Rob Clark <robdclark@gmail.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published by
	7	* the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/spinlock.h>
	19	#include <linux/shmem_fs.h>
05b84911	20	#include <linux/dma-buf.h>
c8afe684 RC	21
	22	#include "msm_drv.h"
	23	#include "msm_gem.h"
7198e6b0	24	#include "msm_gpu.h"
871d812a	25	#include "msm_mmu.h"
c8afe684	26
871d812a RC	27	static dma_addr_t physaddr(struct drm_gem_object *obj)
	28	{
	29	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	30	struct msm_drm_private *priv = obj->dev->dev_private;
	31	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
	32	priv->vram.paddr;
	33	}
	34
	35	/* allocate pages from VRAM carveout, used when no IOMMU: */
	36	static struct page *get_pages_vram(struct drm_gem_object obj,
	37	int npages)
	38	{
	39	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	40	struct msm_drm_private *priv = obj->dev->dev_private;
	41	dma_addr_t paddr;
	42	struct page **p;
	43	int ret, i;
	44
	45	p = drm_malloc_ab(npages, sizeof(struct page *));
	46	if (!p)
	47	return ERR_PTR(-ENOMEM);
	48
	49	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node,
	50	npages, 0, DRM_MM_SEARCH_DEFAULT);
	51	if (ret) {
	52	drm_free_large(p);
	53	return ERR_PTR(ret);
	54	}
	55
	56	paddr = physaddr(obj);
	57	for (i = 0; i < npages; i++) {
	58	p[i] = phys_to_page(paddr);
	59	paddr += PAGE_SIZE;
	60	}
	61
	62	return p;
	63	}
c8afe684 RC	64
	65	/* called with dev->struct_mutex held */
	66	static struct page *get_pages(struct drm_gem_object obj)
	67	{
	68	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	69
	70	if (!msm_obj->pages) {
	71	struct drm_device *dev = obj->dev;
871d812a	72	struct page **p;
c8afe684 RC	73	int npages = obj->size >> PAGE_SHIFT;
c8afe684 RC	74
871d812a RC	75	if (iommu_present(&platform_bus_type))
	76	p = drm_gem_get_pages(obj, 0);
	77	else
	78	p = get_pages_vram(obj, npages);
	79
c8afe684 RC	80	if (IS_ERR(p)) {
	81	dev_err(dev->dev, "could not get pages: %ld\n",
	82	PTR_ERR(p));
	83	return p;
	84	}
	85
	86	msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
1f70e079	87	if (IS_ERR(msm_obj->sgt)) {
c8afe684	88	dev_err(dev->dev, "failed to allocate sgt\n");
1f70e079	89	return ERR_CAST(msm_obj->sgt);
c8afe684 RC	90	}
	91
	92	msm_obj->pages = p;
	93
	94	/* For non-cached buffers, ensure the new pages are clean
	95	* because display controller, GPU, etc. are not coherent:
	96	*/
	97	if (msm_obj->flags & (MSM_BO_WC\|MSM_BO_UNCACHED))
	98	dma_map_sg(dev->dev, msm_obj->sgt->sgl,
	99	msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	100	}
	101
	102	return msm_obj->pages;
	103	}
	104
	105	static void put_pages(struct drm_gem_object *obj)
	106	{
	107	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	108
	109	if (msm_obj->pages) {
	110	/* For non-cached buffers, ensure the new pages are clean
	111	* because display controller, GPU, etc. are not coherent:
	112	*/
	113	if (msm_obj->flags & (MSM_BO_WC\|MSM_BO_UNCACHED))
	114	dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
	115	msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
	116	sg_free_table(msm_obj->sgt);
	117	kfree(msm_obj->sgt);
	118
871d812a RC	119	if (iommu_present(&platform_bus_type))
871d812a RC	120	drm_gem_put_pages(obj, msm_obj->pages, true, false);
1ffa2425	121	else {
871d812a	122	drm_mm_remove_node(msm_obj->vram_node);
1ffa2425 MR	123	drm_free_large(msm_obj->pages);
1ffa2425 MR	124	}
871d812a	125
c8afe684 RC	126	msm_obj->pages = NULL;
	127	}
	128	}
	129
05b84911 RC	130	struct page *msm_gem_get_pages(struct drm_gem_object obj)
	131	{
	132	struct drm_device *dev = obj->dev;
	133	struct page **p;
	134	mutex_lock(&dev->struct_mutex);
	135	p = get_pages(obj);
	136	mutex_unlock(&dev->struct_mutex);
	137	return p;
	138	}
	139
	140	void msm_gem_put_pages(struct drm_gem_object *obj)
	141	{
	142	/* when we start tracking the pin count, then do something here */
	143	}
	144
c8afe684 RC	145	int msm_gem_mmap_obj(struct drm_gem_object *obj,
	146	struct vm_area_struct *vma)
	147	{
	148	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	149
	150	vma->vm_flags &= ~VM_PFNMAP;
	151	vma->vm_flags \|= VM_MIXEDMAP;
	152
	153	if (msm_obj->flags & MSM_BO_WC) {
	154	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	155	} else if (msm_obj->flags & MSM_BO_UNCACHED) {
	156	vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
	157	} else {
	158	/*
	159	* Shunt off cached objs to shmem file so they have their own
	160	* address_space (so unmap_mapping_range does what we want,
	161	* in particular in the case of mmap'd dmabufs)
	162	*/
	163	fput(vma->vm_file);
	164	get_file(obj->filp);
	165	vma->vm_pgoff = 0;
	166	vma->vm_file = obj->filp;
	167
	168	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
	169	}
	170
	171	return 0;
	172	}
	173
	174	int msm_gem_mmap(struct file filp, struct vm_area_struct vma)
	175	{
	176	int ret;
	177
	178	ret = drm_gem_mmap(filp, vma);
	179	if (ret) {
	180	DBG("mmap failed: %d", ret);
	181	return ret;
	182	}
	183
	184	return msm_gem_mmap_obj(vma->vm_private_data, vma);
	185	}
	186
	187	int msm_gem_fault(struct vm_area_struct vma, struct vm_fault vmf)
	188	{
	189	struct drm_gem_object *obj = vma->vm_private_data;
c8afe684 RC	190	struct drm_device *dev = obj->dev;
	191	struct page **pages;
	192	unsigned long pfn;
	193	pgoff_t pgoff;
	194	int ret;
	195
	196	/* Make sure we don't parallel update on a fault, nor move or remove
	197	* something from beneath our feet
	198	*/
	199	ret = mutex_lock_interruptible(&dev->struct_mutex);
	200	if (ret)
	201	goto out;
	202
	203	/* make sure we have pages attached now */
	204	pages = get_pages(obj);
	205	if (IS_ERR(pages)) {
	206	ret = PTR_ERR(pages);
	207	goto out_unlock;
	208	}
	209
	210	/* We don't use vmf->pgoff since that has the fake offset: */
	211	pgoff = ((unsigned long)vmf->virtual_address -
	212	vma->vm_start) >> PAGE_SHIFT;
	213
871d812a	214	pfn = page_to_pfn(pages[pgoff]);
c8afe684 RC	215
	216	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
	217	pfn, pfn << PAGE_SHIFT);
	218
	219	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
	220
	221	out_unlock:
	222	mutex_unlock(&dev->struct_mutex);
	223	out:
	224	switch (ret) {
	225	case -EAGAIN:
c8afe684 RC	226	case 0:
	227	case -ERESTARTSYS:
	228	case -EINTR:
505886d5 RC	229	case -EBUSY:
	230	/*
	231	* EBUSY is ok: this just means that another thread
	232	* already did the job.
	233	*/
c8afe684 RC	234	return VM_FAULT_NOPAGE;
	235	case -ENOMEM:
	236	return VM_FAULT_OOM;
	237	default:
	238	return VM_FAULT_SIGBUS;
	239	}
	240	}
	241
	242	/** get mmap offset */
	243	static uint64_t mmap_offset(struct drm_gem_object *obj)
	244	{
	245	struct drm_device *dev = obj->dev;
	246	int ret;
	247
	248	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	249
	250	/* Make it mmapable */
	251	ret = drm_gem_create_mmap_offset(obj);
	252
	253	if (ret) {
	254	dev_err(dev->dev, "could not allocate mmap offset\n");
	255	return 0;
	256	}
	257
	258	return drm_vma_node_offset_addr(&obj->vma_node);
	259	}
	260
	261	uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
	262	{
	263	uint64_t offset;
	264	mutex_lock(&obj->dev->struct_mutex);
	265	offset = mmap_offset(obj);
	266	mutex_unlock(&obj->dev->struct_mutex);
	267	return offset;
	268	}
	269
c8afe684 RC	270	/* should be called under struct_mutex.. although it can be called
	271	* from atomic context without struct_mutex to acquire an extra
	272	* iova ref if you know one is already held.
	273	*
	274	* That means when I do eventually need to add support for unpinning
	275	* the refcnt counter needs to be atomic_t.
	276	*/
	277	int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
	278	uint32_t *iova)
	279	{
	280	struct msm_gem_object *msm_obj = to_msm_bo(obj);
87e956e9	281	struct drm_device *dev = obj->dev;
c8afe684 RC	282	int ret = 0;
	283
	284	if (!msm_obj->domain[id].iova) {
	285	struct msm_drm_private *priv = obj->dev->dev_private;
871d812a RC	286	struct msm_mmu *mmu = priv->mmus[id];
	287	struct page **pages = get_pages(obj);
	288
87e956e9 SV	289	if (!mmu) {
	290	dev_err(dev->dev, "null MMU pointer\n");
	291	return -EINVAL;
	292	}
	293
c8afe684 RC	294	if (IS_ERR(pages))
c8afe684 RC	295	return PTR_ERR(pages);
871d812a RC	296
	297	if (iommu_present(&platform_bus_type)) {
	298	uint32_t offset = (uint32_t)mmap_offset(obj);
	299	ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
	300	obj->size, IOMMU_READ \| IOMMU_WRITE);
	301	msm_obj->domain[id].iova = offset;
	302	} else {
	303	msm_obj->domain[id].iova = physaddr(obj);
	304	}
c8afe684 RC	305	}
	306
	307	if (!ret)
	308	*iova = msm_obj->domain[id].iova;
	309
	310	return ret;
	311	}
	312
	313	int msm_gem_get_iova(struct drm_gem_object obj, int id, uint32_t iova)
	314	{
edd4fc63	315	struct msm_gem_object *msm_obj = to_msm_bo(obj);
c8afe684	316	int ret;
edd4fc63 RC	317
	318	/* this is safe right now because we don't unmap until the
	319	* bo is deleted:
	320	*/
	321	if (msm_obj->domain[id].iova) {
	322	*iova = msm_obj->domain[id].iova;
	323	return 0;
	324	}
	325
c8afe684 RC	326	mutex_lock(&obj->dev->struct_mutex);
	327	ret = msm_gem_get_iova_locked(obj, id, iova);
	328	mutex_unlock(&obj->dev->struct_mutex);
	329	return ret;
	330	}
	331
	332	void msm_gem_put_iova(struct drm_gem_object *obj, int id)
	333	{
	334	// XXX TODO ..
	335	// NOTE: probably don't need a _locked() version.. we wouldn't
	336	// normally unmap here, but instead just mark that it could be
	337	// unmapped (if the iova refcnt drops to zero), but then later
	338	// if another _get_iova_locked() fails we can start unmapping
	339	// things that are no longer needed..
	340	}
	341
	342	int msm_gem_dumb_create(struct drm_file file, struct drm_device dev,
	343	struct drm_mode_create_dumb *args)
	344	{
	345	args->pitch = align_pitch(args->width, args->bpp);
	346	args->size = PAGE_ALIGN(args->pitch * args->height);
	347	return msm_gem_new_handle(dev, file, args->size,
	348	MSM_BO_SCANOUT \| MSM_BO_WC, &args->handle);
	349	}
	350
c8afe684 RC	351	int msm_gem_dumb_map_offset(struct drm_file file, struct drm_device dev,
	352	uint32_t handle, uint64_t *offset)
	353	{
	354	struct drm_gem_object *obj;
	355	int ret = 0;
	356
	357	/* GEM does all our handle to object mapping */
	358	obj = drm_gem_object_lookup(dev, file, handle);
	359	if (obj == NULL) {
	360	ret = -ENOENT;
	361	goto fail;
	362	}
	363
	364	*offset = msm_gem_mmap_offset(obj);
	365
	366	drm_gem_object_unreference_unlocked(obj);
	367
	368	fail:
	369	return ret;
	370	}
	371
	372	void msm_gem_vaddr_locked(struct drm_gem_object obj)
	373	{
	374	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	375	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
	376	if (!msm_obj->vaddr) {
	377	struct page **pages = get_pages(obj);
	378	if (IS_ERR(pages))
	379	return ERR_CAST(pages);
	380	msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
	381	VM_MAP, pgprot_writecombine(PAGE_KERNEL));
	382	}
	383	return msm_obj->vaddr;
	384	}
	385
	386	void msm_gem_vaddr(struct drm_gem_object obj)
	387	{
	388	void *ret;
	389	mutex_lock(&obj->dev->struct_mutex);
	390	ret = msm_gem_vaddr_locked(obj);
	391	mutex_unlock(&obj->dev->struct_mutex);
	392	return ret;
	393	}
	394
edd4fc63 RC	395	/* setup callback for when bo is no longer busy..
	396	* TODO probably want to differentiate read vs write..
	397	*/
	398	int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
	399	struct msm_fence_cb *cb)
c8afe684 RC	400	{
	401	struct drm_device *dev = obj->dev;
	402	struct msm_drm_private *priv = dev->dev_private;
7198e6b0 RC	403	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	404	int ret = 0;
	405
	406	mutex_lock(&dev->struct_mutex);
edd4fc63	407	if (!list_empty(&cb->work.entry)) {
7198e6b0 RC	408	ret = -EINVAL;
7198e6b0 RC	409	} else if (is_active(msm_obj)) {
edd4fc63 RC	410	cb->fence = max(msm_obj->read_fence, msm_obj->write_fence);
edd4fc63 RC	411	list_add_tail(&cb->work.entry, &priv->fence_cbs);
7198e6b0	412	} else {
edd4fc63	413	queue_work(priv->wq, &cb->work);
7198e6b0 RC	414	}
	415	mutex_unlock(&dev->struct_mutex);
	416
	417	return ret;
	418	}
	419
	420	void msm_gem_move_to_active(struct drm_gem_object *obj,
bf6811f3	421	struct msm_gpu *gpu, bool write, uint32_t fence)
7198e6b0 RC	422	{
	423	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	424	msm_obj->gpu = gpu;
bf6811f3 RC	425	if (write)
	426	msm_obj->write_fence = fence;
	427	else
	428	msm_obj->read_fence = fence;
7198e6b0 RC	429	list_del_init(&msm_obj->mm_list);
	430	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
	431	}
	432
	433	void msm_gem_move_to_inactive(struct drm_gem_object *obj)
	434	{
	435	struct drm_device *dev = obj->dev;
	436	struct msm_drm_private *priv = dev->dev_private;
	437	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	438
	439	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	440
	441	msm_obj->gpu = NULL;
bf6811f3 RC	442	msm_obj->read_fence = 0;
bf6811f3 RC	443	msm_obj->write_fence = 0;
7198e6b0 RC	444	list_del_init(&msm_obj->mm_list);
7198e6b0 RC	445	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
7198e6b0 RC	446	}
	447
	448	int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op,
	449	struct timespec *timeout)
	450	{
	451	struct drm_device *dev = obj->dev;
	452	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	453	int ret = 0;
	454
f816f272	455	if (is_active(msm_obj)) {
bf6811f3	456	uint32_t fence = 0;
f816f272	457
bf6811f3 RC	458	if (op & MSM_PREP_READ)
	459	fence = msm_obj->write_fence;
	460	if (op & MSM_PREP_WRITE)
	461	fence = max(fence, msm_obj->read_fence);
f816f272 RC	462	if (op & MSM_PREP_NOSYNC)
	463	timeout = NULL;
	464
bf6811f3 RC	465	ret = msm_wait_fence_interruptable(dev, fence, timeout);
bf6811f3 RC	466	}
7198e6b0 RC	467
7198e6b0 RC	468	/* TODO cache maintenance */
c8afe684	469
7198e6b0 RC	470	return ret;
7198e6b0 RC	471	}
c8afe684	472
7198e6b0 RC	473	int msm_gem_cpu_fini(struct drm_gem_object *obj)
	474	{
	475	/* TODO cache maintenance */
c8afe684 RC	476	return 0;
	477	}
	478
	479	#ifdef CONFIG_DEBUG_FS
	480	void msm_gem_describe(struct drm_gem_object obj, struct seq_file m)
	481	{
	482	struct drm_device *dev = obj->dev;
	483	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	484	uint64_t off = drm_vma_node_start(&obj->vma_node);
	485
	486	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
bf6811f3	487	seq_printf(m, "%08x: %c(r=%u,w=%u) %2d (%2d) %08llx %p %d\n",
7198e6b0	488	msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
bf6811f3 RC	489	msm_obj->read_fence, msm_obj->write_fence,
bf6811f3 RC	490	obj->name, obj->refcount.refcount.counter,
c8afe684 RC	491	off, msm_obj->vaddr, obj->size);
	492	}
	493
	494	void msm_gem_describe_objects(struct list_head list, struct seq_file m)
	495	{
	496	struct msm_gem_object *msm_obj;
	497	int count = 0;
	498	size_t size = 0;
	499
	500	list_for_each_entry(msm_obj, list, mm_list) {
	501	struct drm_gem_object *obj = &msm_obj->base;
	502	seq_printf(m, " ");
	503	msm_gem_describe(obj, m);
	504	count++;
	505	size += obj->size;
	506	}
	507
	508	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
	509	}
	510	#endif
	511
	512	void msm_gem_free_object(struct drm_gem_object *obj)
	513	{
	514	struct drm_device *dev = obj->dev;
871d812a	515	struct msm_drm_private *priv = obj->dev->dev_private;
c8afe684 RC	516	struct msm_gem_object *msm_obj = to_msm_bo(obj);
	517	int id;
	518
	519	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	520
7198e6b0 RC	521	/* object should not be on active list: */
	522	WARN_ON(is_active(msm_obj));
	523
c8afe684 RC	524	list_del(&msm_obj->mm_list);
	525
	526	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
871d812a RC	527	struct msm_mmu *mmu = priv->mmus[id];
871d812a RC	528	if (mmu && msm_obj->domain[id].iova) {
c8afe684	529	uint32_t offset = (uint32_t)mmap_offset(obj);
871d812a	530	mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
c8afe684 RC	531	}
	532	}
	533
	534	drm_gem_free_mmap_offset(obj);
	535
05b84911 RC	536	if (obj->import_attach) {
	537	if (msm_obj->vaddr)
	538	dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
	539
	540	/* Don't drop the pages for imported dmabuf, as they are not
	541	* ours, just free the array we allocated:
	542	*/
	543	if (msm_obj->pages)
	544	drm_free_large(msm_obj->pages);
c8afe684	545
05b84911 RC	546	} else {
	547	if (msm_obj->vaddr)
	548	vunmap(msm_obj->vaddr);
	549	put_pages(obj);
	550	}
c8afe684	551
7198e6b0 RC	552	if (msm_obj->resv == &msm_obj->_resv)
	553	reservation_object_fini(msm_obj->resv);
	554
c8afe684 RC	555	drm_gem_object_release(obj);
	556
	557	kfree(msm_obj);
	558	}
	559
	560	/* convenience method to construct a GEM buffer object, and userspace handle */
	561	int msm_gem_new_handle(struct drm_device dev, struct drm_file file,
	562	uint32_t size, uint32_t flags, uint32_t *handle)
	563	{
	564	struct drm_gem_object *obj;
	565	int ret;
	566
	567	ret = mutex_lock_interruptible(&dev->struct_mutex);
	568	if (ret)
	569	return ret;
	570
	571	obj = msm_gem_new(dev, size, flags);
	572
	573	mutex_unlock(&dev->struct_mutex);
	574
	575	if (IS_ERR(obj))
	576	return PTR_ERR(obj);
	577
	578	ret = drm_gem_handle_create(file, obj, handle);
	579
	580	/* drop reference from allocate - handle holds it now */
	581	drm_gem_object_unreference_unlocked(obj);
	582
	583	return ret;
	584	}
	585
05b84911 RC	586	static int msm_gem_new_impl(struct drm_device *dev,
	587	uint32_t size, uint32_t flags,
	588	struct drm_gem_object **obj)
c8afe684 RC	589	{
	590	struct msm_drm_private *priv = dev->dev_private;
	591	struct msm_gem_object *msm_obj;
871d812a	592	unsigned sz;
c8afe684 RC	593
	594	switch (flags & MSM_BO_CACHE_MASK) {
	595	case MSM_BO_UNCACHED:
	596	case MSM_BO_CACHED:
	597	case MSM_BO_WC:
	598	break;
	599	default:
	600	dev_err(dev->dev, "invalid cache flag: %x\n",
	601	(flags & MSM_BO_CACHE_MASK));
05b84911	602	return -EINVAL;
c8afe684 RC	603	}
c8afe684 RC	604
871d812a RC	605	sz = sizeof(*msm_obj);
	606	if (!iommu_present(&platform_bus_type))
	607	sz += sizeof(struct drm_mm_node);
	608
	609	msm_obj = kzalloc(sz, GFP_KERNEL);
05b84911 RC	610	if (!msm_obj)
05b84911 RC	611	return -ENOMEM;
c8afe684	612
871d812a RC	613	if (!iommu_present(&platform_bus_type))
	614	msm_obj->vram_node = (void *)&msm_obj[1];
	615
c8afe684 RC	616	msm_obj->flags = flags;
c8afe684 RC	617
7198e6b0 RC	618	msm_obj->resv = &msm_obj->_resv;
7198e6b0 RC	619	reservation_object_init(msm_obj->resv);
c8afe684	620
7198e6b0	621	INIT_LIST_HEAD(&msm_obj->submit_entry);
c8afe684 RC	622	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
c8afe684 RC	623
05b84911 RC	624	*obj = &msm_obj->base;
	625
	626	return 0;
	627	}
	628
	629	struct drm_gem_object msm_gem_new(struct drm_device dev,
	630	uint32_t size, uint32_t flags)
	631	{
871d812a	632	struct drm_gem_object *obj = NULL;
05b84911 RC	633	int ret;
	634
	635	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
	636
	637	size = PAGE_ALIGN(size);
	638
	639	ret = msm_gem_new_impl(dev, size, flags, &obj);
	640	if (ret)
	641	goto fail;
	642
871d812a RC	643	if (iommu_present(&platform_bus_type)) {
	644	ret = drm_gem_object_init(dev, obj, size);
	645	if (ret)
	646	goto fail;
	647	} else {
	648	drm_gem_private_object_init(dev, obj, size);
	649	}
05b84911 RC	650
	651	return obj;
	652
	653	fail:
	654	if (obj)
9999f105	655	drm_gem_object_unreference(obj);
05b84911 RC	656
	657	return ERR_PTR(ret);
	658	}
	659
	660	struct drm_gem_object msm_gem_import(struct drm_device dev,
	661	uint32_t size, struct sg_table *sgt)
	662	{
	663	struct msm_gem_object *msm_obj;
	664	struct drm_gem_object *obj;
	665	int ret, npages;
	666
871d812a RC	667	/* if we don't have IOMMU, don't bother pretending we can import: */
	668	if (!iommu_present(&platform_bus_type)) {
	669	dev_err(dev->dev, "cannot import without IOMMU\n");
	670	return ERR_PTR(-EINVAL);
	671	}
	672
05b84911 RC	673	size = PAGE_ALIGN(size);
	674
	675	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
	676	if (ret)
	677	goto fail;
	678
	679	drm_gem_private_object_init(dev, obj, size);
	680
	681	npages = size / PAGE_SIZE;
	682
	683	msm_obj = to_msm_bo(obj);
	684	msm_obj->sgt = sgt;
	685	msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
	686	if (!msm_obj->pages) {
	687	ret = -ENOMEM;
	688	goto fail;
	689	}
	690
	691	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
	692	if (ret)
	693	goto fail;
	694
c8afe684 RC	695	return obj;
	696
	697	fail:
	698	if (obj)
	699	drm_gem_object_unreference_unlocked(obj);
	700
	701	return ERR_PTR(ret);
	702	}