[deliverable/linux.git] / drivers / gpu / drm / gma500 / gtt.c

/*
 * Copyright (c) 2007, Intel Corporation.
 * All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
 *	    Alan Cox <alan@linux.intel.com>
 */

#include <drm/drmP.h>
#include <linux/shmem_fs.h>
#include "psb_drv.h"
#include "blitter.h"


/*
 *	GTT resource allocator - manage page mappings in GTT space
 */

/**
 *	psb_gtt_mask_pte	-	generate GTT pte entry
 *	@pfn: page number to encode
 *	@type: type of memory in the GTT
 *
 *	Set the GTT entry for the appropriate memory type.
 */
static inline uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
{
	uint32_t mask = PSB_PTE_VALID;

	/* Ensure we explode rather than put an invalid low mapping of
	   a high mapping page into the gtt */
	BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));

	if (type & PSB_MMU_CACHED_MEMORY)
		mask |= PSB_PTE_CACHED;
	if (type & PSB_MMU_RO_MEMORY)
		mask |= PSB_PTE_RO;
	if (type & PSB_MMU_WO_MEMORY)
		mask |= PSB_PTE_WO;

	return (pfn << PAGE_SHIFT) | mask;
}

/**
 *	psb_gtt_entry		-	find the GTT entries for a gtt_range
 *	@dev: our DRM device
 *	@r: our GTT range
 *
 *	Given a gtt_range object return the GTT offset of the page table
 *	entries for this gtt_range
 */
static u32 __iomem *psb_gtt_entry(struct drm_device *dev, struct gtt_range *r)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	unsigned long offset;

	offset = r->resource.start - dev_priv->gtt_mem->start;

	return dev_priv->gtt_map + (offset >> PAGE_SHIFT);
}

/**
 *	psb_gtt_insert	-	put an object into the GTT
 *	@dev: our DRM device
 *	@r: our GTT range
 *
 *	Take our preallocated GTT range and insert the GEM object into
 *	the GTT. This is protected via the gtt mutex which the caller
 *	must hold.
 */
static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
			  int resume)
{
	u32 __iomem *gtt_slot;
	u32 pte;
	struct page **pages;
	int i;

	if (r->pages == NULL) {
		WARN_ON(1);
		return -EINVAL;
	}

	WARN_ON(r->stolen);	/* refcount these maybe ? */

	gtt_slot = psb_gtt_entry(dev, r);
	pages = r->pages;

	if (!resume) {
		/* Make sure changes are visible to the GPU */
		set_pages_array_wc(pages, r->npage);
	}

	/* Write our page entries into the GTT itself */
	for (i = r->roll; i < r->npage; i++) {
		pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
				       PSB_MMU_CACHED_MEMORY);
		iowrite32(pte, gtt_slot++);
	}
	for (i = 0; i < r->roll; i++) {
		pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
				       PSB_MMU_CACHED_MEMORY);
		iowrite32(pte, gtt_slot++);
	}
	/* Make sure all the entries are set before we return */
	ioread32(gtt_slot - 1);

	return 0;
}

/**
 *	psb_gtt_remove	-	remove an object from the GTT
 *	@dev: our DRM device
 *	@r: our GTT range
 *
 *	Remove a preallocated GTT range from the GTT. Overwrite all the
 *	page table entries with the dummy page. This is protected via the gtt
 *	mutex which the caller must hold.
 */
void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	u32 __iomem *gtt_slot;
	u32 pte;
	int i;

	WARN_ON(r->stolen);

	gtt_slot = psb_gtt_entry(dev, r);
	pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page),
			       PSB_MMU_CACHED_MEMORY);

	for (i = 0; i < r->npage; i++)
		iowrite32(pte, gtt_slot++);
	ioread32(gtt_slot - 1);
	set_pages_array_wb(r->pages, r->npage);
}

/**
 *	psb_gtt_roll	-	set scrolling position
 *	@dev: our DRM device
 *	@r: the gtt mapping we are using
 *	@roll: roll offset
 *
 *	Roll an existing pinned mapping by moving the pages through the GTT.
 *	This allows us to implement hardware scrolling on the consoles without
 *	a 2D engine
 */
void psb_gtt_roll(struct drm_device *dev, struct gtt_range *r, int roll)
{
	u32 __iomem *gtt_slot;
	u32 pte;
	int i;

	if (roll >= r->npage) {
		WARN_ON(1);
		return;
	}

	r->roll = roll;

	/* Not currently in the GTT - no worry we will write the mapping at
	   the right position when it gets pinned */
	if (!r->stolen && !r->in_gart)
		return;

	gtt_slot = psb_gtt_entry(dev, r);

	for (i = r->roll; i < r->npage; i++) {
		pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
				       PSB_MMU_CACHED_MEMORY);
		iowrite32(pte, gtt_slot++);
	}
	for (i = 0; i < r->roll; i++) {
		pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
				       PSB_MMU_CACHED_MEMORY);
		iowrite32(pte, gtt_slot++);
	}
	ioread32(gtt_slot - 1);
}

/**
 *	psb_gtt_attach_pages	-	attach and pin GEM pages
 *	@gt: the gtt range
 *
 *	Pin and build an in kernel list of the pages that back our GEM object.
 *	While we hold this the pages cannot be swapped out. This is protected
 *	via the gtt mutex which the caller must hold.
 */
static int psb_gtt_attach_pages(struct gtt_range *gt)
{
	struct page **pages;

	WARN_ON(gt->pages);

	pages = drm_gem_get_pages(&gt->gem);
	if (IS_ERR(pages))
		return PTR_ERR(pages);

	gt->npage = gt->gem.size / PAGE_SIZE;
	gt->pages = pages;

	return 0;
}

/**
 *	psb_gtt_detach_pages	-	attach and pin GEM pages
 *	@gt: the gtt range
 *
 *	Undo the effect of psb_gtt_attach_pages. At this point the pages
 *	must have been removed from the GTT as they could now be paged out
 *	and move bus address. This is protected via the gtt mutex which the
 *	caller must hold.
 */
static void psb_gtt_detach_pages(struct gtt_range *gt)
{
	drm_gem_put_pages(&gt->gem, gt->pages, true, false);
	gt->pages = NULL;
}

/**
 *	psb_gtt_pin		-	pin pages into the GTT
 *	@gt: range to pin
 *
 *	Pin a set of pages into the GTT. The pins are refcounted so that
 *	multiple pins need multiple unpins to undo.
 *
 *	Non GEM backed objects treat this as a no-op as they are always GTT
 *	backed objects.
 */
int psb_gtt_pin(struct gtt_range *gt)
{
	int ret = 0;
	struct drm_device *dev = gt->gem.dev;
	struct drm_psb_private *dev_priv = dev->dev_private;
	u32 gpu_base = dev_priv->gtt.gatt_start;

	mutex_lock(&dev_priv->gtt_mutex);

	if (gt->in_gart == 0 && gt->stolen == 0) {
		ret = psb_gtt_attach_pages(gt);
		if (ret < 0)
			goto out;
		ret = psb_gtt_insert(dev, gt, 0);
		if (ret < 0) {
			psb_gtt_detach_pages(gt);
			goto out;
		}
		psb_mmu_insert_pages(psb_mmu_get_default_pd(dev_priv->mmu),
				     gt->pages, (gpu_base + gt->offset),
				     gt->npage, 0, 0, PSB_MMU_CACHED_MEMORY);
	}
	gt->in_gart++;
out:
	mutex_unlock(&dev_priv->gtt_mutex);
	return ret;
}

/**
 *	psb_gtt_unpin		-	Drop a GTT pin requirement
 *	@gt: range to pin
 *
 *	Undoes the effect of psb_gtt_pin. On the last drop the GEM object
 *	will be removed from the GTT which will also drop the page references
 *	and allow the VM to clean up or page stuff.
 *
 *	Non GEM backed objects treat this as a no-op as they are always GTT
 *	backed objects.
 */
void psb_gtt_unpin(struct gtt_range *gt)
{
	struct drm_device *dev = gt->gem.dev;
	struct drm_psb_private *dev_priv = dev->dev_private;
	u32 gpu_base = dev_priv->gtt.gatt_start;
	int ret;

	/* While holding the gtt_mutex no new blits can be initiated */
	mutex_lock(&dev_priv->gtt_mutex);

	/* Wait for any possible usage of the memory to be finished */
	ret = gma_blt_wait_idle(dev_priv);
	if (ret) {
		DRM_ERROR("Failed to idle the blitter, unpin failed!");
		goto out;
	}

	WARN_ON(!gt->in_gart);

	gt->in_gart--;
	if (gt->in_gart == 0 && gt->stolen == 0) {
		psb_mmu_remove_pages(psb_mmu_get_default_pd(dev_priv->mmu),
				     (gpu_base + gt->offset), gt->npage, 0, 0);
		psb_gtt_remove(dev, gt);
		psb_gtt_detach_pages(gt);
	}

out:
	mutex_unlock(&dev_priv->gtt_mutex);
}

/*
 *	GTT resource allocator - allocate and manage GTT address space
 */

/**
 *	psb_gtt_alloc_range	-	allocate GTT address space
 *	@dev: Our DRM device
 *	@len: length (bytes) of address space required
 *	@name: resource name
 *	@backed: resource should be backed by stolen pages
 *
 *	Ask the kernel core to find us a suitable range of addresses
 *	to use for a GTT mapping.
 *
 *	Returns a gtt_range structure describing the object, or NULL on
 *	error. On successful return the resource is both allocated and marked
 *	as in use.
 */
struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
				      const char *name, int backed, u32 align)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	struct gtt_range *gt;
	struct resource *r = dev_priv->gtt_mem;
	int ret;
	unsigned long start, end;

	if (backed) {
		/* The start of the GTT is the stolen pages */
		start = r->start;
		end = r->start + dev_priv->gtt.stolen_size - 1;
	} else {
		/* The rest we will use for GEM backed objects */
		start = r->start + dev_priv->gtt.stolen_size;
		end = r->end;
	}

	gt = kzalloc(sizeof(struct gtt_range), GFP_KERNEL);
	if (gt == NULL)
		return NULL;
	gt->resource.name = name;
	gt->stolen = backed;
	gt->in_gart = backed;
	gt->roll = 0;
	/* Ensure this is set for non GEM objects */
	gt->gem.dev = dev;
	ret = allocate_resource(dev_priv->gtt_mem, &gt->resource,
				len, start, end, align, NULL, NULL);
	if (ret == 0) {
		gt->offset = gt->resource.start - r->start;
		return gt;
	}
	kfree(gt);
	return NULL;
}

/**
 *	psb_gtt_free_range	-	release GTT address space
 *	@dev: our DRM device
 *	@gt: a mapping created with psb_gtt_alloc_range
 *
 *	Release a resource that was allocated with psb_gtt_alloc_range. If the
 *	object has been pinned by mmap users we clean this up here currently.
 */
void psb_gtt_free_range(struct drm_device *dev, struct gtt_range *gt)
{
	/* Undo the mmap pin if we are destroying the object */
	if (gt->mmapping) {
		psb_gtt_unpin(gt);
		gt->mmapping = 0;
	}
	WARN_ON(gt->in_gart && !gt->stolen);
	release_resource(&gt->resource);
	kfree(gt);
}

static void psb_gtt_alloc(struct drm_device *dev)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	init_rwsem(&dev_priv->gtt.sem);
}

void psb_gtt_takedown(struct drm_device *dev)
{
	struct drm_psb_private *dev_priv = dev->dev_private;

	if (dev_priv->gtt_map) {
		iounmap(dev_priv->gtt_map);
		dev_priv->gtt_map = NULL;
	}
	if (dev_priv->gtt_initialized) {
		pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
				      dev_priv->gmch_ctrl);
		PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
		(void) PSB_RVDC32(PSB_PGETBL_CTL);
	}
	if (dev_priv->vram_addr)
		iounmap(dev_priv->gtt_map);
}

int psb_gtt_init(struct drm_device *dev, int resume)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	unsigned gtt_pages;
	unsigned long stolen_size, vram_stolen_size;
	unsigned i, num_pages;
	unsigned pfn_base;
	struct psb_gtt *pg;

	int ret = 0;
	uint32_t pte;

	if (!resume) {
		mutex_init(&dev_priv->gtt_mutex);
		mutex_init(&dev_priv->mmap_mutex);
		psb_gtt_alloc(dev);
	}

	pg = &dev_priv->gtt;

	/* Enable the GTT */
	pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
	pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
			      dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED);

	dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
	PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
	(void) PSB_RVDC32(PSB_PGETBL_CTL);

	/* The root resource we allocate address space from */
	dev_priv->gtt_initialized = 1;

	pg->gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;

	/*
	 *	The video mmu has a hw bug when accessing 0x0D0000000.
	 *	Make gatt start at 0x0e000,0000. This doesn't actually
	 *	matter for us but may do if the video acceleration ever
	 *	gets opened up.
	 */
	pg->mmu_gatt_start = 0xE0000000;

	pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE);
	gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE)
								>> PAGE_SHIFT;
	/* CDV doesn't report this. In which case the system has 64 gtt pages */
	if (pg->gtt_start == 0 || gtt_pages == 0) {
		dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
		gtt_pages = 64;
		pg->gtt_start = dev_priv->pge_ctl;
	}

	pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE);
	pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE)
								>> PAGE_SHIFT;
	dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE];

	if (pg->gatt_pages == 0 || pg->gatt_start == 0) {
		static struct resource fudge;	/* Preferably peppermint */
		/* This can occur on CDV systems. Fudge it in this case.
		   We really don't care what imaginary space is being allocated
		   at this point */
		dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
		pg->gatt_start = 0x40000000;
		pg->gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
		/* This is a little confusing but in fact the GTT is providing
		   a view from the GPU into memory and not vice versa. As such
		   this is really allocating space that is not the same as the
		   CPU address space on CDV */
		fudge.start = 0x40000000;
		fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
		fudge.name = "fudge";
		fudge.flags = IORESOURCE_MEM;
		dev_priv->gtt_mem = &fudge;
	}

	pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base);
	vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base
								- PAGE_SIZE;

	stolen_size = vram_stolen_size;

	dev_dbg(dev->dev, "Stolen memory base 0x%x, size %luK\n",
			dev_priv->stolen_base, vram_stolen_size / 1024);

	if (resume && (gtt_pages != pg->gtt_pages) &&
	    (stolen_size != pg->stolen_size)) {
		dev_err(dev->dev, "GTT resume error.\n");
		ret = -EINVAL;
		goto out_err;
	}

	pg->gtt_pages = gtt_pages;
	pg->stolen_size = stolen_size;
	dev_priv->vram_stolen_size = vram_stolen_size;

	/*
	 *	Map the GTT and the stolen memory area
	 */
	if (!resume)
		dev_priv->gtt_map = ioremap_nocache(pg->gtt_phys_start,
						gtt_pages << PAGE_SHIFT);
	if (!dev_priv->gtt_map) {
		dev_err(dev->dev, "Failure to map gtt.\n");
		ret = -ENOMEM;
		goto out_err;
	}

	if (!resume)
		dev_priv->vram_addr = ioremap_wc(dev_priv->stolen_base,
						 stolen_size);

	if (!dev_priv->vram_addr) {
		dev_err(dev->dev, "Failure to map stolen base.\n");
		ret = -ENOMEM;
		goto out_err;
	}

	/*
	 * Insert vram stolen pages into the GTT
	 */

	pfn_base = dev_priv->stolen_base >> PAGE_SHIFT;
	num_pages = vram_stolen_size >> PAGE_SHIFT;
	dev_dbg(dev->dev, "Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
		num_pages, pfn_base << PAGE_SHIFT, 0);
	for (i = 0; i < num_pages; ++i) {
		pte = psb_gtt_mask_pte(pfn_base + i, PSB_MMU_CACHED_MEMORY);
		iowrite32(pte, dev_priv->gtt_map + i);
	}

	/*
	 * Init rest of GTT to the scratch page to avoid accidents or scribbles
	 */

	pfn_base = page_to_pfn(dev_priv->scratch_page);
	pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
	for (; i < gtt_pages; ++i)
		iowrite32(pte, dev_priv->gtt_map + i);

	(void) ioread32(dev_priv->gtt_map + i - 1);
	return 0;

out_err:
	psb_gtt_takedown(dev);
	return ret;
}

int psb_gtt_restore(struct drm_device *dev)
{
	struct drm_psb_private *dev_priv = dev->dev_private;
	struct resource *r = dev_priv->gtt_mem->child;
	struct gtt_range *range;
	unsigned int restored = 0, total = 0, size = 0;

	/* On resume, the gtt_mutex is already initialized */
	mutex_lock(&dev_priv->gtt_mutex);
	psb_gtt_init(dev, 1);

	while (r != NULL) {
		range = container_of(r, struct gtt_range, resource);
		if (range->pages) {
			psb_gtt_insert(dev, range, 1);
			size += range->resource.end - range->resource.start;
			restored++;
		}
		r = r->sibling;
		total++;
	}
	mutex_unlock(&dev_priv->gtt_mutex);
	DRM_DEBUG_DRIVER("Restored %u of %u gtt ranges (%u KB)", restored,
			 total, (size / 1024));

	return 0;
}
Commit	Line	Data
8c8f1c95 AC	1	/*
	2	* Copyright (c) 2007, Intel Corporation.
	3	* All Rights Reserved.
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms and conditions of the GNU General Public License,
	7	* version 2, as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program; if not, write to the Free Software Foundation, Inc.,
	16	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	17	*
	18	* Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
	19	* Alan Cox <alan@linux.intel.com>
	20	*/
	21
	22	#include <drm/drmP.h>
e912b6d2	23	#include <linux/shmem_fs.h>
8c8f1c95	24	#include "psb_drv.h"
ae012bdc	25	#include "blitter.h"
8c8f1c95 AC	26
	27
	28	/*
	29	* GTT resource allocator - manage page mappings in GTT space
	30	*/
	31
	32	/**
	33	* psb_gtt_mask_pte - generate GTT pte entry
	34	* @pfn: page number to encode
	35	* @type: type of memory in the GTT
	36	*
	37	* Set the GTT entry for the appropriate memory type.
	38	*/
	39	static inline uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
	40	{
	41	uint32_t mask = PSB_PTE_VALID;
	42
398b4706 AC	43	/* Ensure we explode rather than put an invalid low mapping of
	44	a high mapping page into the gtt */
	45	BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));
	46
8c8f1c95 AC	47	if (type & PSB_MMU_CACHED_MEMORY)
	48	mask \|= PSB_PTE_CACHED;
	49	if (type & PSB_MMU_RO_MEMORY)
	50	mask \|= PSB_PTE_RO;
	51	if (type & PSB_MMU_WO_MEMORY)
	52	mask \|= PSB_PTE_WO;
	53
	54	return (pfn << PAGE_SHIFT) \| mask;
	55	}
	56
	57	/**
	58	* psb_gtt_entry - find the GTT entries for a gtt_range
	59	* @dev: our DRM device
	60	* @r: our GTT range
	61	*
	62	* Given a gtt_range object return the GTT offset of the page table
	63	* entries for this gtt_range
	64	*/
eab37607	65	static u32 __iomem psb_gtt_entry(struct drm_device dev, struct gtt_range *r)
8c8f1c95 AC	66	{
	67	struct drm_psb_private *dev_priv = dev->dev_private;
	68	unsigned long offset;
	69
	70	offset = r->resource.start - dev_priv->gtt_mem->start;
	71
	72	return dev_priv->gtt_map + (offset >> PAGE_SHIFT);
	73	}
	74
	75	/**
	76	* psb_gtt_insert - put an object into the GTT
	77	* @dev: our DRM device
	78	* @r: our GTT range
	79	*
	80	* Take our preallocated GTT range and insert the GEM object into
a746092b AC	81	* the GTT. This is protected via the gtt mutex which the caller
a746092b AC	82	* must hold.
8c8f1c95	83	*/
070839ea PJ	84	static int psb_gtt_insert(struct drm_device dev, struct gtt_range r,
070839ea PJ	85	int resume)
8c8f1c95	86	{
eab37607 KS	87	u32 __iomem *gtt_slot;
eab37607 KS	88	u32 pte;
8c8f1c95 AC	89	struct page **pages;
	90	int i;
	91
	92	if (r->pages == NULL) {
	93	WARN_ON(1);
	94	return -EINVAL;
	95	}
	96
	97	WARN_ON(r->stolen); /* refcount these maybe ? */
	98
	99	gtt_slot = psb_gtt_entry(dev, r);
	100	pages = r->pages;
	101
070839ea PJ	102	if (!resume) {
	103	/* Make sure changes are visible to the GPU */
	104	set_pages_array_wc(pages, r->npage);
	105	}
8c8f1c95 AC	106
8c8f1c95 AC	107	/* Write our page entries into the GTT itself */
a6ba582d	108	for (i = r->roll; i < r->npage; i++) {
ae012bdc PJ	109	pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
ae012bdc PJ	110	PSB_MMU_CACHED_MEMORY);
a6ba582d AC	111	iowrite32(pte, gtt_slot++);
	112	}
	113	for (i = 0; i < r->roll; i++) {
ae012bdc PJ	114	pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
ae012bdc PJ	115	PSB_MMU_CACHED_MEMORY);
8c8f1c95 AC	116	iowrite32(pte, gtt_slot++);
	117	}
	118	/* Make sure all the entries are set before we return */
	119	ioread32(gtt_slot - 1);
a6ba582d	120
8c8f1c95 AC	121	return 0;
	122	}
	123
	124	/**
	125	* psb_gtt_remove - remove an object from the GTT
	126	* @dev: our DRM device
	127	* @r: our GTT range
	128	*
	129	* Remove a preallocated GTT range from the GTT. Overwrite all the
a746092b AC	130	* page table entries with the dummy page. This is protected via the gtt
a746092b AC	131	* mutex which the caller must hold.
8c8f1c95	132	*/
ae012bdc	133	void psb_gtt_remove(struct drm_device dev, struct gtt_range r)
8c8f1c95 AC	134	{
8c8f1c95 AC	135	struct drm_psb_private *dev_priv = dev->dev_private;
eab37607 KS	136	u32 __iomem *gtt_slot;
eab37607 KS	137	u32 pte;
8c8f1c95 AC	138	int i;
	139
	140	WARN_ON(r->stolen);
	141
	142	gtt_slot = psb_gtt_entry(dev, r);
ae012bdc PJ	143	pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page),
ae012bdc PJ	144	PSB_MMU_CACHED_MEMORY);
8c8f1c95 AC	145
	146	for (i = 0; i < r->npage; i++)
	147	iowrite32(pte, gtt_slot++);
	148	ioread32(gtt_slot - 1);
	149	set_pages_array_wb(r->pages, r->npage);
	150	}
	151
a6ba582d AC	152	/**
	153	* psb_gtt_roll - set scrolling position
	154	* @dev: our DRM device
	155	* @r: the gtt mapping we are using
	156	* @roll: roll offset
	157	*
	158	* Roll an existing pinned mapping by moving the pages through the GTT.
	159	* This allows us to implement hardware scrolling on the consoles without
	160	* a 2D engine
	161	*/
	162	void psb_gtt_roll(struct drm_device dev, struct gtt_range r, int roll)
	163	{
eab37607 KS	164	u32 __iomem *gtt_slot;
eab37607 KS	165	u32 pte;
a6ba582d AC	166	int i;
	167
	168	if (roll >= r->npage) {
	169	WARN_ON(1);
	170	return;
	171	}
	172
	173	r->roll = roll;
	174
	175	/* Not currently in the GTT - no worry we will write the mapping at
	176	the right position when it gets pinned */
	177	if (!r->stolen && !r->in_gart)
	178	return;
	179
	180	gtt_slot = psb_gtt_entry(dev, r);
	181
	182	for (i = r->roll; i < r->npage; i++) {
ae012bdc PJ	183	pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
ae012bdc PJ	184	PSB_MMU_CACHED_MEMORY);
a6ba582d AC	185	iowrite32(pte, gtt_slot++);
	186	}
	187	for (i = 0; i < r->roll; i++) {
ae012bdc PJ	188	pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
ae012bdc PJ	189	PSB_MMU_CACHED_MEMORY);
a6ba582d AC	190	iowrite32(pte, gtt_slot++);
	191	}
	192	ioread32(gtt_slot - 1);
	193	}
	194
8c8f1c95 AC	195	/**
	196	* psb_gtt_attach_pages - attach and pin GEM pages
	197	* @gt: the gtt range
	198	*
	199	* Pin and build an in kernel list of the pages that back our GEM object.
a746092b AC	200	* While we hold this the pages cannot be swapped out. This is protected
a746092b AC	201	* via the gtt mutex which the caller must hold.
8c8f1c95 AC	202	*/
	203	static int psb_gtt_attach_pages(struct gtt_range *gt)
	204	{
8b9ba7a3	205	struct page **pages;
8c8f1c95 AC	206
	207	WARN_ON(gt->pages);
	208
0cdbe8ac	209	pages = drm_gem_get_pages(&gt->gem);
8b9ba7a3 RC	210	if (IS_ERR(pages))
8b9ba7a3 RC	211	return PTR_ERR(pages);
8c8f1c95	212
b9aa8510	213	gt->npage = gt->gem.size / PAGE_SIZE;
8b9ba7a3	214	gt->pages = pages;
8c8f1c95	215
8c8f1c95	216	return 0;
8c8f1c95 AC	217	}
	218
	219	/**
	220	* psb_gtt_detach_pages - attach and pin GEM pages
	221	* @gt: the gtt range
	222	*
	223	* Undo the effect of psb_gtt_attach_pages. At this point the pages
	224	* must have been removed from the GTT as they could now be paged out
a746092b AC	225	* and move bus address. This is protected via the gtt mutex which the
a746092b AC	226	* caller must hold.
8c8f1c95 AC	227	*/
	228	static void psb_gtt_detach_pages(struct gtt_range *gt)
	229	{
8b9ba7a3	230	drm_gem_put_pages(&gt->gem, gt->pages, true, false);
8c8f1c95 AC	231	gt->pages = NULL;
	232	}
	233
	234	/**
	235	* psb_gtt_pin - pin pages into the GTT
	236	* @gt: range to pin
	237	*
	238	* Pin a set of pages into the GTT. The pins are refcounted so that
	239	* multiple pins need multiple unpins to undo.
	240	*
	241	* Non GEM backed objects treat this as a no-op as they are always GTT
	242	* backed objects.
	243	*/
	244	int psb_gtt_pin(struct gtt_range *gt)
	245	{
	246	int ret = 0;
	247	struct drm_device *dev = gt->gem.dev;
	248	struct drm_psb_private *dev_priv = dev->dev_private;
ae012bdc	249	u32 gpu_base = dev_priv->gtt.gatt_start;
8c8f1c95 AC	250
	251	mutex_lock(&dev_priv->gtt_mutex);
	252
	253	if (gt->in_gart == 0 && gt->stolen == 0) {
	254	ret = psb_gtt_attach_pages(gt);
	255	if (ret < 0)
	256	goto out;
070839ea	257	ret = psb_gtt_insert(dev, gt, 0);
8c8f1c95 AC	258	if (ret < 0) {
	259	psb_gtt_detach_pages(gt);
	260	goto out;
	261	}
ae012bdc PJ	262	psb_mmu_insert_pages(psb_mmu_get_default_pd(dev_priv->mmu),
	263	gt->pages, (gpu_base + gt->offset),
	264	gt->npage, 0, 0, PSB_MMU_CACHED_MEMORY);
8c8f1c95 AC	265	}
	266	gt->in_gart++;
	267	out:
	268	mutex_unlock(&dev_priv->gtt_mutex);
	269	return ret;
	270	}
	271
	272	/**
	273	* psb_gtt_unpin - Drop a GTT pin requirement
	274	* @gt: range to pin
	275	*
	276	* Undoes the effect of psb_gtt_pin. On the last drop the GEM object
	277	* will be removed from the GTT which will also drop the page references
	278	* and allow the VM to clean up or page stuff.
	279	*
	280	* Non GEM backed objects treat this as a no-op as they are always GTT
	281	* backed objects.
	282	*/
	283	void psb_gtt_unpin(struct gtt_range *gt)
	284	{
	285	struct drm_device *dev = gt->gem.dev;
	286	struct drm_psb_private *dev_priv = dev->dev_private;
ae012bdc PJ	287	u32 gpu_base = dev_priv->gtt.gatt_start;
ae012bdc PJ	288	int ret;
8c8f1c95	289
ae012bdc	290	/* While holding the gtt_mutex no new blits can be initiated */
8c8f1c95 AC	291	mutex_lock(&dev_priv->gtt_mutex);
8c8f1c95 AC	292
ae012bdc PJ	293	/* Wait for any possible usage of the memory to be finished */
	294	ret = gma_blt_wait_idle(dev_priv);
	295	if (ret) {
	296	DRM_ERROR("Failed to idle the blitter, unpin failed!");
	297	goto out;
	298	}
	299
8c8f1c95 AC	300	WARN_ON(!gt->in_gart);
	301
	302	gt->in_gart--;
	303	if (gt->in_gart == 0 && gt->stolen == 0) {
ae012bdc PJ	304	psb_mmu_remove_pages(psb_mmu_get_default_pd(dev_priv->mmu),
ae012bdc PJ	305	(gpu_base + gt->offset), gt->npage, 0, 0);
8c8f1c95 AC	306	psb_gtt_remove(dev, gt);
	307	psb_gtt_detach_pages(gt);
	308	}
ae012bdc PJ	309
ae012bdc PJ	310	out:
8c8f1c95 AC	311	mutex_unlock(&dev_priv->gtt_mutex);
	312	}
	313
	314	/*
	315	* GTT resource allocator - allocate and manage GTT address space
	316	*/
	317
	318	/**
	319	* psb_gtt_alloc_range - allocate GTT address space
	320	* @dev: Our DRM device
	321	* @len: length (bytes) of address space required
	322	* @name: resource name
	323	* @backed: resource should be backed by stolen pages
	324	*
	325	* Ask the kernel core to find us a suitable range of addresses
	326	* to use for a GTT mapping.
	327	*
	328	* Returns a gtt_range structure describing the object, or NULL on
	329	* error. On successful return the resource is both allocated and marked
	330	* as in use.
	331	*/
	332	struct gtt_range psb_gtt_alloc_range(struct drm_device dev, int len,
c269c685	333	const char *name, int backed, u32 align)
8c8f1c95 AC	334	{
	335	struct drm_psb_private *dev_priv = dev->dev_private;
	336	struct gtt_range *gt;
	337	struct resource *r = dev_priv->gtt_mem;
	338	int ret;
	339	unsigned long start, end;
	340
	341	if (backed) {
	342	/* The start of the GTT is the stolen pages */
	343	start = r->start;
	344	end = r->start + dev_priv->gtt.stolen_size - 1;
	345	} else {
	346	/* The rest we will use for GEM backed objects */
	347	start = r->start + dev_priv->gtt.stolen_size;
	348	end = r->end;
	349	}
	350
	351	gt = kzalloc(sizeof(struct gtt_range), GFP_KERNEL);
	352	if (gt == NULL)
	353	return NULL;
	354	gt->resource.name = name;
	355	gt->stolen = backed;
	356	gt->in_gart = backed;
a6ba582d	357	gt->roll = 0;
8c8f1c95 AC	358	/* Ensure this is set for non GEM objects */
	359	gt->gem.dev = dev;
	360	ret = allocate_resource(dev_priv->gtt_mem, &gt->resource,
c269c685	361	len, start, end, align, NULL, NULL);
8c8f1c95 AC	362	if (ret == 0) {
	363	gt->offset = gt->resource.start - r->start;
	364	return gt;
	365	}
	366	kfree(gt);
	367	return NULL;
	368	}
	369
	370	/**
	371	* psb_gtt_free_range - release GTT address space
	372	* @dev: our DRM device
	373	* @gt: a mapping created with psb_gtt_alloc_range
	374	*
	375	* Release a resource that was allocated with psb_gtt_alloc_range. If the
	376	* object has been pinned by mmap users we clean this up here currently.
	377	*/
	378	void psb_gtt_free_range(struct drm_device dev, struct gtt_range gt)
	379	{
	380	/* Undo the mmap pin if we are destroying the object */
	381	if (gt->mmapping) {
	382	psb_gtt_unpin(gt);
	383	gt->mmapping = 0;
	384	}
	385	WARN_ON(gt->in_gart && !gt->stolen);
	386	release_resource(&gt->resource);
	387	kfree(gt);
	388	}
	389
ffe94d9c	390	static void psb_gtt_alloc(struct drm_device *dev)
8c8f1c95 AC	391	{
	392	struct drm_psb_private *dev_priv = dev->dev_private;
	393	init_rwsem(&dev_priv->gtt.sem);
	394	}
	395
	396	void psb_gtt_takedown(struct drm_device *dev)
	397	{
	398	struct drm_psb_private *dev_priv = dev->dev_private;
	399
	400	if (dev_priv->gtt_map) {
	401	iounmap(dev_priv->gtt_map);
	402	dev_priv->gtt_map = NULL;
	403	}
	404	if (dev_priv->gtt_initialized) {
	405	pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
	406	dev_priv->gmch_ctrl);
	407	PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
	408	(void) PSB_RVDC32(PSB_PGETBL_CTL);
	409	}
	410	if (dev_priv->vram_addr)
	411	iounmap(dev_priv->gtt_map);
	412	}
	413
	414	int psb_gtt_init(struct drm_device *dev, int resume)
	415	{
	416	struct drm_psb_private *dev_priv = dev->dev_private;
	417	unsigned gtt_pages;
	418	unsigned long stolen_size, vram_stolen_size;
	419	unsigned i, num_pages;
	420	unsigned pfn_base;
8c8f1c95 AC	421	struct psb_gtt *pg;
	422
	423	int ret = 0;
	424	uint32_t pte;
	425
070839ea PJ	426	if (!resume) {
070839ea PJ	427	mutex_init(&dev_priv->gtt_mutex);
737292a3	428	mutex_init(&dev_priv->mmap_mutex);
070839ea PJ	429	psb_gtt_alloc(dev);
070839ea PJ	430	}
8c8f1c95	431
8c8f1c95 AC	432	pg = &dev_priv->gtt;
	433
	434	/* Enable the GTT */
	435	pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
	436	pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
	437	dev_priv->gmch_ctrl \| _PSB_GMCH_ENABLED);
	438
	439	dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
	440	PSB_WVDC32(dev_priv->pge_ctl \| _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
	441	(void) PSB_RVDC32(PSB_PGETBL_CTL);
	442
	443	/* The root resource we allocate address space from */
	444	dev_priv->gtt_initialized = 1;
	445
	446	pg->gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;
	447
	448	/*
a746092b AC	449	* The video mmu has a hw bug when accessing 0x0D0000000.
	450	* Make gatt start at 0x0e000,0000. This doesn't actually
	451	* matter for us but may do if the video acceleration ever
	452	* gets opened up.
8c8f1c95 AC	453	*/
	454	pg->mmu_gatt_start = 0xE0000000;
	455
	456	pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE);
	457	gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE)
	458	>> PAGE_SHIFT;
055bf38d	459	/* CDV doesn't report this. In which case the system has 64 gtt pages */
8c8f1c95	460	if (pg->gtt_start == 0 \|\| gtt_pages == 0) {
055bf38d	461	dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
8c8f1c95 AC	462	gtt_pages = 64;
	463	pg->gtt_start = dev_priv->pge_ctl;
	464	}
	465
	466	pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE);
	467	pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE)
	468	>> PAGE_SHIFT;
	469	dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE];
	470
	471	if (pg->gatt_pages == 0 \|\| pg->gatt_start == 0) {
	472	static struct resource fudge; /* Preferably peppermint */
055bf38d	473	/* This can occur on CDV systems. Fudge it in this case.
8c8f1c95 AC	474	We really don't care what imaginary space is being allocated
8c8f1c95 AC	475	at this point */
055bf38d	476	dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
8c8f1c95 AC	477	pg->gatt_start = 0x40000000;
8c8f1c95 AC	478	pg->gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
a746092b AC	479	/* This is a little confusing but in fact the GTT is providing
	480	a view from the GPU into memory and not vice versa. As such
	481	this is really allocating space that is not the same as the
	482	CPU address space on CDV */
8c8f1c95 AC	483	fudge.start = 0x40000000;
	484	fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
	485	fudge.name = "fudge";
	486	fudge.flags = IORESOURCE_MEM;
	487	dev_priv->gtt_mem = &fudge;
	488	}
	489
	490	pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base);
	491	vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base
	492	- PAGE_SIZE;
	493
	494	stolen_size = vram_stolen_size;
	495
31a0685a AC	496	dev_dbg(dev->dev, "Stolen memory base 0x%x, size %luK\n",
31a0685a AC	497	dev_priv->stolen_base, vram_stolen_size / 1024);
8c8f1c95 AC	498
	499	if (resume && (gtt_pages != pg->gtt_pages) &&
	500	(stolen_size != pg->stolen_size)) {
	501	dev_err(dev->dev, "GTT resume error.\n");
	502	ret = -EINVAL;
	503	goto out_err;
	504	}
	505
	506	pg->gtt_pages = gtt_pages;
	507	pg->stolen_size = stolen_size;
	508	dev_priv->vram_stolen_size = vram_stolen_size;
	509
	510	/*
	511	* Map the GTT and the stolen memory area
	512	*/
070839ea PJ	513	if (!resume)
070839ea PJ	514	dev_priv->gtt_map = ioremap_nocache(pg->gtt_phys_start,
8c8f1c95 AC	515	gtt_pages << PAGE_SHIFT);
	516	if (!dev_priv->gtt_map) {
	517	dev_err(dev->dev, "Failure to map gtt.\n");
	518	ret = -ENOMEM;
	519	goto out_err;
	520	}
	521
070839ea PJ	522	if (!resume)
	523	dev_priv->vram_addr = ioremap_wc(dev_priv->stolen_base,
	524	stolen_size);
ae012bdc	525
8c8f1c95 AC	526	if (!dev_priv->vram_addr) {
	527	dev_err(dev->dev, "Failure to map stolen base.\n");
	528	ret = -ENOMEM;
	529	goto out_err;
	530	}
	531
	532	/*
	533	* Insert vram stolen pages into the GTT
	534	*/
	535
	536	pfn_base = dev_priv->stolen_base >> PAGE_SHIFT;
f728bd1a	537	num_pages = vram_stolen_size >> PAGE_SHIFT;
31a0685a	538	dev_dbg(dev->dev, "Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
8c8f1c95 AC	539	num_pages, pfn_base << PAGE_SHIFT, 0);
8c8f1c95 AC	540	for (i = 0; i < num_pages; ++i) {
ae012bdc	541	pte = psb_gtt_mask_pte(pfn_base + i, PSB_MMU_CACHED_MEMORY);
8c8f1c95 AC	542	iowrite32(pte, dev_priv->gtt_map + i);
	543	}
	544
	545	/*
	546	* Init rest of GTT to the scratch page to avoid accidents or scribbles
	547	*/
	548
	549	pfn_base = page_to_pfn(dev_priv->scratch_page);
ae012bdc	550	pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
8c8f1c95 AC	551	for (; i < gtt_pages; ++i)
	552	iowrite32(pte, dev_priv->gtt_map + i);
	553
	554	(void) ioread32(dev_priv->gtt_map + i - 1);
	555	return 0;
	556
	557	out_err:
	558	psb_gtt_takedown(dev);
	559	return ret;
	560	}
070839ea PJ	561
	562	int psb_gtt_restore(struct drm_device *dev)
	563	{
	564	struct drm_psb_private *dev_priv = dev->dev_private;
	565	struct resource *r = dev_priv->gtt_mem->child;
	566	struct gtt_range *range;
1611f845	567	unsigned int restored = 0, total = 0, size = 0;
070839ea PJ	568
	569	/* On resume, the gtt_mutex is already initialized */
	570	mutex_lock(&dev_priv->gtt_mutex);
	571	psb_gtt_init(dev, 1);
	572
	573	while (r != NULL) {
	574	range = container_of(r, struct gtt_range, resource);
1611f845	575	if (range->pages) {
070839ea	576	psb_gtt_insert(dev, range, 1);
1611f845 PJ	577	size += range->resource.end - range->resource.start;
	578	restored++;
	579	}
070839ea	580	r = r->sibling;
1611f845	581	total++;
070839ea PJ	582	}
070839ea PJ	583	mutex_unlock(&dev_priv->gtt_mutex);
1611f845 PJ	584	DRM_DEBUG_DRIVER("Restored %u of %u gtt ranges (%u KB)", restored,
1611f845 PJ	585	total, (size / 1024));
070839ea PJ	586
	587	return 0;
	588	}