[deliverable/linux.git] / drivers / gpu / drm / radeon / radeon_ring.c

/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
#include <linux/seq_file.h>
#include <linux/slab.h>
#include "drmP.h"
#include "radeon_drm.h"
#include "radeon_reg.h"
#include "radeon.h"
#include "atom.h"

int radeon_debugfs_ib_init(struct radeon_device *rdev);

u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
{
	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
	u32 pg_idx, pg_offset;
	u32 idx_value = 0;
	int new_page;

	pg_idx = (idx * 4) / PAGE_SIZE;
	pg_offset = (idx * 4) % PAGE_SIZE;

	if (ibc->kpage_idx[0] == pg_idx)
		return ibc->kpage[0][pg_offset/4];
	if (ibc->kpage_idx[1] == pg_idx)
		return ibc->kpage[1][pg_offset/4];

	new_page = radeon_cs_update_pages(p, pg_idx);
	if (new_page < 0) {
		p->parser_error = new_page;
		return 0;
	}

	idx_value = ibc->kpage[new_page][pg_offset/4];
	return idx_value;
}

void radeon_ring_write(struct radeon_cp *cp, uint32_t v)
{
#if DRM_DEBUG_CODE
	if (cp->count_dw <= 0) {
		DRM_ERROR("radeon: writting more dword to ring than expected !\n");
	}
#endif
	cp->ring[cp->wptr++] = v;
	cp->wptr &= cp->ptr_mask;
	cp->count_dw--;
	cp->ring_free_dw--;
}

void radeon_ib_bogus_cleanup(struct radeon_device *rdev)
{
	struct radeon_ib *ib, *n;

	list_for_each_entry_safe(ib, n, &rdev->ib_pool.bogus_ib, list) {
		list_del(&ib->list);
		vfree(ib->ptr);
		kfree(ib);
	}
}

void radeon_ib_bogus_add(struct radeon_device *rdev, struct radeon_ib *ib)
{
	struct radeon_ib *bib;

	bib = kmalloc(sizeof(*bib), GFP_KERNEL);
	if (bib == NULL)
		return;
	bib->ptr = vmalloc(ib->length_dw * 4);
	if (bib->ptr == NULL) {
		kfree(bib);
		return;
	}
	memcpy(bib->ptr, ib->ptr, ib->length_dw * 4);
	bib->length_dw = ib->length_dw;
	mutex_lock(&rdev->ib_pool.mutex);
	list_add_tail(&bib->list, &rdev->ib_pool.bogus_ib);
	mutex_unlock(&rdev->ib_pool.mutex);
}

/*
 * IB.
 */
int radeon_ib_get(struct radeon_device *rdev, int ring, struct radeon_ib **ib)
{
	struct radeon_fence *fence;
	struct radeon_ib *nib;
	int r = 0, i, c;

	*ib = NULL;
	r = radeon_fence_create(rdev, &fence, ring);
	if (r) {
		dev_err(rdev->dev, "failed to create fence for new IB\n");
		return r;
	}
	mutex_lock(&rdev->ib_pool.mutex);
	for (i = rdev->ib_pool.head_id, c = 0, nib = NULL; c < RADEON_IB_POOL_SIZE; c++, i++) {
		i &= (RADEON_IB_POOL_SIZE - 1);
		if (rdev->ib_pool.ibs[i].free) {
			nib = &rdev->ib_pool.ibs[i];
			break;
		}
	}
	if (nib == NULL) {
		/* This should never happen, it means we allocated all
		 * IB and haven't scheduled one yet, return EBUSY to
		 * userspace hoping that on ioctl recall we get better
		 * luck
		 */
		dev_err(rdev->dev, "no free indirect buffer !\n");
		mutex_unlock(&rdev->ib_pool.mutex);
		radeon_fence_unref(&fence);
		return -EBUSY;
	}
	rdev->ib_pool.head_id = (nib->idx + 1) & (RADEON_IB_POOL_SIZE - 1);
	nib->free = false;
	if (nib->fence) {
		mutex_unlock(&rdev->ib_pool.mutex);
		r = radeon_fence_wait(nib->fence, false);
		if (r) {
			dev_err(rdev->dev, "error waiting fence of IB(%u:0x%016lX:%u)\n",
				nib->idx, (unsigned long)nib->gpu_addr, nib->length_dw);
			mutex_lock(&rdev->ib_pool.mutex);
			nib->free = true;
			mutex_unlock(&rdev->ib_pool.mutex);
			radeon_fence_unref(&fence);
			return r;
		}
		mutex_lock(&rdev->ib_pool.mutex);
	}
	radeon_fence_unref(&nib->fence);
	nib->fence = fence;
	nib->length_dw = 0;
	mutex_unlock(&rdev->ib_pool.mutex);
	*ib = nib;
	return 0;
}

void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
{
	struct radeon_ib *tmp = *ib;

	*ib = NULL;
	if (tmp == NULL) {
		return;
	}
	if (!tmp->fence->emitted)
		radeon_fence_unref(&tmp->fence);
	mutex_lock(&rdev->ib_pool.mutex);
	tmp->free = true;
	mutex_unlock(&rdev->ib_pool.mutex);
}

int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)
{
	struct radeon_cp *cp = &rdev->cp[ib->fence->ring];
	int r = 0;

	if (!ib->length_dw || !cp->ready) {
		/* TODO: Nothings in the ib we should report. */
		DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx);
		return -EINVAL;
	}

	/* 64 dwords should be enough for fence too */
	r = radeon_ring_lock(rdev, cp, 64);
	if (r) {
		DRM_ERROR("radeon: scheduling IB failed (%d).\n", r);
		return r;
	}
	radeon_ring_ib_execute(rdev, ib->fence->ring, ib);
	radeon_fence_emit(rdev, ib->fence);
	mutex_lock(&rdev->ib_pool.mutex);
	/* once scheduled IB is considered free and protected by the fence */
	ib->free = true;
	mutex_unlock(&rdev->ib_pool.mutex);
	radeon_ring_unlock_commit(rdev, cp);
	return 0;
}

int radeon_ib_pool_init(struct radeon_device *rdev)
{
	void *ptr;
	uint64_t gpu_addr;
	int i;
	int r = 0;

	if (rdev->ib_pool.robj)
		return 0;
	INIT_LIST_HEAD(&rdev->ib_pool.bogus_ib);
	/* Allocate 1M object buffer */
	r = radeon_bo_create(rdev, RADEON_IB_POOL_SIZE*64*1024,
			     PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
			     &rdev->ib_pool.robj);
	if (r) {
		DRM_ERROR("radeon: failed to ib pool (%d).\n", r);
		return r;
	}
	r = radeon_bo_reserve(rdev->ib_pool.robj, false);
	if (unlikely(r != 0))
		return r;
	r = radeon_bo_pin(rdev->ib_pool.robj, RADEON_GEM_DOMAIN_GTT, &gpu_addr);
	if (r) {
		radeon_bo_unreserve(rdev->ib_pool.robj);
		DRM_ERROR("radeon: failed to pin ib pool (%d).\n", r);
		return r;
	}
	r = radeon_bo_kmap(rdev->ib_pool.robj, &ptr);
	radeon_bo_unreserve(rdev->ib_pool.robj);
	if (r) {
		DRM_ERROR("radeon: failed to map ib pool (%d).\n", r);
		return r;
	}
	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
		unsigned offset;

		offset = i * 64 * 1024;
		rdev->ib_pool.ibs[i].gpu_addr = gpu_addr + offset;
		rdev->ib_pool.ibs[i].ptr = ptr + offset;
		rdev->ib_pool.ibs[i].idx = i;
		rdev->ib_pool.ibs[i].length_dw = 0;
		rdev->ib_pool.ibs[i].free = true;
	}
	rdev->ib_pool.head_id = 0;
	rdev->ib_pool.ready = true;
	DRM_INFO("radeon: ib pool ready.\n");
	if (radeon_debugfs_ib_init(rdev)) {
		DRM_ERROR("Failed to register debugfs file for IB !\n");
	}
	return r;
}

void radeon_ib_pool_fini(struct radeon_device *rdev)
{
	int r;
	struct radeon_bo *robj;

	if (!rdev->ib_pool.ready) {
		return;
	}
	mutex_lock(&rdev->ib_pool.mutex);
	radeon_ib_bogus_cleanup(rdev);
	robj = rdev->ib_pool.robj;
	rdev->ib_pool.robj = NULL;
	mutex_unlock(&rdev->ib_pool.mutex);

	if (robj) {
		r = radeon_bo_reserve(robj, false);
		if (likely(r == 0)) {
			radeon_bo_kunmap(robj);
			radeon_bo_unpin(robj);
			radeon_bo_unreserve(robj);
		}
		radeon_bo_unref(&robj);
	}
}


/*
 * Ring.
 */
int radeon_ring_index(struct radeon_device *rdev, struct radeon_cp *cp)
{
	/* r1xx-r5xx only has CP ring */
	if (rdev->family < CHIP_R600)
		return RADEON_RING_TYPE_GFX_INDEX;

	if (rdev->family >= CHIP_CAYMAN) {
		if (cp == &rdev->cp[CAYMAN_RING_TYPE_CP1_INDEX])
			return CAYMAN_RING_TYPE_CP1_INDEX;
		else if (cp == &rdev->cp[CAYMAN_RING_TYPE_CP2_INDEX])
			return CAYMAN_RING_TYPE_CP2_INDEX;
	}
	return RADEON_RING_TYPE_GFX_INDEX;
}

void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_cp *cp)
{
	if (rdev->wb.enabled)
		cp->rptr = le32_to_cpu(rdev->wb.wb[cp->rptr_offs/4]);
	else
		cp->rptr = RREG32(cp->rptr_reg);
	/* This works because ring_size is a power of 2 */
	cp->ring_free_dw = (cp->rptr + (cp->ring_size / 4));
	cp->ring_free_dw -= cp->wptr;
	cp->ring_free_dw &= cp->ptr_mask;
	if (!cp->ring_free_dw) {
		cp->ring_free_dw = cp->ring_size / 4;
	}
}


int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_cp *cp, unsigned ndw)
{
	int r;

	/* Align requested size with padding so unlock_commit can
	 * pad safely */
	ndw = (ndw + cp->align_mask) & ~cp->align_mask;
	while (ndw > (cp->ring_free_dw - 1)) {
		radeon_ring_free_size(rdev, cp);
		if (ndw < cp->ring_free_dw) {
			break;
		}
		r = radeon_fence_wait_next(rdev, radeon_ring_index(rdev, cp));
		if (r)
			return r;
	}
	cp->count_dw = ndw;
	cp->wptr_old = cp->wptr;
	return 0;
}

int radeon_ring_lock(struct radeon_device *rdev, struct radeon_cp *cp, unsigned ndw)
{
	int r;

	mutex_lock(&cp->mutex);
	r = radeon_ring_alloc(rdev, cp, ndw);
	if (r) {
		mutex_unlock(&cp->mutex);
		return r;
	}
	return 0;
}

void radeon_ring_commit(struct radeon_device *rdev, struct radeon_cp *cp)
{
	unsigned count_dw_pad;
	unsigned i;

	/* We pad to match fetch size */
	count_dw_pad = (cp->align_mask + 1) -
		       (cp->wptr & cp->align_mask);
	for (i = 0; i < count_dw_pad; i++) {
		radeon_ring_write(cp, 2 << 30);
	}
	DRM_MEMORYBARRIER();
	WREG32(cp->wptr_reg, cp->wptr);
	(void)RREG32(cp->wptr_reg);
}

void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_cp *cp)
{
	radeon_ring_commit(rdev, cp);
	mutex_unlock(&cp->mutex);
}

void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_cp *cp)
{
	cp->wptr = cp->wptr_old;
	mutex_unlock(&cp->mutex);
}

int radeon_ring_init(struct radeon_device *rdev, struct radeon_cp *cp, unsigned ring_size,
		     unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg)
{
	int r;

	cp->ring_size = ring_size;
	cp->rptr_offs = rptr_offs;
	cp->rptr_reg = rptr_reg;
	cp->wptr_reg = wptr_reg;
	/* Allocate ring buffer */
	if (cp->ring_obj == NULL) {
		r = radeon_bo_create(rdev, cp->ring_size, PAGE_SIZE, true,
					RADEON_GEM_DOMAIN_GTT,
					&cp->ring_obj);
		if (r) {
			dev_err(rdev->dev, "(%d) ring create failed\n", r);
			return r;
		}
		r = radeon_bo_reserve(cp->ring_obj, false);
		if (unlikely(r != 0))
			return r;
		r = radeon_bo_pin(cp->ring_obj, RADEON_GEM_DOMAIN_GTT,
					&cp->gpu_addr);
		if (r) {
			radeon_bo_unreserve(cp->ring_obj);
			dev_err(rdev->dev, "(%d) ring pin failed\n", r);
			return r;
		}
		r = radeon_bo_kmap(cp->ring_obj,
				       (void **)&cp->ring);
		radeon_bo_unreserve(cp->ring_obj);
		if (r) {
			dev_err(rdev->dev, "(%d) ring map failed\n", r);
			return r;
		}
	}
	cp->ptr_mask = (cp->ring_size / 4) - 1;
	cp->ring_free_dw = cp->ring_size / 4;
	return 0;
}

void radeon_ring_fini(struct radeon_device *rdev, struct radeon_cp *cp)
{
	int r;
	struct radeon_bo *ring_obj;

	mutex_lock(&cp->mutex);
	ring_obj = cp->ring_obj;
	cp->ring = NULL;
	cp->ring_obj = NULL;
	mutex_unlock(&cp->mutex);

	if (ring_obj) {
		r = radeon_bo_reserve(ring_obj, false);
		if (likely(r == 0)) {
			radeon_bo_kunmap(ring_obj);
			radeon_bo_unpin(ring_obj);
			radeon_bo_unreserve(ring_obj);
		}
		radeon_bo_unref(&ring_obj);
	}
}

/*
 * Debugfs info
 */
#if defined(CONFIG_DEBUG_FS)
static int radeon_debugfs_ib_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *) m->private;
	struct radeon_ib *ib = node->info_ent->data;
	unsigned i;

	if (ib == NULL) {
		return 0;
	}
	seq_printf(m, "IB %04u\n", ib->idx);
	seq_printf(m, "IB fence %p\n", ib->fence);
	seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
	for (i = 0; i < ib->length_dw; i++) {
		seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
	}
	return 0;
}

static int radeon_debugfs_ib_bogus_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *) m->private;
	struct radeon_device *rdev = node->info_ent->data;
	struct radeon_ib *ib;
	unsigned i;

	mutex_lock(&rdev->ib_pool.mutex);
	if (list_empty(&rdev->ib_pool.bogus_ib)) {
		mutex_unlock(&rdev->ib_pool.mutex);
		seq_printf(m, "no bogus IB recorded\n");
		return 0;
	}
	ib = list_first_entry(&rdev->ib_pool.bogus_ib, struct radeon_ib, list);
	list_del_init(&ib->list);
	mutex_unlock(&rdev->ib_pool.mutex);
	seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
	for (i = 0; i < ib->length_dw; i++) {
		seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
	}
	vfree(ib->ptr);
	kfree(ib);
	return 0;
}

static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE];
static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32];

static struct drm_info_list radeon_debugfs_ib_bogus_info_list[] = {
	{"radeon_ib_bogus", radeon_debugfs_ib_bogus_info, 0, NULL},
};
#endif

int radeon_debugfs_ib_init(struct radeon_device *rdev)
{
#if defined(CONFIG_DEBUG_FS)
	unsigned i;
	int r;

	radeon_debugfs_ib_bogus_info_list[0].data = rdev;
	r = radeon_debugfs_add_files(rdev, radeon_debugfs_ib_bogus_info_list, 1);
	if (r)
		return r;
	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
		sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i);
		radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i];
		radeon_debugfs_ib_list[i].show = &radeon_debugfs_ib_info;
		radeon_debugfs_ib_list[i].driver_features = 0;
		radeon_debugfs_ib_list[i].data = &rdev->ib_pool.ibs[i];
	}
	return radeon_debugfs_add_files(rdev, radeon_debugfs_ib_list,
					RADEON_IB_POOL_SIZE);
#else
	return 0;
#endif
}
Commit	Line	Data
771fe6b9 JG	1	/*
	2	* Copyright 2008 Advanced Micro Devices, Inc.
	3	* Copyright 2008 Red Hat Inc.
	4	* Copyright 2009 Jerome Glisse.
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining a
	7	* copy of this software and associated documentation files (the "Software"),
	8	* to deal in the Software without restriction, including without limitation
	9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	10	* and/or sell copies of the Software, and to permit persons to whom the
	11	* Software is furnished to do so, subject to the following conditions:
	12	*
	13	* The above copyright notice and this permission notice shall be included in
	14	* all copies or substantial portions of the Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	22	* OTHER DEALINGS IN THE SOFTWARE.
	23	*
	24	* Authors: Dave Airlie
	25	* Alex Deucher
	26	* Jerome Glisse
	27	*/
	28	#include <linux/seq_file.h>
5a0e3ad6	29	#include <linux/slab.h>
771fe6b9 JG	30	#include "drmP.h"
	31	#include "radeon_drm.h"
	32	#include "radeon_reg.h"
	33	#include "radeon.h"
	34	#include "atom.h"
	35
	36	int radeon_debugfs_ib_init(struct radeon_device *rdev);
	37
ce580fab AK	38	u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
	39	{
	40	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
	41	u32 pg_idx, pg_offset;
	42	u32 idx_value = 0;
	43	int new_page;
	44
	45	pg_idx = (idx * 4) / PAGE_SIZE;
	46	pg_offset = (idx * 4) % PAGE_SIZE;
	47
	48	if (ibc->kpage_idx[0] == pg_idx)
	49	return ibc->kpage[0][pg_offset/4];
	50	if (ibc->kpage_idx[1] == pg_idx)
	51	return ibc->kpage[1][pg_offset/4];
	52
	53	new_page = radeon_cs_update_pages(p, pg_idx);
	54	if (new_page < 0) {
	55	p->parser_error = new_page;
	56	return 0;
	57	}
	58
	59	idx_value = ibc->kpage[new_page][pg_offset/4];
	60	return idx_value;
	61	}
	62
7b1f2485	63	void radeon_ring_write(struct radeon_cp *cp, uint32_t v)
ce580fab AK	64	{
ce580fab AK	65	#if DRM_DEBUG_CODE
7b1f2485	66	if (cp->count_dw <= 0) {
ce580fab AK	67	DRM_ERROR("radeon: writting more dword to ring than expected !\n");
	68	}
	69	#endif
7b1f2485 CK	70	cp->ring[cp->wptr++] = v;
	71	cp->wptr &= cp->ptr_mask;
	72	cp->count_dw--;
	73	cp->ring_free_dw--;
ce580fab AK	74	}
ce580fab AK	75
9f93ed39 JG	76	void radeon_ib_bogus_cleanup(struct radeon_device *rdev)
	77	{
	78	struct radeon_ib ib, n;
	79
	80	list_for_each_entry_safe(ib, n, &rdev->ib_pool.bogus_ib, list) {
	81	list_del(&ib->list);
	82	vfree(ib->ptr);
	83	kfree(ib);
	84	}
	85	}
	86
	87	void radeon_ib_bogus_add(struct radeon_device rdev, struct radeon_ib ib)
	88	{
	89	struct radeon_ib *bib;
	90
	91	bib = kmalloc(sizeof(*bib), GFP_KERNEL);
	92	if (bib == NULL)
	93	return;
	94	bib->ptr = vmalloc(ib->length_dw * 4);
	95	if (bib->ptr == NULL) {
	96	kfree(bib);
	97	return;
	98	}
	99	memcpy(bib->ptr, ib->ptr, ib->length_dw * 4);
	100	bib->length_dw = ib->length_dw;
	101	mutex_lock(&rdev->ib_pool.mutex);
	102	list_add_tail(&bib->list, &rdev->ib_pool.bogus_ib);
	103	mutex_unlock(&rdev->ib_pool.mutex);
	104	}
	105
771fe6b9 JG	106	/*
	107	* IB.
	108	*/
7b1f2485	109	int radeon_ib_get(struct radeon_device rdev, int ring, struct radeon_ib *ib)
771fe6b9 JG	110	{
	111	struct radeon_fence *fence;
	112	struct radeon_ib *nib;
91cb91be	113	int r = 0, i, c;
771fe6b9 JG	114
771fe6b9 JG	115	*ib = NULL;
7b1f2485	116	r = radeon_fence_create(rdev, &fence, ring);
771fe6b9	117	if (r) {
91cb91be	118	dev_err(rdev->dev, "failed to create fence for new IB\n");
771fe6b9 JG	119	return r;
	120	}
	121	mutex_lock(&rdev->ib_pool.mutex);
91cb91be JG	122	for (i = rdev->ib_pool.head_id, c = 0, nib = NULL; c < RADEON_IB_POOL_SIZE; c++, i++) {
	123	i &= (RADEON_IB_POOL_SIZE - 1);
	124	if (rdev->ib_pool.ibs[i].free) {
	125	nib = &rdev->ib_pool.ibs[i];
	126	break;
	127	}
771fe6b9	128	}
91cb91be JG	129	if (nib == NULL) {
	130	/* This should never happen, it means we allocated all
	131	* IB and haven't scheduled one yet, return EBUSY to
	132	* userspace hoping that on ioctl recall we get better
	133	* luck
	134	*/
	135	dev_err(rdev->dev, "no free indirect buffer !\n");
ecb114a1	136	mutex_unlock(&rdev->ib_pool.mutex);
91cb91be JG	137	radeon_fence_unref(&fence);
91cb91be JG	138	return -EBUSY;
771fe6b9	139	}
91cb91be JG	140	rdev->ib_pool.head_id = (nib->idx + 1) & (RADEON_IB_POOL_SIZE - 1);
	141	nib->free = false;
	142	if (nib->fence) {
ecb114a1	143	mutex_unlock(&rdev->ib_pool.mutex);
91cb91be JG	144	r = radeon_fence_wait(nib->fence, false);
	145	if (r) {
	146	dev_err(rdev->dev, "error waiting fence of IB(%u:0x%016lX:%u)\n",
	147	nib->idx, (unsigned long)nib->gpu_addr, nib->length_dw);
	148	mutex_lock(&rdev->ib_pool.mutex);
	149	nib->free = true;
	150	mutex_unlock(&rdev->ib_pool.mutex);
	151	radeon_fence_unref(&fence);
	152	return r;
	153	}
	154	mutex_lock(&rdev->ib_pool.mutex);
771fe6b9 JG	155	}
771fe6b9 JG	156	radeon_fence_unref(&nib->fence);
91cb91be	157	nib->fence = fence;
771fe6b9	158	nib->length_dw = 0;
ecb114a1	159	mutex_unlock(&rdev->ib_pool.mutex);
771fe6b9	160	*ib = nib;
91cb91be	161	return 0;
771fe6b9 JG	162	}
	163
	164	void radeon_ib_free(struct radeon_device rdev, struct radeon_ib *ib)
	165	{
	166	struct radeon_ib tmp = ib;
	167
	168	*ib = NULL;
	169	if (tmp == NULL) {
	170	return;
	171	}
851a6bd9	172	if (!tmp->fence->emitted)
7d404c7b	173	radeon_fence_unref(&tmp->fence);
771fe6b9	174	mutex_lock(&rdev->ib_pool.mutex);
91cb91be	175	tmp->free = true;
771fe6b9 JG	176	mutex_unlock(&rdev->ib_pool.mutex);
	177	}
	178
771fe6b9 JG	179	int radeon_ib_schedule(struct radeon_device rdev, struct radeon_ib ib)
771fe6b9 JG	180	{
bf852799	181	struct radeon_cp *cp = &rdev->cp[ib->fence->ring];
771fe6b9 JG	182	int r = 0;
771fe6b9 JG	183
7b1f2485	184	if (!ib->length_dw \|\| !cp->ready) {
771fe6b9	185	/* TODO: Nothings in the ib we should report. */
91cb91be	186	DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx);
771fe6b9 JG	187	return -EINVAL;
771fe6b9 JG	188	}
ecb114a1	189
6cdf6585	190	/* 64 dwords should be enough for fence too */
7b1f2485	191	r = radeon_ring_lock(rdev, cp, 64);
771fe6b9	192	if (r) {
ec4f2ac4	193	DRM_ERROR("radeon: scheduling IB failed (%d).\n", r);
771fe6b9 JG	194	return r;
771fe6b9 JG	195	}
4c87bc26	196	radeon_ring_ib_execute(rdev, ib->fence->ring, ib);
771fe6b9	197	radeon_fence_emit(rdev, ib->fence);
ecb114a1	198	mutex_lock(&rdev->ib_pool.mutex);
91cb91be JG	199	/* once scheduled IB is considered free and protected by the fence */
91cb91be JG	200	ib->free = true;
771fe6b9	201	mutex_unlock(&rdev->ib_pool.mutex);
7b1f2485	202	radeon_ring_unlock_commit(rdev, cp);
771fe6b9 JG	203	return 0;
	204	}
	205
	206	int radeon_ib_pool_init(struct radeon_device *rdev)
	207	{
	208	void *ptr;
	209	uint64_t gpu_addr;
	210	int i;
	211	int r = 0;
	212
9f022ddf JG	213	if (rdev->ib_pool.robj)
9f022ddf JG	214	return 0;
9f93ed39	215	INIT_LIST_HEAD(&rdev->ib_pool.bogus_ib);
771fe6b9	216	/* Allocate 1M object buffer */
441921d5	217	r = radeon_bo_create(rdev, RADEON_IB_POOL_SIZE641024,
268b2510 AD	218	PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
268b2510 AD	219	&rdev->ib_pool.robj);
771fe6b9 JG	220	if (r) {
	221	DRM_ERROR("radeon: failed to ib pool (%d).\n", r);
	222	return r;
	223	}
4c788679 JG	224	r = radeon_bo_reserve(rdev->ib_pool.robj, false);
	225	if (unlikely(r != 0))
	226	return r;
	227	r = radeon_bo_pin(rdev->ib_pool.robj, RADEON_GEM_DOMAIN_GTT, &gpu_addr);
771fe6b9	228	if (r) {
4c788679	229	radeon_bo_unreserve(rdev->ib_pool.robj);
771fe6b9 JG	230	DRM_ERROR("radeon: failed to pin ib pool (%d).\n", r);
	231	return r;
	232	}
4c788679 JG	233	r = radeon_bo_kmap(rdev->ib_pool.robj, &ptr);
4c788679 JG	234	radeon_bo_unreserve(rdev->ib_pool.robj);
771fe6b9	235	if (r) {
205a44a4	236	DRM_ERROR("radeon: failed to map ib pool (%d).\n", r);
771fe6b9 JG	237	return r;
	238	}
	239	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
	240	unsigned offset;
	241
	242	offset = i * 64 * 1024;
	243	rdev->ib_pool.ibs[i].gpu_addr = gpu_addr + offset;
	244	rdev->ib_pool.ibs[i].ptr = ptr + offset;
	245	rdev->ib_pool.ibs[i].idx = i;
	246	rdev->ib_pool.ibs[i].length_dw = 0;
91cb91be	247	rdev->ib_pool.ibs[i].free = true;
771fe6b9	248	}
91cb91be	249	rdev->ib_pool.head_id = 0;
771fe6b9 JG	250	rdev->ib_pool.ready = true;
	251	DRM_INFO("radeon: ib pool ready.\n");
	252	if (radeon_debugfs_ib_init(rdev)) {
	253	DRM_ERROR("Failed to register debugfs file for IB !\n");
	254	}
	255	return r;
	256	}
	257
	258	void radeon_ib_pool_fini(struct radeon_device *rdev)
	259	{
4c788679	260	int r;
ca2af923	261	struct radeon_bo *robj;
4c788679	262
771fe6b9 JG	263	if (!rdev->ib_pool.ready) {
	264	return;
	265	}
	266	mutex_lock(&rdev->ib_pool.mutex);
9f93ed39	267	radeon_ib_bogus_cleanup(rdev);
ca2af923 AD	268	robj = rdev->ib_pool.robj;
	269	rdev->ib_pool.robj = NULL;
	270	mutex_unlock(&rdev->ib_pool.mutex);
eb6b6d7c	271
ca2af923 AD	272	if (robj) {
ca2af923 AD	273	r = radeon_bo_reserve(robj, false);
4c788679	274	if (likely(r == 0)) {
ca2af923 AD	275	radeon_bo_kunmap(robj);
	276	radeon_bo_unpin(robj);
	277	radeon_bo_unreserve(robj);
4c788679	278	}
ca2af923	279	radeon_bo_unref(&robj);
771fe6b9	280	}
771fe6b9 JG	281	}
771fe6b9 JG	282
771fe6b9 JG	283
	284	/*
	285	* Ring.
	286	*/
bf852799 CK	287	int radeon_ring_index(struct radeon_device rdev, struct radeon_cp cp)
	288	{
	289	/* r1xx-r5xx only has CP ring */
	290	if (rdev->family < CHIP_R600)
	291	return RADEON_RING_TYPE_GFX_INDEX;
	292
	293	if (rdev->family >= CHIP_CAYMAN) {
	294	if (cp == &rdev->cp[CAYMAN_RING_TYPE_CP1_INDEX])
	295	return CAYMAN_RING_TYPE_CP1_INDEX;
	296	else if (cp == &rdev->cp[CAYMAN_RING_TYPE_CP2_INDEX])
	297	return CAYMAN_RING_TYPE_CP2_INDEX;
	298	}
	299	return RADEON_RING_TYPE_GFX_INDEX;
	300	}
	301
7b1f2485	302	void radeon_ring_free_size(struct radeon_device rdev, struct radeon_cp cp)
771fe6b9	303	{
724c80e1	304	if (rdev->wb.enabled)
5596a9db CK	305	cp->rptr = le32_to_cpu(rdev->wb.wb[cp->rptr_offs/4]);
	306	else
	307	cp->rptr = RREG32(cp->rptr_reg);
771fe6b9	308	/* This works because ring_size is a power of 2 */
7b1f2485 CK	309	cp->ring_free_dw = (cp->rptr + (cp->ring_size / 4));
	310	cp->ring_free_dw -= cp->wptr;
	311	cp->ring_free_dw &= cp->ptr_mask;
	312	if (!cp->ring_free_dw) {
	313	cp->ring_free_dw = cp->ring_size / 4;
771fe6b9 JG	314	}
	315	}
	316
7b1f2485 CK	317
7b1f2485 CK	318	int radeon_ring_alloc(struct radeon_device rdev, struct radeon_cp cp, unsigned ndw)
771fe6b9 JG	319	{
	320	int r;
	321
	322	/* Align requested size with padding so unlock_commit can
	323	* pad safely */
7b1f2485 CK	324	ndw = (ndw + cp->align_mask) & ~cp->align_mask;
	325	while (ndw > (cp->ring_free_dw - 1)) {
	326	radeon_ring_free_size(rdev, cp);
	327	if (ndw < cp->ring_free_dw) {
771fe6b9 JG	328	break;
771fe6b9 JG	329	}
bf852799	330	r = radeon_fence_wait_next(rdev, radeon_ring_index(rdev, cp));
91700f3c	331	if (r)
771fe6b9	332	return r;
771fe6b9	333	}
7b1f2485 CK	334	cp->count_dw = ndw;
7b1f2485 CK	335	cp->wptr_old = cp->wptr;
771fe6b9 JG	336	return 0;
	337	}
	338
7b1f2485	339	int radeon_ring_lock(struct radeon_device rdev, struct radeon_cp cp, unsigned ndw)
91700f3c MG	340	{
	341	int r;
	342
7b1f2485 CK	343	mutex_lock(&cp->mutex);
7b1f2485 CK	344	r = radeon_ring_alloc(rdev, cp, ndw);
91700f3c	345	if (r) {
7b1f2485	346	mutex_unlock(&cp->mutex);
91700f3c MG	347	return r;
	348	}
	349	return 0;
	350	}
	351
7b1f2485	352	void radeon_ring_commit(struct radeon_device rdev, struct radeon_cp cp)
771fe6b9 JG	353	{
	354	unsigned count_dw_pad;
	355	unsigned i;
	356
	357	/* We pad to match fetch size */
7b1f2485 CK	358	count_dw_pad = (cp->align_mask + 1) -
7b1f2485 CK	359	(cp->wptr & cp->align_mask);
771fe6b9	360	for (i = 0; i < count_dw_pad; i++) {
7b1f2485	361	radeon_ring_write(cp, 2 << 30);
771fe6b9 JG	362	}
771fe6b9 JG	363	DRM_MEMORYBARRIER();
5596a9db CK	364	WREG32(cp->wptr_reg, cp->wptr);
5596a9db CK	365	(void)RREG32(cp->wptr_reg);
91700f3c MG	366	}
91700f3c MG	367
7b1f2485	368	void radeon_ring_unlock_commit(struct radeon_device rdev, struct radeon_cp cp)
91700f3c	369	{
7b1f2485 CK	370	radeon_ring_commit(rdev, cp);
7b1f2485 CK	371	mutex_unlock(&cp->mutex);
771fe6b9 JG	372	}
771fe6b9 JG	373
7b1f2485	374	void radeon_ring_unlock_undo(struct radeon_device rdev, struct radeon_cp cp)
771fe6b9	375	{
7b1f2485 CK	376	cp->wptr = cp->wptr_old;
7b1f2485 CK	377	mutex_unlock(&cp->mutex);
771fe6b9 JG	378	}
771fe6b9 JG	379
5596a9db CK	380	int radeon_ring_init(struct radeon_device rdev, struct radeon_cp cp, unsigned ring_size,
5596a9db CK	381	unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg)
771fe6b9 JG	382	{
	383	int r;
	384
7b1f2485	385	cp->ring_size = ring_size;
5596a9db CK	386	cp->rptr_offs = rptr_offs;
	387	cp->rptr_reg = rptr_reg;
	388	cp->wptr_reg = wptr_reg;
771fe6b9	389	/* Allocate ring buffer */
7b1f2485 CK	390	if (cp->ring_obj == NULL) {
7b1f2485 CK	391	r = radeon_bo_create(rdev, cp->ring_size, PAGE_SIZE, true,
4c788679	392	RADEON_GEM_DOMAIN_GTT,
7b1f2485	393	&cp->ring_obj);
771fe6b9	394	if (r) {
4c788679	395	dev_err(rdev->dev, "(%d) ring create failed\n", r);
771fe6b9 JG	396	return r;
771fe6b9 JG	397	}
7b1f2485	398	r = radeon_bo_reserve(cp->ring_obj, false);
4c788679 JG	399	if (unlikely(r != 0))
4c788679 JG	400	return r;
7b1f2485 CK	401	r = radeon_bo_pin(cp->ring_obj, RADEON_GEM_DOMAIN_GTT,
7b1f2485 CK	402	&cp->gpu_addr);
771fe6b9	403	if (r) {
7b1f2485	404	radeon_bo_unreserve(cp->ring_obj);
4c788679	405	dev_err(rdev->dev, "(%d) ring pin failed\n", r);
771fe6b9 JG	406	return r;
771fe6b9 JG	407	}
7b1f2485 CK	408	r = radeon_bo_kmap(cp->ring_obj,
	409	(void **)&cp->ring);
	410	radeon_bo_unreserve(cp->ring_obj);
771fe6b9	411	if (r) {
4c788679	412	dev_err(rdev->dev, "(%d) ring map failed\n", r);
771fe6b9 JG	413	return r;
	414	}
	415	}
7b1f2485 CK	416	cp->ptr_mask = (cp->ring_size / 4) - 1;
7b1f2485 CK	417	cp->ring_free_dw = cp->ring_size / 4;
771fe6b9 JG	418	return 0;
	419	}
	420
7b1f2485	421	void radeon_ring_fini(struct radeon_device rdev, struct radeon_cp cp)
771fe6b9	422	{
4c788679	423	int r;
ca2af923	424	struct radeon_bo *ring_obj;
4c788679	425
7b1f2485 CK	426	mutex_lock(&cp->mutex);
	427	ring_obj = cp->ring_obj;
	428	cp->ring = NULL;
	429	cp->ring_obj = NULL;
	430	mutex_unlock(&cp->mutex);
ca2af923 AD	431
	432	if (ring_obj) {
	433	r = radeon_bo_reserve(ring_obj, false);
4c788679	434	if (likely(r == 0)) {
ca2af923 AD	435	radeon_bo_kunmap(ring_obj);
	436	radeon_bo_unpin(ring_obj);
	437	radeon_bo_unreserve(ring_obj);
4c788679	438	}
ca2af923	439	radeon_bo_unref(&ring_obj);
771fe6b9	440	}
771fe6b9 JG	441	}
771fe6b9 JG	442
771fe6b9 JG	443	/*
	444	* Debugfs info
	445	*/
	446	#if defined(CONFIG_DEBUG_FS)
	447	static int radeon_debugfs_ib_info(struct seq_file m, void data)
	448	{
	449	struct drm_info_node node = (struct drm_info_node ) m->private;
	450	struct radeon_ib *ib = node->info_ent->data;
	451	unsigned i;
	452
	453	if (ib == NULL) {
	454	return 0;
	455	}
91cb91be	456	seq_printf(m, "IB %04u\n", ib->idx);
771fe6b9 JG	457	seq_printf(m, "IB fence %p\n", ib->fence);
	458	seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
	459	for (i = 0; i < ib->length_dw; i++) {
	460	seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
	461	}
	462	return 0;
	463	}
	464
9f93ed39 JG	465	static int radeon_debugfs_ib_bogus_info(struct seq_file m, void data)
	466	{
	467	struct drm_info_node node = (struct drm_info_node ) m->private;
	468	struct radeon_device *rdev = node->info_ent->data;
	469	struct radeon_ib *ib;
	470	unsigned i;
	471
	472	mutex_lock(&rdev->ib_pool.mutex);
	473	if (list_empty(&rdev->ib_pool.bogus_ib)) {
	474	mutex_unlock(&rdev->ib_pool.mutex);
	475	seq_printf(m, "no bogus IB recorded\n");
	476	return 0;
	477	}
	478	ib = list_first_entry(&rdev->ib_pool.bogus_ib, struct radeon_ib, list);
	479	list_del_init(&ib->list);
	480	mutex_unlock(&rdev->ib_pool.mutex);
	481	seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
	482	for (i = 0; i < ib->length_dw; i++) {
	483	seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
	484	}
	485	vfree(ib->ptr);
	486	kfree(ib);
	487	return 0;
	488	}
	489
771fe6b9 JG	490	static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE];
771fe6b9 JG	491	static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32];
9f93ed39 JG	492
	493	static struct drm_info_list radeon_debugfs_ib_bogus_info_list[] = {
	494	{"radeon_ib_bogus", radeon_debugfs_ib_bogus_info, 0, NULL},
	495	};
771fe6b9 JG	496	#endif
	497
	498	int radeon_debugfs_ib_init(struct radeon_device *rdev)
	499	{
	500	#if defined(CONFIG_DEBUG_FS)
	501	unsigned i;
9f93ed39	502	int r;
771fe6b9	503
9f93ed39 JG	504	radeon_debugfs_ib_bogus_info_list[0].data = rdev;
	505	r = radeon_debugfs_add_files(rdev, radeon_debugfs_ib_bogus_info_list, 1);
	506	if (r)
	507	return r;
771fe6b9 JG	508	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
	509	sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i);
	510	radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i];
	511	radeon_debugfs_ib_list[i].show = &radeon_debugfs_ib_info;
	512	radeon_debugfs_ib_list[i].driver_features = 0;
	513	radeon_debugfs_ib_list[i].data = &rdev->ib_pool.ibs[i];
	514	}
	515	return radeon_debugfs_add_files(rdev, radeon_debugfs_ib_list,
	516	RADEON_IB_POOL_SIZE);
	517	#else
	518	return 0;
	519	#endif
	520	}