[deliverable/linux.git] / drivers / gpu / drm / radeon / radeon_fence.c

/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Dave Airlie
 */
#include <linux/seq_file.h>
#include <linux/atomic.h>
#include <linux/wait.h>
#include <linux/list.h>
#include <linux/kref.h>
#include <linux/slab.h>
#include "drmP.h"
#include "drm.h"
#include "radeon_reg.h"
#include "radeon.h"
#include "radeon_trace.h"

static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
{
	if (rdev->wb.enabled) {
		*rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
	} else {
		WREG32(rdev->fence_drv[ring].scratch_reg, seq);
	}
}

static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
{
	u32 seq = 0;

	if (rdev->wb.enabled) {
		seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
	} else {
		seq = RREG32(rdev->fence_drv[ring].scratch_reg);
	}
	return seq;
}

int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
{
	unsigned long irq_flags;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (fence->emitted) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}
	fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
	radeon_fence_ring_emit(rdev, fence->ring, fence);
	trace_radeon_fence_emit(rdev->ddev, fence->seq);
	fence->emitted = true;
	/* are we the first fence on a previusly idle ring? */
	if (list_empty(&rdev->fence_drv[fence->ring].emitted)) {
		rdev->fence_drv[fence->ring].last_activity = jiffies;
	}
	list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return 0;
}

static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
{
	struct radeon_fence *fence;
	struct list_head *i, *n;
	uint32_t seq;
	bool wake = false;

	seq = radeon_fence_read(rdev, ring);
	if (seq == rdev->fence_drv[ring].last_seq)
		return false;

	rdev->fence_drv[ring].last_seq = seq;
	rdev->fence_drv[ring].last_activity = jiffies;

	n = NULL;
	list_for_each(i, &rdev->fence_drv[ring].emitted) {
		fence = list_entry(i, struct radeon_fence, list);
		if (fence->seq == seq) {
			n = i;
			break;
		}
	}
	/* all fence previous to this one are considered as signaled */
	if (n) {
		i = n;
		do {
			n = i->prev;
			list_move_tail(i, &rdev->fence_drv[ring].signaled);
			fence = list_entry(i, struct radeon_fence, list);
			fence->signaled = true;
			i = n;
		} while (i != &rdev->fence_drv[ring].emitted);
		wake = true;
	}
	return wake;
}

static void radeon_fence_destroy(struct kref *kref)
{
	unsigned long irq_flags;
        struct radeon_fence *fence;

	fence = container_of(kref, struct radeon_fence, kref);
	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
	list_del(&fence->list);
	fence->emitted = false;
	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
	if (fence->semaphore)
		radeon_semaphore_free(fence->rdev, fence->semaphore);
	kfree(fence);
}

int radeon_fence_create(struct radeon_device *rdev,
			struct radeon_fence **fence,
			int ring)
{
	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
	if ((*fence) == NULL) {
		return -ENOMEM;
	}
	kref_init(&((*fence)->kref));
	(*fence)->rdev = rdev;
	(*fence)->emitted = false;
	(*fence)->signaled = false;
	(*fence)->seq = 0;
	(*fence)->ring = ring;
	(*fence)->semaphore = NULL;
	INIT_LIST_HEAD(&(*fence)->list);
	return 0;
}

bool radeon_fence_signaled(struct radeon_fence *fence)
{
	unsigned long irq_flags;
	bool signaled = false;

	if (!fence)
		return true;

	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
	signaled = fence->signaled;
	/* if we are shuting down report all fence as signaled */
	if (fence->rdev->shutdown) {
		signaled = true;
	}
	if (!fence->emitted) {
		WARN(1, "Querying an unemitted fence : %p !\n", fence);
		signaled = true;
	}
	if (!signaled) {
		radeon_fence_poll_locked(fence->rdev, fence->ring);
		signaled = fence->signaled;
	}
	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
	return signaled;
}

int radeon_fence_wait(struct radeon_fence *fence, bool intr)
{
	struct radeon_device *rdev;
	unsigned long irq_flags, timeout;
	u32 seq;
	int i, r;
	bool signaled;

	if (fence == NULL) {
		WARN(1, "Querying an invalid fence : %p !\n", fence);
		return -EINVAL;
	}

	rdev = fence->rdev;
	signaled = radeon_fence_signaled(fence);
	while (!signaled) {
		read_lock_irqsave(&rdev->fence_lock, irq_flags);
		timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
		if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) {
			/* the normal case, timeout is somewhere before last_activity */
			timeout = rdev->fence_drv[fence->ring].last_activity - timeout;
		} else {
			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
			 * anyway we will just wait for the minimum amount and then check for a lockup */
			timeout = 1;
		}
		/* save current sequence value used to check for GPU lockups */
		seq = rdev->fence_drv[fence->ring].last_seq;
		read_unlock_irqrestore(&rdev->fence_lock, irq_flags);

		trace_radeon_fence_wait_begin(rdev->ddev, seq);
		radeon_irq_kms_sw_irq_get(rdev, fence->ring);
		if (intr) {
			r = wait_event_interruptible_timeout(
				rdev->fence_drv[fence->ring].queue,
				(signaled = radeon_fence_signaled(fence)), timeout);
		} else {
			r = wait_event_timeout(
				rdev->fence_drv[fence->ring].queue,
				(signaled = radeon_fence_signaled(fence)), timeout);
		}
		radeon_irq_kms_sw_irq_put(rdev, fence->ring);
		if (unlikely(r < 0)) {
			return r;
		}
		trace_radeon_fence_wait_end(rdev->ddev, seq);

		if (unlikely(!signaled)) {
			/* we were interrupted for some reason and fence
			 * isn't signaled yet, resume waiting */
			if (r) {
				continue;
			}

			write_lock_irqsave(&rdev->fence_lock, irq_flags);
			/* check if sequence value has changed since last_activity */
			if (seq != rdev->fence_drv[fence->ring].last_seq) {
				write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
				continue;
			}

			/* change sequence value on all rings, so nobody else things there is a lockup */
			for (i = 0; i < RADEON_NUM_RINGS; ++i)
				rdev->fence_drv[i].last_seq -= 0x10000;

			rdev->fence_drv[fence->ring].last_activity = jiffies;
			write_unlock_irqrestore(&rdev->fence_lock, irq_flags);

			if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {

				/* good news we believe it's a lockup */
				printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
				     fence->seq, seq);

				/* mark the ring as not ready any more */
				rdev->ring[fence->ring].ready = false;
				return -EDEADLK;
			}
		}
	}
	return 0;
}

int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	struct radeon_fence *fence;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->ring[ring].ready) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -EBUSY;
	}
	if (list_empty(&rdev->fence_drv[ring].emitted)) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -ENOENT;
	}
	fence = list_entry(rdev->fence_drv[ring].emitted.next,
			   struct radeon_fence, list);
	radeon_fence_ref(fence);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	r = radeon_fence_wait(fence, false);
	radeon_fence_unref(&fence);
	return r;
}

int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	struct radeon_fence *fence;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->ring[ring].ready) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return -EBUSY;
	}
	if (list_empty(&rdev->fence_drv[ring].emitted)) {
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}
	fence = list_entry(rdev->fence_drv[ring].emitted.prev,
			   struct radeon_fence, list);
	radeon_fence_ref(fence);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	r = radeon_fence_wait(fence, false);
	radeon_fence_unref(&fence);
	return r;
}

struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
{
	kref_get(&fence->kref);
	return fence;
}

void radeon_fence_unref(struct radeon_fence **fence)
{
	struct radeon_fence *tmp = *fence;

	*fence = NULL;
	if (tmp) {
		kref_put(&tmp->kref, radeon_fence_destroy);
	}
}

void radeon_fence_process(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	bool wake;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	wake = radeon_fence_poll_locked(rdev, ring);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	if (wake) {
		wake_up_all(&rdev->fence_drv[ring].queue);
	}
}

int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	int not_processed = 0;

	read_lock_irqsave(&rdev->fence_lock, irq_flags);
	if (!rdev->fence_drv[ring].initialized) {
		read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		return 0;
	}

	if (!list_empty(&rdev->fence_drv[ring].emitted)) {
		struct list_head *ptr;
		list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
			/* count up to 3, that's enought info */
			if (++not_processed >= 3)
				break;
		}
	}
	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return not_processed;
}

int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
{
	unsigned long irq_flags;
	uint64_t index;
	int r;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	if (rdev->wb.use_event) {
		rdev->fence_drv[ring].scratch_reg = 0;
		index = R600_WB_EVENT_OFFSET + ring * 4;
	} else {
		r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
		if (r) {
			dev_err(rdev->dev, "fence failed to get scratch register\n");
			write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
			return r;
		}
		index = RADEON_WB_SCRATCH_OFFSET +
			rdev->fence_drv[ring].scratch_reg -
			rdev->scratch.reg_base;
	}
	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
	radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
	rdev->fence_drv[ring].initialized = true;
	DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
		 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	return 0;
}

static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
{
	rdev->fence_drv[ring].scratch_reg = -1;
	rdev->fence_drv[ring].cpu_addr = NULL;
	rdev->fence_drv[ring].gpu_addr = 0;
	atomic_set(&rdev->fence_drv[ring].seq, 0);
	INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
	INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
	init_waitqueue_head(&rdev->fence_drv[ring].queue);
	rdev->fence_drv[ring].initialized = false;
}

int radeon_fence_driver_init(struct radeon_device *rdev)
{
	unsigned long irq_flags;
	int ring;

	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
		radeon_fence_driver_init_ring(rdev, ring);
	}
	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	if (radeon_debugfs_fence_init(rdev)) {
		dev_err(rdev->dev, "fence debugfs file creation failed\n");
	}
	return 0;
}

void radeon_fence_driver_fini(struct radeon_device *rdev)
{
	unsigned long irq_flags;
	int ring;

	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
		if (!rdev->fence_drv[ring].initialized)
			continue;
		radeon_fence_wait_empty(rdev, ring);
		wake_up_all(&rdev->fence_drv[ring].queue);
		write_lock_irqsave(&rdev->fence_lock, irq_flags);
		radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
		write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
		rdev->fence_drv[ring].initialized = false;
	}
}


/*
 * Fence debugfs
 */
#if defined(CONFIG_DEBUG_FS)
static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *)m->private;
	struct drm_device *dev = node->minor->dev;
	struct radeon_device *rdev = dev->dev_private;
	struct radeon_fence *fence;
	int i;

	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
		if (!rdev->fence_drv[i].initialized)
			continue;

		seq_printf(m, "--- ring %d ---\n", i);
		seq_printf(m, "Last signaled fence 0x%08X\n",
			   radeon_fence_read(rdev, i));
		if (!list_empty(&rdev->fence_drv[i].emitted)) {
			fence = list_entry(rdev->fence_drv[i].emitted.prev,
					   struct radeon_fence, list);
			seq_printf(m, "Last emitted fence %p with 0x%08X\n",
				   fence,  fence->seq);
		}
	}
	return 0;
}

static struct drm_info_list radeon_debugfs_fence_list[] = {
	{"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
};
#endif

int radeon_debugfs_fence_init(struct radeon_device *rdev)
{
#if defined(CONFIG_DEBUG_FS)
	return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
#else
	return 0;
#endif
}
Commit	Line	Data
771fe6b9 JG	1	/*
	2	* Copyright 2009 Jerome Glisse.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sub license, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	20	*
	21	* The above copyright notice and this permission notice (including the
	22	* next paragraph) shall be included in all copies or substantial portions
	23	* of the Software.
	24	*
	25	*/
	26	/*
	27	* Authors:
	28	* Jerome Glisse <glisse@freedesktop.org>
	29	* Dave Airlie
	30	*/
	31	#include <linux/seq_file.h>
60063497	32	#include <linux/atomic.h>
771fe6b9 JG	33	#include <linux/wait.h>
	34	#include <linux/list.h>
	35	#include <linux/kref.h>
5a0e3ad6	36	#include <linux/slab.h>
771fe6b9 JG	37	#include "drmP.h"
	38	#include "drm.h"
	39	#include "radeon_reg.h"
	40	#include "radeon.h"
99ee7fac	41	#include "radeon_trace.h"
771fe6b9	42
7465280c	43	static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
b81157d0 AD	44	{
b81157d0 AD	45	if (rdev->wb.enabled) {
30eb77f4 JG	46	*rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
30eb77f4 JG	47	} else {
7465280c	48	WREG32(rdev->fence_drv[ring].scratch_reg, seq);
30eb77f4	49	}
b81157d0 AD	50	}
b81157d0 AD	51
7465280c	52	static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
b81157d0	53	{
7465280c	54	u32 seq = 0;
b81157d0 AD	55
b81157d0 AD	56	if (rdev->wb.enabled) {
30eb77f4 JG	57	seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
30eb77f4 JG	58	} else {
7465280c	59	seq = RREG32(rdev->fence_drv[ring].scratch_reg);
30eb77f4	60	}
b81157d0 AD	61	return seq;
	62	}
	63
771fe6b9 JG	64	int radeon_fence_emit(struct radeon_device rdev, struct radeon_fence fence)
	65	{
	66	unsigned long irq_flags;
	67
7465280c	68	write_lock_irqsave(&rdev->fence_lock, irq_flags);
851a6bd9	69	if (fence->emitted) {
7465280c	70	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	71	return 0;
771fe6b9 JG	72	}
7465280c	73	fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
25a9e352	74	radeon_fence_ring_emit(rdev, fence->ring, fence);
99ee7fac	75	trace_radeon_fence_emit(rdev->ddev, fence->seq);
851a6bd9	76	fence->emitted = true;
36abacae CK	77	/* are we the first fence on a previusly idle ring? */
	78	if (list_empty(&rdev->fence_drv[fence->ring].emitted)) {
	79	rdev->fence_drv[fence->ring].last_activity = jiffies;
	80	}
7465280c AD	81	list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
7465280c AD	82	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	83	return 0;
	84	}
	85
7465280c	86	static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
771fe6b9 JG	87	{
	88	struct radeon_fence *fence;
	89	struct list_head i, n;
	90	uint32_t seq;
	91	bool wake = false;
	92
7465280c	93	seq = radeon_fence_read(rdev, ring);
36abacae	94	if (seq == rdev->fence_drv[ring].last_seq)
225758d8	95	return false;
36abacae CK	96
	97	rdev->fence_drv[ring].last_seq = seq;
	98	rdev->fence_drv[ring].last_activity = jiffies;
	99
771fe6b9	100	n = NULL;
7465280c	101	list_for_each(i, &rdev->fence_drv[ring].emitted) {
771fe6b9 JG	102	fence = list_entry(i, struct radeon_fence, list);
	103	if (fence->seq == seq) {
	104	n = i;
	105	break;
	106	}
	107	}
	108	/* all fence previous to this one are considered as signaled */
	109	if (n) {
	110	i = n;
	111	do {
	112	n = i->prev;
7465280c	113	list_move_tail(i, &rdev->fence_drv[ring].signaled);
771fe6b9 JG	114	fence = list_entry(i, struct radeon_fence, list);
	115	fence->signaled = true;
	116	i = n;
7465280c	117	} while (i != &rdev->fence_drv[ring].emitted);
771fe6b9 JG	118	wake = true;
	119	}
	120	return wake;
	121	}
	122
	123	static void radeon_fence_destroy(struct kref *kref)
	124	{
	125	unsigned long irq_flags;
	126	struct radeon_fence *fence;
	127
	128	fence = container_of(kref, struct radeon_fence, kref);
7465280c	129	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
771fe6b9	130	list_del(&fence->list);
851a6bd9	131	fence->emitted = false;
7465280c	132	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
93504fce CK	133	if (fence->semaphore)
93504fce CK	134	radeon_semaphore_free(fence->rdev, fence->semaphore);
771fe6b9 JG	135	kfree(fence);
	136	}
	137
7465280c AD	138	int radeon_fence_create(struct radeon_device *rdev,
	139	struct radeon_fence **fence,
	140	int ring)
771fe6b9	141	{
771fe6b9 JG	142	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
	143	if ((*fence) == NULL) {
	144	return -ENOMEM;
	145	}
	146	kref_init(&((*fence)->kref));
	147	(*fence)->rdev = rdev;
851a6bd9	148	(*fence)->emitted = false;
771fe6b9 JG	149	(*fence)->signaled = false;
771fe6b9 JG	150	(*fence)->seq = 0;
7465280c	151	(*fence)->ring = ring;
93504fce	152	(*fence)->semaphore = NULL;
771fe6b9	153	INIT_LIST_HEAD(&(*fence)->list);
771fe6b9 JG	154	return 0;
	155	}
	156
771fe6b9 JG	157	bool radeon_fence_signaled(struct radeon_fence *fence)
771fe6b9 JG	158	{
771fe6b9 JG	159	unsigned long irq_flags;
	160	bool signaled = false;
	161
3655d54a	162	if (!fence)
771fe6b9	163	return true;
3655d54a	164
7465280c	165	write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
771fe6b9 JG	166	signaled = fence->signaled;
	167	/* if we are shuting down report all fence as signaled */
	168	if (fence->rdev->shutdown) {
	169	signaled = true;
	170	}
851a6bd9 CK	171	if (!fence->emitted) {
851a6bd9 CK	172	WARN(1, "Querying an unemitted fence : %p !\n", fence);
771fe6b9 JG	173	signaled = true;
	174	}
	175	if (!signaled) {
7465280c	176	radeon_fence_poll_locked(fence->rdev, fence->ring);
771fe6b9 JG	177	signaled = fence->signaled;
771fe6b9 JG	178	}
7465280c	179	write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
771fe6b9 JG	180	return signaled;
	181	}
	182
3ce0a23d	183	int radeon_fence_wait(struct radeon_fence *fence, bool intr)
771fe6b9 JG	184	{
771fe6b9 JG	185	struct radeon_device *rdev;
225758d8 JG	186	unsigned long irq_flags, timeout;
225758d8 JG	187	u32 seq;
36abacae CK	188	int i, r;
36abacae CK	189	bool signaled;
771fe6b9	190
771fe6b9 JG	191	if (fence == NULL) {
771fe6b9 JG	192	WARN(1, "Querying an invalid fence : %p !\n", fence);
36abacae	193	return -EINVAL;
771fe6b9	194	}
36abacae	195
771fe6b9	196	rdev = fence->rdev;
36abacae CK	197	signaled = radeon_fence_signaled(fence);
	198	while (!signaled) {
	199	read_lock_irqsave(&rdev->fence_lock, irq_flags);
	200	timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
	201	if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) {
	202	/* the normal case, timeout is somewhere before last_activity */
	203	timeout = rdev->fence_drv[fence->ring].last_activity - timeout;
	204	} else {
	205	/* either jiffies wrapped around, or no fence was signaled in the last 500ms
	206	* anyway we will just wait for the minimum amount and then check for a lockup */
	207	timeout = 1;
	208	}
	209	/* save current sequence value used to check for GPU lockups */
	210	seq = rdev->fence_drv[fence->ring].last_seq;
	211	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	212
	213	trace_radeon_fence_wait_begin(rdev->ddev, seq);
1b37078b	214	radeon_irq_kms_sw_irq_get(rdev, fence->ring);
36abacae CK	215	if (intr) {
	216	r = wait_event_interruptible_timeout(
	217	rdev->fence_drv[fence->ring].queue,
	218	(signaled = radeon_fence_signaled(fence)), timeout);
	219	} else {
	220	r = wait_event_timeout(
	221	rdev->fence_drv[fence->ring].queue,
	222	(signaled = radeon_fence_signaled(fence)), timeout);
	223	}
1b37078b	224	radeon_irq_kms_sw_irq_put(rdev, fence->ring);
90aca4d2	225	if (unlikely(r < 0)) {
5cc6fbab	226	return r;
90aca4d2	227	}
36abacae	228	trace_radeon_fence_wait_end(rdev->ddev, seq);
25a9e352	229
36abacae CK	230	if (unlikely(!signaled)) {
	231	/* we were interrupted for some reason and fence
	232	* isn't signaled yet, resume waiting */
	233	if (r) {
	234	continue;
	235	}
25a9e352	236
36abacae CK	237	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	238	/* check if sequence value has changed since last_activity */
	239	if (seq != rdev->fence_drv[fence->ring].last_seq) {
	240	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	241	continue;
	242	}
	243
	244	/* change sequence value on all rings, so nobody else things there is a lockup */
	245	for (i = 0; i < RADEON_NUM_RINGS; ++i)
	246	rdev->fence_drv[i].last_seq -= 0x10000;
6c6f4783 CK	247
6c6f4783 CK	248	rdev->fence_drv[fence->ring].last_activity = jiffies;
36abacae CK	249	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	250
	251	if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {
	252
	253	/* good news we believe it's a lockup */
	254	printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
	255	fence->seq, seq);
	256
	257	/* mark the ring as not ready any more */
	258	rdev->ring[fence->ring].ready = false;
6c6f4783	259	return -EDEADLK;
36abacae	260	}
771fe6b9	261	}
771fe6b9	262	}
771fe6b9 JG	263	return 0;
	264	}
	265
7465280c	266	int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
771fe6b9 JG	267	{
	268	unsigned long irq_flags;
	269	struct radeon_fence *fence;
	270	int r;
	271
7465280c	272	write_lock_irqsave(&rdev->fence_lock, irq_flags);
25a9e352 CK	273	if (!rdev->ring[ring].ready) {
	274	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	275	return -EBUSY;
	276	}
7465280c AD	277	if (list_empty(&rdev->fence_drv[ring].emitted)) {
7465280c AD	278	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
2f6bfe11	279	return -ENOENT;
771fe6b9	280	}
7465280c	281	fence = list_entry(rdev->fence_drv[ring].emitted.next,
771fe6b9 JG	282	struct radeon_fence, list);
771fe6b9 JG	283	radeon_fence_ref(fence);
7465280c	284	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	285	r = radeon_fence_wait(fence, false);
	286	radeon_fence_unref(&fence);
	287	return r;
	288	}
	289
adea5c27	290	int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
771fe6b9 JG	291	{
	292	unsigned long irq_flags;
	293	struct radeon_fence *fence;
	294	int r;
	295
7465280c	296	write_lock_irqsave(&rdev->fence_lock, irq_flags);
25a9e352 CK	297	if (!rdev->ring[ring].ready) {
	298	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	299	return -EBUSY;
	300	}
7465280c AD	301	if (list_empty(&rdev->fence_drv[ring].emitted)) {
7465280c AD	302	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	303	return 0;
771fe6b9 JG	304	}
7465280c	305	fence = list_entry(rdev->fence_drv[ring].emitted.prev,
771fe6b9 JG	306	struct radeon_fence, list);
771fe6b9 JG	307	radeon_fence_ref(fence);
7465280c	308	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9 JG	309	r = radeon_fence_wait(fence, false);
	310	radeon_fence_unref(&fence);
	311	return r;
	312	}
	313
	314	struct radeon_fence radeon_fence_ref(struct radeon_fence fence)
	315	{
	316	kref_get(&fence->kref);
	317	return fence;
	318	}
	319
	320	void radeon_fence_unref(struct radeon_fence **fence)
	321	{
	322	struct radeon_fence tmp = fence;
	323
	324	*fence = NULL;
	325	if (tmp) {
cdb650a4	326	kref_put(&tmp->kref, radeon_fence_destroy);
771fe6b9 JG	327	}
	328	}
	329
7465280c	330	void radeon_fence_process(struct radeon_device *rdev, int ring)
771fe6b9 JG	331	{
	332	unsigned long irq_flags;
	333	bool wake;
	334
7465280c AD	335	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	336	wake = radeon_fence_poll_locked(rdev, ring);
	337	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9	338	if (wake) {
7465280c	339	wake_up_all(&rdev->fence_drv[ring].queue);
771fe6b9 JG	340	}
	341	}
	342
47492a23 CK	343	int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
	344	{
	345	unsigned long irq_flags;
	346	int not_processed = 0;
	347
	348	read_lock_irqsave(&rdev->fence_lock, irq_flags);
40e8c738 DA	349	if (!rdev->fence_drv[ring].initialized) {
40e8c738 DA	350	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
47492a23	351	return 0;
40e8c738	352	}
47492a23 CK	353
	354	if (!list_empty(&rdev->fence_drv[ring].emitted)) {
	355	struct list_head *ptr;
	356	list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
	357	/* count up to 3, that's enought info */
	358	if (++not_processed >= 3)
	359	break;
	360	}
	361	}
	362	read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	363	return not_processed;
	364	}
	365
30eb77f4	366	int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
771fe6b9 JG	367	{
771fe6b9 JG	368	unsigned long irq_flags;
30eb77f4 JG	369	uint64_t index;
30eb77f4 JG	370	int r;
771fe6b9	371
30eb77f4 JG	372	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	373	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	374	if (rdev->wb.use_event) {
	375	rdev->fence_drv[ring].scratch_reg = 0;
	376	index = R600_WB_EVENT_OFFSET + ring * 4;
	377	} else {
7465280c AD	378	r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
	379	if (r) {
	380	dev_err(rdev->dev, "fence failed to get scratch register\n");
	381	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	382	return r;
	383	}
30eb77f4 JG	384	index = RADEON_WB_SCRATCH_OFFSET +
	385	rdev->fence_drv[ring].scratch_reg -
	386	rdev->scratch.reg_base;
7465280c	387	}
30eb77f4 JG	388	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
	389	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
	390	radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
	391	rdev->fence_drv[ring].initialized = true;
	392	DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
	393	ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
	394	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	395	return 0;
	396	}
	397
	398	static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
	399	{
	400	rdev->fence_drv[ring].scratch_reg = -1;
	401	rdev->fence_drv[ring].cpu_addr = NULL;
	402	rdev->fence_drv[ring].gpu_addr = 0;
	403	atomic_set(&rdev->fence_drv[ring].seq, 0);
30eb77f4 JG	404	INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
	405	INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
	406	init_waitqueue_head(&rdev->fence_drv[ring].queue);
	407	rdev->fence_drv[ring].initialized = false;
	408	}
	409
	410	int radeon_fence_driver_init(struct radeon_device *rdev)
	411	{
	412	unsigned long irq_flags;
	413	int ring;
	414
	415	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	416	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
	417	radeon_fence_driver_init_ring(rdev, ring);
771fe6b9	418	}
30eb77f4	419	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
771fe6b9	420	if (radeon_debugfs_fence_init(rdev)) {
0a0c7596	421	dev_err(rdev->dev, "fence debugfs file creation failed\n");
771fe6b9 JG	422	}
	423	return 0;
	424	}
	425
	426	void radeon_fence_driver_fini(struct radeon_device *rdev)
	427	{
	428	unsigned long irq_flags;
7465280c AD	429	int ring;
	430
	431	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
	432	if (!rdev->fence_drv[ring].initialized)
	433	continue;
adea5c27	434	radeon_fence_wait_empty(rdev, ring);
7465280c AD	435	wake_up_all(&rdev->fence_drv[ring].queue);
	436	write_lock_irqsave(&rdev->fence_lock, irq_flags);
	437	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
	438	write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
	439	rdev->fence_drv[ring].initialized = false;
	440	}
771fe6b9 JG	441	}
	442
	443
	444	/*
	445	* Fence debugfs
	446	*/
	447	#if defined(CONFIG_DEBUG_FS)
	448	static int radeon_debugfs_fence_info(struct seq_file m, void data)
	449	{
	450	struct drm_info_node node = (struct drm_info_node )m->private;
	451	struct drm_device *dev = node->minor->dev;
	452	struct radeon_device *rdev = dev->dev_private;
	453	struct radeon_fence *fence;
7465280c AD	454	int i;
	455
	456	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
	457	if (!rdev->fence_drv[i].initialized)
	458	continue;
	459
	460	seq_printf(m, "--- ring %d ---\n", i);
	461	seq_printf(m, "Last signaled fence 0x%08X\n",
	462	radeon_fence_read(rdev, i));
	463	if (!list_empty(&rdev->fence_drv[i].emitted)) {
	464	fence = list_entry(rdev->fence_drv[i].emitted.prev,
	465	struct radeon_fence, list);
	466	seq_printf(m, "Last emitted fence %p with 0x%08X\n",
	467	fence, fence->seq);
	468	}
771fe6b9 JG	469	}
	470	return 0;
	471	}
	472
	473	static struct drm_info_list radeon_debugfs_fence_list[] = {
	474	{"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
	475	};
	476	#endif
	477
	478	int radeon_debugfs_fence_init(struct radeon_device *rdev)
	479	{
	480	#if defined(CONFIG_DEBUG_FS)
	481	return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
	482	#else
	483	return 0;
	484	#endif
	485	}