[deliverable/linux.git] / drivers / gpu / drm / nouveau / nouveau_fence.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <trace/events/fence.h>

#include <nvif/cl826e.h>
#include <nvif/notify.h>
#include <nvif/event.h>

#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"

static const struct fence_ops nouveau_fence_ops_uevent;
static const struct fence_ops nouveau_fence_ops_legacy;

static inline struct nouveau_fence *
from_fence(struct fence *fence)
{
	return container_of(fence, struct nouveau_fence, base);
}

static inline struct nouveau_fence_chan *
nouveau_fctx(struct nouveau_fence *fence)
{
	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
}

static int
nouveau_fence_signal(struct nouveau_fence *fence)
{
	int drop = 0;

	fence_signal_locked(&fence->base);
	list_del(&fence->head);
	rcu_assign_pointer(fence->channel, NULL);

	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

		if (!--fctx->notify_ref)
			drop = 1;
	}

	fence_put(&fence->base);
	return drop;
}

static struct nouveau_fence *
nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
	struct nouveau_fence_priv *priv = (void*)drm->fence;

	if (fence->ops != &nouveau_fence_ops_legacy &&
	    fence->ops != &nouveau_fence_ops_uevent)
		return NULL;

	if (fence->context < priv->context_base ||
	    fence->context >= priv->context_base + priv->contexts)
		return NULL;

	return from_fence(fence);
}

void
nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	spin_lock_irq(&fctx->lock);
	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if (nouveau_fence_signal(fence))
			nvif_notify_put(&fctx->notify);
	}
	spin_unlock_irq(&fctx->lock);

	nvif_notify_fini(&fctx->notify);
	fctx->dead = 1;

	/*
	 * Ensure that all accesses to fence->channel complete before freeing
	 * the channel.
	 */
	synchronize_rcu();
}

static void
nouveau_fence_context_put(struct kref *fence_ref)
{
	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
}

void
nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
{
	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
}

static int
nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;
	int drop = 0;
	u32 seq = fctx->read(chan);

	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if ((int)(seq - fence->base.seqno) < 0)
			break;

		drop |= nouveau_fence_signal(fence);
	}

	return drop;
}

static int
nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
{
	struct nouveau_fence_chan *fctx =
		container_of(notify, typeof(*fctx), notify);
	unsigned long flags;
	int ret = NVIF_NOTIFY_KEEP;

	spin_lock_irqsave(&fctx->lock, flags);
	if (!list_empty(&fctx->pending)) {
		struct nouveau_fence *fence;
		struct nouveau_channel *chan;

		fence = list_entry(fctx->pending.next, typeof(*fence), head);
		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
		if (nouveau_fence_update(fence->channel, fctx))
			ret = NVIF_NOTIFY_DROP;
	}
	spin_unlock_irqrestore(&fctx->lock, flags);

	return ret;
}

void
nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	struct nouveau_cli *cli = (void *)chan->user.client;
	int ret;

	INIT_LIST_HEAD(&fctx->flip);
	INIT_LIST_HEAD(&fctx->pending);
	spin_lock_init(&fctx->lock);
	fctx->context = priv->context_base + chan->chid;

	if (chan == chan->drm->cechan)
		strcpy(fctx->name, "copy engine channel");
	else if (chan == chan->drm->channel)
		strcpy(fctx->name, "generic kernel channel");
	else
		strcpy(fctx->name, nvxx_client(&cli->base)->name);

	kref_init(&fctx->fence_ref);
	if (!priv->uevent)
		return;

	ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
			       false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
			       &(struct nvif_notify_uevent_req) { },
			       sizeof(struct nvif_notify_uevent_req),
			       sizeof(struct nvif_notify_uevent_rep),
			       &fctx->notify);

	WARN_ON(ret);
}

struct nouveau_fence_work {
	struct work_struct work;
	struct fence_cb cb;
	void (*func)(void *);
	void *data;
};

static void
nouveau_fence_work_handler(struct work_struct *kwork)
{
	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
	work->func(work->data);
	kfree(work);
}

static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
{
	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);

	schedule_work(&work->work);
}

void
nouveau_fence_work(struct fence *fence,
		   void (*func)(void *), void *data)
{
	struct nouveau_fence_work *work;

	if (fence_is_signaled(fence))
		goto err;

	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work) {
		/*
		 * this might not be a nouveau fence any more,
		 * so force a lazy wait here
		 */
		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
					   true, false));
		goto err;
	}

	INIT_WORK(&work->work, nouveau_fence_work_handler);
	work->func = func;
	work->data = data;

	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
		goto err_free;
	return;

err_free:
	kfree(work);
err:
	func(data);
}

int
nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	fence->channel  = chan;
	fence->timeout  = jiffies + (15 * HZ);

	if (priv->uevent)
		fence_init(&fence->base, &nouveau_fence_ops_uevent,
			   &fctx->lock, fctx->context, ++fctx->sequence);
	else
		fence_init(&fence->base, &nouveau_fence_ops_legacy,
			   &fctx->lock, fctx->context, ++fctx->sequence);
	kref_get(&fctx->fence_ref);

	trace_fence_emit(&fence->base);
	ret = fctx->emit(fence);
	if (!ret) {
		fence_get(&fence->base);
		spin_lock_irq(&fctx->lock);

		if (nouveau_fence_update(chan, fctx))
			nvif_notify_put(&fctx->notify);

		list_add_tail(&fence->head, &fctx->pending);
		spin_unlock_irq(&fctx->lock);
	}

	return ret;
}

bool
nouveau_fence_done(struct nouveau_fence *fence)
{
	if (fence->base.ops == &nouveau_fence_ops_legacy ||
	    fence->base.ops == &nouveau_fence_ops_uevent) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
		struct nouveau_channel *chan;
		unsigned long flags;

		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
			return true;

		spin_lock_irqsave(&fctx->lock, flags);
		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
		if (chan && nouveau_fence_update(chan, fctx))
			nvif_notify_put(&fctx->notify);
		spin_unlock_irqrestore(&fctx->lock, flags);
	}
	return fence_is_signaled(&fence->base);
}

static long
nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
{
	struct nouveau_fence *fence = from_fence(f);
	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	unsigned long t = jiffies, timeout = t + wait;

	while (!nouveau_fence_done(fence)) {
		ktime_t kt;

		t = jiffies;

		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
			__set_current_state(TASK_RUNNING);
			return 0;
		}

		__set_current_state(intr ? TASK_INTERRUPTIBLE :
					   TASK_UNINTERRUPTIBLE);

		kt = ktime_set(0, sleep_time);
		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
		sleep_time *= 2;
		if (sleep_time > NSEC_PER_MSEC)
			sleep_time = NSEC_PER_MSEC;

		if (intr && signal_pending(current))
			return -ERESTARTSYS;
	}

	__set_current_state(TASK_RUNNING);

	return timeout - t;
}

static int
nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
{
	int ret = 0;

	while (!nouveau_fence_done(fence)) {
		if (time_after_eq(jiffies, fence->timeout)) {
			ret = -EBUSY;
			break;
		}

		__set_current_state(intr ?
				    TASK_INTERRUPTIBLE :
				    TASK_UNINTERRUPTIBLE);

		if (intr && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	}

	__set_current_state(TASK_RUNNING);
	return ret;
}

int
nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
{
	long ret;

	if (!lazy)
		return nouveau_fence_wait_busy(fence, intr);

	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	if (ret < 0)
		return ret;
	else if (!ret)
		return -EBUSY;
	else
		return 0;
}

int
nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct fence *fence;
	struct reservation_object *resv = nvbo->bo.resv;
	struct reservation_object_list *fobj;
	struct nouveau_fence *f;
	int ret = 0, i;

	if (!exclusive) {
		ret = reservation_object_reserve_shared(resv);

		if (ret)
			return ret;
	}

	fobj = reservation_object_get_list(resv);
	fence = reservation_object_get_excl(resv);

	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
		struct nouveau_channel *prev = NULL;
		bool must_wait = true;

		f = nouveau_local_fence(fence, chan->drm);
		if (f) {
			rcu_read_lock();
			prev = rcu_dereference(f->channel);
			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
				must_wait = false;
			rcu_read_unlock();
		}

		if (must_wait)
			ret = fence_wait(fence, intr);

		return ret;
	}

	if (!exclusive || !fobj)
		return ret;

	for (i = 0; i < fobj->shared_count && !ret; ++i) {
		struct nouveau_channel *prev = NULL;
		bool must_wait = true;

		fence = rcu_dereference_protected(fobj->shared[i],
						reservation_object_held(resv));

		f = nouveau_local_fence(fence, chan->drm);
		if (f) {
			rcu_read_lock();
			prev = rcu_dereference(f->channel);
			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
				must_wait = false;
			rcu_read_unlock();
		}

		if (must_wait)
			ret = fence_wait(fence, intr);
	}

	return ret;
}

void
nouveau_fence_unref(struct nouveau_fence **pfence)
{
	if (*pfence)
		fence_put(&(*pfence)->base);
	*pfence = NULL;
}

int
nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
		  struct nouveau_fence **pfence)
{
	struct nouveau_fence *fence;
	int ret = 0;

	if (unlikely(!chan->fence))
		return -ENODEV;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
		return -ENOMEM;

	fence->sysmem = sysmem;

	ret = nouveau_fence_emit(fence, chan);
	if (ret)
		nouveau_fence_unref(&fence);

	*pfence = fence;
	return ret;
}

static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
{
	return "nouveau";
}

static const char *nouveau_fence_get_timeline_name(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	return !fctx->dead ? fctx->name : "dead channel";
}

/*
 * In an ideal world, read would not assume the channel context is still alive.
 * This function may be called from another device, running into free memory as a
 * result. The drm node should still be there, so we can derive the index from
 * the fence context.
 */
static bool nouveau_fence_is_signaled(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	struct nouveau_channel *chan;
	bool ret = false;

	rcu_read_lock();
	chan = rcu_dereference(fence->channel);
	if (chan)
		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	rcu_read_unlock();

	return ret;
}

static bool nouveau_fence_no_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);

	/*
	 * caller should have a reference on the fence,
	 * else fence could get freed here
	 */
	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);

	/*
	 * This needs uevents to work correctly, but fence_add_callback relies on
	 * being able to enable signaling. It will still get signaled eventually,
	 * just not right away.
	 */
	if (nouveau_fence_is_signaled(f)) {
		list_del(&fence->head);

		fence_put(&fence->base);
		return false;
	}

	return true;
}

static void nouveau_fence_release(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	fence_free(&fence->base);
}

static const struct fence_ops nouveau_fence_ops_legacy = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_no_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = nouveau_fence_wait_legacy,
	.release = nouveau_fence_release
};

static bool nouveau_fence_enable_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	bool ret;

	if (!fctx->notify_ref++)
		nvif_notify_get(&fctx->notify);

	ret = nouveau_fence_no_signaling(f);
	if (ret)
		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	else if (!--fctx->notify_ref)
		nvif_notify_put(&fctx->notify);

	return ret;
}

static const struct fence_ops nouveau_fence_ops_uevent = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_enable_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = fence_default_wait,
	.release = NULL
};
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
760285e7	27	#include <drm/drmP.h>
6ee73861	28
bd35fe5a MS	29	#include <linux/ktime.h>
bd35fe5a MS	30	#include <linux/hrtimer.h>
29ba89b2	31	#include <trace/events/fence.h>
bd35fe5a	32
8ed1730c	33	#include <nvif/cl826e.h>
867920f8 BS	34	#include <nvif/notify.h>
	35	#include <nvif/event.h>
	36
ebb945a9	37	#include "nouveau_drm.h"
6ee73861	38	#include "nouveau_dma.h"
ebb945a9	39	#include "nouveau_fence.h"
6ee73861	40
29ba89b2 ML	41	static const struct fence_ops nouveau_fence_ops_uevent;
	42	static const struct fence_ops nouveau_fence_ops_legacy;
	43
	44	static inline struct nouveau_fence *
	45	from_fence(struct fence *fence)
	46	{
	47	return container_of(fence, struct nouveau_fence, base);
	48	}
	49
	50	static inline struct nouveau_fence_chan *
	51	nouveau_fctx(struct nouveau_fence *fence)
	52	{
	53	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
	54	}
c4c7044f	55
0ec5f02f	56	static int
c4c7044f BS	57	nouveau_fence_signal(struct nouveau_fence *fence)
c4c7044f BS	58	{
0ec5f02f ML	59	int drop = 0;
0ec5f02f ML	60
29ba89b2 ML	61	fence_signal_locked(&fence->base);
29ba89b2 ML	62	list_del(&fence->head);
0ec5f02f	63	rcu_assign_pointer(fence->channel, NULL);
29ba89b2 ML	64
	65	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
	66	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
c4c7044f	67
29ba89b2	68	if (!--fctx->notify_ref)
0ec5f02f	69	drop = 1;
c4c7044f BS	70	}
c4c7044f BS	71
29ba89b2	72	fence_put(&fence->base);
0ec5f02f	73	return drop;
29ba89b2 ML	74	}
	75
	76	static struct nouveau_fence *
	77	nouveau_local_fence(struct fence fence, struct nouveau_drm drm) {
	78	struct nouveau_fence_priv priv = (void)drm->fence;
	79
	80	if (fence->ops != &nouveau_fence_ops_legacy &&
	81	fence->ops != &nouveau_fence_ops_uevent)
	82	return NULL;
	83
	84	if (fence->context < priv->context_base \|\|
	85	fence->context >= priv->context_base + priv->contexts)
	86	return NULL;
	87
	88	return from_fence(fence);
c4c7044f BS	89	}
c4c7044f BS	90
5e120f6e BS	91	void
	92	nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
	93	{
29ba89b2 ML	94	struct nouveau_fence *fence;
29ba89b2 ML	95
29ba89b2 ML	96	spin_lock_irq(&fctx->lock);
	97	while (!list_empty(&fctx->pending)) {
	98	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	99
0ec5f02f ML	100	if (nouveau_fence_signal(fence))
0ec5f02f ML	101	nvif_notify_put(&fctx->notify);
5e120f6e	102	}
29ba89b2	103	spin_unlock_irq(&fctx->lock);
0ec5f02f ML	104
	105	nvif_notify_fini(&fctx->notify);
	106	fctx->dead = 1;
	107
	108	/*
	109	* Ensure that all accesses to fence->channel complete before freeing
	110	* the channel.
	111	*/
	112	synchronize_rcu();
29ba89b2 ML	113	}
29ba89b2 ML	114
15a996bb ML	115	static void
	116	nouveau_fence_context_put(struct kref *fence_ref)
	117	{
	118	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
	119	}
	120
	121	void
	122	nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
	123	{
	124	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	125	}
	126
0ec5f02f	127	static int
29ba89b2 ML	128	nouveau_fence_update(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
	129	{
	130	struct nouveau_fence *fence;
0ec5f02f	131	int drop = 0;
29ba89b2 ML	132	u32 seq = fctx->read(chan);
	133
	134	while (!list_empty(&fctx->pending)) {
	135	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	136
	137	if ((int)(seq - fence->base.seqno) < 0)
0ec5f02f	138	break;
29ba89b2	139
0ec5f02f	140	drop \|= nouveau_fence_signal(fence);
29ba89b2	141	}
0ec5f02f ML	142
0ec5f02f ML	143	return drop;
29ba89b2 ML	144	}
	145
	146	static int
	147	nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
	148	{
	149	struct nouveau_fence_chan *fctx =
	150	container_of(notify, typeof(*fctx), notify);
	151	unsigned long flags;
0ec5f02f	152	int ret = NVIF_NOTIFY_KEEP;
29ba89b2 ML	153
	154	spin_lock_irqsave(&fctx->lock, flags);
	155	if (!list_empty(&fctx->pending)) {
	156	struct nouveau_fence *fence;
0ec5f02f	157	struct nouveau_channel *chan;
29ba89b2 ML	158
29ba89b2 ML	159	fence = list_entry(fctx->pending.next, typeof(*fence), head);
0ec5f02f ML	160	chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
	161	if (nouveau_fence_update(fence->channel, fctx))
	162	ret = NVIF_NOTIFY_DROP;
29ba89b2 ML	163	}
	164	spin_unlock_irqrestore(&fctx->lock, flags);
	165
0ec5f02f	166	return ret;
5e120f6e BS	167	}
	168
	169	void
29ba89b2	170	nouveau_fence_context_new(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
5e120f6e	171	{
29ba89b2	172	struct nouveau_fence_priv priv = (void)chan->drm->fence;
a01ca78c	173	struct nouveau_cli cli = (void )chan->user.client;
29ba89b2 ML	174	int ret;
29ba89b2 ML	175
f589be88	176	INIT_LIST_HEAD(&fctx->flip);
5e120f6e BS	177	INIT_LIST_HEAD(&fctx->pending);
5e120f6e BS	178	spin_lock_init(&fctx->lock);
29ba89b2 ML	179	fctx->context = priv->context_base + chan->chid;
29ba89b2 ML	180
15a996bb ML	181	if (chan == chan->drm->cechan)
	182	strcpy(fctx->name, "copy engine channel");
	183	else if (chan == chan->drm->channel)
	184	strcpy(fctx->name, "generic kernel channel");
	185	else
989aa5b7	186	strcpy(fctx->name, nvxx_client(&cli->base)->name);
15a996bb ML	187
15a996bb ML	188	kref_init(&fctx->fence_ref);
29ba89b2 ML	189	if (!priv->uevent)
	190	return;
	191
a01ca78c BS	192	ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
	193	false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
	194	&(struct nvif_notify_uevent_req) { },
	195	sizeof(struct nvif_notify_uevent_req),
	196	sizeof(struct nvif_notify_uevent_rep),
	197	&fctx->notify);
29ba89b2 ML	198
29ba89b2 ML	199	WARN_ON(ret);
5e120f6e	200	}
6ee73861	201
29ba89b2 ML	202	struct nouveau_fence_work {
	203	struct work_struct work;
	204	struct fence_cb cb;
	205	void (func)(void );
	206	void *data;
	207	};
	208
c4c7044f BS	209	static void
	210	nouveau_fence_work_handler(struct work_struct *kwork)
	211	{
29ba89b2	212	struct nouveau_fence_work work = container_of(kwork, typeof(work), work);
c4c7044f BS	213	work->func(work->data);
	214	kfree(work);
	215	}
	216
29ba89b2 ML	217	static void nouveau_fence_work_cb(struct fence fence, struct fence_cb cb)
	218	{
	219	struct nouveau_fence_work work = container_of(cb, typeof(work), cb);
	220
	221	schedule_work(&work->work);
	222	}
	223
c4c7044f	224	void
f2c24b83	225	nouveau_fence_work(struct fence *fence,
c4c7044f BS	226	void (func)(void ), void *data)
c4c7044f BS	227	{
29ba89b2	228	struct nouveau_fence_work *work;
c4c7044f	229
f2c24b83	230	if (fence_is_signaled(fence))
29ba89b2	231	goto err;
c4c7044f	232
c4c7044f BS	233	work = kmalloc(sizeof(*work), GFP_KERNEL);
c4c7044f BS	234	if (!work) {
bb6178b0 ML	235	/*
	236	* this might not be a nouveau fence any more,
	237	* so force a lazy wait here
	238	*/
f2c24b83	239	WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
bb6178b0	240	true, false));
29ba89b2	241	goto err;
c4c7044f BS	242	}
c4c7044f BS	243
29ba89b2	244	INIT_WORK(&work->work, nouveau_fence_work_handler);
c4c7044f BS	245	work->func = func;
c4c7044f BS	246	work->data = data;
6ee73861	247
f2c24b83	248	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
29ba89b2 ML	249	goto err_free;
29ba89b2 ML	250	return;
b08abd4e	251
29ba89b2 ML	252	err_free:
	253	kfree(work);
	254	err:
	255	func(data);
6ee73861 BS	256	}
	257
	258	int
d375e7d5	259	nouveau_fence_emit(struct nouveau_fence fence, struct nouveau_channel chan)
6ee73861	260	{
e193b1d4	261	struct nouveau_fence_chan *fctx = chan->fence;
29ba89b2	262	struct nouveau_fence_priv priv = (void)chan->drm->fence;
6ee73861 BS	263	int ret;
6ee73861 BS	264
5e120f6e	265	fence->channel = chan;
bfd8303a	266	fence->timeout = jiffies + (15 * HZ);
6ee73861	267
29ba89b2 ML	268	if (priv->uevent)
29ba89b2 ML	269	fence_init(&fence->base, &nouveau_fence_ops_uevent,
e3be4c23	270	&fctx->lock, fctx->context, ++fctx->sequence);
29ba89b2 ML	271	else
29ba89b2 ML	272	fence_init(&fence->base, &nouveau_fence_ops_legacy,
e3be4c23	273	&fctx->lock, fctx->context, ++fctx->sequence);
15a996bb	274	kref_get(&fctx->fence_ref);
29ba89b2 ML	275
29ba89b2 ML	276	trace_fence_emit(&fence->base);
827520ce	277	ret = fctx->emit(fence);
5e120f6e	278	if (!ret) {
29ba89b2 ML	279	fence_get(&fence->base);
29ba89b2 ML	280	spin_lock_irq(&fctx->lock);
0ec5f02f ML	281
	282	if (nouveau_fence_update(chan, fctx))
	283	nvif_notify_put(&fctx->notify);
	284
5e120f6e	285	list_add_tail(&fence->head, &fctx->pending);
29ba89b2	286	spin_unlock_irq(&fctx->lock);
529c4959	287	}
6ee73861	288
5e120f6e	289	return ret;
6ee73861 BS	290	}
6ee73861 BS	291
6ee73861	292	bool
d375e7d5	293	nouveau_fence_done(struct nouveau_fence *fence)
6ee73861	294	{
29ba89b2 ML	295	if (fence->base.ops == &nouveau_fence_ops_legacy \|\|
	296	fence->base.ops == &nouveau_fence_ops_uevent) {
	297	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
0ec5f02f	298	struct nouveau_channel *chan;
29ba89b2	299	unsigned long flags;
6ee73861	300
29ba89b2 ML	301	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
29ba89b2 ML	302	return true;
79ca2770	303
29ba89b2	304	spin_lock_irqsave(&fctx->lock, flags);
0ec5f02f ML	305	chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
	306	if (chan && nouveau_fence_update(chan, fctx))
	307	nvif_notify_put(&fctx->notify);
29ba89b2 ML	308	spin_unlock_irqrestore(&fctx->lock, flags);
	309	}
	310	return fence_is_signaled(&fence->base);
e18c080f BS	311	}
e18c080f BS	312
29ba89b2 ML	313	static long
29ba89b2 ML	314	nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
e18c080f	315	{
29ba89b2 ML	316	struct nouveau_fence *fence = from_fence(f);
	317	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	318	unsigned long t = jiffies, timeout = t + wait;
e18c080f	319
29ba89b2 ML	320	while (!nouveau_fence_done(fence)) {
29ba89b2 ML	321	ktime_t kt;
51cb4b39	322
29ba89b2	323	t = jiffies;
e18c080f	324
29ba89b2 ML	325	if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
	326	__set_current_state(TASK_RUNNING);
	327	return 0;
e18c080f	328	}
29ba89b2 ML	329
	330	__set_current_state(intr ? TASK_INTERRUPTIBLE :
	331	TASK_UNINTERRUPTIBLE);
	332
	333	kt = ktime_set(0, sleep_time);
	334	schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
	335	sleep_time *= 2;
	336	if (sleep_time > NSEC_PER_MSEC)
	337	sleep_time = NSEC_PER_MSEC;
	338
	339	if (intr && signal_pending(current))
	340	return -ERESTARTSYS;
e18c080f BS	341	}
e18c080f BS	342
29ba89b2	343	__set_current_state(TASK_RUNNING);
e18c080f	344
29ba89b2	345	return timeout - t;
e18c080f BS	346	}
e18c080f BS	347
29ba89b2 ML	348	static int
29ba89b2 ML	349	nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
6ee73861	350	{
6ee73861 BS	351	int ret = 0;
6ee73861 BS	352
d375e7d5	353	while (!nouveau_fence_done(fence)) {
29ba89b2	354	if (time_after_eq(jiffies, fence->timeout)) {
6ee73861 BS	355	ret = -EBUSY;
	356	break;
	357	}
	358
29ba89b2 ML	359	__set_current_state(intr ?
	360	TASK_INTERRUPTIBLE :
	361	TASK_UNINTERRUPTIBLE);
6ee73861 BS	362
6ee73861 BS	363	if (intr && signal_pending(current)) {
9ddc8c52	364	ret = -ERESTARTSYS;
6ee73861 BS	365	break;
	366	}
	367	}
	368
	369	__set_current_state(TASK_RUNNING);
d375e7d5 BS	370	return ret;
	371	}
	372
5e120f6e	373	int
29ba89b2 ML	374	nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
	375	{
	376	long ret;
	377
	378	if (!lazy)
	379	return nouveau_fence_wait_busy(fence, intr);
	380
	381	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	382	if (ret < 0)
	383	return ret;
	384	else if (!ret)
	385	return -EBUSY;
	386	else
	387	return 0;
	388	}
	389
	390	int
e3be4c23	391	nouveau_fence_sync(struct nouveau_bo nvbo, struct nouveau_channel chan, bool exclusive, bool intr)
5e120f6e	392	{
827520ce	393	struct nouveau_fence_chan *fctx = chan->fence;
809e9447	394	struct fence *fence;
29ba89b2 ML	395	struct reservation_object *resv = nvbo->bo.resv;
29ba89b2 ML	396	struct reservation_object_list *fobj;
809e9447	397	struct nouveau_fence *f;
29ba89b2 ML	398	int ret = 0, i;
29ba89b2 ML	399
809e9447 ML	400	if (!exclusive) {
	401	ret = reservation_object_reserve_shared(resv);
	402
	403	if (ret)
	404	return ret;
	405	}
	406
	407	fobj = reservation_object_get_list(resv);
f2c24b83	408	fence = reservation_object_get_excl(resv);
29ba89b2	409
809e9447 ML	410	if (fence && (!exclusive \|\| !fobj \|\| !fobj->shared_count)) {
809e9447 ML	411	struct nouveau_channel *prev = NULL;
0ec5f02f	412	bool must_wait = true;
5e120f6e	413
809e9447	414	f = nouveau_local_fence(fence, chan->drm);
0ec5f02f ML	415	if (f) {
	416	rcu_read_lock();
	417	prev = rcu_dereference(f->channel);
	418	if (prev && (prev == chan \|\| fctx->sync(f, prev, chan) == 0))
	419	must_wait = false;
	420	rcu_read_unlock();
	421	}
809e9447	422
0ec5f02f	423	if (must_wait)
e3be4c23	424	ret = fence_wait(fence, intr);
5e120f6e	425
29ba89b2	426	return ret;
809e9447	427	}
5e120f6e	428
809e9447	429	if (!exclusive \|\| !fobj)
29ba89b2 ML	430	return ret;
	431
	432	for (i = 0; i < fobj->shared_count && !ret; ++i) {
809e9447	433	struct nouveau_channel *prev = NULL;
0ec5f02f	434	bool must_wait = true;
809e9447	435
29ba89b2 ML	436	fence = rcu_dereference_protected(fobj->shared[i],
	437	reservation_object_held(resv));
	438
809e9447	439	f = nouveau_local_fence(fence, chan->drm);
0ec5f02f ML	440	if (f) {
	441	rcu_read_lock();
	442	prev = rcu_dereference(f->channel);
	443	if (prev && (prev == chan \|\| fctx->sync(f, prev, chan) == 0))
	444	must_wait = false;
	445	rcu_read_unlock();
	446	}
809e9447	447
0ec5f02f	448	if (must_wait)
e3be4c23	449	ret = fence_wait(fence, intr);
29ba89b2 ML	450	}
	451
	452	return ret;
d375e7d5 BS	453	}
	454
	455	void
	456	nouveau_fence_unref(struct nouveau_fence **pfence)
	457	{
	458	if (*pfence)
29ba89b2	459	fence_put(&(*pfence)->base);
d375e7d5 BS	460	*pfence = NULL;
	461	}
	462
d375e7d5	463	int
264ce192 BS	464	nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
264ce192 BS	465	struct nouveau_fence **pfence)
d375e7d5 BS	466	{
	467	struct nouveau_fence *fence;
	468	int ret = 0;
6ee73861	469
e193b1d4	470	if (unlikely(!chan->fence))
5e120f6e BS	471	return -ENODEV;
5e120f6e BS	472
d375e7d5 BS	473	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	474	if (!fence)
	475	return -ENOMEM;
264ce192 BS	476
264ce192 BS	477	fence->sysmem = sysmem;
d375e7d5	478
b5d8f052 CD	479	ret = nouveau_fence_emit(fence, chan);
	480	if (ret)
	481	nouveau_fence_unref(&fence);
d375e7d5 BS	482
d375e7d5 BS	483	*pfence = fence;
6ee73861 BS	484	return ret;
6ee73861 BS	485	}
29ba89b2 ML	486
	487	static const char nouveau_fence_get_get_driver_name(struct fence fence)
	488	{
	489	return "nouveau";
	490	}
	491
	492	static const char nouveau_fence_get_timeline_name(struct fence f)
	493	{
	494	struct nouveau_fence *fence = from_fence(f);
	495	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	496
0ec5f02f	497	return !fctx->dead ? fctx->name : "dead channel";
29ba89b2 ML	498	}
	499
	500	/*
	501	* In an ideal world, read would not assume the channel context is still alive.
	502	* This function may be called from another device, running into free memory as a
	503	* result. The drm node should still be there, so we can derive the index from
	504	* the fence context.
	505	*/
	506	static bool nouveau_fence_is_signaled(struct fence *f)
	507	{
	508	struct nouveau_fence *fence = from_fence(f);
	509	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
0ec5f02f ML	510	struct nouveau_channel *chan;
	511	bool ret = false;
	512
	513	rcu_read_lock();
	514	chan = rcu_dereference(fence->channel);
	515	if (chan)
	516	ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	517	rcu_read_unlock();
29ba89b2	518
0ec5f02f	519	return ret;
29ba89b2 ML	520	}
	521
	522	static bool nouveau_fence_no_signaling(struct fence *f)
	523	{
	524	struct nouveau_fence *fence = from_fence(f);
	525
	526	/*
	527	* caller should have a reference on the fence,
	528	* else fence could get freed here
	529	*/
	530	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
	531
	532	/*
	533	* This needs uevents to work correctly, but fence_add_callback relies on
	534	* being able to enable signaling. It will still get signaled eventually,
	535	* just not right away.
	536	*/
	537	if (nouveau_fence_is_signaled(f)) {
	538	list_del(&fence->head);
	539
	540	fence_put(&fence->base);
	541	return false;
	542	}
	543
	544	return true;
	545	}
	546
15a996bb ML	547	static void nouveau_fence_release(struct fence *f)
	548	{
	549	struct nouveau_fence *fence = from_fence(f);
	550	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	551
	552	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	553	fence_free(&fence->base);
	554	}
	555
29ba89b2 ML	556	static const struct fence_ops nouveau_fence_ops_legacy = {
	557	.get_driver_name = nouveau_fence_get_get_driver_name,
	558	.get_timeline_name = nouveau_fence_get_timeline_name,
	559	.enable_signaling = nouveau_fence_no_signaling,
	560	.signaled = nouveau_fence_is_signaled,
	561	.wait = nouveau_fence_wait_legacy,
15a996bb	562	.release = nouveau_fence_release
29ba89b2 ML	563	};
	564
	565	static bool nouveau_fence_enable_signaling(struct fence *f)
	566	{
	567	struct nouveau_fence *fence = from_fence(f);
	568	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	569	bool ret;
	570
	571	if (!fctx->notify_ref++)
	572	nvif_notify_get(&fctx->notify);
	573
	574	ret = nouveau_fence_no_signaling(f);
	575	if (ret)
	576	set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	577	else if (!--fctx->notify_ref)
	578	nvif_notify_put(&fctx->notify);
	579
	580	return ret;
	581	}
	582
	583	static const struct fence_ops nouveau_fence_ops_uevent = {
	584	.get_driver_name = nouveau_fence_get_get_driver_name,
	585	.get_timeline_name = nouveau_fence_get_timeline_name,
	586	.enable_signaling = nouveau_fence_enable_signaling,
	587	.signaled = nouveau_fence_is_signaled,
	588	.wait = fence_default_wait,
	589	.release = NULL
	590	};