[deliverable/linux.git] / drivers / gpu / drm / nouveau / nouveau_fence.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <trace/events/fence.h>

#include <nvif/notify.h>
#include <nvif/event.h>

#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"

static const struct fence_ops nouveau_fence_ops_uevent;
static const struct fence_ops nouveau_fence_ops_legacy;

static inline struct nouveau_fence *
from_fence(struct fence *fence)
{
	return container_of(fence, struct nouveau_fence, base);
}

static inline struct nouveau_fence_chan *
nouveau_fctx(struct nouveau_fence *fence)
{
	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
}

static int
nouveau_fence_signal(struct nouveau_fence *fence)
{
	int drop = 0;

	fence_signal_locked(&fence->base);
	list_del(&fence->head);
	rcu_assign_pointer(fence->channel, NULL);

	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

		if (!--fctx->notify_ref)
			drop = 1;
	}

	fence_put(&fence->base);
	return drop;
}

static struct nouveau_fence *
nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
	struct nouveau_fence_priv *priv = (void*)drm->fence;

	if (fence->ops != &nouveau_fence_ops_legacy &&
	    fence->ops != &nouveau_fence_ops_uevent)
		return NULL;

	if (fence->context < priv->context_base ||
	    fence->context >= priv->context_base + priv->contexts)
		return NULL;

	return from_fence(fence);
}

void
nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	spin_lock_irq(&fctx->lock);
	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if (nouveau_fence_signal(fence))
			nvif_notify_put(&fctx->notify);
	}
	spin_unlock_irq(&fctx->lock);

	nvif_notify_fini(&fctx->notify);
	fctx->dead = 1;

	/*
	 * Ensure that all accesses to fence->channel complete before freeing
	 * the channel.
	 */
	synchronize_rcu();
}

static void
nouveau_fence_context_put(struct kref *fence_ref)
{
	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
}

void
nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
{
	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
}

static int
nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;
	int drop = 0;
	u32 seq = fctx->read(chan);

	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if ((int)(seq - fence->base.seqno) < 0)
			break;

		drop |= nouveau_fence_signal(fence);
	}

	return drop;
}

static int
nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
{
	struct nouveau_fence_chan *fctx =
		container_of(notify, typeof(*fctx), notify);
	unsigned long flags;
	int ret = NVIF_NOTIFY_KEEP;

	spin_lock_irqsave(&fctx->lock, flags);
	if (!list_empty(&fctx->pending)) {
		struct nouveau_fence *fence;
		struct nouveau_channel *chan;

		fence = list_entry(fctx->pending.next, typeof(*fence), head);
		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
		if (nouveau_fence_update(fence->channel, fctx))
			ret = NVIF_NOTIFY_DROP;
	}
	spin_unlock_irqrestore(&fctx->lock, flags);

	return ret;
}

void
nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	struct nouveau_cli *cli = (void *)chan->user.client;
	int ret;

	INIT_LIST_HEAD(&fctx->flip);
	INIT_LIST_HEAD(&fctx->pending);
	spin_lock_init(&fctx->lock);
	fctx->context = priv->context_base + chan->chid;

	if (chan == chan->drm->cechan)
		strcpy(fctx->name, "copy engine channel");
	else if (chan == chan->drm->channel)
		strcpy(fctx->name, "generic kernel channel");
	else
		strcpy(fctx->name, nvxx_client(&cli->base)->name);

	kref_init(&fctx->fence_ref);
	if (!priv->uevent)
		return;

	ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
			       false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
			       &(struct nvif_notify_uevent_req) { },
			       sizeof(struct nvif_notify_uevent_req),
			       sizeof(struct nvif_notify_uevent_rep),
			       &fctx->notify);

	WARN_ON(ret);
}

struct nouveau_fence_work {
	struct work_struct work;
	struct fence_cb cb;
	void (*func)(void *);
	void *data;
};

static void
nouveau_fence_work_handler(struct work_struct *kwork)
{
	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
	work->func(work->data);
	kfree(work);
}

static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
{
	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);

	schedule_work(&work->work);
}

void
nouveau_fence_work(struct fence *fence,
		   void (*func)(void *), void *data)
{
	struct nouveau_fence_work *work;

	if (fence_is_signaled(fence))
		goto err;

	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work) {
		/*
		 * this might not be a nouveau fence any more,
		 * so force a lazy wait here
		 */
		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
					   true, false));
		goto err;
	}

	INIT_WORK(&work->work, nouveau_fence_work_handler);
	work->func = func;
	work->data = data;

	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
		goto err_free;
	return;

err_free:
	kfree(work);
err:
	func(data);
}

int
nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	fence->channel  = chan;
	fence->timeout  = jiffies + (15 * HZ);

	if (priv->uevent)
		fence_init(&fence->base, &nouveau_fence_ops_uevent,
			   &fctx->lock, fctx->context, ++fctx->sequence);
	else
		fence_init(&fence->base, &nouveau_fence_ops_legacy,
			   &fctx->lock, fctx->context, ++fctx->sequence);
	kref_get(&fctx->fence_ref);

	trace_fence_emit(&fence->base);
	ret = fctx->emit(fence);
	if (!ret) {
		fence_get(&fence->base);
		spin_lock_irq(&fctx->lock);

		if (nouveau_fence_update(chan, fctx))
			nvif_notify_put(&fctx->notify);

		list_add_tail(&fence->head, &fctx->pending);
		spin_unlock_irq(&fctx->lock);
	}

	return ret;
}

bool
nouveau_fence_done(struct nouveau_fence *fence)
{
	if (fence->base.ops == &nouveau_fence_ops_legacy ||
	    fence->base.ops == &nouveau_fence_ops_uevent) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
		struct nouveau_channel *chan;
		unsigned long flags;

		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
			return true;

		spin_lock_irqsave(&fctx->lock, flags);
		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
		if (chan && nouveau_fence_update(chan, fctx))
			nvif_notify_put(&fctx->notify);
		spin_unlock_irqrestore(&fctx->lock, flags);
	}
	return fence_is_signaled(&fence->base);
}

static long
nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
{
	struct nouveau_fence *fence = from_fence(f);
	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	unsigned long t = jiffies, timeout = t + wait;

	while (!nouveau_fence_done(fence)) {
		ktime_t kt;

		t = jiffies;

		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
			__set_current_state(TASK_RUNNING);
			return 0;
		}

		__set_current_state(intr ? TASK_INTERRUPTIBLE :
					   TASK_UNINTERRUPTIBLE);

		kt = ktime_set(0, sleep_time);
		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
		sleep_time *= 2;
		if (sleep_time > NSEC_PER_MSEC)
			sleep_time = NSEC_PER_MSEC;

		if (intr && signal_pending(current))
			return -ERESTARTSYS;
	}

	__set_current_state(TASK_RUNNING);

	return timeout - t;
}

static int
nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
{
	int ret = 0;

	while (!nouveau_fence_done(fence)) {
		if (time_after_eq(jiffies, fence->timeout)) {
			ret = -EBUSY;
			break;
		}

		__set_current_state(intr ?
				    TASK_INTERRUPTIBLE :
				    TASK_UNINTERRUPTIBLE);

		if (intr && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	}

	__set_current_state(TASK_RUNNING);
	return ret;
}

int
nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
{
	long ret;

	if (!lazy)
		return nouveau_fence_wait_busy(fence, intr);

	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	if (ret < 0)
		return ret;
	else if (!ret)
		return -EBUSY;
	else
		return 0;
}

int
nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct fence *fence;
	struct reservation_object *resv = nvbo->bo.resv;
	struct reservation_object_list *fobj;
	struct nouveau_fence *f;
	int ret = 0, i;

	if (!exclusive) {
		ret = reservation_object_reserve_shared(resv);

		if (ret)
			return ret;
	}

	fobj = reservation_object_get_list(resv);
	fence = reservation_object_get_excl(resv);

	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
		struct nouveau_channel *prev = NULL;
		bool must_wait = true;

		f = nouveau_local_fence(fence, chan->drm);
		if (f) {
			rcu_read_lock();
			prev = rcu_dereference(f->channel);
			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
				must_wait = false;
			rcu_read_unlock();
		}

		if (must_wait)
			ret = fence_wait(fence, intr);

		return ret;
	}

	if (!exclusive || !fobj)
		return ret;

	for (i = 0; i < fobj->shared_count && !ret; ++i) {
		struct nouveau_channel *prev = NULL;
		bool must_wait = true;

		fence = rcu_dereference_protected(fobj->shared[i],
						reservation_object_held(resv));

		f = nouveau_local_fence(fence, chan->drm);
		if (f) {
			rcu_read_lock();
			prev = rcu_dereference(f->channel);
			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
				must_wait = false;
			rcu_read_unlock();
		}

		if (must_wait)
			ret = fence_wait(fence, intr);
	}

	return ret;
}

void
nouveau_fence_unref(struct nouveau_fence **pfence)
{
	if (*pfence)
		fence_put(&(*pfence)->base);
	*pfence = NULL;
}

int
nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
		  struct nouveau_fence **pfence)
{
	struct nouveau_fence *fence;
	int ret = 0;

	if (unlikely(!chan->fence))
		return -ENODEV;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
		return -ENOMEM;

	fence->sysmem = sysmem;

	ret = nouveau_fence_emit(fence, chan);
	if (ret)
		nouveau_fence_unref(&fence);

	*pfence = fence;
	return ret;
}

static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
{
	return "nouveau";
}

static const char *nouveau_fence_get_timeline_name(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	return !fctx->dead ? fctx->name : "dead channel";
}

/*
 * In an ideal world, read would not assume the channel context is still alive.
 * This function may be called from another device, running into free memory as a
 * result. The drm node should still be there, so we can derive the index from
 * the fence context.
 */
static bool nouveau_fence_is_signaled(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	struct nouveau_channel *chan;
	bool ret = false;

	rcu_read_lock();
	chan = rcu_dereference(fence->channel);
	if (chan)
		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	rcu_read_unlock();

	return ret;
}

static bool nouveau_fence_no_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);

	/*
	 * caller should have a reference on the fence,
	 * else fence could get freed here
	 */
	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);

	/*
	 * This needs uevents to work correctly, but fence_add_callback relies on
	 * being able to enable signaling. It will still get signaled eventually,
	 * just not right away.
	 */
	if (nouveau_fence_is_signaled(f)) {
		list_del(&fence->head);

		fence_put(&fence->base);
		return false;
	}

	return true;
}

static void nouveau_fence_release(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	fence_free(&fence->base);
}

static const struct fence_ops nouveau_fence_ops_legacy = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_no_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = nouveau_fence_wait_legacy,
	.release = nouveau_fence_release
};

static bool nouveau_fence_enable_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	bool ret;

	if (!fctx->notify_ref++)
		nvif_notify_get(&fctx->notify);

	ret = nouveau_fence_no_signaling(f);
	if (ret)
		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	else if (!--fctx->notify_ref)
		nvif_notify_put(&fctx->notify);

	return ret;
}

static const struct fence_ops nouveau_fence_ops_uevent = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_enable_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = fence_default_wait,
	.release = NULL
};
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
760285e7	27	#include <drm/drmP.h>
6ee73861	28
bd35fe5a MS	29	#include <linux/ktime.h>
bd35fe5a MS	30	#include <linux/hrtimer.h>
29ba89b2	31	#include <trace/events/fence.h>
bd35fe5a	32
867920f8 BS	33	#include <nvif/notify.h>
	34	#include <nvif/event.h>
	35
ebb945a9	36	#include "nouveau_drm.h"
6ee73861	37	#include "nouveau_dma.h"
ebb945a9	38	#include "nouveau_fence.h"
6ee73861	39
29ba89b2 ML	40	static const struct fence_ops nouveau_fence_ops_uevent;
	41	static const struct fence_ops nouveau_fence_ops_legacy;
	42
	43	static inline struct nouveau_fence *
	44	from_fence(struct fence *fence)
	45	{
	46	return container_of(fence, struct nouveau_fence, base);
	47	}
	48
	49	static inline struct nouveau_fence_chan *
	50	nouveau_fctx(struct nouveau_fence *fence)
	51	{
	52	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
	53	}
c4c7044f	54
0ec5f02f	55	static int
c4c7044f BS	56	nouveau_fence_signal(struct nouveau_fence *fence)
c4c7044f BS	57	{
0ec5f02f ML	58	int drop = 0;
0ec5f02f ML	59
29ba89b2 ML	60	fence_signal_locked(&fence->base);
29ba89b2 ML	61	list_del(&fence->head);
0ec5f02f	62	rcu_assign_pointer(fence->channel, NULL);
29ba89b2 ML	63
	64	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
	65	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
c4c7044f	66
29ba89b2	67	if (!--fctx->notify_ref)
0ec5f02f	68	drop = 1;
c4c7044f BS	69	}
c4c7044f BS	70
29ba89b2	71	fence_put(&fence->base);
0ec5f02f	72	return drop;
29ba89b2 ML	73	}
	74
	75	static struct nouveau_fence *
	76	nouveau_local_fence(struct fence fence, struct nouveau_drm drm) {
	77	struct nouveau_fence_priv priv = (void)drm->fence;
	78
	79	if (fence->ops != &nouveau_fence_ops_legacy &&
	80	fence->ops != &nouveau_fence_ops_uevent)
	81	return NULL;
	82
	83	if (fence->context < priv->context_base \|\|
	84	fence->context >= priv->context_base + priv->contexts)
	85	return NULL;
	86
	87	return from_fence(fence);
c4c7044f BS	88	}
c4c7044f BS	89
5e120f6e BS	90	void
	91	nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
	92	{
29ba89b2 ML	93	struct nouveau_fence *fence;
29ba89b2 ML	94
29ba89b2 ML	95	spin_lock_irq(&fctx->lock);
	96	while (!list_empty(&fctx->pending)) {
	97	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	98
0ec5f02f ML	99	if (nouveau_fence_signal(fence))
0ec5f02f ML	100	nvif_notify_put(&fctx->notify);
5e120f6e	101	}
29ba89b2	102	spin_unlock_irq(&fctx->lock);
0ec5f02f ML	103
	104	nvif_notify_fini(&fctx->notify);
	105	fctx->dead = 1;
	106
	107	/*
	108	* Ensure that all accesses to fence->channel complete before freeing
	109	* the channel.
	110	*/
	111	synchronize_rcu();
29ba89b2 ML	112	}
29ba89b2 ML	113
15a996bb ML	114	static void
	115	nouveau_fence_context_put(struct kref *fence_ref)
	116	{
	117	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
	118	}
	119
	120	void
	121	nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
	122	{
	123	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	124	}
	125
0ec5f02f	126	static int
29ba89b2 ML	127	nouveau_fence_update(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
	128	{
	129	struct nouveau_fence *fence;
0ec5f02f	130	int drop = 0;
29ba89b2 ML	131	u32 seq = fctx->read(chan);
	132
	133	while (!list_empty(&fctx->pending)) {
	134	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	135
	136	if ((int)(seq - fence->base.seqno) < 0)
0ec5f02f	137	break;
29ba89b2	138
0ec5f02f	139	drop \|= nouveau_fence_signal(fence);
29ba89b2	140	}
0ec5f02f ML	141
0ec5f02f ML	142	return drop;
29ba89b2 ML	143	}
	144
	145	static int
	146	nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
	147	{
	148	struct nouveau_fence_chan *fctx =
	149	container_of(notify, typeof(*fctx), notify);
	150	unsigned long flags;
0ec5f02f	151	int ret = NVIF_NOTIFY_KEEP;
29ba89b2 ML	152
	153	spin_lock_irqsave(&fctx->lock, flags);
	154	if (!list_empty(&fctx->pending)) {
	155	struct nouveau_fence *fence;
0ec5f02f	156	struct nouveau_channel *chan;
29ba89b2 ML	157
29ba89b2 ML	158	fence = list_entry(fctx->pending.next, typeof(*fence), head);
0ec5f02f ML	159	chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
	160	if (nouveau_fence_update(fence->channel, fctx))
	161	ret = NVIF_NOTIFY_DROP;
29ba89b2 ML	162	}
	163	spin_unlock_irqrestore(&fctx->lock, flags);
	164
0ec5f02f	165	return ret;
5e120f6e BS	166	}
	167
	168	void
29ba89b2	169	nouveau_fence_context_new(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
5e120f6e	170	{
29ba89b2	171	struct nouveau_fence_priv priv = (void)chan->drm->fence;
a01ca78c	172	struct nouveau_cli cli = (void )chan->user.client;
29ba89b2 ML	173	int ret;
29ba89b2 ML	174
f589be88	175	INIT_LIST_HEAD(&fctx->flip);
5e120f6e BS	176	INIT_LIST_HEAD(&fctx->pending);
5e120f6e BS	177	spin_lock_init(&fctx->lock);
29ba89b2 ML	178	fctx->context = priv->context_base + chan->chid;
29ba89b2 ML	179
15a996bb ML	180	if (chan == chan->drm->cechan)
	181	strcpy(fctx->name, "copy engine channel");
	182	else if (chan == chan->drm->channel)
	183	strcpy(fctx->name, "generic kernel channel");
	184	else
989aa5b7	185	strcpy(fctx->name, nvxx_client(&cli->base)->name);
15a996bb ML	186
15a996bb ML	187	kref_init(&fctx->fence_ref);
29ba89b2 ML	188	if (!priv->uevent)
	189	return;
	190
a01ca78c BS	191	ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
	192	false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
	193	&(struct nvif_notify_uevent_req) { },
	194	sizeof(struct nvif_notify_uevent_req),
	195	sizeof(struct nvif_notify_uevent_rep),
	196	&fctx->notify);
29ba89b2 ML	197
29ba89b2 ML	198	WARN_ON(ret);
5e120f6e	199	}
6ee73861	200
29ba89b2 ML	201	struct nouveau_fence_work {
	202	struct work_struct work;
	203	struct fence_cb cb;
	204	void (func)(void );
	205	void *data;
	206	};
	207
c4c7044f BS	208	static void
	209	nouveau_fence_work_handler(struct work_struct *kwork)
	210	{
29ba89b2	211	struct nouveau_fence_work work = container_of(kwork, typeof(work), work);
c4c7044f BS	212	work->func(work->data);
	213	kfree(work);
	214	}
	215
29ba89b2 ML	216	static void nouveau_fence_work_cb(struct fence fence, struct fence_cb cb)
	217	{
	218	struct nouveau_fence_work work = container_of(cb, typeof(work), cb);
	219
	220	schedule_work(&work->work);
	221	}
	222
c4c7044f	223	void
f2c24b83	224	nouveau_fence_work(struct fence *fence,
c4c7044f BS	225	void (func)(void ), void *data)
c4c7044f BS	226	{
29ba89b2	227	struct nouveau_fence_work *work;
c4c7044f	228
f2c24b83	229	if (fence_is_signaled(fence))
29ba89b2	230	goto err;
c4c7044f	231
c4c7044f BS	232	work = kmalloc(sizeof(*work), GFP_KERNEL);
c4c7044f BS	233	if (!work) {
bb6178b0 ML	234	/*
	235	* this might not be a nouveau fence any more,
	236	* so force a lazy wait here
	237	*/
f2c24b83	238	WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
bb6178b0	239	true, false));
29ba89b2	240	goto err;
c4c7044f BS	241	}
c4c7044f BS	242
29ba89b2	243	INIT_WORK(&work->work, nouveau_fence_work_handler);
c4c7044f BS	244	work->func = func;
c4c7044f BS	245	work->data = data;
6ee73861	246
f2c24b83	247	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
29ba89b2 ML	248	goto err_free;
29ba89b2 ML	249	return;
b08abd4e	250
29ba89b2 ML	251	err_free:
	252	kfree(work);
	253	err:
	254	func(data);
6ee73861 BS	255	}
	256
	257	int
d375e7d5	258	nouveau_fence_emit(struct nouveau_fence fence, struct nouveau_channel chan)
6ee73861	259	{
e193b1d4	260	struct nouveau_fence_chan *fctx = chan->fence;
29ba89b2	261	struct nouveau_fence_priv priv = (void)chan->drm->fence;
6ee73861 BS	262	int ret;
6ee73861 BS	263
5e120f6e	264	fence->channel = chan;
bfd8303a	265	fence->timeout = jiffies + (15 * HZ);
6ee73861	266
29ba89b2 ML	267	if (priv->uevent)
29ba89b2 ML	268	fence_init(&fence->base, &nouveau_fence_ops_uevent,
e3be4c23	269	&fctx->lock, fctx->context, ++fctx->sequence);
29ba89b2 ML	270	else
29ba89b2 ML	271	fence_init(&fence->base, &nouveau_fence_ops_legacy,
e3be4c23	272	&fctx->lock, fctx->context, ++fctx->sequence);
15a996bb	273	kref_get(&fctx->fence_ref);
29ba89b2 ML	274
29ba89b2 ML	275	trace_fence_emit(&fence->base);
827520ce	276	ret = fctx->emit(fence);
5e120f6e	277	if (!ret) {
29ba89b2 ML	278	fence_get(&fence->base);
29ba89b2 ML	279	spin_lock_irq(&fctx->lock);
0ec5f02f ML	280
	281	if (nouveau_fence_update(chan, fctx))
	282	nvif_notify_put(&fctx->notify);
	283
5e120f6e	284	list_add_tail(&fence->head, &fctx->pending);
29ba89b2	285	spin_unlock_irq(&fctx->lock);
529c4959	286	}
6ee73861	287
5e120f6e	288	return ret;
6ee73861 BS	289	}
6ee73861 BS	290
6ee73861	291	bool
d375e7d5	292	nouveau_fence_done(struct nouveau_fence *fence)
6ee73861	293	{
29ba89b2 ML	294	if (fence->base.ops == &nouveau_fence_ops_legacy \|\|
	295	fence->base.ops == &nouveau_fence_ops_uevent) {
	296	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
0ec5f02f	297	struct nouveau_channel *chan;
29ba89b2	298	unsigned long flags;
6ee73861	299
29ba89b2 ML	300	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
29ba89b2 ML	301	return true;
79ca2770	302
29ba89b2	303	spin_lock_irqsave(&fctx->lock, flags);
0ec5f02f ML	304	chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
	305	if (chan && nouveau_fence_update(chan, fctx))
	306	nvif_notify_put(&fctx->notify);
29ba89b2 ML	307	spin_unlock_irqrestore(&fctx->lock, flags);
	308	}
	309	return fence_is_signaled(&fence->base);
e18c080f BS	310	}
e18c080f BS	311
29ba89b2 ML	312	static long
29ba89b2 ML	313	nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
e18c080f	314	{
29ba89b2 ML	315	struct nouveau_fence *fence = from_fence(f);
	316	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	317	unsigned long t = jiffies, timeout = t + wait;
e18c080f	318
29ba89b2 ML	319	while (!nouveau_fence_done(fence)) {
29ba89b2 ML	320	ktime_t kt;
51cb4b39	321
29ba89b2	322	t = jiffies;
e18c080f	323
29ba89b2 ML	324	if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
	325	__set_current_state(TASK_RUNNING);
	326	return 0;
e18c080f	327	}
29ba89b2 ML	328
	329	__set_current_state(intr ? TASK_INTERRUPTIBLE :
	330	TASK_UNINTERRUPTIBLE);
	331
	332	kt = ktime_set(0, sleep_time);
	333	schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
	334	sleep_time *= 2;
	335	if (sleep_time > NSEC_PER_MSEC)
	336	sleep_time = NSEC_PER_MSEC;
	337
	338	if (intr && signal_pending(current))
	339	return -ERESTARTSYS;
e18c080f BS	340	}
e18c080f BS	341
29ba89b2	342	__set_current_state(TASK_RUNNING);
e18c080f	343
29ba89b2	344	return timeout - t;
e18c080f BS	345	}
e18c080f BS	346
29ba89b2 ML	347	static int
29ba89b2 ML	348	nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
6ee73861	349	{
6ee73861 BS	350	int ret = 0;
6ee73861 BS	351
d375e7d5	352	while (!nouveau_fence_done(fence)) {
29ba89b2	353	if (time_after_eq(jiffies, fence->timeout)) {
6ee73861 BS	354	ret = -EBUSY;
	355	break;
	356	}
	357
29ba89b2 ML	358	__set_current_state(intr ?
	359	TASK_INTERRUPTIBLE :
	360	TASK_UNINTERRUPTIBLE);
6ee73861 BS	361
6ee73861 BS	362	if (intr && signal_pending(current)) {
9ddc8c52	363	ret = -ERESTARTSYS;
6ee73861 BS	364	break;
	365	}
	366	}
	367
	368	__set_current_state(TASK_RUNNING);
d375e7d5 BS	369	return ret;
	370	}
	371
5e120f6e	372	int
29ba89b2 ML	373	nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
	374	{
	375	long ret;
	376
	377	if (!lazy)
	378	return nouveau_fence_wait_busy(fence, intr);
	379
	380	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	381	if (ret < 0)
	382	return ret;
	383	else if (!ret)
	384	return -EBUSY;
	385	else
	386	return 0;
	387	}
	388
	389	int
e3be4c23	390	nouveau_fence_sync(struct nouveau_bo nvbo, struct nouveau_channel chan, bool exclusive, bool intr)
5e120f6e	391	{
827520ce	392	struct nouveau_fence_chan *fctx = chan->fence;
809e9447	393	struct fence *fence;
29ba89b2 ML	394	struct reservation_object *resv = nvbo->bo.resv;
29ba89b2 ML	395	struct reservation_object_list *fobj;
809e9447	396	struct nouveau_fence *f;
29ba89b2 ML	397	int ret = 0, i;
29ba89b2 ML	398
809e9447 ML	399	if (!exclusive) {
	400	ret = reservation_object_reserve_shared(resv);
	401
	402	if (ret)
	403	return ret;
	404	}
	405
	406	fobj = reservation_object_get_list(resv);
f2c24b83	407	fence = reservation_object_get_excl(resv);
29ba89b2	408
809e9447 ML	409	if (fence && (!exclusive \|\| !fobj \|\| !fobj->shared_count)) {
809e9447 ML	410	struct nouveau_channel *prev = NULL;
0ec5f02f	411	bool must_wait = true;
5e120f6e	412
809e9447	413	f = nouveau_local_fence(fence, chan->drm);
0ec5f02f ML	414	if (f) {
	415	rcu_read_lock();
	416	prev = rcu_dereference(f->channel);
	417	if (prev && (prev == chan \|\| fctx->sync(f, prev, chan) == 0))
	418	must_wait = false;
	419	rcu_read_unlock();
	420	}
809e9447	421
0ec5f02f	422	if (must_wait)
e3be4c23	423	ret = fence_wait(fence, intr);
5e120f6e	424
29ba89b2	425	return ret;
809e9447	426	}
5e120f6e	427
809e9447	428	if (!exclusive \|\| !fobj)
29ba89b2 ML	429	return ret;
	430
	431	for (i = 0; i < fobj->shared_count && !ret; ++i) {
809e9447	432	struct nouveau_channel *prev = NULL;
0ec5f02f	433	bool must_wait = true;
809e9447	434
29ba89b2 ML	435	fence = rcu_dereference_protected(fobj->shared[i],
	436	reservation_object_held(resv));
	437
809e9447	438	f = nouveau_local_fence(fence, chan->drm);
0ec5f02f ML	439	if (f) {
	440	rcu_read_lock();
	441	prev = rcu_dereference(f->channel);
	442	if (prev && (prev == chan \|\| fctx->sync(f, prev, chan) == 0))
	443	must_wait = false;
	444	rcu_read_unlock();
	445	}
809e9447	446
0ec5f02f	447	if (must_wait)
e3be4c23	448	ret = fence_wait(fence, intr);
29ba89b2 ML	449	}
	450
	451	return ret;
d375e7d5 BS	452	}
	453
	454	void
	455	nouveau_fence_unref(struct nouveau_fence **pfence)
	456	{
	457	if (*pfence)
29ba89b2	458	fence_put(&(*pfence)->base);
d375e7d5 BS	459	*pfence = NULL;
	460	}
	461
d375e7d5	462	int
264ce192 BS	463	nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
264ce192 BS	464	struct nouveau_fence **pfence)
d375e7d5 BS	465	{
	466	struct nouveau_fence *fence;
	467	int ret = 0;
6ee73861	468
e193b1d4	469	if (unlikely(!chan->fence))
5e120f6e BS	470	return -ENODEV;
5e120f6e BS	471
d375e7d5 BS	472	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	473	if (!fence)
	474	return -ENOMEM;
264ce192 BS	475
264ce192 BS	476	fence->sysmem = sysmem;
d375e7d5	477
b5d8f052 CD	478	ret = nouveau_fence_emit(fence, chan);
	479	if (ret)
	480	nouveau_fence_unref(&fence);
d375e7d5 BS	481
d375e7d5 BS	482	*pfence = fence;
6ee73861 BS	483	return ret;
6ee73861 BS	484	}
29ba89b2 ML	485
	486	static const char nouveau_fence_get_get_driver_name(struct fence fence)
	487	{
	488	return "nouveau";
	489	}
	490
	491	static const char nouveau_fence_get_timeline_name(struct fence f)
	492	{
	493	struct nouveau_fence *fence = from_fence(f);
	494	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	495
0ec5f02f	496	return !fctx->dead ? fctx->name : "dead channel";
29ba89b2 ML	497	}
	498
	499	/*
	500	* In an ideal world, read would not assume the channel context is still alive.
	501	* This function may be called from another device, running into free memory as a
	502	* result. The drm node should still be there, so we can derive the index from
	503	* the fence context.
	504	*/
	505	static bool nouveau_fence_is_signaled(struct fence *f)
	506	{
	507	struct nouveau_fence *fence = from_fence(f);
	508	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
0ec5f02f ML	509	struct nouveau_channel *chan;
	510	bool ret = false;
	511
	512	rcu_read_lock();
	513	chan = rcu_dereference(fence->channel);
	514	if (chan)
	515	ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	516	rcu_read_unlock();
29ba89b2	517
0ec5f02f	518	return ret;
29ba89b2 ML	519	}
	520
	521	static bool nouveau_fence_no_signaling(struct fence *f)
	522	{
	523	struct nouveau_fence *fence = from_fence(f);
	524
	525	/*
	526	* caller should have a reference on the fence,
	527	* else fence could get freed here
	528	*/
	529	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
	530
	531	/*
	532	* This needs uevents to work correctly, but fence_add_callback relies on
	533	* being able to enable signaling. It will still get signaled eventually,
	534	* just not right away.
	535	*/
	536	if (nouveau_fence_is_signaled(f)) {
	537	list_del(&fence->head);
	538
	539	fence_put(&fence->base);
	540	return false;
	541	}
	542
	543	return true;
	544	}
	545
15a996bb ML	546	static void nouveau_fence_release(struct fence *f)
	547	{
	548	struct nouveau_fence *fence = from_fence(f);
	549	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	550
	551	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
	552	fence_free(&fence->base);
	553	}
	554
29ba89b2 ML	555	static const struct fence_ops nouveau_fence_ops_legacy = {
	556	.get_driver_name = nouveau_fence_get_get_driver_name,
	557	.get_timeline_name = nouveau_fence_get_timeline_name,
	558	.enable_signaling = nouveau_fence_no_signaling,
	559	.signaled = nouveau_fence_is_signaled,
	560	.wait = nouveau_fence_wait_legacy,
15a996bb	561	.release = nouveau_fence_release
29ba89b2 ML	562	};
	563
	564	static bool nouveau_fence_enable_signaling(struct fence *f)
	565	{
	566	struct nouveau_fence *fence = from_fence(f);
	567	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	568	bool ret;
	569
	570	if (!fctx->notify_ref++)
	571	nvif_notify_get(&fctx->notify);
	572
	573	ret = nouveau_fence_no_signaling(f);
	574	if (ret)
	575	set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	576	else if (!--fctx->notify_ref)
	577	nvif_notify_put(&fctx->notify);
	578
	579	return ret;
	580	}
	581
	582	static const struct fence_ops nouveau_fence_ops_uevent = {
	583	.get_driver_name = nouveau_fence_get_get_driver_name,
	584	.get_timeline_name = nouveau_fence_get_timeline_name,
	585	.enable_signaling = nouveau_fence_enable_signaling,
	586	.signaled = nouveau_fence_is_signaled,
	587	.wait = fence_default_wait,
	588	.release = NULL
	589	};