[deliverable/linux.git] / drivers / gpu / drm / nouveau / nouveau_fence.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <trace/events/fence.h>

#include <nvif/notify.h>
#include <nvif/event.h>

#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"

static const struct fence_ops nouveau_fence_ops_uevent;
static const struct fence_ops nouveau_fence_ops_legacy;

static inline struct nouveau_fence *
from_fence(struct fence *fence)
{
	return container_of(fence, struct nouveau_fence, base);
}

static inline struct nouveau_fence_chan *
nouveau_fctx(struct nouveau_fence *fence)
{
	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
}

static void
nouveau_fence_signal(struct nouveau_fence *fence)
{
	fence_signal_locked(&fence->base);
	list_del(&fence->head);

	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

		if (!--fctx->notify_ref)
			nvif_notify_put(&fctx->notify);
	}

	fence_put(&fence->base);
}

static struct nouveau_fence *
nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
	struct nouveau_fence_priv *priv = (void*)drm->fence;

	if (fence->ops != &nouveau_fence_ops_legacy &&
	    fence->ops != &nouveau_fence_ops_uevent)
		return NULL;

	if (fence->context < priv->context_base ||
	    fence->context >= priv->context_base + priv->contexts)
		return NULL;

	return from_fence(fence);
}

void
nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	nvif_notify_fini(&fctx->notify);

	spin_lock_irq(&fctx->lock);
	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		nouveau_fence_signal(fence);
		fence->channel = NULL;
	}
	spin_unlock_irq(&fctx->lock);
}

static void
nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	u32 seq = fctx->read(chan);

	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if ((int)(seq - fence->base.seqno) < 0)
			return;

		nouveau_fence_signal(fence);
	}
}

static int
nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
{
	struct nouveau_fence_chan *fctx =
		container_of(notify, typeof(*fctx), notify);
	unsigned long flags;

	spin_lock_irqsave(&fctx->lock, flags);
	if (!list_empty(&fctx->pending)) {
		struct nouveau_fence *fence;

		fence = list_entry(fctx->pending.next, typeof(*fence), head);
		nouveau_fence_update(fence->channel, fctx);
	}
	spin_unlock_irqrestore(&fctx->lock, flags);

	/* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
	return NVIF_NOTIFY_KEEP;
}

void
nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	INIT_LIST_HEAD(&fctx->flip);
	INIT_LIST_HEAD(&fctx->pending);
	spin_lock_init(&fctx->lock);
	fctx->context = priv->context_base + chan->chid;

	if (!priv->uevent)
		return;

	ret = nvif_notify_init(chan->object, NULL,
			 nouveau_fence_wait_uevent_handler, false,
			 G82_CHANNEL_DMA_V0_NTFY_UEVENT,
			 &(struct nvif_notify_uevent_req) { },
			 sizeof(struct nvif_notify_uevent_req),
			 sizeof(struct nvif_notify_uevent_rep),
			 &fctx->notify);

	WARN_ON(ret);
}

struct nouveau_fence_work {
	struct work_struct work;
	struct fence_cb cb;
	void (*func)(void *);
	void *data;
};

static void
nouveau_fence_work_handler(struct work_struct *kwork)
{
	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
	work->func(work->data);
	kfree(work);
}

static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
{
	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);

	schedule_work(&work->work);
}

void
nouveau_fence_work(struct fence *fence,
		   void (*func)(void *), void *data)
{
	struct nouveau_fence_work *work;

	if (fence_is_signaled(fence))
		goto err;

	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work) {
		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
					   false, false));
		goto err;
	}

	INIT_WORK(&work->work, nouveau_fence_work_handler);
	work->func = func;
	work->data = data;

	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
		goto err_free;
	return;

err_free:
	kfree(work);
err:
	func(data);
}

int
nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	fence->channel  = chan;
	fence->timeout  = jiffies + (15 * HZ);

	if (priv->uevent)
		fence_init(&fence->base, &nouveau_fence_ops_uevent,
			   &fctx->lock,
			   priv->context_base + chan->chid, ++fctx->sequence);
	else
		fence_init(&fence->base, &nouveau_fence_ops_legacy,
			   &fctx->lock,
			   priv->context_base + chan->chid, ++fctx->sequence);

	trace_fence_emit(&fence->base);
	ret = fctx->emit(fence);
	if (!ret) {
		fence_get(&fence->base);
		spin_lock_irq(&fctx->lock);
		nouveau_fence_update(chan, fctx);
		list_add_tail(&fence->head, &fctx->pending);
		spin_unlock_irq(&fctx->lock);
	}

	return ret;
}

bool
nouveau_fence_done(struct nouveau_fence *fence)
{
	if (fence->base.ops == &nouveau_fence_ops_legacy ||
	    fence->base.ops == &nouveau_fence_ops_uevent) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
		unsigned long flags;

		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
			return true;

		spin_lock_irqsave(&fctx->lock, flags);
		nouveau_fence_update(fence->channel, fctx);
		spin_unlock_irqrestore(&fctx->lock, flags);
	}
	return fence_is_signaled(&fence->base);
}

static long
nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
{
	struct nouveau_fence *fence = from_fence(f);
	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	unsigned long t = jiffies, timeout = t + wait;

	while (!nouveau_fence_done(fence)) {
		ktime_t kt;

		t = jiffies;

		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
			__set_current_state(TASK_RUNNING);
			return 0;
		}

		__set_current_state(intr ? TASK_INTERRUPTIBLE :
					   TASK_UNINTERRUPTIBLE);

		kt = ktime_set(0, sleep_time);
		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
		sleep_time *= 2;
		if (sleep_time > NSEC_PER_MSEC)
			sleep_time = NSEC_PER_MSEC;

		if (intr && signal_pending(current))
			return -ERESTARTSYS;
	}

	__set_current_state(TASK_RUNNING);

	return timeout - t;
}

static int
nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
{
	int ret = 0;

	while (!nouveau_fence_done(fence)) {
		if (time_after_eq(jiffies, fence->timeout)) {
			ret = -EBUSY;
			break;
		}

		__set_current_state(intr ?
				    TASK_INTERRUPTIBLE :
				    TASK_UNINTERRUPTIBLE);

		if (intr && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	}

	__set_current_state(TASK_RUNNING);
	return ret;
}

int
nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
{
	long ret;

	if (!lazy)
		return nouveau_fence_wait_busy(fence, intr);

	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	if (ret < 0)
		return ret;
	else if (!ret)
		return -EBUSY;
	else
		return 0;
}

int
nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct fence *fence = NULL;
	struct reservation_object *resv = nvbo->bo.resv;
	struct reservation_object_list *fobj;
	int ret = 0, i;

	fence = reservation_object_get_excl(resv);

	if (fence && !fence_is_signaled(fence)) {
		struct nouveau_fence *f = from_fence(fence);
		struct nouveau_channel *prev = f->channel;

		if (prev != chan) {
			ret = fctx->sync(f, prev, chan);
			if (unlikely(ret))
				ret = nouveau_fence_wait(f, true, true);
		}
	}

	if (ret)
		return ret;

	fobj = reservation_object_get_list(resv);
	if (!fobj)
		return ret;

	for (i = 0; i < fobj->shared_count && !ret; ++i) {
		fence = rcu_dereference_protected(fobj->shared[i],
						reservation_object_held(resv));

		/* should always be true, for now */
		if (!nouveau_local_fence(fence, chan->drm))
			ret = fence_wait(fence, true);
	}

	return ret;
}

void
nouveau_fence_unref(struct nouveau_fence **pfence)
{
	if (*pfence)
		fence_put(&(*pfence)->base);
	*pfence = NULL;
}

struct nouveau_fence *
nouveau_fence_ref(struct nouveau_fence *fence)
{
	if (fence)
		fence_get(&fence->base);
	return fence;
}

int
nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
		  struct nouveau_fence **pfence)
{
	struct nouveau_fence *fence;
	int ret = 0;

	if (unlikely(!chan->fence))
		return -ENODEV;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
		return -ENOMEM;

	fence->sysmem = sysmem;

	ret = nouveau_fence_emit(fence, chan);
	if (ret)
		nouveau_fence_unref(&fence);

	*pfence = fence;
	return ret;
}

static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
{
	return "nouveau";
}

static const char *nouveau_fence_get_timeline_name(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	return fence->channel ? fctx->name : "dead channel";
}

/*
 * In an ideal world, read would not assume the channel context is still alive.
 * This function may be called from another device, running into free memory as a
 * result. The drm node should still be there, so we can derive the index from
 * the fence context.
 */
static bool nouveau_fence_is_signaled(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	struct nouveau_channel *chan = fence->channel;

	return (int)(fctx->read(chan) - fence->base.seqno) >= 0;
}

static bool nouveau_fence_no_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);

	/*
	 * caller should have a reference on the fence,
	 * else fence could get freed here
	 */
	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);

	/*
	 * This needs uevents to work correctly, but fence_add_callback relies on
	 * being able to enable signaling. It will still get signaled eventually,
	 * just not right away.
	 */
	if (nouveau_fence_is_signaled(f)) {
		list_del(&fence->head);

		fence_put(&fence->base);
		return false;
	}

	return true;
}

static const struct fence_ops nouveau_fence_ops_legacy = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_no_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = nouveau_fence_wait_legacy,
	.release = NULL
};

static bool nouveau_fence_enable_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	bool ret;

	if (!fctx->notify_ref++)
		nvif_notify_get(&fctx->notify);

	ret = nouveau_fence_no_signaling(f);
	if (ret)
		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	else if (!--fctx->notify_ref)
		nvif_notify_put(&fctx->notify);

	return ret;
}

static const struct fence_ops nouveau_fence_ops_uevent = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_enable_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = fence_default_wait,
	.release = NULL
};
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
760285e7	27	#include <drm/drmP.h>
6ee73861	28
bd35fe5a MS	29	#include <linux/ktime.h>
bd35fe5a MS	30	#include <linux/hrtimer.h>
29ba89b2	31	#include <trace/events/fence.h>
bd35fe5a	32
867920f8 BS	33	#include <nvif/notify.h>
	34	#include <nvif/event.h>
	35
ebb945a9	36	#include "nouveau_drm.h"
6ee73861	37	#include "nouveau_dma.h"
ebb945a9	38	#include "nouveau_fence.h"
6ee73861	39
29ba89b2 ML	40	static const struct fence_ops nouveau_fence_ops_uevent;
	41	static const struct fence_ops nouveau_fence_ops_legacy;
	42
	43	static inline struct nouveau_fence *
	44	from_fence(struct fence *fence)
	45	{
	46	return container_of(fence, struct nouveau_fence, base);
	47	}
	48
	49	static inline struct nouveau_fence_chan *
	50	nouveau_fctx(struct nouveau_fence *fence)
	51	{
	52	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
	53	}
c4c7044f BS	54
	55	static void
	56	nouveau_fence_signal(struct nouveau_fence *fence)
	57	{
29ba89b2 ML	58	fence_signal_locked(&fence->base);
	59	list_del(&fence->head);
	60
	61	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
	62	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
c4c7044f	63
29ba89b2 ML	64	if (!--fctx->notify_ref)
29ba89b2 ML	65	nvif_notify_put(&fctx->notify);
c4c7044f BS	66	}
c4c7044f BS	67
29ba89b2 ML	68	fence_put(&fence->base);
	69	}
	70
	71	static struct nouveau_fence *
	72	nouveau_local_fence(struct fence fence, struct nouveau_drm drm) {
	73	struct nouveau_fence_priv priv = (void)drm->fence;
	74
	75	if (fence->ops != &nouveau_fence_ops_legacy &&
	76	fence->ops != &nouveau_fence_ops_uevent)
	77	return NULL;
	78
	79	if (fence->context < priv->context_base \|\|
	80	fence->context >= priv->context_base + priv->contexts)
	81	return NULL;
	82
	83	return from_fence(fence);
c4c7044f BS	84	}
c4c7044f BS	85
5e120f6e BS	86	void
	87	nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
	88	{
29ba89b2 ML	89	struct nouveau_fence *fence;
	90
	91	nvif_notify_fini(&fctx->notify);
	92
	93	spin_lock_irq(&fctx->lock);
	94	while (!list_empty(&fctx->pending)) {
	95	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	96
c4c7044f	97	nouveau_fence_signal(fence);
29ba89b2	98	fence->channel = NULL;
5e120f6e	99	}
29ba89b2 ML	100	spin_unlock_irq(&fctx->lock);
	101	}
	102
	103	static void
	104	nouveau_fence_update(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
	105	{
	106	struct nouveau_fence *fence;
	107
	108	u32 seq = fctx->read(chan);
	109
	110	while (!list_empty(&fctx->pending)) {
	111	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	112
	113	if ((int)(seq - fence->base.seqno) < 0)
	114	return;
	115
	116	nouveau_fence_signal(fence);
	117	}
	118	}
	119
	120	static int
	121	nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
	122	{
	123	struct nouveau_fence_chan *fctx =
	124	container_of(notify, typeof(*fctx), notify);
	125	unsigned long flags;
	126
	127	spin_lock_irqsave(&fctx->lock, flags);
	128	if (!list_empty(&fctx->pending)) {
	129	struct nouveau_fence *fence;
	130
	131	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	132	nouveau_fence_update(fence->channel, fctx);
	133	}
	134	spin_unlock_irqrestore(&fctx->lock, flags);
	135
	136	/* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
	137	return NVIF_NOTIFY_KEEP;
5e120f6e BS	138	}
	139
	140	void
29ba89b2	141	nouveau_fence_context_new(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
5e120f6e	142	{
29ba89b2 ML	143	struct nouveau_fence_priv priv = (void)chan->drm->fence;
	144	int ret;
	145
f589be88	146	INIT_LIST_HEAD(&fctx->flip);
5e120f6e BS	147	INIT_LIST_HEAD(&fctx->pending);
5e120f6e BS	148	spin_lock_init(&fctx->lock);
29ba89b2 ML	149	fctx->context = priv->context_base + chan->chid;
	150
	151	if (!priv->uevent)
	152	return;
	153
	154	ret = nvif_notify_init(chan->object, NULL,
	155	nouveau_fence_wait_uevent_handler, false,
	156	G82_CHANNEL_DMA_V0_NTFY_UEVENT,
	157	&(struct nvif_notify_uevent_req) { },
	158	sizeof(struct nvif_notify_uevent_req),
	159	sizeof(struct nvif_notify_uevent_rep),
	160	&fctx->notify);
	161
	162	WARN_ON(ret);
5e120f6e	163	}
6ee73861	164
29ba89b2 ML	165	struct nouveau_fence_work {
	166	struct work_struct work;
	167	struct fence_cb cb;
	168	void (func)(void );
	169	void *data;
	170	};
	171
c4c7044f BS	172	static void
	173	nouveau_fence_work_handler(struct work_struct *kwork)
	174	{
29ba89b2	175	struct nouveau_fence_work work = container_of(kwork, typeof(work), work);
c4c7044f BS	176	work->func(work->data);
	177	kfree(work);
	178	}
	179
29ba89b2 ML	180	static void nouveau_fence_work_cb(struct fence fence, struct fence_cb cb)
	181	{
	182	struct nouveau_fence_work work = container_of(cb, typeof(work), cb);
	183
	184	schedule_work(&work->work);
	185	}
	186
c4c7044f	187	void
f2c24b83	188	nouveau_fence_work(struct fence *fence,
c4c7044f BS	189	void (func)(void ), void *data)
c4c7044f BS	190	{
29ba89b2	191	struct nouveau_fence_work *work;
c4c7044f	192
f2c24b83	193	if (fence_is_signaled(fence))
29ba89b2	194	goto err;
c4c7044f	195
c4c7044f BS	196	work = kmalloc(sizeof(*work), GFP_KERNEL);
c4c7044f BS	197	if (!work) {
f2c24b83 ML	198	WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
f2c24b83 ML	199	false, false));
29ba89b2	200	goto err;
c4c7044f BS	201	}
c4c7044f BS	202
29ba89b2	203	INIT_WORK(&work->work, nouveau_fence_work_handler);
c4c7044f BS	204	work->func = func;
c4c7044f BS	205	work->data = data;
6ee73861	206
f2c24b83	207	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
29ba89b2 ML	208	goto err_free;
29ba89b2 ML	209	return;
b08abd4e	210
29ba89b2 ML	211	err_free:
	212	kfree(work);
	213	err:
	214	func(data);
6ee73861 BS	215	}
	216
	217	int
d375e7d5	218	nouveau_fence_emit(struct nouveau_fence fence, struct nouveau_channel chan)
6ee73861	219	{
e193b1d4	220	struct nouveau_fence_chan *fctx = chan->fence;
29ba89b2	221	struct nouveau_fence_priv priv = (void)chan->drm->fence;
6ee73861 BS	222	int ret;
6ee73861 BS	223
5e120f6e	224	fence->channel = chan;
bfd8303a	225	fence->timeout = jiffies + (15 * HZ);
6ee73861	226
29ba89b2 ML	227	if (priv->uevent)
	228	fence_init(&fence->base, &nouveau_fence_ops_uevent,
	229	&fctx->lock,
	230	priv->context_base + chan->chid, ++fctx->sequence);
	231	else
	232	fence_init(&fence->base, &nouveau_fence_ops_legacy,
	233	&fctx->lock,
	234	priv->context_base + chan->chid, ++fctx->sequence);
	235
	236	trace_fence_emit(&fence->base);
827520ce	237	ret = fctx->emit(fence);
5e120f6e	238	if (!ret) {
29ba89b2 ML	239	fence_get(&fence->base);
	240	spin_lock_irq(&fctx->lock);
	241	nouveau_fence_update(chan, fctx);
5e120f6e	242	list_add_tail(&fence->head, &fctx->pending);
29ba89b2	243	spin_unlock_irq(&fctx->lock);
529c4959	244	}
6ee73861	245
5e120f6e	246	return ret;
6ee73861 BS	247	}
6ee73861 BS	248
6ee73861	249	bool
d375e7d5	250	nouveau_fence_done(struct nouveau_fence *fence)
6ee73861	251	{
29ba89b2 ML	252	if (fence->base.ops == &nouveau_fence_ops_legacy \|\|
	253	fence->base.ops == &nouveau_fence_ops_uevent) {
	254	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	255	unsigned long flags;
6ee73861	256
29ba89b2 ML	257	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
29ba89b2 ML	258	return true;
79ca2770	259
29ba89b2 ML	260	spin_lock_irqsave(&fctx->lock, flags);
	261	nouveau_fence_update(fence->channel, fctx);
	262	spin_unlock_irqrestore(&fctx->lock, flags);
	263	}
	264	return fence_is_signaled(&fence->base);
e18c080f BS	265	}
e18c080f BS	266
29ba89b2 ML	267	static long
29ba89b2 ML	268	nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
e18c080f	269	{
29ba89b2 ML	270	struct nouveau_fence *fence = from_fence(f);
	271	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	272	unsigned long t = jiffies, timeout = t + wait;
e18c080f	273
29ba89b2 ML	274	while (!nouveau_fence_done(fence)) {
29ba89b2 ML	275	ktime_t kt;
51cb4b39	276
29ba89b2	277	t = jiffies;
e18c080f	278
29ba89b2 ML	279	if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
	280	__set_current_state(TASK_RUNNING);
	281	return 0;
e18c080f	282	}
29ba89b2 ML	283
	284	__set_current_state(intr ? TASK_INTERRUPTIBLE :
	285	TASK_UNINTERRUPTIBLE);
	286
	287	kt = ktime_set(0, sleep_time);
	288	schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
	289	sleep_time *= 2;
	290	if (sleep_time > NSEC_PER_MSEC)
	291	sleep_time = NSEC_PER_MSEC;
	292
	293	if (intr && signal_pending(current))
	294	return -ERESTARTSYS;
e18c080f BS	295	}
e18c080f BS	296
29ba89b2	297	__set_current_state(TASK_RUNNING);
e18c080f	298
29ba89b2	299	return timeout - t;
e18c080f BS	300	}
e18c080f BS	301
29ba89b2 ML	302	static int
29ba89b2 ML	303	nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
6ee73861	304	{
6ee73861 BS	305	int ret = 0;
6ee73861 BS	306
d375e7d5	307	while (!nouveau_fence_done(fence)) {
29ba89b2	308	if (time_after_eq(jiffies, fence->timeout)) {
6ee73861 BS	309	ret = -EBUSY;
	310	break;
	311	}
	312
29ba89b2 ML	313	__set_current_state(intr ?
	314	TASK_INTERRUPTIBLE :
	315	TASK_UNINTERRUPTIBLE);
6ee73861 BS	316
6ee73861 BS	317	if (intr && signal_pending(current)) {
9ddc8c52	318	ret = -ERESTARTSYS;
6ee73861 BS	319	break;
	320	}
	321	}
	322
	323	__set_current_state(TASK_RUNNING);
d375e7d5 BS	324	return ret;
	325	}
	326
5e120f6e	327	int
29ba89b2 ML	328	nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
	329	{
	330	long ret;
	331
	332	if (!lazy)
	333	return nouveau_fence_wait_busy(fence, intr);
	334
	335	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	336	if (ret < 0)
	337	return ret;
	338	else if (!ret)
	339	return -EBUSY;
	340	else
	341	return 0;
	342	}
	343
	344	int
	345	nouveau_fence_sync(struct nouveau_bo nvbo, struct nouveau_channel chan)
5e120f6e	346	{
827520ce	347	struct nouveau_fence_chan *fctx = chan->fence;
29ba89b2 ML	348	struct fence *fence = NULL;
	349	struct reservation_object *resv = nvbo->bo.resv;
	350	struct reservation_object_list *fobj;
	351	int ret = 0, i;
	352
f2c24b83	353	fence = reservation_object_get_excl(resv);
29ba89b2	354
f2c24b83	355	if (fence && !fence_is_signaled(fence)) {
29ba89b2 ML	356	struct nouveau_fence *f = from_fence(fence);
29ba89b2 ML	357	struct nouveau_channel *prev = f->channel;
5e120f6e	358
29ba89b2 ML	359	if (prev != chan) {
29ba89b2 ML	360	ret = fctx->sync(f, prev, chan);
906c033e	361	if (unlikely(ret))
29ba89b2	362	ret = nouveau_fence_wait(f, true, true);
906c033e	363	}
5e120f6e BS	364	}
5e120f6e BS	365
29ba89b2 ML	366	if (ret)
29ba89b2 ML	367	return ret;
5e120f6e	368
29ba89b2	369	fobj = reservation_object_get_list(resv);
f2c24b83	370	if (!fobj)
29ba89b2 ML	371	return ret;
	372
	373	for (i = 0; i < fobj->shared_count && !ret; ++i) {
	374	fence = rcu_dereference_protected(fobj->shared[i],
	375	reservation_object_held(resv));
	376
	377	/* should always be true, for now */
	378	if (!nouveau_local_fence(fence, chan->drm))
	379	ret = fence_wait(fence, true);
	380	}
	381
	382	return ret;
d375e7d5 BS	383	}
	384
	385	void
	386	nouveau_fence_unref(struct nouveau_fence **pfence)
	387	{
	388	if (*pfence)
29ba89b2	389	fence_put(&(*pfence)->base);
d375e7d5 BS	390	*pfence = NULL;
	391	}
	392
	393	struct nouveau_fence *
	394	nouveau_fence_ref(struct nouveau_fence *fence)
	395	{
5d216f60	396	if (fence)
29ba89b2	397	fence_get(&fence->base);
d375e7d5 BS	398	return fence;
	399	}
	400
	401	int
264ce192 BS	402	nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
264ce192 BS	403	struct nouveau_fence **pfence)
d375e7d5 BS	404	{
	405	struct nouveau_fence *fence;
	406	int ret = 0;
6ee73861	407
e193b1d4	408	if (unlikely(!chan->fence))
5e120f6e BS	409	return -ENODEV;
5e120f6e BS	410
d375e7d5 BS	411	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	412	if (!fence)
	413	return -ENOMEM;
264ce192 BS	414
264ce192 BS	415	fence->sysmem = sysmem;
d375e7d5	416
b5d8f052 CD	417	ret = nouveau_fence_emit(fence, chan);
	418	if (ret)
	419	nouveau_fence_unref(&fence);
d375e7d5 BS	420
d375e7d5 BS	421	*pfence = fence;
6ee73861 BS	422	return ret;
6ee73861 BS	423	}
29ba89b2 ML	424
	425	static const char nouveau_fence_get_get_driver_name(struct fence fence)
	426	{
	427	return "nouveau";
	428	}
	429
	430	static const char nouveau_fence_get_timeline_name(struct fence f)
	431	{
	432	struct nouveau_fence *fence = from_fence(f);
	433	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	434
	435	return fence->channel ? fctx->name : "dead channel";
	436	}
	437
	438	/*
	439	* In an ideal world, read would not assume the channel context is still alive.
	440	* This function may be called from another device, running into free memory as a
	441	* result. The drm node should still be there, so we can derive the index from
	442	* the fence context.
	443	*/
	444	static bool nouveau_fence_is_signaled(struct fence *f)
	445	{
	446	struct nouveau_fence *fence = from_fence(f);
	447	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	448	struct nouveau_channel *chan = fence->channel;
	449
	450	return (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	451	}
	452
	453	static bool nouveau_fence_no_signaling(struct fence *f)
	454	{
	455	struct nouveau_fence *fence = from_fence(f);
	456
	457	/*
	458	* caller should have a reference on the fence,
	459	* else fence could get freed here
	460	*/
	461	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
	462
	463	/*
	464	* This needs uevents to work correctly, but fence_add_callback relies on
	465	* being able to enable signaling. It will still get signaled eventually,
	466	* just not right away.
	467	*/
	468	if (nouveau_fence_is_signaled(f)) {
	469	list_del(&fence->head);
	470
	471	fence_put(&fence->base);
	472	return false;
	473	}
	474
	475	return true;
	476	}
	477
	478	static const struct fence_ops nouveau_fence_ops_legacy = {
	479	.get_driver_name = nouveau_fence_get_get_driver_name,
	480	.get_timeline_name = nouveau_fence_get_timeline_name,
	481	.enable_signaling = nouveau_fence_no_signaling,
	482	.signaled = nouveau_fence_is_signaled,
	483	.wait = nouveau_fence_wait_legacy,
	484	.release = NULL
	485	};
	486
	487	static bool nouveau_fence_enable_signaling(struct fence *f)
488	{
489	struct nouveau_fence *fence = from_fence(f);
490	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
491	bool ret;
492
493	if (!fctx->notify_ref++)
494	nvif_notify_get(&fctx->notify);
495
496	ret = nouveau_fence_no_signaling(f);
497	if (ret)
498	set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
499	else if (!--fctx->notify_ref)
500	nvif_notify_put(&fctx->notify);
501
502	return ret;
503	}
504
505	static const struct fence_ops nouveau_fence_ops_uevent = {
506	.get_driver_name = nouveau_fence_get_get_driver_name,
507	.get_timeline_name = nouveau_fence_get_timeline_name,
508	.enable_signaling = nouveau_fence_enable_signaling,
509	.signaled = nouveau_fence_is_signaled,
510	.wait = fence_default_wait,
511	.release = NULL
512	};