[deliverable/linux.git] / drivers / gpu / drm / nouveau / nouveau_fence.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <trace/events/fence.h>

#include <nvif/notify.h>
#include <nvif/event.h>

#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"

static const struct fence_ops nouveau_fence_ops_uevent;
static const struct fence_ops nouveau_fence_ops_legacy;

static inline struct nouveau_fence *
from_fence(struct fence *fence)
{
	return container_of(fence, struct nouveau_fence, base);
}

static inline struct nouveau_fence_chan *
nouveau_fctx(struct nouveau_fence *fence)
{
	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
}

static void
nouveau_fence_signal(struct nouveau_fence *fence)
{
	fence_signal_locked(&fence->base);
	list_del(&fence->head);

	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

		if (!--fctx->notify_ref)
			nvif_notify_put(&fctx->notify);
	}

	fence_put(&fence->base);
}

static struct nouveau_fence *
nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
	struct nouveau_fence_priv *priv = (void*)drm->fence;

	if (fence->ops != &nouveau_fence_ops_legacy &&
	    fence->ops != &nouveau_fence_ops_uevent)
		return NULL;

	if (fence->context < priv->context_base ||
	    fence->context >= priv->context_base + priv->contexts)
		return NULL;

	return from_fence(fence);
}

void
nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	nvif_notify_fini(&fctx->notify);

	spin_lock_irq(&fctx->lock);
	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		nouveau_fence_signal(fence);
		fence->channel = NULL;
	}
	spin_unlock_irq(&fctx->lock);
}

static void
nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence;

	u32 seq = fctx->read(chan);

	while (!list_empty(&fctx->pending)) {
		fence = list_entry(fctx->pending.next, typeof(*fence), head);

		if ((int)(seq - fence->base.seqno) < 0)
			return;

		nouveau_fence_signal(fence);
	}
}

static int
nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
{
	struct nouveau_fence_chan *fctx =
		container_of(notify, typeof(*fctx), notify);
	unsigned long flags;

	spin_lock_irqsave(&fctx->lock, flags);
	if (!list_empty(&fctx->pending)) {
		struct nouveau_fence *fence;

		fence = list_entry(fctx->pending.next, typeof(*fence), head);
		nouveau_fence_update(fence->channel, fctx);
	}
	spin_unlock_irqrestore(&fctx->lock, flags);

	/* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
	return NVIF_NOTIFY_KEEP;
}

void
nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	INIT_LIST_HEAD(&fctx->flip);
	INIT_LIST_HEAD(&fctx->pending);
	spin_lock_init(&fctx->lock);
	fctx->context = priv->context_base + chan->chid;

	if (!priv->uevent)
		return;

	ret = nvif_notify_init(chan->object, NULL,
			 nouveau_fence_wait_uevent_handler, false,
			 G82_CHANNEL_DMA_V0_NTFY_UEVENT,
			 &(struct nvif_notify_uevent_req) { },
			 sizeof(struct nvif_notify_uevent_req),
			 sizeof(struct nvif_notify_uevent_rep),
			 &fctx->notify);

	WARN_ON(ret);
}

struct nouveau_fence_work {
	struct work_struct work;
	struct fence_cb cb;
	void (*func)(void *);
	void *data;
};

static void
nouveau_fence_work_handler(struct work_struct *kwork)
{
	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
	work->func(work->data);
	kfree(work);
}

static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
{
	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);

	schedule_work(&work->work);
}

void
nouveau_fence_work(struct fence *fence,
		   void (*func)(void *), void *data)
{
	struct nouveau_fence_work *work;

	if (fence_is_signaled(fence))
		goto err;

	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work) {
		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
					   false, false));
		goto err;
	}

	INIT_WORK(&work->work, nouveau_fence_work_handler);
	work->func = func;
	work->data = data;

	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
		goto err_free;
	return;

err_free:
	kfree(work);
err:
	func(data);
}

int
nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
	int ret;

	fence->channel  = chan;
	fence->timeout  = jiffies + (15 * HZ);

	if (priv->uevent)
		fence_init(&fence->base, &nouveau_fence_ops_uevent,
			   &fctx->lock, fctx->context, ++fctx->sequence);
	else
		fence_init(&fence->base, &nouveau_fence_ops_legacy,
			   &fctx->lock, fctx->context, ++fctx->sequence);

	trace_fence_emit(&fence->base);
	ret = fctx->emit(fence);
	if (!ret) {
		fence_get(&fence->base);
		spin_lock_irq(&fctx->lock);
		nouveau_fence_update(chan, fctx);
		list_add_tail(&fence->head, &fctx->pending);
		spin_unlock_irq(&fctx->lock);
	}

	return ret;
}

bool
nouveau_fence_done(struct nouveau_fence *fence)
{
	if (fence->base.ops == &nouveau_fence_ops_legacy ||
	    fence->base.ops == &nouveau_fence_ops_uevent) {
		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
		unsigned long flags;

		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
			return true;

		spin_lock_irqsave(&fctx->lock, flags);
		nouveau_fence_update(fence->channel, fctx);
		spin_unlock_irqrestore(&fctx->lock, flags);
	}
	return fence_is_signaled(&fence->base);
}

static long
nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
{
	struct nouveau_fence *fence = from_fence(f);
	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	unsigned long t = jiffies, timeout = t + wait;

	while (!nouveau_fence_done(fence)) {
		ktime_t kt;

		t = jiffies;

		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
			__set_current_state(TASK_RUNNING);
			return 0;
		}

		__set_current_state(intr ? TASK_INTERRUPTIBLE :
					   TASK_UNINTERRUPTIBLE);

		kt = ktime_set(0, sleep_time);
		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
		sleep_time *= 2;
		if (sleep_time > NSEC_PER_MSEC)
			sleep_time = NSEC_PER_MSEC;

		if (intr && signal_pending(current))
			return -ERESTARTSYS;
	}

	__set_current_state(TASK_RUNNING);

	return timeout - t;
}

static int
nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
{
	int ret = 0;

	while (!nouveau_fence_done(fence)) {
		if (time_after_eq(jiffies, fence->timeout)) {
			ret = -EBUSY;
			break;
		}

		__set_current_state(intr ?
				    TASK_INTERRUPTIBLE :
				    TASK_UNINTERRUPTIBLE);

		if (intr && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	}

	__set_current_state(TASK_RUNNING);
	return ret;
}

int
nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
{
	long ret;

	if (!lazy)
		return nouveau_fence_wait_busy(fence, intr);

	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	if (ret < 0)
		return ret;
	else if (!ret)
		return -EBUSY;
	else
		return 0;
}

int
nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct fence *fence;
	struct reservation_object *resv = nvbo->bo.resv;
	struct reservation_object_list *fobj;
	struct nouveau_fence *f;
	int ret = 0, i;

	if (!exclusive) {
		ret = reservation_object_reserve_shared(resv);

		if (ret)
			return ret;
	}

	fobj = reservation_object_get_list(resv);
	fence = reservation_object_get_excl(resv);

	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
		struct nouveau_channel *prev = NULL;

		f = nouveau_local_fence(fence, chan->drm);
		if (f)
			prev = f->channel;

		if (!prev || (prev != chan && (ret = fctx->sync(f, prev, chan))))
			ret = fence_wait(fence, intr);

		return ret;
	}

	if (!exclusive || !fobj)
		return ret;

	for (i = 0; i < fobj->shared_count && !ret; ++i) {
		struct nouveau_channel *prev = NULL;

		fence = rcu_dereference_protected(fobj->shared[i],
						reservation_object_held(resv));

		f = nouveau_local_fence(fence, chan->drm);
		if (f)
			prev = f->channel;

		if (!prev || (prev != chan && (ret = fctx->sync(f, prev, chan))))
			ret = fence_wait(fence, intr);

		if (ret)
			break;
	}

	return ret;
}

void
nouveau_fence_unref(struct nouveau_fence **pfence)
{
	if (*pfence)
		fence_put(&(*pfence)->base);
	*pfence = NULL;
}

int
nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
		  struct nouveau_fence **pfence)
{
	struct nouveau_fence *fence;
	int ret = 0;

	if (unlikely(!chan->fence))
		return -ENODEV;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
		return -ENOMEM;

	fence->sysmem = sysmem;

	ret = nouveau_fence_emit(fence, chan);
	if (ret)
		nouveau_fence_unref(&fence);

	*pfence = fence;
	return ret;
}

static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
{
	return "nouveau";
}

static const char *nouveau_fence_get_timeline_name(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);

	return fence->channel ? fctx->name : "dead channel";
}

/*
 * In an ideal world, read would not assume the channel context is still alive.
 * This function may be called from another device, running into free memory as a
 * result. The drm node should still be there, so we can derive the index from
 * the fence context.
 */
static bool nouveau_fence_is_signaled(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	struct nouveau_channel *chan = fence->channel;

	return (int)(fctx->read(chan) - fence->base.seqno) >= 0;
}

static bool nouveau_fence_no_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);

	/*
	 * caller should have a reference on the fence,
	 * else fence could get freed here
	 */
	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);

	/*
	 * This needs uevents to work correctly, but fence_add_callback relies on
	 * being able to enable signaling. It will still get signaled eventually,
	 * just not right away.
	 */
	if (nouveau_fence_is_signaled(f)) {
		list_del(&fence->head);

		fence_put(&fence->base);
		return false;
	}

	return true;
}

static const struct fence_ops nouveau_fence_ops_legacy = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_no_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = nouveau_fence_wait_legacy,
	.release = NULL
};

static bool nouveau_fence_enable_signaling(struct fence *f)
{
	struct nouveau_fence *fence = from_fence(f);
	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	bool ret;

	if (!fctx->notify_ref++)
		nvif_notify_get(&fctx->notify);

	ret = nouveau_fence_no_signaling(f);
	if (ret)
		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
	else if (!--fctx->notify_ref)
		nvif_notify_put(&fctx->notify);

	return ret;
}

static const struct fence_ops nouveau_fence_ops_uevent = {
	.get_driver_name = nouveau_fence_get_get_driver_name,
	.get_timeline_name = nouveau_fence_get_timeline_name,
	.enable_signaling = nouveau_fence_enable_signaling,
	.signaled = nouveau_fence_is_signaled,
	.wait = fence_default_wait,
	.release = NULL
};
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
760285e7	27	#include <drm/drmP.h>
6ee73861	28
bd35fe5a MS	29	#include <linux/ktime.h>
bd35fe5a MS	30	#include <linux/hrtimer.h>
29ba89b2	31	#include <trace/events/fence.h>
bd35fe5a	32
867920f8 BS	33	#include <nvif/notify.h>
	34	#include <nvif/event.h>
	35
ebb945a9	36	#include "nouveau_drm.h"
6ee73861	37	#include "nouveau_dma.h"
ebb945a9	38	#include "nouveau_fence.h"
6ee73861	39
29ba89b2 ML	40	static const struct fence_ops nouveau_fence_ops_uevent;
	41	static const struct fence_ops nouveau_fence_ops_legacy;
	42
	43	static inline struct nouveau_fence *
	44	from_fence(struct fence *fence)
	45	{
	46	return container_of(fence, struct nouveau_fence, base);
	47	}
	48
	49	static inline struct nouveau_fence_chan *
	50	nouveau_fctx(struct nouveau_fence *fence)
	51	{
	52	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
	53	}
c4c7044f BS	54
	55	static void
	56	nouveau_fence_signal(struct nouveau_fence *fence)
	57	{
29ba89b2 ML	58	fence_signal_locked(&fence->base);
	59	list_del(&fence->head);
	60
	61	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
	62	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
c4c7044f	63
29ba89b2 ML	64	if (!--fctx->notify_ref)
29ba89b2 ML	65	nvif_notify_put(&fctx->notify);
c4c7044f BS	66	}
c4c7044f BS	67
29ba89b2 ML	68	fence_put(&fence->base);
	69	}
	70
	71	static struct nouveau_fence *
	72	nouveau_local_fence(struct fence fence, struct nouveau_drm drm) {
	73	struct nouveau_fence_priv priv = (void)drm->fence;
	74
	75	if (fence->ops != &nouveau_fence_ops_legacy &&
	76	fence->ops != &nouveau_fence_ops_uevent)
	77	return NULL;
	78
	79	if (fence->context < priv->context_base \|\|
	80	fence->context >= priv->context_base + priv->contexts)
	81	return NULL;
	82
	83	return from_fence(fence);
c4c7044f BS	84	}
c4c7044f BS	85
5e120f6e BS	86	void
	87	nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
	88	{
29ba89b2 ML	89	struct nouveau_fence *fence;
	90
	91	nvif_notify_fini(&fctx->notify);
	92
	93	spin_lock_irq(&fctx->lock);
	94	while (!list_empty(&fctx->pending)) {
	95	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	96
c4c7044f	97	nouveau_fence_signal(fence);
29ba89b2	98	fence->channel = NULL;
5e120f6e	99	}
29ba89b2 ML	100	spin_unlock_irq(&fctx->lock);
	101	}
	102
	103	static void
	104	nouveau_fence_update(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
	105	{
	106	struct nouveau_fence *fence;
	107
	108	u32 seq = fctx->read(chan);
	109
	110	while (!list_empty(&fctx->pending)) {
	111	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	112
	113	if ((int)(seq - fence->base.seqno) < 0)
	114	return;
	115
	116	nouveau_fence_signal(fence);
	117	}
	118	}
	119
	120	static int
	121	nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
	122	{
	123	struct nouveau_fence_chan *fctx =
	124	container_of(notify, typeof(*fctx), notify);
	125	unsigned long flags;
	126
	127	spin_lock_irqsave(&fctx->lock, flags);
	128	if (!list_empty(&fctx->pending)) {
	129	struct nouveau_fence *fence;
	130
	131	fence = list_entry(fctx->pending.next, typeof(*fence), head);
	132	nouveau_fence_update(fence->channel, fctx);
	133	}
	134	spin_unlock_irqrestore(&fctx->lock, flags);
	135
	136	/* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
	137	return NVIF_NOTIFY_KEEP;
5e120f6e BS	138	}
	139
	140	void
29ba89b2	141	nouveau_fence_context_new(struct nouveau_channel chan, struct nouveau_fence_chan fctx)
5e120f6e	142	{
29ba89b2 ML	143	struct nouveau_fence_priv priv = (void)chan->drm->fence;
	144	int ret;
	145
f589be88	146	INIT_LIST_HEAD(&fctx->flip);
5e120f6e BS	147	INIT_LIST_HEAD(&fctx->pending);
5e120f6e BS	148	spin_lock_init(&fctx->lock);
29ba89b2 ML	149	fctx->context = priv->context_base + chan->chid;
	150
	151	if (!priv->uevent)
	152	return;
	153
	154	ret = nvif_notify_init(chan->object, NULL,
	155	nouveau_fence_wait_uevent_handler, false,
	156	G82_CHANNEL_DMA_V0_NTFY_UEVENT,
	157	&(struct nvif_notify_uevent_req) { },
	158	sizeof(struct nvif_notify_uevent_req),
	159	sizeof(struct nvif_notify_uevent_rep),
	160	&fctx->notify);
	161
	162	WARN_ON(ret);
5e120f6e	163	}
6ee73861	164
29ba89b2 ML	165	struct nouveau_fence_work {
	166	struct work_struct work;
	167	struct fence_cb cb;
	168	void (func)(void );
	169	void *data;
	170	};
	171
c4c7044f BS	172	static void
	173	nouveau_fence_work_handler(struct work_struct *kwork)
	174	{
29ba89b2	175	struct nouveau_fence_work work = container_of(kwork, typeof(work), work);
c4c7044f BS	176	work->func(work->data);
	177	kfree(work);
	178	}
	179
29ba89b2 ML	180	static void nouveau_fence_work_cb(struct fence fence, struct fence_cb cb)
	181	{
	182	struct nouveau_fence_work work = container_of(cb, typeof(work), cb);
	183
	184	schedule_work(&work->work);
	185	}
	186
c4c7044f	187	void
f2c24b83	188	nouveau_fence_work(struct fence *fence,
c4c7044f BS	189	void (func)(void ), void *data)
c4c7044f BS	190	{
29ba89b2	191	struct nouveau_fence_work *work;
c4c7044f	192
f2c24b83	193	if (fence_is_signaled(fence))
29ba89b2	194	goto err;
c4c7044f	195
c4c7044f BS	196	work = kmalloc(sizeof(*work), GFP_KERNEL);
c4c7044f BS	197	if (!work) {
f2c24b83 ML	198	WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
f2c24b83 ML	199	false, false));
29ba89b2	200	goto err;
c4c7044f BS	201	}
c4c7044f BS	202
29ba89b2	203	INIT_WORK(&work->work, nouveau_fence_work_handler);
c4c7044f BS	204	work->func = func;
c4c7044f BS	205	work->data = data;
6ee73861	206
f2c24b83	207	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
29ba89b2 ML	208	goto err_free;
29ba89b2 ML	209	return;
b08abd4e	210
29ba89b2 ML	211	err_free:
	212	kfree(work);
	213	err:
	214	func(data);
6ee73861 BS	215	}
	216
	217	int
d375e7d5	218	nouveau_fence_emit(struct nouveau_fence fence, struct nouveau_channel chan)
6ee73861	219	{
e193b1d4	220	struct nouveau_fence_chan *fctx = chan->fence;
29ba89b2	221	struct nouveau_fence_priv priv = (void)chan->drm->fence;
6ee73861 BS	222	int ret;
6ee73861 BS	223
5e120f6e	224	fence->channel = chan;
bfd8303a	225	fence->timeout = jiffies + (15 * HZ);
6ee73861	226
29ba89b2 ML	227	if (priv->uevent)
29ba89b2 ML	228	fence_init(&fence->base, &nouveau_fence_ops_uevent,
e3be4c23	229	&fctx->lock, fctx->context, ++fctx->sequence);
29ba89b2 ML	230	else
29ba89b2 ML	231	fence_init(&fence->base, &nouveau_fence_ops_legacy,
e3be4c23	232	&fctx->lock, fctx->context, ++fctx->sequence);
29ba89b2 ML	233
29ba89b2 ML	234	trace_fence_emit(&fence->base);
827520ce	235	ret = fctx->emit(fence);
5e120f6e	236	if (!ret) {
29ba89b2 ML	237	fence_get(&fence->base);
	238	spin_lock_irq(&fctx->lock);
	239	nouveau_fence_update(chan, fctx);
5e120f6e	240	list_add_tail(&fence->head, &fctx->pending);
29ba89b2	241	spin_unlock_irq(&fctx->lock);
529c4959	242	}
6ee73861	243
5e120f6e	244	return ret;
6ee73861 BS	245	}
6ee73861 BS	246
6ee73861	247	bool
d375e7d5	248	nouveau_fence_done(struct nouveau_fence *fence)
6ee73861	249	{
29ba89b2 ML	250	if (fence->base.ops == &nouveau_fence_ops_legacy \|\|
	251	fence->base.ops == &nouveau_fence_ops_uevent) {
	252	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	253	unsigned long flags;
6ee73861	254
29ba89b2 ML	255	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
29ba89b2 ML	256	return true;
79ca2770	257
29ba89b2 ML	258	spin_lock_irqsave(&fctx->lock, flags);
	259	nouveau_fence_update(fence->channel, fctx);
	260	spin_unlock_irqrestore(&fctx->lock, flags);
	261	}
	262	return fence_is_signaled(&fence->base);
e18c080f BS	263	}
e18c080f BS	264
29ba89b2 ML	265	static long
29ba89b2 ML	266	nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
e18c080f	267	{
29ba89b2 ML	268	struct nouveau_fence *fence = from_fence(f);
	269	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	270	unsigned long t = jiffies, timeout = t + wait;
e18c080f	271
29ba89b2 ML	272	while (!nouveau_fence_done(fence)) {
29ba89b2 ML	273	ktime_t kt;
51cb4b39	274
29ba89b2	275	t = jiffies;
e18c080f	276
29ba89b2 ML	277	if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
	278	__set_current_state(TASK_RUNNING);
	279	return 0;
e18c080f	280	}
29ba89b2 ML	281
	282	__set_current_state(intr ? TASK_INTERRUPTIBLE :
	283	TASK_UNINTERRUPTIBLE);
	284
	285	kt = ktime_set(0, sleep_time);
	286	schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
	287	sleep_time *= 2;
	288	if (sleep_time > NSEC_PER_MSEC)
	289	sleep_time = NSEC_PER_MSEC;
	290
	291	if (intr && signal_pending(current))
	292	return -ERESTARTSYS;
e18c080f BS	293	}
e18c080f BS	294
29ba89b2	295	__set_current_state(TASK_RUNNING);
e18c080f	296
29ba89b2	297	return timeout - t;
e18c080f BS	298	}
e18c080f BS	299
29ba89b2 ML	300	static int
29ba89b2 ML	301	nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
6ee73861	302	{
6ee73861 BS	303	int ret = 0;
6ee73861 BS	304
d375e7d5	305	while (!nouveau_fence_done(fence)) {
29ba89b2	306	if (time_after_eq(jiffies, fence->timeout)) {
6ee73861 BS	307	ret = -EBUSY;
	308	break;
	309	}
	310
29ba89b2 ML	311	__set_current_state(intr ?
	312	TASK_INTERRUPTIBLE :
	313	TASK_UNINTERRUPTIBLE);
6ee73861 BS	314
6ee73861 BS	315	if (intr && signal_pending(current)) {
9ddc8c52	316	ret = -ERESTARTSYS;
6ee73861 BS	317	break;
	318	}
	319	}
	320
	321	__set_current_state(TASK_RUNNING);
d375e7d5 BS	322	return ret;
	323	}
	324
5e120f6e	325	int
29ba89b2 ML	326	nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
	327	{
	328	long ret;
	329
	330	if (!lazy)
	331	return nouveau_fence_wait_busy(fence, intr);
	332
	333	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
	334	if (ret < 0)
	335	return ret;
	336	else if (!ret)
	337	return -EBUSY;
	338	else
	339	return 0;
	340	}
	341
	342	int
e3be4c23	343	nouveau_fence_sync(struct nouveau_bo nvbo, struct nouveau_channel chan, bool exclusive, bool intr)
5e120f6e	344	{
827520ce	345	struct nouveau_fence_chan *fctx = chan->fence;
809e9447	346	struct fence *fence;
29ba89b2 ML	347	struct reservation_object *resv = nvbo->bo.resv;
29ba89b2 ML	348	struct reservation_object_list *fobj;
809e9447	349	struct nouveau_fence *f;
29ba89b2 ML	350	int ret = 0, i;
29ba89b2 ML	351
809e9447 ML	352	if (!exclusive) {
	353	ret = reservation_object_reserve_shared(resv);
	354
	355	if (ret)
	356	return ret;
	357	}
	358
	359	fobj = reservation_object_get_list(resv);
f2c24b83	360	fence = reservation_object_get_excl(resv);
29ba89b2	361
809e9447 ML	362	if (fence && (!exclusive \|\| !fobj \|\| !fobj->shared_count)) {
809e9447 ML	363	struct nouveau_channel *prev = NULL;
5e120f6e	364
809e9447 ML	365	f = nouveau_local_fence(fence, chan->drm);
	366	if (f)
	367	prev = f->channel;
	368
	369	if (!prev \|\| (prev != chan && (ret = fctx->sync(f, prev, chan))))
e3be4c23	370	ret = fence_wait(fence, intr);
5e120f6e	371
29ba89b2	372	return ret;
809e9447	373	}
5e120f6e	374
809e9447	375	if (!exclusive \|\| !fobj)
29ba89b2 ML	376	return ret;
	377
	378	for (i = 0; i < fobj->shared_count && !ret; ++i) {
809e9447 ML	379	struct nouveau_channel *prev = NULL;
809e9447 ML	380
29ba89b2 ML	381	fence = rcu_dereference_protected(fobj->shared[i],
	382	reservation_object_held(resv));
	383
809e9447 ML	384	f = nouveau_local_fence(fence, chan->drm);
	385	if (f)
	386	prev = f->channel;
	387
e3be4c23 ML	388	if (!prev \|\| (prev != chan && (ret = fctx->sync(f, prev, chan))))
e3be4c23 ML	389	ret = fence_wait(fence, intr);
809e9447 ML	390
	391	if (ret)
	392	break;
29ba89b2 ML	393	}
	394
	395	return ret;
d375e7d5 BS	396	}
	397
	398	void
	399	nouveau_fence_unref(struct nouveau_fence **pfence)
	400	{
	401	if (*pfence)
29ba89b2	402	fence_put(&(*pfence)->base);
d375e7d5 BS	403	*pfence = NULL;
	404	}
	405
d375e7d5	406	int
264ce192 BS	407	nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
264ce192 BS	408	struct nouveau_fence **pfence)
d375e7d5 BS	409	{
	410	struct nouveau_fence *fence;
	411	int ret = 0;
6ee73861	412
e193b1d4	413	if (unlikely(!chan->fence))
5e120f6e BS	414	return -ENODEV;
5e120f6e BS	415
d375e7d5 BS	416	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	417	if (!fence)
	418	return -ENOMEM;
264ce192 BS	419
264ce192 BS	420	fence->sysmem = sysmem;
d375e7d5	421
b5d8f052 CD	422	ret = nouveau_fence_emit(fence, chan);
	423	if (ret)
	424	nouveau_fence_unref(&fence);
d375e7d5 BS	425
d375e7d5 BS	426	*pfence = fence;
6ee73861 BS	427	return ret;
6ee73861 BS	428	}
29ba89b2 ML	429
	430	static const char nouveau_fence_get_get_driver_name(struct fence fence)
	431	{
	432	return "nouveau";
	433	}
	434
	435	static const char nouveau_fence_get_timeline_name(struct fence f)
	436	{
	437	struct nouveau_fence *fence = from_fence(f);
	438	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	439
	440	return fence->channel ? fctx->name : "dead channel";
	441	}
	442
	443	/*
	444	* In an ideal world, read would not assume the channel context is still alive.
	445	* This function may be called from another device, running into free memory as a
	446	* result. The drm node should still be there, so we can derive the index from
	447	* the fence context.
	448	*/
	449	static bool nouveau_fence_is_signaled(struct fence *f)
	450	{
	451	struct nouveau_fence *fence = from_fence(f);
	452	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
	453	struct nouveau_channel *chan = fence->channel;
	454
	455	return (int)(fctx->read(chan) - fence->base.seqno) >= 0;
	456	}
	457
	458	static bool nouveau_fence_no_signaling(struct fence *f)
	459	{
	460	struct nouveau_fence *fence = from_fence(f);
	461
	462	/*
	463	* caller should have a reference on the fence,
	464	* else fence could get freed here
	465	*/
	466	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
	467
	468	/*
	469	* This needs uevents to work correctly, but fence_add_callback relies on
	470	* being able to enable signaling. It will still get signaled eventually,
	471	* just not right away.
	472	*/
	473	if (nouveau_fence_is_signaled(f)) {
	474	list_del(&fence->head);
	475
	476	fence_put(&fence->base);
	477	return false;
	478	}
	479
	480	return true;
	481	}
	482
	483	static const struct fence_ops nouveau_fence_ops_legacy = {
	484	.get_driver_name = nouveau_fence_get_get_driver_name,
	485	.get_timeline_name = nouveau_fence_get_timeline_name,
	486	.enable_signaling = nouveau_fence_no_signaling,
	487	.signaled = nouveau_fence_is_signaled,
	488	.wait = nouveau_fence_wait_legacy,
	489	.release = NULL
	490	};
	491
	492	static bool nouveau_fence_enable_signaling(struct fence *f)
493	{
494	struct nouveau_fence *fence = from_fence(f);
495	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
496	bool ret;
497
498	if (!fctx->notify_ref++)
499	nvif_notify_get(&fctx->notify);
500
501	ret = nouveau_fence_no_signaling(f);
502	if (ret)
503	set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
504	else if (!--fctx->notify_ref)
505	nvif_notify_put(&fctx->notify);
506
507	return ret;
508	}
509
510	static const struct fence_ops nouveau_fence_ops_uevent = {
511	.get_driver_name = nouveau_fence_get_get_driver_name,
512	.get_timeline_name = nouveau_fence_get_timeline_name,
513	.enable_signaling = nouveau_fence_enable_signaling,
514	.signaled = nouveau_fence_is_signaled,
515	.wait = fence_default_wait,
516	.release = NULL
517	};