[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_evict.c

/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Chris Wilson <chris@chris-wilson.co.uuk>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drv.h"
#include "i915_drm.h"

static struct drm_i915_gem_object *
i915_gem_next_active_object(struct drm_device *dev,
			    struct list_head **render_iter,
			    struct list_head **bsd_iter)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *render_obj = NULL, *bsd_obj = NULL;

	if (*render_iter != &dev_priv->render_ring.active_list)
		render_obj = list_entry(*render_iter,
					struct drm_i915_gem_object,
					list);

	if (HAS_BSD(dev)) {
		if (*bsd_iter != &dev_priv->bsd_ring.active_list)
			bsd_obj = list_entry(*bsd_iter,
					     struct drm_i915_gem_object,
					     list);

		if (render_obj == NULL) {
			*bsd_iter = (*bsd_iter)->next;
			return bsd_obj;
		}

		if (bsd_obj == NULL) {
			*render_iter = (*render_iter)->next;
			return render_obj;
		}

		/* XXX can we handle seqno wrapping? */
		if (render_obj->last_rendering_seqno < bsd_obj->last_rendering_seqno) {
			*render_iter = (*render_iter)->next;
			return render_obj;
		} else {
			*bsd_iter = (*bsd_iter)->next;
			return bsd_obj;
		}
	} else {
		*render_iter = (*render_iter)->next;
		return render_obj;
	}
}

static bool
mark_free(struct drm_i915_gem_object *obj_priv,
	   struct list_head *unwind)
{
	list_add(&obj_priv->evict_list, unwind);
	return drm_mm_scan_add_block(obj_priv->gtt_space);
}

#define i915_for_each_active_object(OBJ, R, B) \
	*(R) = dev_priv->render_ring.active_list.next; \
	*(B) = dev_priv->bsd_ring.active_list.next; \
	while (((OBJ) = i915_gem_next_active_object(dev, (R), (B))) != NULL)

int
i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct list_head eviction_list, unwind_list;
	struct drm_i915_gem_object *obj_priv, *tmp_obj_priv;
	struct list_head *render_iter, *bsd_iter;
	int ret = 0;

	i915_gem_retire_requests(dev);

	/* Re-check for free space after retiring requests */
	if (drm_mm_search_free(&dev_priv->mm.gtt_space,
			       min_size, alignment, 0))
		return 0;

	/*
	 * The goal is to evict objects and amalgamate space in LRU order.
	 * The oldest idle objects reside on the inactive list, which is in
	 * retirement order. The next objects to retire are those on the (per
	 * ring) active list that do not have an outstanding flush. Once the
	 * hardware reports completion (the seqno is updated after the
	 * batchbuffer has been finished) the clean buffer objects would
	 * be retired to the inactive list. Any dirty objects would be added
	 * to the tail of the flushing list. So after processing the clean
	 * active objects we need to emit a MI_FLUSH to retire the flushing
	 * list, hence the retirement order of the flushing list is in
	 * advance of the dirty objects on the active lists.
	 *
	 * The retirement sequence is thus:
	 *   1. Inactive objects (already retired)
	 *   2. Clean active objects
	 *   3. Flushing list
	 *   4. Dirty active objects.
	 *
	 * On each list, the oldest objects lie at the HEAD with the freshest
	 * object on the TAIL.
	 */

	INIT_LIST_HEAD(&unwind_list);
	drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);

	/* First see if there is a large enough contiguous idle region... */
	list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
		if (mark_free(obj_priv, &unwind_list))
			goto found;
	}

	/* Now merge in the soon-to-be-expired objects... */
	i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
		/* Does the object require an outstanding flush? */
		if (obj_priv->base.write_domain || obj_priv->pin_count)
			continue;

		if (mark_free(obj_priv, &unwind_list))
			goto found;
	}

	/* Finally add anything with a pending flush (in order of retirement) */
	list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
		if (obj_priv->pin_count)
			continue;

		if (mark_free(obj_priv, &unwind_list))
			goto found;
	}
	i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
		if (! obj_priv->base.write_domain || obj_priv->pin_count)
			continue;

		if (mark_free(obj_priv, &unwind_list))
			goto found;
	}

	/* Nothing found, clean up and bail out! */
	list_for_each_entry(obj_priv, &unwind_list, evict_list) {
		ret = drm_mm_scan_remove_block(obj_priv->gtt_space);
		BUG_ON(ret);
	}

	/* We expect the caller to unpin, evict all and try again, or give up.
	 * So calling i915_gem_evict_everything() is unnecessary.
	 */
	return -ENOSPC;

found:
	INIT_LIST_HEAD(&eviction_list);
	list_for_each_entry_safe(obj_priv, tmp_obj_priv,
				 &unwind_list, evict_list) {
		if (drm_mm_scan_remove_block(obj_priv->gtt_space)) {
			/* drm_mm doesn't allow any other other operations while
			 * scanning, therefore store to be evicted objects on a
			 * temporary list. */
			list_move(&obj_priv->evict_list, &eviction_list);
		}
	}

	/* Unbinding will emit any required flushes */
	list_for_each_entry_safe(obj_priv, tmp_obj_priv,
				 &eviction_list, evict_list) {
#if WATCH_LRU
		DRM_INFO("%s: evicting %p\n", __func__, obj);
#endif
		ret = i915_gem_object_unbind(&obj_priv->base);
		if (ret)
			return ret;
	}

	/* The just created free hole should be on the top of the free stack
	 * maintained by drm_mm, so this BUG_ON actually executes in O(1).
	 * Furthermore all accessed data has just recently been used, so it
	 * should be really fast, too. */
	BUG_ON(!drm_mm_search_free(&dev_priv->mm.gtt_space, min_size,
				   alignment, 0));

	return 0;
}

int
i915_gem_evict_everything(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
	bool lists_empty;

	spin_lock(&dev_priv->mm.active_list_lock);
	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
		       list_empty(&dev_priv->mm.flushing_list) &&
		       list_empty(&dev_priv->render_ring.active_list) &&
		       (!HAS_BSD(dev)
			|| list_empty(&dev_priv->bsd_ring.active_list)));
	spin_unlock(&dev_priv->mm.active_list_lock);

	if (lists_empty)
		return -ENOSPC;

	/* Flush everything (on to the inactive lists) and evict */
	ret = i915_gpu_idle(dev);
	if (ret)
		return ret;

	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));

	ret = i915_gem_evict_inactive(dev);
	if (ret)
		return ret;

	spin_lock(&dev_priv->mm.active_list_lock);
	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
		       list_empty(&dev_priv->mm.flushing_list) &&
		       list_empty(&dev_priv->render_ring.active_list) &&
		       (!HAS_BSD(dev)
			|| list_empty(&dev_priv->bsd_ring.active_list)));
	spin_unlock(&dev_priv->mm.active_list_lock);
	BUG_ON(!lists_empty);

	return 0;
}

/** Unbinds all inactive objects. */
int
i915_gem_evict_inactive(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

	while (!list_empty(&dev_priv->mm.inactive_list)) {
		struct drm_gem_object *obj;
		int ret;

		obj = &list_first_entry(&dev_priv->mm.inactive_list,
					struct drm_i915_gem_object,
					list)->base;

		ret = i915_gem_object_unbind(obj);
		if (ret != 0) {
			DRM_ERROR("Error unbinding object: %d\n", ret);
			return ret;
		}
	}

	return 0;
}
Commit	Line	Data
b47eb4a2 CW	1	/*
	2	* Copyright © 2008-2010 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	* Authors:
	24	* Eric Anholt <eric@anholt.net>
	25	* Chris Wilson <chris@chris-wilson.co.uuk>
	26	*
	27	*/
	28
	29	#include "drmP.h"
	30	#include "drm.h"
	31	#include "i915_drv.h"
	32	#include "i915_drm.h"
	33
cd377ea9 CW	34	static struct drm_i915_gem_object *
	35	i915_gem_next_active_object(struct drm_device *dev,
	36	struct list_head **render_iter,
	37	struct list_head **bsd_iter)
b47eb4a2 CW	38	{
b47eb4a2 CW	39	drm_i915_private_t *dev_priv = dev->dev_private;
cd377ea9 CW	40	struct drm_i915_gem_object render_obj = NULL, bsd_obj = NULL;
	41
	42	if (*render_iter != &dev_priv->render_ring.active_list)
	43	render_obj = list_entry(*render_iter,
	44	struct drm_i915_gem_object,
	45	list);
	46
	47	if (HAS_BSD(dev)) {
	48	if (*bsd_iter != &dev_priv->bsd_ring.active_list)
	49	bsd_obj = list_entry(*bsd_iter,
	50	struct drm_i915_gem_object,
	51	list);
	52
	53	if (render_obj == NULL) {
	54	bsd_iter = (bsd_iter)->next;
	55	return bsd_obj;
b47eb4a2	56	}
b47eb4a2	57
cd377ea9 CW	58	if (bsd_obj == NULL) {
	59	render_iter = (render_iter)->next;
	60	return render_obj;
	61	}
b47eb4a2	62
cd377ea9 CW	63	/* XXX can we handle seqno wrapping? */
	64	if (render_obj->last_rendering_seqno < bsd_obj->last_rendering_seqno) {
	65	render_iter = (render_iter)->next;
	66	return render_obj;
	67	} else {
	68	bsd_iter = (bsd_iter)->next;
	69	return bsd_obj;
	70	}
	71	} else {
	72	render_iter = (render_iter)->next;
	73	return render_obj;
b47eb4a2	74	}
b47eb4a2 CW	75	}
b47eb4a2 CW	76
cd377ea9 CW	77	static bool
	78	mark_free(struct drm_i915_gem_object *obj_priv,
	79	struct list_head *unwind)
b47eb4a2	80	{
cd377ea9 CW	81	list_add(&obj_priv->evict_list, unwind);
cd377ea9 CW	82	return drm_mm_scan_add_block(obj_priv->gtt_space);
b47eb4a2 CW	83	}
b47eb4a2 CW	84
cd377ea9 CW	85	#define i915_for_each_active_object(OBJ, R, B) \
	86	*(R) = dev_priv->render_ring.active_list.next; \
	87	*(B) = dev_priv->bsd_ring.active_list.next; \
	88	while (((OBJ) = i915_gem_next_active_object(dev, (R), (B))) != NULL)
	89
b47eb4a2	90	int
cd377ea9	91	i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment)
b47eb4a2 CW	92	{
b47eb4a2 CW	93	drm_i915_private_t *dev_priv = dev->dev_private;
cd377ea9 CW	94	struct list_head eviction_list, unwind_list;
	95	struct drm_i915_gem_object obj_priv, tmp_obj_priv;
	96	struct list_head render_iter, bsd_iter;
	97	int ret = 0;
b47eb4a2	98
cd377ea9	99	i915_gem_retire_requests(dev);
b47eb4a2	100
cd377ea9 CW	101	/* Re-check for free space after retiring requests */
	102	if (drm_mm_search_free(&dev_priv->mm.gtt_space,
	103	min_size, alignment, 0))
	104	return 0;
b47eb4a2	105
cd377ea9 CW	106	/*
	107	* The goal is to evict objects and amalgamate space in LRU order.
	108	* The oldest idle objects reside on the inactive list, which is in
	109	* retirement order. The next objects to retire are those on the (per
	110	* ring) active list that do not have an outstanding flush. Once the
	111	* hardware reports completion (the seqno is updated after the
	112	* batchbuffer has been finished) the clean buffer objects would
	113	* be retired to the inactive list. Any dirty objects would be added
	114	* to the tail of the flushing list. So after processing the clean
	115	* active objects we need to emit a MI_FLUSH to retire the flushing
	116	* list, hence the retirement order of the flushing list is in
	117	* advance of the dirty objects on the active lists.
	118	*
	119	* The retirement sequence is thus:
	120	* 1. Inactive objects (already retired)
	121	* 2. Clean active objects
	122	* 3. Flushing list
	123	* 4. Dirty active objects.
	124	*
	125	* On each list, the oldest objects lie at the HEAD with the freshest
	126	* object on the TAIL.
	127	*/
	128
	129	INIT_LIST_HEAD(&unwind_list);
	130	drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
	131
	132	/* First see if there is a large enough contiguous idle region... */
	133	list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
	134	if (mark_free(obj_priv, &unwind_list))
	135	goto found;
	136	}
b47eb4a2	137
cd377ea9 CW	138	/* Now merge in the soon-to-be-expired objects... */
	139	i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
	140	/* Does the object require an outstanding flush? */
	141	if (obj_priv->base.write_domain \|\| obj_priv->pin_count)
b47eb4a2	142	continue;
b47eb4a2	143
cd377ea9 CW	144	if (mark_free(obj_priv, &unwind_list))
	145	goto found;
	146	}
b47eb4a2	147
cd377ea9 CW	148	/* Finally add anything with a pending flush (in order of retirement) */
	149	list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
	150	if (obj_priv->pin_count)
	151	continue;
b47eb4a2	152
cd377ea9 CW	153	if (mark_free(obj_priv, &unwind_list))
	154	goto found;
	155	}
	156	i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
	157	if (! obj_priv->base.write_domain \|\| obj_priv->pin_count)
b47eb4a2	158	continue;
b47eb4a2	159
cd377ea9 CW	160	if (mark_free(obj_priv, &unwind_list))
	161	goto found;
	162	}
	163
	164	/* Nothing found, clean up and bail out! */
	165	list_for_each_entry(obj_priv, &unwind_list, evict_list) {
	166	ret = drm_mm_scan_remove_block(obj_priv->gtt_space);
	167	BUG_ON(ret);
	168	}
	169
	170	/* We expect the caller to unpin, evict all and try again, or give up.
	171	* So calling i915_gem_evict_everything() is unnecessary.
	172	*/
	173	return -ENOSPC;
	174
	175	found:
	176	INIT_LIST_HEAD(&eviction_list);
	177	list_for_each_entry_safe(obj_priv, tmp_obj_priv,
	178	&unwind_list, evict_list) {
	179	if (drm_mm_scan_remove_block(obj_priv->gtt_space)) {
	180	/* drm_mm doesn't allow any other other operations while
	181	* scanning, therefore store to be evicted objects on a
	182	* temporary list. */
	183	list_move(&obj_priv->evict_list, &eviction_list);
b47eb4a2	184	}
cd377ea9	185	}
b47eb4a2	186
cd377ea9 CW	187	/* Unbinding will emit any required flushes */
	188	list_for_each_entry_safe(obj_priv, tmp_obj_priv,
	189	&eviction_list, evict_list) {
	190	#if WATCH_LRU
	191	DRM_INFO("%s: evicting %p\n", __func__, obj);
	192	#endif
	193	ret = i915_gem_object_unbind(&obj_priv->base);
	194	if (ret)
	195	return ret;
b47eb4a2	196	}
cd377ea9 CW	197
	198	/* The just created free hole should be on the top of the free stack
	199	* maintained by drm_mm, so this BUG_ON actually executes in O(1).
	200	* Furthermore all accessed data has just recently been used, so it
	201	* should be really fast, too. */
	202	BUG_ON(!drm_mm_search_free(&dev_priv->mm.gtt_space, min_size,
	203	alignment, 0));
	204
	205	return 0;
b47eb4a2 CW	206	}
	207
	208	int
	209	i915_gem_evict_everything(struct drm_device *dev)
	210	{
	211	drm_i915_private_t *dev_priv = dev->dev_private;
	212	int ret;
	213	bool lists_empty;
	214
	215	spin_lock(&dev_priv->mm.active_list_lock);
	216	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
	217	list_empty(&dev_priv->mm.flushing_list) &&
	218	list_empty(&dev_priv->render_ring.active_list) &&
	219	(!HAS_BSD(dev)
	220	\|\| list_empty(&dev_priv->bsd_ring.active_list)));
	221	spin_unlock(&dev_priv->mm.active_list_lock);
	222
	223	if (lists_empty)
	224	return -ENOSPC;
	225
	226	/* Flush everything (on to the inactive lists) and evict */
	227	ret = i915_gpu_idle(dev);
	228	if (ret)
	229	return ret;
	230
	231	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
	232
	233	ret = i915_gem_evict_inactive(dev);
	234	if (ret)
	235	return ret;
	236
	237	spin_lock(&dev_priv->mm.active_list_lock);
	238	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
	239	list_empty(&dev_priv->mm.flushing_list) &&
	240	list_empty(&dev_priv->render_ring.active_list) &&
	241	(!HAS_BSD(dev)
	242	\|\| list_empty(&dev_priv->bsd_ring.active_list)));
	243	spin_unlock(&dev_priv->mm.active_list_lock);
	244	BUG_ON(!lists_empty);
	245
	246	return 0;
	247	}
	248
	249	/** Unbinds all inactive objects. */
	250	int
	251	i915_gem_evict_inactive(struct drm_device *dev)
	252	{
	253	drm_i915_private_t *dev_priv = dev->dev_private;
	254
	255	while (!list_empty(&dev_priv->mm.inactive_list)) {
	256	struct drm_gem_object *obj;
	257	int ret;
	258
	259	obj = &list_first_entry(&dev_priv->mm.inactive_list,
	260	struct drm_i915_gem_object,
	261	list)->base;
	262
	263	ret = i915_gem_object_unbind(obj);
	264	if (ret != 0) {
	265	DRM_ERROR("Error unbinding object: %d\n", ret);
	266	return ret;
	267	}
	268	}
	269
270	return 0;
271	}