2 * Copyright © 2008-2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/prefetch.h>
29 static const char *i915_fence_get_driver_name(struct fence
*fence
)
34 static const char *i915_fence_get_timeline_name(struct fence
*fence
)
36 /* Timelines are bound by eviction to a VM. However, since
37 * we only have a global seqno at the moment, we only have
38 * a single timeline. Note that each timeline will have
39 * multiple execution contexts (fence contexts) as we allow
40 * engines within a single timeline to execute in parallel.
45 static bool i915_fence_signaled(struct fence
*fence
)
47 return i915_gem_request_completed(to_request(fence
));
50 static bool i915_fence_enable_signaling(struct fence
*fence
)
52 if (i915_fence_signaled(fence
))
55 intel_engine_enable_signaling(to_request(fence
));
59 static signed long i915_fence_wait(struct fence
*fence
,
61 signed long timeout_jiffies
)
63 s64 timeout_ns
, *timeout
;
66 if (timeout_jiffies
!= MAX_SCHEDULE_TIMEOUT
) {
67 timeout_ns
= jiffies_to_nsecs(timeout_jiffies
);
68 timeout
= &timeout_ns
;
73 ret
= i915_wait_request(to_request(fence
),
74 interruptible
, timeout
,
82 if (timeout_jiffies
!= MAX_SCHEDULE_TIMEOUT
)
83 timeout_jiffies
= nsecs_to_jiffies(timeout_ns
);
85 return timeout_jiffies
;
88 static void i915_fence_value_str(struct fence
*fence
, char *str
, int size
)
90 snprintf(str
, size
, "%u", fence
->seqno
);
93 static void i915_fence_timeline_value_str(struct fence
*fence
, char *str
,
96 snprintf(str
, size
, "%u",
97 intel_engine_get_seqno(to_request(fence
)->engine
));
100 static void i915_fence_release(struct fence
*fence
)
102 struct drm_i915_gem_request
*req
= to_request(fence
);
104 kmem_cache_free(req
->i915
->requests
, req
);
107 const struct fence_ops i915_fence_ops
= {
108 .get_driver_name
= i915_fence_get_driver_name
,
109 .get_timeline_name
= i915_fence_get_timeline_name
,
110 .enable_signaling
= i915_fence_enable_signaling
,
111 .signaled
= i915_fence_signaled
,
112 .wait
= i915_fence_wait
,
113 .release
= i915_fence_release
,
114 .fence_value_str
= i915_fence_value_str
,
115 .timeline_value_str
= i915_fence_timeline_value_str
,
118 int i915_gem_request_add_to_client(struct drm_i915_gem_request
*req
,
119 struct drm_file
*file
)
121 struct drm_i915_private
*dev_private
;
122 struct drm_i915_file_private
*file_priv
;
124 WARN_ON(!req
|| !file
|| req
->file_priv
);
132 dev_private
= req
->i915
;
133 file_priv
= file
->driver_priv
;
135 spin_lock(&file_priv
->mm
.lock
);
136 req
->file_priv
= file_priv
;
137 list_add_tail(&req
->client_list
, &file_priv
->mm
.request_list
);
138 spin_unlock(&file_priv
->mm
.lock
);
140 req
->pid
= get_pid(task_pid(current
));
146 i915_gem_request_remove_from_client(struct drm_i915_gem_request
*request
)
148 struct drm_i915_file_private
*file_priv
= request
->file_priv
;
153 spin_lock(&file_priv
->mm
.lock
);
154 list_del(&request
->client_list
);
155 request
->file_priv
= NULL
;
156 spin_unlock(&file_priv
->mm
.lock
);
158 put_pid(request
->pid
);
162 void i915_gem_retire_noop(struct i915_gem_active
*active
,
163 struct drm_i915_gem_request
*request
)
165 /* Space left intentionally blank */
168 static void i915_gem_request_retire(struct drm_i915_gem_request
*request
)
170 struct i915_gem_active
*active
, *next
;
172 trace_i915_gem_request_retire(request
);
173 list_del(&request
->link
);
175 /* We know the GPU must have read the request to have
176 * sent us the seqno + interrupt, so use the position
177 * of tail of the request to update the last known position
180 * Note this requires that we are always called in request
183 list_del(&request
->ring_link
);
184 request
->ring
->last_retired_head
= request
->postfix
;
186 /* Walk through the active list, calling retire on each. This allows
187 * objects to track their GPU activity and mark themselves as idle
188 * when their *last* active request is completed (updating state
189 * tracking lists for eviction, active references for GEM, etc).
191 * As the ->retire() may free the node, we decouple it first and
192 * pass along the auxiliary information (to avoid dereferencing
193 * the node after the callback).
195 list_for_each_entry_safe(active
, next
, &request
->active_list
, link
) {
196 /* In microbenchmarks or focusing upon time inside the kernel,
197 * we may spend an inordinate amount of time simply handling
198 * the retirement of requests and processing their callbacks.
199 * Of which, this loop itself is particularly hot due to the
200 * cache misses when jumping around the list of i915_gem_active.
201 * So we try to keep this loop as streamlined as possible and
202 * also prefetch the next i915_gem_active to try and hide
203 * the likely cache miss.
207 INIT_LIST_HEAD(&active
->link
);
208 RCU_INIT_POINTER(active
->request
, NULL
);
210 active
->retire(active
, request
);
213 i915_gem_request_remove_from_client(request
);
215 if (request
->previous_context
) {
216 if (i915
.enable_execlists
)
217 intel_lr_context_unpin(request
->previous_context
,
221 i915_gem_context_put(request
->ctx
);
222 i915_gem_request_put(request
);
225 void i915_gem_request_retire_upto(struct drm_i915_gem_request
*req
)
227 struct intel_engine_cs
*engine
= req
->engine
;
228 struct drm_i915_gem_request
*tmp
;
230 lockdep_assert_held(&req
->i915
->drm
.struct_mutex
);
231 GEM_BUG_ON(list_empty(&req
->link
));
234 tmp
= list_first_entry(&engine
->request_list
,
237 i915_gem_request_retire(tmp
);
238 } while (tmp
!= req
);
241 static int i915_gem_check_wedge(unsigned int reset_counter
, bool interruptible
)
243 if (__i915_terminally_wedged(reset_counter
))
246 if (__i915_reset_in_progress(reset_counter
)) {
247 /* Non-interruptible callers can't handle -EAGAIN, hence return
248 * -EIO unconditionally for these.
259 static int i915_gem_init_seqno(struct drm_i915_private
*dev_priv
, u32 seqno
)
261 struct intel_engine_cs
*engine
;
264 /* Carefully retire all requests without writing to the rings */
265 for_each_engine(engine
, dev_priv
) {
266 ret
= intel_engine_idle(engine
, true);
270 i915_gem_retire_requests(dev_priv
);
272 /* If the seqno wraps around, we need to clear the breadcrumb rbtree */
273 if (!i915_seqno_passed(seqno
, dev_priv
->next_seqno
)) {
274 while (intel_kick_waiters(dev_priv
) ||
275 intel_kick_signalers(dev_priv
))
279 /* Finally reset hw state */
280 for_each_engine(engine
, dev_priv
)
281 intel_engine_init_seqno(engine
, seqno
);
286 int i915_gem_set_seqno(struct drm_device
*dev
, u32 seqno
)
288 struct drm_i915_private
*dev_priv
= to_i915(dev
);
294 /* HWS page needs to be set less than what we
295 * will inject to ring
297 ret
= i915_gem_init_seqno(dev_priv
, seqno
- 1);
301 dev_priv
->next_seqno
= seqno
;
305 static int i915_gem_get_seqno(struct drm_i915_private
*dev_priv
, u32
*seqno
)
307 /* reserve 0 for non-seqno */
308 if (unlikely(dev_priv
->next_seqno
== 0)) {
311 ret
= i915_gem_init_seqno(dev_priv
, 0);
315 dev_priv
->next_seqno
= 1;
318 *seqno
= dev_priv
->next_seqno
++;
323 * i915_gem_request_alloc - allocate a request structure
325 * @engine: engine that we wish to issue the request on.
326 * @ctx: context that the request will be associated with.
327 * This can be NULL if the request is not directly related to
328 * any specific user context, in which case this function will
329 * choose an appropriate context to use.
331 * Returns a pointer to the allocated request if successful,
332 * or an error code if not.
334 struct drm_i915_gem_request
*
335 i915_gem_request_alloc(struct intel_engine_cs
*engine
,
336 struct i915_gem_context
*ctx
)
338 struct drm_i915_private
*dev_priv
= engine
->i915
;
339 unsigned int reset_counter
= i915_reset_counter(&dev_priv
->gpu_error
);
340 struct drm_i915_gem_request
*req
;
344 /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
345 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
348 ret
= i915_gem_check_wedge(reset_counter
, dev_priv
->mm
.interruptible
);
352 /* Move the oldest request to the slab-cache (if not in use!) */
353 req
= list_first_entry_or_null(&engine
->request_list
,
355 if (req
&& i915_gem_request_completed(req
))
356 i915_gem_request_retire(req
);
358 /* Beware: Dragons be flying overhead.
360 * We use RCU to look up requests in flight. The lookups may
361 * race with the request being allocated from the slab freelist.
362 * That is the request we are writing to here, may be in the process
363 * of being read by __i915_gem_active_get_rcu(). As such,
364 * we have to be very careful when overwriting the contents. During
365 * the RCU lookup, we change chase the request->engine pointer,
366 * read the request->fence.seqno and increment the reference count.
368 * The reference count is incremented atomically. If it is zero,
369 * the lookup knows the request is unallocated and complete. Otherwise,
370 * it is either still in use, or has been reallocated and reset
371 * with fence_init(). This increment is safe for release as we check
372 * that the request we have a reference to and matches the active
375 * Before we increment the refcount, we chase the request->engine
376 * pointer. We must not call kmem_cache_zalloc() or else we set
377 * that pointer to NULL and cause a crash during the lookup. If
378 * we see the request is completed (based on the value of the
379 * old engine and seqno), the lookup is complete and reports NULL.
380 * If we decide the request is not completed (new engine or seqno),
381 * then we grab a reference and double check that it is still the
382 * active request - which it won't be and restart the lookup.
384 * Do not use kmem_cache_zalloc() here!
386 req
= kmem_cache_alloc(dev_priv
->requests
, GFP_KERNEL
);
388 return ERR_PTR(-ENOMEM
);
390 ret
= i915_gem_get_seqno(dev_priv
, &seqno
);
394 spin_lock_init(&req
->lock
);
395 fence_init(&req
->fence
,
398 engine
->fence_context
,
401 INIT_LIST_HEAD(&req
->active_list
);
402 req
->i915
= dev_priv
;
403 req
->engine
= engine
;
404 req
->ctx
= i915_gem_context_get(ctx
);
406 /* No zalloc, must clear what we need by hand */
407 req
->previous_context
= NULL
;
408 req
->file_priv
= NULL
;
409 req
->batch_obj
= NULL
;
411 req
->elsp_submitted
= 0;
414 * Reserve space in the ring buffer for all the commands required to
415 * eventually emit this request. This is to guarantee that the
416 * i915_add_request() call can't fail. Note that the reserve may need
417 * to be redone if the request is not actually submitted straight
418 * away, e.g. because a GPU scheduler has deferred it.
420 req
->reserved_space
= MIN_SPACE_FOR_ADD_REQUEST
;
422 if (i915
.enable_execlists
)
423 ret
= intel_logical_ring_alloc_request_extras(req
);
425 ret
= intel_ring_alloc_request_extras(req
);
432 i915_gem_context_put(ctx
);
434 kmem_cache_free(dev_priv
->requests
, req
);
438 static void i915_gem_mark_busy(const struct intel_engine_cs
*engine
)
440 struct drm_i915_private
*dev_priv
= engine
->i915
;
442 dev_priv
->gt
.active_engines
|= intel_engine_flag(engine
);
443 if (dev_priv
->gt
.awake
)
446 intel_runtime_pm_get_noresume(dev_priv
);
447 dev_priv
->gt
.awake
= true;
449 intel_enable_gt_powersave(dev_priv
);
450 i915_update_gfx_val(dev_priv
);
451 if (INTEL_GEN(dev_priv
) >= 6)
452 gen6_rps_busy(dev_priv
);
454 queue_delayed_work(dev_priv
->wq
,
455 &dev_priv
->gt
.retire_work
,
456 round_jiffies_up_relative(HZ
));
460 * NB: This function is not allowed to fail. Doing so would mean the the
461 * request is not being tracked for completion but the work itself is
462 * going to happen on the hardware. This would be a Bad Thing(tm).
464 void __i915_add_request(struct drm_i915_gem_request
*request
, bool flush_caches
)
466 struct intel_engine_cs
*engine
;
467 struct intel_ring
*ring
;
472 if (WARN_ON(!request
))
475 engine
= request
->engine
;
476 ring
= request
->ring
;
479 * To ensure that this call will not fail, space for its emissions
480 * should already have been reserved in the ring buffer. Let the ring
481 * know that it is time to use that space up.
483 request_start
= ring
->tail
;
484 reserved_tail
= request
->reserved_space
;
485 request
->reserved_space
= 0;
488 * Emit any outstanding flushes - execbuf can fail to emit the flush
489 * after having emitted the batchbuffer command. Hence we need to fix
490 * things up similar to emitting the lazy request. The difference here
491 * is that the flush _must_ happen before the next request, no matter
495 ret
= engine
->emit_flush(request
, EMIT_FLUSH
);
497 /* Not allowed to fail! */
498 WARN(ret
, "engine->emit_flush() failed: %d!\n", ret
);
501 trace_i915_gem_request_add(request
);
503 request
->head
= request_start
;
505 /* Seal the request and mark it as pending execution. Note that
506 * we may inspect this state, without holding any locks, during
507 * hangcheck. Hence we apply the barrier to ensure that we do not
508 * see a more recent value in the hws than we are tracking.
510 request
->emitted_jiffies
= jiffies
;
511 request
->previous_seqno
= engine
->last_submitted_seqno
;
512 engine
->last_submitted_seqno
= request
->fence
.seqno
;
513 i915_gem_active_set(&engine
->last_request
, request
);
514 list_add_tail(&request
->link
, &engine
->request_list
);
515 list_add_tail(&request
->ring_link
, &ring
->request_list
);
517 /* Record the position of the start of the request so that
518 * should we detect the updated seqno part-way through the
519 * GPU processing the request, we never over-estimate the
520 * position of the head.
522 request
->postfix
= ring
->tail
;
524 /* Not allowed to fail! */
525 ret
= engine
->emit_request(request
);
526 WARN(ret
, "(%s)->emit_request failed: %d!\n", engine
->name
, ret
);
528 /* Sanity check that the reserved size was large enough. */
529 ret
= ring
->tail
- request_start
;
532 WARN_ONCE(ret
> reserved_tail
,
533 "Not enough space reserved (%d bytes) "
534 "for adding the request (%d bytes)\n",
537 i915_gem_mark_busy(engine
);
538 engine
->submit_request(request
);
541 static unsigned long local_clock_us(unsigned int *cpu
)
545 /* Cheaply and approximately convert from nanoseconds to microseconds.
546 * The result and subsequent calculations are also defined in the same
547 * approximate microseconds units. The principal source of timing
548 * error here is from the simple truncation.
550 * Note that local_clock() is only defined wrt to the current CPU;
551 * the comparisons are no longer valid if we switch CPUs. Instead of
552 * blocking preemption for the entire busywait, we can detect the CPU
553 * switch and use that as indicator of system load and a reason to
554 * stop busywaiting, see busywait_stop().
557 t
= local_clock() >> 10;
563 static bool busywait_stop(unsigned long timeout
, unsigned int cpu
)
565 unsigned int this_cpu
;
567 if (time_after(local_clock_us(&this_cpu
), timeout
))
570 return this_cpu
!= cpu
;
573 bool __i915_spin_request(const struct drm_i915_gem_request
*req
,
574 int state
, unsigned long timeout_us
)
578 /* When waiting for high frequency requests, e.g. during synchronous
579 * rendering split between the CPU and GPU, the finite amount of time
580 * required to set up the irq and wait upon it limits the response
581 * rate. By busywaiting on the request completion for a short while we
582 * can service the high frequency waits as quick as possible. However,
583 * if it is a slow request, we want to sleep as quickly as possible.
584 * The tradeoff between waiting and sleeping is roughly the time it
585 * takes to sleep on a request, on the order of a microsecond.
588 timeout_us
+= local_clock_us(&cpu
);
590 if (i915_gem_request_completed(req
))
593 if (signal_pending_state(state
, current
))
596 if (busywait_stop(timeout_us
, cpu
))
599 cpu_relax_lowlatency();
600 } while (!need_resched());
606 * i915_wait_request - wait until execution of request has finished
608 * @interruptible: do an interruptible wait (normally yes)
609 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
610 * @rps: client to charge for RPS boosting
612 * Note: It is of utmost importance that the passed in seqno and reset_counter
613 * values have been read by the caller in an smp safe manner. Where read-side
614 * locks are involved, it is sufficient to read the reset_counter before
615 * unlocking the lock that protects the seqno. For lockless tricks, the
616 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
619 * Returns 0 if the request was found within the alloted time. Else returns the
620 * errno with remaining time filled in timeout argument.
622 int i915_wait_request(struct drm_i915_gem_request
*req
,
625 struct intel_rps_client
*rps
)
627 int state
= interruptible
? TASK_INTERRUPTIBLE
: TASK_UNINTERRUPTIBLE
;
629 struct intel_wait wait
;
630 unsigned long timeout_remain
;
635 if (i915_gem_request_completed(req
))
638 timeout_remain
= MAX_SCHEDULE_TIMEOUT
;
640 if (WARN_ON(*timeout
< 0))
646 /* Record current time in case interrupted, or wedged */
647 timeout_remain
= nsecs_to_jiffies_timeout(*timeout
);
648 *timeout
+= ktime_get_raw_ns();
651 trace_i915_gem_request_wait_begin(req
);
653 /* This client is about to stall waiting for the GPU. In many cases
654 * this is undesirable and limits the throughput of the system, as
655 * many clients cannot continue processing user input/output whilst
656 * blocked. RPS autotuning may take tens of milliseconds to respond
657 * to the GPU load and thus incurs additional latency for the client.
658 * We can circumvent that by promoting the GPU frequency to maximum
659 * before we wait. This makes the GPU throttle up much more quickly
660 * (good for benchmarks and user experience, e.g. window animations),
661 * but at a cost of spending more power processing the workload
662 * (bad for battery). Not all clients even want their results
663 * immediately and for them we should just let the GPU select its own
664 * frequency to maximise efficiency. To prevent a single client from
665 * forcing the clocks too high for the whole system, we only allow
666 * each client to waitboost once in a busy period.
668 if (IS_RPS_CLIENT(rps
) && INTEL_GEN(req
->i915
) >= 6)
669 gen6_rps_boost(req
->i915
, rps
, req
->emitted_jiffies
);
671 /* Optimistic short spin before touching IRQs */
672 if (i915_spin_request(req
, state
, 5))
675 set_current_state(state
);
676 add_wait_queue(&req
->i915
->gpu_error
.wait_queue
, &reset
);
678 intel_wait_init(&wait
, req
->fence
.seqno
);
679 if (intel_engine_add_wait(req
->engine
, &wait
))
680 /* In order to check that we haven't missed the interrupt
681 * as we enabled it, we need to kick ourselves to do a
682 * coherent check on the seqno before we sleep.
687 if (signal_pending_state(state
, current
)) {
692 timeout_remain
= io_schedule_timeout(timeout_remain
);
693 if (timeout_remain
== 0) {
698 if (intel_wait_complete(&wait
))
701 set_current_state(state
);
704 /* Carefully check if the request is complete, giving time
705 * for the seqno to be visible following the interrupt.
706 * We also have to check in case we are kicked by the GPU
707 * reset in order to drop the struct_mutex.
709 if (__i915_request_irq_complete(req
))
712 /* Only spin if we know the GPU is processing this request */
713 if (i915_spin_request(req
, state
, 2))
716 remove_wait_queue(&req
->i915
->gpu_error
.wait_queue
, &reset
);
718 intel_engine_remove_wait(req
->engine
, &wait
);
719 __set_current_state(TASK_RUNNING
);
721 trace_i915_gem_request_wait_end(req
);
724 *timeout
-= ktime_get_raw_ns();
729 * Apparently ktime isn't accurate enough and occasionally has a
730 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
731 * things up to make the test happy. We allow up to 1 jiffy.
733 * This is a regrssion from the timespec->ktime conversion.
735 if (ret
== -ETIME
&& *timeout
< jiffies_to_usecs(1)*1000)
739 if (IS_RPS_USER(rps
) &&
740 req
->fence
.seqno
== req
->engine
->last_submitted_seqno
) {
741 /* The GPU is now idle and this client has stalled.
742 * Since no other client has submitted a request in the
743 * meantime, assume that this client is the only one
744 * supplying work to the GPU but is unable to keep that
745 * work supplied because it is waiting. Since the GPU is
746 * then never kept fully busy, RPS autoclocking will
747 * keep the clocks relatively low, causing further delays.
748 * Compensate by giving the synchronous client credit for
749 * a waitboost next time.
751 spin_lock(&req
->i915
->rps
.client_lock
);
752 list_del_init(&rps
->link
);
753 spin_unlock(&req
->i915
->rps
.client_lock
);
759 static void engine_retire_requests(struct intel_engine_cs
*engine
)
761 struct drm_i915_gem_request
*request
, *next
;
763 list_for_each_entry_safe(request
, next
, &engine
->request_list
, link
) {
764 if (!i915_gem_request_completed(request
))
767 i915_gem_request_retire(request
);
771 void i915_gem_retire_requests(struct drm_i915_private
*dev_priv
)
773 struct intel_engine_cs
*engine
;
775 lockdep_assert_held(&dev_priv
->drm
.struct_mutex
);
777 if (dev_priv
->gt
.active_engines
== 0)
780 GEM_BUG_ON(!dev_priv
->gt
.awake
);
782 for_each_engine(engine
, dev_priv
) {
783 engine_retire_requests(engine
);
784 if (!intel_engine_is_active(engine
))
785 dev_priv
->gt
.active_engines
&= ~intel_engine_flag(engine
);
788 if (dev_priv
->gt
.active_engines
== 0)
789 queue_delayed_work(dev_priv
->wq
,
790 &dev_priv
->gt
.idle_work
,
791 msecs_to_jiffies(100));