2 * Copyright © 2008-2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 int i915_gem_request_add_to_client(struct drm_i915_gem_request
*req
,
28 struct drm_file
*file
)
30 struct drm_i915_private
*dev_private
;
31 struct drm_i915_file_private
*file_priv
;
33 WARN_ON(!req
|| !file
|| req
->file_priv
);
41 dev_private
= req
->i915
;
42 file_priv
= file
->driver_priv
;
44 spin_lock(&file_priv
->mm
.lock
);
45 req
->file_priv
= file_priv
;
46 list_add_tail(&req
->client_list
, &file_priv
->mm
.request_list
);
47 spin_unlock(&file_priv
->mm
.lock
);
49 req
->pid
= get_pid(task_pid(current
));
55 i915_gem_request_remove_from_client(struct drm_i915_gem_request
*request
)
57 struct drm_i915_file_private
*file_priv
= request
->file_priv
;
62 spin_lock(&file_priv
->mm
.lock
);
63 list_del(&request
->client_list
);
64 request
->file_priv
= NULL
;
65 spin_unlock(&file_priv
->mm
.lock
);
67 put_pid(request
->pid
);
71 static void i915_gem_request_retire(struct drm_i915_gem_request
*request
)
73 trace_i915_gem_request_retire(request
);
74 list_del_init(&request
->list
);
76 /* We know the GPU must have read the request to have
77 * sent us the seqno + interrupt, so use the position
78 * of tail of the request to update the last known position
81 * Note this requires that we are always called in request
84 request
->ringbuf
->last_retired_head
= request
->postfix
;
86 i915_gem_request_remove_from_client(request
);
88 if (request
->previous_context
) {
89 if (i915
.enable_execlists
)
90 intel_lr_context_unpin(request
->previous_context
,
94 i915_gem_context_unreference(request
->ctx
);
95 i915_gem_request_unreference(request
);
98 void i915_gem_request_retire_upto(struct drm_i915_gem_request
*req
)
100 struct intel_engine_cs
*engine
= req
->engine
;
101 struct drm_i915_gem_request
*tmp
;
103 lockdep_assert_held(&req
->i915
->drm
.struct_mutex
);
105 if (list_empty(&req
->list
))
109 tmp
= list_first_entry(&engine
->request_list
,
112 i915_gem_request_retire(tmp
);
113 } while (tmp
!= req
);
115 WARN_ON(i915_verify_lists(engine
->dev
));
118 static int i915_gem_check_wedge(unsigned int reset_counter
, bool interruptible
)
120 if (__i915_terminally_wedged(reset_counter
))
123 if (__i915_reset_in_progress(reset_counter
)) {
124 /* Non-interruptible callers can't handle -EAGAIN, hence return
125 * -EIO unconditionally for these.
136 static int i915_gem_init_seqno(struct drm_i915_private
*dev_priv
, u32 seqno
)
138 struct intel_engine_cs
*engine
;
141 /* Carefully retire all requests without writing to the rings */
142 for_each_engine(engine
, dev_priv
) {
143 ret
= intel_engine_idle(engine
);
147 i915_gem_retire_requests(dev_priv
);
149 /* If the seqno wraps around, we need to clear the breadcrumb rbtree */
150 if (!i915_seqno_passed(seqno
, dev_priv
->next_seqno
)) {
151 while (intel_kick_waiters(dev_priv
) ||
152 intel_kick_signalers(dev_priv
))
156 /* Finally reset hw state */
157 for_each_engine(engine
, dev_priv
)
158 intel_ring_init_seqno(engine
, seqno
);
163 int i915_gem_set_seqno(struct drm_device
*dev
, u32 seqno
)
165 struct drm_i915_private
*dev_priv
= to_i915(dev
);
171 /* HWS page needs to be set less than what we
172 * will inject to ring
174 ret
= i915_gem_init_seqno(dev_priv
, seqno
- 1);
178 /* Carefully set the last_seqno value so that wrap
179 * detection still works
181 dev_priv
->next_seqno
= seqno
;
182 dev_priv
->last_seqno
= seqno
- 1;
183 if (dev_priv
->last_seqno
== 0)
184 dev_priv
->last_seqno
--;
189 static int i915_gem_get_seqno(struct drm_i915_private
*dev_priv
, u32
*seqno
)
191 /* reserve 0 for non-seqno */
192 if (unlikely(dev_priv
->next_seqno
== 0)) {
195 ret
= i915_gem_init_seqno(dev_priv
, 0);
199 dev_priv
->next_seqno
= 1;
202 *seqno
= dev_priv
->last_seqno
= dev_priv
->next_seqno
++;
207 __i915_gem_request_alloc(struct intel_engine_cs
*engine
,
208 struct i915_gem_context
*ctx
,
209 struct drm_i915_gem_request
**req_out
)
211 struct drm_i915_private
*dev_priv
= engine
->i915
;
212 unsigned int reset_counter
= i915_reset_counter(&dev_priv
->gpu_error
);
213 struct drm_i915_gem_request
*req
;
221 /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
222 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
225 ret
= i915_gem_check_wedge(reset_counter
, dev_priv
->mm
.interruptible
);
229 /* Move the oldest request to the slab-cache (if not in use!) */
230 if (!list_empty(&engine
->request_list
)) {
231 req
= list_first_entry(&engine
->request_list
,
233 if (i915_gem_request_completed(req
))
234 i915_gem_request_retire(req
);
237 req
= kmem_cache_zalloc(dev_priv
->requests
, GFP_KERNEL
);
241 ret
= i915_gem_get_seqno(dev_priv
, &req
->seqno
);
245 kref_init(&req
->ref
);
246 req
->i915
= dev_priv
;
247 req
->engine
= engine
;
249 i915_gem_context_reference(ctx
);
252 * Reserve space in the ring buffer for all the commands required to
253 * eventually emit this request. This is to guarantee that the
254 * i915_add_request() call can't fail. Note that the reserve may need
255 * to be redone if the request is not actually submitted straight
256 * away, e.g. because a GPU scheduler has deferred it.
258 req
->reserved_space
= MIN_SPACE_FOR_ADD_REQUEST
;
260 if (i915
.enable_execlists
)
261 ret
= intel_logical_ring_alloc_request_extras(req
);
263 ret
= intel_ring_alloc_request_extras(req
);
271 i915_gem_context_unreference(ctx
);
273 kmem_cache_free(dev_priv
->requests
, req
);
278 * i915_gem_request_alloc - allocate a request structure
280 * @engine: engine that we wish to issue the request on.
281 * @ctx: context that the request will be associated with.
282 * This can be NULL if the request is not directly related to
283 * any specific user context, in which case this function will
284 * choose an appropriate context to use.
286 * Returns a pointer to the allocated request if successful,
287 * or an error code if not.
289 struct drm_i915_gem_request
*
290 i915_gem_request_alloc(struct intel_engine_cs
*engine
,
291 struct i915_gem_context
*ctx
)
293 struct drm_i915_gem_request
*req
;
297 ctx
= engine
->i915
->kernel_context
;
298 err
= __i915_gem_request_alloc(engine
, ctx
, &req
);
299 return err
? ERR_PTR(err
) : req
;
302 static void i915_gem_mark_busy(const struct intel_engine_cs
*engine
)
304 struct drm_i915_private
*dev_priv
= engine
->i915
;
306 dev_priv
->gt
.active_engines
|= intel_engine_flag(engine
);
307 if (dev_priv
->gt
.awake
)
310 intel_runtime_pm_get_noresume(dev_priv
);
311 dev_priv
->gt
.awake
= true;
313 intel_enable_gt_powersave(dev_priv
);
314 i915_update_gfx_val(dev_priv
);
315 if (INTEL_GEN(dev_priv
) >= 6)
316 gen6_rps_busy(dev_priv
);
318 queue_delayed_work(dev_priv
->wq
,
319 &dev_priv
->gt
.retire_work
,
320 round_jiffies_up_relative(HZ
));
324 * NB: This function is not allowed to fail. Doing so would mean the the
325 * request is not being tracked for completion but the work itself is
326 * going to happen on the hardware. This would be a Bad Thing(tm).
328 void __i915_add_request(struct drm_i915_gem_request
*request
,
329 struct drm_i915_gem_object
*obj
,
332 struct intel_engine_cs
*engine
;
333 struct intel_ringbuffer
*ringbuf
;
338 if (WARN_ON(!request
))
341 engine
= request
->engine
;
342 ringbuf
= request
->ringbuf
;
345 * To ensure that this call will not fail, space for its emissions
346 * should already have been reserved in the ring buffer. Let the ring
347 * know that it is time to use that space up.
349 request_start
= intel_ring_get_tail(ringbuf
);
350 reserved_tail
= request
->reserved_space
;
351 request
->reserved_space
= 0;
354 * Emit any outstanding flushes - execbuf can fail to emit the flush
355 * after having emitted the batchbuffer command. Hence we need to fix
356 * things up similar to emitting the lazy request. The difference here
357 * is that the flush _must_ happen before the next request, no matter
361 if (i915
.enable_execlists
)
362 ret
= logical_ring_flush_all_caches(request
);
364 ret
= intel_ring_flush_all_caches(request
);
365 /* Not allowed to fail! */
366 WARN(ret
, "*_ring_flush_all_caches failed: %d!\n", ret
);
369 trace_i915_gem_request_add(request
);
371 request
->head
= request_start
;
373 /* Whilst this request exists, batch_obj will be on the
374 * active_list, and so will hold the active reference. Only when this
375 * request is retired will the the batch_obj be moved onto the
376 * inactive_list and lose its active reference. Hence we do not need
377 * to explicitly hold another reference here.
379 request
->batch_obj
= obj
;
381 /* Seal the request and mark it as pending execution. Note that
382 * we may inspect this state, without holding any locks, during
383 * hangcheck. Hence we apply the barrier to ensure that we do not
384 * see a more recent value in the hws than we are tracking.
386 request
->emitted_jiffies
= jiffies
;
387 request
->previous_seqno
= engine
->last_submitted_seqno
;
388 smp_store_mb(engine
->last_submitted_seqno
, request
->seqno
);
389 list_add_tail(&request
->list
, &engine
->request_list
);
391 /* Record the position of the start of the request so that
392 * should we detect the updated seqno part-way through the
393 * GPU processing the request, we never over-estimate the
394 * position of the head.
396 request
->postfix
= intel_ring_get_tail(ringbuf
);
398 if (i915
.enable_execlists
) {
399 ret
= engine
->emit_request(request
);
401 ret
= engine
->add_request(request
);
403 request
->tail
= intel_ring_get_tail(ringbuf
);
405 /* Not allowed to fail! */
406 WARN(ret
, "emit|add_request failed: %d!\n", ret
);
407 /* Sanity check that the reserved size was large enough. */
408 ret
= intel_ring_get_tail(ringbuf
) - request_start
;
410 ret
+= ringbuf
->size
;
411 WARN_ONCE(ret
> reserved_tail
,
412 "Not enough space reserved (%d bytes) "
413 "for adding the request (%d bytes)\n",
416 i915_gem_mark_busy(engine
);
419 static unsigned long local_clock_us(unsigned int *cpu
)
423 /* Cheaply and approximately convert from nanoseconds to microseconds.
424 * The result and subsequent calculations are also defined in the same
425 * approximate microseconds units. The principal source of timing
426 * error here is from the simple truncation.
428 * Note that local_clock() is only defined wrt to the current CPU;
429 * the comparisons are no longer valid if we switch CPUs. Instead of
430 * blocking preemption for the entire busywait, we can detect the CPU
431 * switch and use that as indicator of system load and a reason to
432 * stop busywaiting, see busywait_stop().
435 t
= local_clock() >> 10;
441 static bool busywait_stop(unsigned long timeout
, unsigned int cpu
)
443 unsigned int this_cpu
;
445 if (time_after(local_clock_us(&this_cpu
), timeout
))
448 return this_cpu
!= cpu
;
451 bool __i915_spin_request(const struct drm_i915_gem_request
*req
,
452 int state
, unsigned long timeout_us
)
456 /* When waiting for high frequency requests, e.g. during synchronous
457 * rendering split between the CPU and GPU, the finite amount of time
458 * required to set up the irq and wait upon it limits the response
459 * rate. By busywaiting on the request completion for a short while we
460 * can service the high frequency waits as quick as possible. However,
461 * if it is a slow request, we want to sleep as quickly as possible.
462 * The tradeoff between waiting and sleeping is roughly the time it
463 * takes to sleep on a request, on the order of a microsecond.
466 timeout_us
+= local_clock_us(&cpu
);
468 if (i915_gem_request_completed(req
))
471 if (signal_pending_state(state
, current
))
474 if (busywait_stop(timeout_us
, cpu
))
477 cpu_relax_lowlatency();
478 } while (!need_resched());
484 * __i915_wait_request - wait until execution of request has finished
486 * @interruptible: do an interruptible wait (normally yes)
487 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
488 * @rps: client to charge for RPS boosting
490 * Note: It is of utmost importance that the passed in seqno and reset_counter
491 * values have been read by the caller in an smp safe manner. Where read-side
492 * locks are involved, it is sufficient to read the reset_counter before
493 * unlocking the lock that protects the seqno. For lockless tricks, the
494 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
497 * Returns 0 if the request was found within the alloted time. Else returns the
498 * errno with remaining time filled in timeout argument.
500 int __i915_wait_request(struct drm_i915_gem_request
*req
,
503 struct intel_rps_client
*rps
)
505 int state
= interruptible
? TASK_INTERRUPTIBLE
: TASK_UNINTERRUPTIBLE
;
507 struct intel_wait wait
;
508 unsigned long timeout_remain
;
513 if (list_empty(&req
->list
))
516 if (i915_gem_request_completed(req
))
519 timeout_remain
= MAX_SCHEDULE_TIMEOUT
;
521 if (WARN_ON(*timeout
< 0))
527 /* Record current time in case interrupted, or wedged */
528 timeout_remain
= nsecs_to_jiffies_timeout(*timeout
);
529 *timeout
+= ktime_get_raw_ns();
532 trace_i915_gem_request_wait_begin(req
);
534 /* This client is about to stall waiting for the GPU. In many cases
535 * this is undesirable and limits the throughput of the system, as
536 * many clients cannot continue processing user input/output whilst
537 * blocked. RPS autotuning may take tens of milliseconds to respond
538 * to the GPU load and thus incurs additional latency for the client.
539 * We can circumvent that by promoting the GPU frequency to maximum
540 * before we wait. This makes the GPU throttle up much more quickly
541 * (good for benchmarks and user experience, e.g. window animations),
542 * but at a cost of spending more power processing the workload
543 * (bad for battery). Not all clients even want their results
544 * immediately and for them we should just let the GPU select its own
545 * frequency to maximise efficiency. To prevent a single client from
546 * forcing the clocks too high for the whole system, we only allow
547 * each client to waitboost once in a busy period.
549 if (INTEL_GEN(req
->i915
) >= 6)
550 gen6_rps_boost(req
->i915
, rps
, req
->emitted_jiffies
);
552 /* Optimistic spin for the next ~jiffie before touching IRQs */
553 if (i915_spin_request(req
, state
, 5))
556 set_current_state(state
);
557 add_wait_queue(&req
->i915
->gpu_error
.wait_queue
, &reset
);
559 intel_wait_init(&wait
, req
->seqno
);
560 if (intel_engine_add_wait(req
->engine
, &wait
))
561 /* In order to check that we haven't missed the interrupt
562 * as we enabled it, we need to kick ourselves to do a
563 * coherent check on the seqno before we sleep.
568 if (signal_pending_state(state
, current
)) {
573 timeout_remain
= io_schedule_timeout(timeout_remain
);
574 if (timeout_remain
== 0) {
579 if (intel_wait_complete(&wait
))
582 set_current_state(state
);
585 /* Carefully check if the request is complete, giving time
586 * for the seqno to be visible following the interrupt.
587 * We also have to check in case we are kicked by the GPU
588 * reset in order to drop the struct_mutex.
590 if (__i915_request_irq_complete(req
))
593 /* Only spin if we know the GPU is processing this request */
594 if (i915_spin_request(req
, state
, 2))
597 remove_wait_queue(&req
->i915
->gpu_error
.wait_queue
, &reset
);
599 intel_engine_remove_wait(req
->engine
, &wait
);
600 __set_current_state(TASK_RUNNING
);
602 trace_i915_gem_request_wait_end(req
);
605 *timeout
-= ktime_get_raw_ns();
610 * Apparently ktime isn't accurate enough and occasionally has a
611 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
612 * things up to make the test happy. We allow up to 1 jiffy.
614 * This is a regrssion from the timespec->ktime conversion.
616 if (ret
== -ETIME
&& *timeout
< jiffies_to_usecs(1)*1000)
620 if (rps
&& req
->seqno
== req
->engine
->last_submitted_seqno
) {
621 /* The GPU is now idle and this client has stalled.
622 * Since no other client has submitted a request in the
623 * meantime, assume that this client is the only one
624 * supplying work to the GPU but is unable to keep that
625 * work supplied because it is waiting. Since the GPU is
626 * then never kept fully busy, RPS autoclocking will
627 * keep the clocks relatively low, causing further delays.
628 * Compensate by giving the synchronous client credit for
629 * a waitboost next time.
631 spin_lock(&req
->i915
->rps
.client_lock
);
632 list_del_init(&rps
->link
);
633 spin_unlock(&req
->i915
->rps
.client_lock
);
640 * Waits for a request to be signaled, and cleans up the
641 * request and object lists appropriately for that event.
643 int i915_wait_request(struct drm_i915_gem_request
*req
)
648 lockdep_assert_held(&req
->i915
->drm
.struct_mutex
);
650 ret
= __i915_wait_request(req
, req
->i915
->mm
.interruptible
, NULL
, NULL
);
654 /* If the GPU hung, we want to keep the requests to find the guilty. */
655 if (!i915_reset_in_progress(&req
->i915
->gpu_error
))
656 i915_gem_request_retire_upto(req
);
661 void i915_gem_request_free(struct kref
*req_ref
)
663 struct drm_i915_gem_request
*req
=
664 container_of(req_ref
, typeof(*req
), ref
);
665 kmem_cache_free(req
->i915
->requests
, req
);