2 * Tegra host1x Command DMA
4 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <asm/cacheflush.h>
21 #include <linux/device.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/host1x.h>
24 #include <linux/interrupt.h>
25 #include <linux/kernel.h>
26 #include <linux/kfifo.h>
27 #include <linux/slab.h>
28 #include <trace/events/host1x.h>
39 * The push buffer is a circular array of words to be fetched by command DMA.
40 * Note that it works slightly differently to the sync queue; fence == pos
41 * means that the push buffer is full, not empty.
44 #define HOST1X_PUSHBUFFER_SLOTS 512
47 * Clean up push buffer resources
49 static void host1x_pushbuffer_destroy(struct push_buffer
*pb
)
51 struct host1x_cdma
*cdma
= pb_to_cdma(pb
);
52 struct host1x
*host1x
= cdma_to_host1x(cdma
);
55 dma_free_wc(host1x
->dev
, pb
->size_bytes
+ 4, pb
->mapped
,
63 * Init push buffer resources
65 static int host1x_pushbuffer_init(struct push_buffer
*pb
)
67 struct host1x_cdma
*cdma
= pb_to_cdma(pb
);
68 struct host1x
*host1x
= cdma_to_host1x(cdma
);
72 pb
->size_bytes
= HOST1X_PUSHBUFFER_SLOTS
* 8;
74 /* initialize buffer pointers */
75 pb
->fence
= pb
->size_bytes
- 8;
78 /* allocate and map pushbuffer memory */
79 pb
->mapped
= dma_alloc_wc(host1x
->dev
, pb
->size_bytes
+ 4, &pb
->phys
,
84 host1x_hw_pushbuffer_init(host1x
, pb
);
89 host1x_pushbuffer_destroy(pb
);
94 * Push two words to the push buffer
95 * Caller must ensure push buffer is not full
97 static void host1x_pushbuffer_push(struct push_buffer
*pb
, u32 op1
, u32 op2
)
99 u32
*p
= (u32
*)((void *)pb
->mapped
+ pb
->pos
);
101 WARN_ON(pb
->pos
== pb
->fence
);
104 pb
->pos
= (pb
->pos
+ 8) & (pb
->size_bytes
- 1);
108 * Pop a number of two word slots from the push buffer
109 * Caller must ensure push buffer is not empty
111 static void host1x_pushbuffer_pop(struct push_buffer
*pb
, unsigned int slots
)
113 /* Advance the next write position */
114 pb
->fence
= (pb
->fence
+ slots
* 8) & (pb
->size_bytes
- 1);
118 * Return the number of two word slots free in the push buffer
120 static u32
host1x_pushbuffer_space(struct push_buffer
*pb
)
122 return ((pb
->fence
- pb
->pos
) & (pb
->size_bytes
- 1)) / 8;
126 * Sleep (if necessary) until the requested event happens
127 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
129 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
130 * - Return the amount of space (> 0)
131 * Must be called with the cdma lock held.
133 unsigned int host1x_cdma_wait_locked(struct host1x_cdma
*cdma
,
134 enum cdma_event event
)
139 if (event
== CDMA_EVENT_SYNC_QUEUE_EMPTY
)
140 space
= list_empty(&cdma
->sync_queue
) ? 1 : 0;
141 else if (event
== CDMA_EVENT_PUSH_BUFFER_SPACE
) {
142 struct push_buffer
*pb
= &cdma
->push_buffer
;
143 space
= host1x_pushbuffer_space(pb
);
152 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma
)->dev
),
155 /* If somebody has managed to already start waiting, yield */
156 if (cdma
->event
!= CDMA_EVENT_NONE
) {
157 mutex_unlock(&cdma
->lock
);
159 mutex_lock(&cdma
->lock
);
164 mutex_unlock(&cdma
->lock
);
166 mutex_lock(&cdma
->lock
);
172 * Start timer that tracks the time spent by the job.
173 * Must be called with the cdma lock held.
175 static void cdma_start_timer_locked(struct host1x_cdma
*cdma
,
176 struct host1x_job
*job
)
178 struct host1x
*host
= cdma_to_host1x(cdma
);
180 if (cdma
->timeout
.client
) {
181 /* timer already started */
185 cdma
->timeout
.client
= job
->client
;
186 cdma
->timeout
.syncpt
= host1x_syncpt_get(host
, job
->syncpt_id
);
187 cdma
->timeout
.syncpt_val
= job
->syncpt_end
;
188 cdma
->timeout
.start_ktime
= ktime_get();
190 schedule_delayed_work(&cdma
->timeout
.wq
,
191 msecs_to_jiffies(job
->timeout
));
195 * Stop timer when a buffer submission completes.
196 * Must be called with the cdma lock held.
198 static void stop_cdma_timer_locked(struct host1x_cdma
*cdma
)
200 cancel_delayed_work(&cdma
->timeout
.wq
);
201 cdma
->timeout
.client
= 0;
205 * For all sync queue entries that have already finished according to the
206 * current sync point registers:
207 * - unpin & unref their mems
208 * - pop their push buffer slots
209 * - remove them from the sync queue
210 * This is normally called from the host code's worker thread, but can be
211 * called manually if necessary.
212 * Must be called with the cdma lock held.
214 static void update_cdma_locked(struct host1x_cdma
*cdma
)
217 struct host1x
*host1x
= cdma_to_host1x(cdma
);
218 struct host1x_job
*job
, *n
;
220 /* If CDMA is stopped, queue is cleared and we can return */
225 * Walk the sync queue, reading the sync point registers as necessary,
226 * to consume as many sync queue entries as possible without blocking
228 list_for_each_entry_safe(job
, n
, &cdma
->sync_queue
, list
) {
229 struct host1x_syncpt
*sp
=
230 host1x_syncpt_get(host1x
, job
->syncpt_id
);
232 /* Check whether this syncpt has completed, and bail if not */
233 if (!host1x_syncpt_is_expired(sp
, job
->syncpt_end
)) {
234 /* Start timer on next pending syncpt */
236 cdma_start_timer_locked(cdma
, job
);
240 /* Cancel timeout, when a buffer completes */
241 if (cdma
->timeout
.client
)
242 stop_cdma_timer_locked(cdma
);
244 /* Unpin the memory */
245 host1x_job_unpin(job
);
247 /* Pop push buffer slots */
248 if (job
->num_slots
) {
249 struct push_buffer
*pb
= &cdma
->push_buffer
;
251 host1x_pushbuffer_pop(pb
, job
->num_slots
);
252 if (cdma
->event
== CDMA_EVENT_PUSH_BUFFER_SPACE
)
256 list_del(&job
->list
);
260 if (cdma
->event
== CDMA_EVENT_SYNC_QUEUE_EMPTY
&&
261 list_empty(&cdma
->sync_queue
))
265 cdma
->event
= CDMA_EVENT_NONE
;
270 void host1x_cdma_update_sync_queue(struct host1x_cdma
*cdma
,
275 struct host1x_job
*job
= NULL
;
277 struct host1x
*host1x
= cdma_to_host1x(cdma
);
279 syncpt_val
= host1x_syncpt_load(cdma
->timeout
.syncpt
);
281 dev_dbg(dev
, "%s: starting cleanup (thresh %d)\n",
282 __func__
, syncpt_val
);
285 * Move the sync_queue read pointer to the first entry that hasn't
286 * completed based on the current HW syncpt value. It's likely there
287 * won't be any (i.e. we're still at the head), but covers the case
288 * where a syncpt incr happens just prior/during the teardown.
291 dev_dbg(dev
, "%s: skip completed buffers still in sync_queue\n",
294 list_for_each_entry(job
, &cdma
->sync_queue
, list
) {
295 if (syncpt_val
< job
->syncpt_end
)
298 host1x_job_dump(dev
, job
);
302 * Walk the sync_queue, first incrementing with the CPU syncpts that
303 * are partially executed (the first buffer) or fully skipped while
304 * still in the current context (slots are also NOP-ed).
306 * At the point contexts are interleaved, syncpt increments must be
307 * done inline with the pushbuffer from a GATHER buffer to maintain
308 * the order (slots are modified to be a GATHER of syncpt incrs).
310 * Note: save in restart_addr the location where the timed out buffer
311 * started in the PB, so we can start the refetch from there (with the
312 * modified NOP-ed PB slots). This lets things appear to have completed
313 * properly for this buffer and resources are freed.
316 dev_dbg(dev
, "%s: perform CPU incr on pending same ctx buffers\n",
319 if (!list_empty(&cdma
->sync_queue
))
320 restart_addr
= job
->first_get
;
322 restart_addr
= cdma
->last_pos
;
324 /* do CPU increments as long as this context continues */
325 list_for_each_entry_from(job
, &cdma
->sync_queue
, list
) {
326 /* different context, gets us out of this loop */
327 if (job
->client
!= cdma
->timeout
.client
)
330 /* won't need a timeout when replayed */
333 syncpt_incrs
= job
->syncpt_end
- syncpt_val
;
334 dev_dbg(dev
, "%s: CPU incr (%d)\n", __func__
, syncpt_incrs
);
336 host1x_job_dump(dev
, job
);
338 /* safe to use CPU to incr syncpts */
339 host1x_hw_cdma_timeout_cpu_incr(host1x
, cdma
, job
->first_get
,
340 syncpt_incrs
, job
->syncpt_end
,
343 syncpt_val
+= syncpt_incrs
;
347 * The following sumbits from the same client may be dependent on the
348 * failed submit and therefore they may fail. Force a small timeout
349 * to make the queue cleanup faster.
352 list_for_each_entry_from(job
, &cdma
->sync_queue
, list
)
353 if (job
->client
== cdma
->timeout
.client
)
354 job
->timeout
= min_t(unsigned int, job
->timeout
, 500);
356 dev_dbg(dev
, "%s: finished sync_queue modification\n", __func__
);
358 /* roll back DMAGET and start up channel again */
359 host1x_hw_cdma_resume(host1x
, cdma
, restart_addr
);
365 int host1x_cdma_init(struct host1x_cdma
*cdma
)
369 mutex_init(&cdma
->lock
);
370 sema_init(&cdma
->sem
, 0);
372 INIT_LIST_HEAD(&cdma
->sync_queue
);
374 cdma
->event
= CDMA_EVENT_NONE
;
375 cdma
->running
= false;
376 cdma
->torndown
= false;
378 err
= host1x_pushbuffer_init(&cdma
->push_buffer
);
387 int host1x_cdma_deinit(struct host1x_cdma
*cdma
)
389 struct push_buffer
*pb
= &cdma
->push_buffer
;
390 struct host1x
*host1x
= cdma_to_host1x(cdma
);
393 pr_warn("%s: CDMA still running\n", __func__
);
397 host1x_pushbuffer_destroy(pb
);
398 host1x_hw_cdma_timeout_destroy(host1x
, cdma
);
404 * Begin a cdma submit
406 int host1x_cdma_begin(struct host1x_cdma
*cdma
, struct host1x_job
*job
)
408 struct host1x
*host1x
= cdma_to_host1x(cdma
);
410 mutex_lock(&cdma
->lock
);
413 /* init state on first submit with timeout value */
414 if (!cdma
->timeout
.initialized
) {
417 err
= host1x_hw_cdma_timeout_init(host1x
, cdma
,
420 mutex_unlock(&cdma
->lock
);
426 host1x_hw_cdma_start(host1x
, cdma
);
428 cdma
->slots_free
= 0;
429 cdma
->slots_used
= 0;
430 cdma
->first_get
= cdma
->push_buffer
.pos
;
432 trace_host1x_cdma_begin(dev_name(job
->channel
->dev
));
437 * Push two words into a push buffer slot
438 * Blocks as necessary if the push buffer is full.
440 void host1x_cdma_push(struct host1x_cdma
*cdma
, u32 op1
, u32 op2
)
442 struct host1x
*host1x
= cdma_to_host1x(cdma
);
443 struct push_buffer
*pb
= &cdma
->push_buffer
;
444 u32 slots_free
= cdma
->slots_free
;
446 if (host1x_debug_trace_cmdbuf
)
447 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma
)->dev
),
450 if (slots_free
== 0) {
451 host1x_hw_cdma_flush(host1x
, cdma
);
452 slots_free
= host1x_cdma_wait_locked(cdma
,
453 CDMA_EVENT_PUSH_BUFFER_SPACE
);
455 cdma
->slots_free
= slots_free
- 1;
457 host1x_pushbuffer_push(pb
, op1
, op2
);
462 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
463 * from the pushbuffer. The handles for a submit must all be pinned at the same
464 * time, but they can be unpinned in smaller chunks.
466 void host1x_cdma_end(struct host1x_cdma
*cdma
,
467 struct host1x_job
*job
)
469 struct host1x
*host1x
= cdma_to_host1x(cdma
);
470 bool idle
= list_empty(&cdma
->sync_queue
);
472 host1x_hw_cdma_flush(host1x
, cdma
);
474 job
->first_get
= cdma
->first_get
;
475 job
->num_slots
= cdma
->slots_used
;
477 list_add_tail(&job
->list
, &cdma
->sync_queue
);
479 /* start timer on idle -> active transitions */
480 if (job
->timeout
&& idle
)
481 cdma_start_timer_locked(cdma
, job
);
483 trace_host1x_cdma_end(dev_name(job
->channel
->dev
));
484 mutex_unlock(&cdma
->lock
);
488 * Update cdma state according to current sync point values
490 void host1x_cdma_update(struct host1x_cdma
*cdma
)
492 mutex_lock(&cdma
->lock
);
493 update_cdma_locked(cdma
);
494 mutex_unlock(&cdma
->lock
);