Commit | Line | Data |
---|---|---|
6579324a TB |
1 | /* |
2 | * Tegra host1x Command DMA | |
3 | * | |
4 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms and conditions of the GNU General Public License, | |
8 | * version 2, as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 | */ | |
18 | ||
19 | #include <linux/slab.h> | |
20 | #include <linux/scatterlist.h> | |
21 | #include <linux/dma-mapping.h> | |
22 | ||
fc3be3e8 TR |
23 | #include "../cdma.h" |
24 | #include "../channel.h" | |
25 | #include "../dev.h" | |
26 | #include "../debug.h" | |
6579324a TB |
27 | |
28 | /* | |
7f27d60b | 29 | * Put the restart at the end of pushbuffer memory |
6579324a TB |
30 | */ |
31 | static void push_buffer_init(struct push_buffer *pb) | |
32 | { | |
0169b93f | 33 | *(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0); |
6579324a TB |
34 | } |
35 | ||
36 | /* | |
37 | * Increment timedout buffer's syncpt via CPU. | |
38 | */ | |
39 | static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, | |
40 | u32 syncpt_incrs, u32 syncval, u32 nr_slots) | |
41 | { | |
42 | struct host1x *host1x = cdma_to_host1x(cdma); | |
43 | struct push_buffer *pb = &cdma->push_buffer; | |
14c95fc8 | 44 | unsigned int i; |
6579324a TB |
45 | |
46 | for (i = 0; i < syncpt_incrs; i++) | |
ebae30b1 | 47 | host1x_syncpt_incr(cdma->timeout.syncpt); |
6579324a TB |
48 | |
49 | /* after CPU incr, ensure shadow is up to date */ | |
50 | host1x_syncpt_load(cdma->timeout.syncpt); | |
51 | ||
52 | /* NOP all the PB slots */ | |
53 | while (nr_slots--) { | |
0169b93f | 54 | u32 *p = (u32 *)(pb->mapped + getptr); |
6579324a TB |
55 | *(p++) = HOST1X_OPCODE_NOP; |
56 | *(p++) = HOST1X_OPCODE_NOP; | |
ba73fbc2 TR |
57 | dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__, |
58 | &pb->phys, getptr); | |
6579324a TB |
59 | getptr = (getptr + 8) & (pb->size_bytes - 1); |
60 | } | |
0b8070d1 | 61 | |
6579324a TB |
62 | wmb(); |
63 | } | |
64 | ||
65 | /* | |
66 | * Start channel DMA | |
67 | */ | |
68 | static void cdma_start(struct host1x_cdma *cdma) | |
69 | { | |
70 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
71 | ||
72 | if (cdma->running) | |
73 | return; | |
74 | ||
75 | cdma->last_pos = cdma->push_buffer.pos; | |
76 | ||
77 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, | |
78 | HOST1X_CHANNEL_DMACTRL); | |
79 | ||
80 | /* set base, put and end pointer */ | |
81 | host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); | |
82 | host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); | |
83 | host1x_ch_writel(ch, cdma->push_buffer.phys + | |
84 | cdma->push_buffer.size_bytes + 4, | |
85 | HOST1X_CHANNEL_DMAEND); | |
86 | ||
87 | /* reset GET */ | |
88 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | | |
89 | HOST1X_CHANNEL_DMACTRL_DMAGETRST | | |
90 | HOST1X_CHANNEL_DMACTRL_DMAINITGET, | |
91 | HOST1X_CHANNEL_DMACTRL); | |
92 | ||
93 | /* start the command DMA */ | |
94 | host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); | |
95 | ||
96 | cdma->running = true; | |
97 | } | |
98 | ||
99 | /* | |
100 | * Similar to cdma_start(), but rather than starting from an idle | |
101 | * state (where DMA GET is set to DMA PUT), on a timeout we restore | |
102 | * DMA GET from an explicit value (so DMA may again be pending). | |
103 | */ | |
104 | static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) | |
105 | { | |
106 | struct host1x *host1x = cdma_to_host1x(cdma); | |
107 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
108 | ||
109 | if (cdma->running) | |
110 | return; | |
111 | ||
112 | cdma->last_pos = cdma->push_buffer.pos; | |
113 | ||
114 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, | |
115 | HOST1X_CHANNEL_DMACTRL); | |
116 | ||
117 | /* set base, end pointer (all of memory) */ | |
118 | host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); | |
119 | host1x_ch_writel(ch, cdma->push_buffer.phys + | |
120 | cdma->push_buffer.size_bytes, | |
121 | HOST1X_CHANNEL_DMAEND); | |
122 | ||
123 | /* set GET, by loading the value in PUT (then reset GET) */ | |
124 | host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); | |
125 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | | |
126 | HOST1X_CHANNEL_DMACTRL_DMAGETRST | | |
127 | HOST1X_CHANNEL_DMACTRL_DMAINITGET, | |
128 | HOST1X_CHANNEL_DMACTRL); | |
129 | ||
130 | dev_dbg(host1x->dev, | |
131 | "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, | |
132 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), | |
133 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), | |
134 | cdma->last_pos); | |
135 | ||
136 | /* deassert GET reset and set PUT */ | |
137 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, | |
138 | HOST1X_CHANNEL_DMACTRL); | |
139 | host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); | |
140 | ||
141 | /* start the command DMA */ | |
142 | host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); | |
143 | ||
144 | cdma->running = true; | |
145 | } | |
146 | ||
147 | /* | |
148 | * Kick channel DMA into action by writing its PUT offset (if it has changed) | |
149 | */ | |
150 | static void cdma_flush(struct host1x_cdma *cdma) | |
151 | { | |
152 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
153 | ||
154 | if (cdma->push_buffer.pos != cdma->last_pos) { | |
155 | host1x_ch_writel(ch, cdma->push_buffer.pos, | |
156 | HOST1X_CHANNEL_DMAPUT); | |
157 | cdma->last_pos = cdma->push_buffer.pos; | |
158 | } | |
159 | } | |
160 | ||
161 | static void cdma_stop(struct host1x_cdma *cdma) | |
162 | { | |
163 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
164 | ||
165 | mutex_lock(&cdma->lock); | |
0b8070d1 | 166 | |
6579324a TB |
167 | if (cdma->running) { |
168 | host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); | |
169 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, | |
170 | HOST1X_CHANNEL_DMACTRL); | |
171 | cdma->running = false; | |
172 | } | |
0b8070d1 | 173 | |
6579324a TB |
174 | mutex_unlock(&cdma->lock); |
175 | } | |
176 | ||
177 | /* | |
178 | * Stops both channel's command processor and CDMA immediately. | |
179 | * Also, tears down the channel and resets corresponding module. | |
180 | */ | |
181 | static void cdma_freeze(struct host1x_cdma *cdma) | |
182 | { | |
183 | struct host1x *host = cdma_to_host1x(cdma); | |
184 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
185 | u32 cmdproc_stop; | |
186 | ||
187 | if (cdma->torndown && !cdma->running) { | |
188 | dev_warn(host->dev, "Already torn down\n"); | |
189 | return; | |
190 | } | |
191 | ||
192 | dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); | |
193 | ||
194 | cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); | |
195 | cmdproc_stop |= BIT(ch->id); | |
196 | host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); | |
197 | ||
198 | dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", | |
199 | __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), | |
200 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), | |
201 | cdma->last_pos); | |
202 | ||
203 | host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, | |
204 | HOST1X_CHANNEL_DMACTRL); | |
205 | ||
206 | host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); | |
207 | ||
208 | cdma->running = false; | |
209 | cdma->torndown = true; | |
210 | } | |
211 | ||
212 | static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) | |
213 | { | |
214 | struct host1x *host1x = cdma_to_host1x(cdma); | |
215 | struct host1x_channel *ch = cdma_to_channel(cdma); | |
216 | u32 cmdproc_stop; | |
217 | ||
218 | dev_dbg(host1x->dev, | |
5c0d8d38 | 219 | "resuming channel (id %u, DMAGET restart = 0x%x)\n", |
6579324a TB |
220 | ch->id, getptr); |
221 | ||
222 | cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); | |
813a9d4e | 223 | cmdproc_stop &= ~BIT(ch->id); |
6579324a TB |
224 | host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); |
225 | ||
226 | cdma->torndown = false; | |
227 | cdma_timeout_restart(cdma, getptr); | |
228 | } | |
229 | ||
230 | /* | |
231 | * If this timeout fires, it indicates the current sync_queue entry has | |
232 | * exceeded its TTL and the userctx should be timed out and remaining | |
233 | * submits already issued cleaned up (future submits return an error). | |
234 | */ | |
235 | static void cdma_timeout_handler(struct work_struct *work) | |
236 | { | |
0b8070d1 | 237 | u32 prev_cmdproc, cmdproc_stop, syncpt_val; |
6579324a TB |
238 | struct host1x_cdma *cdma; |
239 | struct host1x *host1x; | |
240 | struct host1x_channel *ch; | |
241 | ||
6579324a TB |
242 | cdma = container_of(to_delayed_work(work), struct host1x_cdma, |
243 | timeout.wq); | |
244 | host1x = cdma_to_host1x(cdma); | |
245 | ch = cdma_to_channel(cdma); | |
246 | ||
6236451d TB |
247 | host1x_debug_dump(cdma_to_host1x(cdma)); |
248 | ||
6579324a TB |
249 | mutex_lock(&cdma->lock); |
250 | ||
251 | if (!cdma->timeout.client) { | |
252 | dev_dbg(host1x->dev, | |
253 | "cdma_timeout: expired, but has no clientid\n"); | |
254 | mutex_unlock(&cdma->lock); | |
255 | return; | |
256 | } | |
257 | ||
258 | /* stop processing to get a clean snapshot */ | |
259 | prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); | |
260 | cmdproc_stop = prev_cmdproc | BIT(ch->id); | |
261 | host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); | |
262 | ||
263 | dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n", | |
264 | prev_cmdproc, cmdproc_stop); | |
265 | ||
266 | syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); | |
267 | ||
268 | /* has buffer actually completed? */ | |
269 | if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { | |
270 | dev_dbg(host1x->dev, | |
271 | "cdma_timeout: expired, but buffer had completed\n"); | |
272 | /* restore */ | |
273 | cmdproc_stop = prev_cmdproc & ~(BIT(ch->id)); | |
274 | host1x_sync_writel(host1x, cmdproc_stop, | |
275 | HOST1X_SYNC_CMDPROC_STOP); | |
276 | mutex_unlock(&cdma->lock); | |
277 | return; | |
278 | } | |
279 | ||
5c0d8d38 | 280 | dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", |
0b8070d1 TR |
281 | __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, |
282 | syncpt_val, cdma->timeout.syncpt_val); | |
6579324a TB |
283 | |
284 | /* stop HW, resetting channel/module */ | |
285 | host1x_hw_cdma_freeze(host1x, cdma); | |
286 | ||
287 | host1x_cdma_update_sync_queue(cdma, ch->dev); | |
288 | mutex_unlock(&cdma->lock); | |
289 | } | |
290 | ||
291 | /* | |
292 | * Init timeout resources | |
293 | */ | |
5c0d8d38 | 294 | static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt) |
6579324a TB |
295 | { |
296 | INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); | |
297 | cdma->timeout.initialized = true; | |
298 | ||
299 | return 0; | |
300 | } | |
301 | ||
302 | /* | |
303 | * Clean up timeout resources | |
304 | */ | |
305 | static void cdma_timeout_destroy(struct host1x_cdma *cdma) | |
306 | { | |
307 | if (cdma->timeout.initialized) | |
308 | cancel_delayed_work(&cdma->timeout.wq); | |
0b8070d1 | 309 | |
6579324a TB |
310 | cdma->timeout.initialized = false; |
311 | } | |
312 | ||
313 | static const struct host1x_cdma_ops host1x_cdma_ops = { | |
314 | .start = cdma_start, | |
315 | .stop = cdma_stop, | |
316 | .flush = cdma_flush, | |
317 | ||
318 | .timeout_init = cdma_timeout_init, | |
319 | .timeout_destroy = cdma_timeout_destroy, | |
320 | .freeze = cdma_freeze, | |
321 | .resume = cdma_resume, | |
322 | .timeout_cpu_incr = cdma_timeout_cpu_incr, | |
323 | }; | |
324 | ||
325 | static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { | |
326 | .init = push_buffer_init, | |
327 | }; |