Commit | Line | Data |
---|---|---|
351e3db2 BV |
1 | /* |
2 | * Copyright © 2013 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Brad Volkin <bradley.d.volkin@intel.com> | |
25 | * | |
26 | */ | |
27 | ||
28 | #include "i915_drv.h" | |
29 | ||
30 | /** | |
122b2505 | 31 | * DOC: batch buffer command parser |
351e3db2 BV |
32 | * |
33 | * Motivation: | |
34 | * Certain OpenGL features (e.g. transform feedback, performance monitoring) | |
35 | * require userspace code to submit batches containing commands such as | |
36 | * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some | |
37 | * generations of the hardware will noop these commands in "unsecure" batches | |
38 | * (which includes all userspace batches submitted via i915) even though the | |
39 | * commands may be safe and represent the intended programming model of the | |
40 | * device. | |
41 | * | |
42 | * The software command parser is similar in operation to the command parsing | |
43 | * done in hardware for unsecure batches. However, the software parser allows | |
44 | * some operations that would be noop'd by hardware, if the parser determines | |
45 | * the operation is safe, and submits the batch as "secure" to prevent hardware | |
46 | * parsing. | |
47 | * | |
48 | * Threats: | |
49 | * At a high level, the hardware (and software) checks attempt to prevent | |
50 | * granting userspace undue privileges. There are three categories of privilege. | |
51 | * | |
52 | * First, commands which are explicitly defined as privileged or which should | |
53 | * only be used by the kernel driver. The parser generally rejects such | |
54 | * commands, though it may allow some from the drm master process. | |
55 | * | |
56 | * Second, commands which access registers. To support correct/enhanced | |
57 | * userspace functionality, particularly certain OpenGL extensions, the parser | |
58 | * provides a whitelist of registers which userspace may safely access (for both | |
59 | * normal and drm master processes). | |
60 | * | |
61 | * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc). | |
62 | * The parser always rejects such commands. | |
63 | * | |
64 | * The majority of the problematic commands fall in the MI_* range, with only a | |
33a051a5 | 65 | * few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW). |
351e3db2 BV |
66 | * |
67 | * Implementation: | |
33a051a5 CW |
68 | * Each engine maintains tables of commands and registers which the parser |
69 | * uses in scanning batch buffers submitted to that engine. | |
351e3db2 BV |
70 | * |
71 | * Since the set of commands that the parser must check for is significantly | |
72 | * smaller than the number of commands supported, the parser tables contain only | |
73 | * those commands required by the parser. This generally works because command | |
74 | * opcode ranges have standard command length encodings. So for commands that | |
75 | * the parser does not need to check, it can easily skip them. This is | |
33a051a5 | 76 | * implemented via a per-engine length decoding vfunc. |
351e3db2 BV |
77 | * |
78 | * Unfortunately, there are a number of commands that do not follow the standard | |
79 | * length encoding for their opcode range, primarily amongst the MI_* commands. | |
80 | * To handle this, the parser provides a way to define explicit "skip" entries | |
33a051a5 | 81 | * in the per-engine command tables. |
351e3db2 BV |
82 | * |
83 | * Other command table entries map fairly directly to high level categories | |
84 | * mentioned above: rejected, master-only, register whitelist. The parser | |
85 | * implements a number of checks, including the privileged memory checks, via a | |
86 | * general bitmasking mechanism. | |
87 | */ | |
88 | ||
3a6fa984 BV |
89 | #define STD_MI_OPCODE_MASK 0xFF800000 |
90 | #define STD_3D_OPCODE_MASK 0xFFFF0000 | |
91 | #define STD_2D_OPCODE_MASK 0xFFC00000 | |
92 | #define STD_MFX_OPCODE_MASK 0xFFFF0000 | |
93 | ||
94 | #define CMD(op, opm, f, lm, fl, ...) \ | |
95 | { \ | |
96 | .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ | |
f1afe24f | 97 | .cmd = { (op), (opm) }, \ |
3a6fa984 BV |
98 | .length = { (lm) }, \ |
99 | __VA_ARGS__ \ | |
100 | } | |
101 | ||
102 | /* Convenience macros to compress the tables */ | |
103 | #define SMI STD_MI_OPCODE_MASK | |
104 | #define S3D STD_3D_OPCODE_MASK | |
105 | #define S2D STD_2D_OPCODE_MASK | |
106 | #define SMFX STD_MFX_OPCODE_MASK | |
107 | #define F true | |
108 | #define S CMD_DESC_SKIP | |
109 | #define R CMD_DESC_REJECT | |
110 | #define W CMD_DESC_REGISTER | |
111 | #define B CMD_DESC_BITMASK | |
112 | #define M CMD_DESC_MASTER | |
113 | ||
114 | /* Command Mask Fixed Len Action | |
115 | ---------------------------------------------------------- */ | |
116 | static const struct drm_i915_cmd_descriptor common_cmds[] = { | |
117 | CMD( MI_NOOP, SMI, F, 1, S ), | |
b18b396b | 118 | CMD( MI_USER_INTERRUPT, SMI, F, 1, R ), |
17c1eb15 | 119 | CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, M ), |
3a6fa984 BV |
120 | CMD( MI_ARB_CHECK, SMI, F, 1, S ), |
121 | CMD( MI_REPORT_HEAD, SMI, F, 1, S ), | |
122 | CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), | |
9c640d1d BV |
123 | CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ), |
124 | CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ), | |
f0a346bd | 125 | CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, |
6a65c5b9 | 126 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ), |
614f4ad7 | 127 | CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B, |
d4d48035 BV |
128 | .reg = { .offset = 1, .mask = 0x007FFFFC }, |
129 | .bits = {{ | |
130 | .offset = 0, | |
131 | .mask = MI_GLOBAL_GTT, | |
132 | .expected = 0, | |
133 | }}, ), | |
614f4ad7 | 134 | CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B, |
d4d48035 BV |
135 | .reg = { .offset = 1, .mask = 0x007FFFFC }, |
136 | .bits = {{ | |
137 | .offset = 0, | |
138 | .mask = MI_GLOBAL_GTT, | |
139 | .expected = 0, | |
140 | }}, ), | |
42c7156a BV |
141 | /* |
142 | * MI_BATCH_BUFFER_START requires some special handling. It's not | |
143 | * really a 'skip' action but it doesn't seem like it's worth adding | |
144 | * a new action. See i915_parse_cmds(). | |
145 | */ | |
3a6fa984 BV |
146 | CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ), |
147 | }; | |
148 | ||
149 | static const struct drm_i915_cmd_descriptor render_cmds[] = { | |
150 | CMD( MI_FLUSH, SMI, F, 1, S ), | |
9c640d1d | 151 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
3a6fa984 BV |
152 | CMD( MI_PREDICATE, SMI, F, 1, S ), |
153 | CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ), | |
86ef630d | 154 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
9f58582c | 155 | CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), |
9c640d1d | 156 | CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ), |
3a6fa984 | 157 | CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ), |
d4d48035 BV |
158 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B, |
159 | .bits = {{ | |
160 | .offset = 0, | |
161 | .mask = MI_GLOBAL_GTT, | |
162 | .expected = 0, | |
163 | }}, ), | |
9c640d1d | 164 | CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ), |
d4d48035 BV |
165 | CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B, |
166 | .bits = {{ | |
167 | .offset = 0, | |
168 | .mask = MI_GLOBAL_GTT, | |
169 | .expected = 0, | |
170 | }}, ), | |
171 | CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B, | |
172 | .bits = {{ | |
173 | .offset = 1, | |
174 | .mask = MI_REPORT_PERF_COUNT_GGTT, | |
175 | .expected = 0, | |
176 | }}, ), | |
177 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, | |
178 | .bits = {{ | |
179 | .offset = 0, | |
180 | .mask = MI_GLOBAL_GTT, | |
181 | .expected = 0, | |
182 | }}, ), | |
3a6fa984 BV |
183 | CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ), |
184 | CMD( PIPELINE_SELECT, S3D, F, 1, S ), | |
f0a346bd BV |
185 | CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B, |
186 | .bits = {{ | |
187 | .offset = 2, | |
188 | .mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK, | |
189 | .expected = 0, | |
190 | }}, ), | |
3a6fa984 BV |
191 | CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ), |
192 | CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ), | |
193 | CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ), | |
f0a346bd BV |
194 | CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B, |
195 | .bits = {{ | |
196 | .offset = 1, | |
b18b396b | 197 | .mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY), |
f0a346bd | 198 | .expected = 0, |
d4d48035 BV |
199 | }, |
200 | { | |
201 | .offset = 1, | |
114d4f70 BV |
202 | .mask = (PIPE_CONTROL_GLOBAL_GTT_IVB | |
203 | PIPE_CONTROL_STORE_DATA_INDEX), | |
d4d48035 BV |
204 | .expected = 0, |
205 | .condition_offset = 1, | |
206 | .condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK, | |
f0a346bd | 207 | }}, ), |
3a6fa984 BV |
208 | }; |
209 | ||
210 | static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = { | |
211 | CMD( MI_SET_PREDICATE, SMI, F, 1, S ), | |
212 | CMD( MI_RS_CONTROL, SMI, F, 1, S ), | |
213 | CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ), | |
86ef630d | 214 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
3a6fa984 | 215 | CMD( MI_RS_CONTEXT, SMI, F, 1, S ), |
17c1eb15 | 216 | CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), |
9c640d1d | 217 | CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), |
6761d0a1 KG |
218 | CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, |
219 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), | |
3a6fa984 BV |
220 | CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ), |
221 | CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ), | |
222 | CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ), | |
223 | CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ), | |
224 | CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ), | |
225 | ||
226 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ), | |
227 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ), | |
228 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ), | |
229 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ), | |
230 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ), | |
231 | }; | |
232 | ||
233 | static const struct drm_i915_cmd_descriptor video_cmds[] = { | |
9c640d1d | 234 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
86ef630d | 235 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
d4d48035 BV |
236 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, |
237 | .bits = {{ | |
238 | .offset = 0, | |
239 | .mask = MI_GLOBAL_GTT, | |
240 | .expected = 0, | |
241 | }}, ), | |
9c640d1d | 242 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
b18b396b BV |
243 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
244 | .bits = {{ | |
245 | .offset = 0, | |
246 | .mask = MI_FLUSH_DW_NOTIFY, | |
247 | .expected = 0, | |
d4d48035 BV |
248 | }, |
249 | { | |
250 | .offset = 1, | |
251 | .mask = MI_FLUSH_DW_USE_GTT, | |
252 | .expected = 0, | |
253 | .condition_offset = 0, | |
254 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
114d4f70 BV |
255 | }, |
256 | { | |
257 | .offset = 0, | |
258 | .mask = MI_FLUSH_DW_STORE_INDEX, | |
259 | .expected = 0, | |
260 | .condition_offset = 0, | |
261 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
d4d48035 BV |
262 | }}, ), |
263 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, | |
264 | .bits = {{ | |
265 | .offset = 0, | |
266 | .mask = MI_GLOBAL_GTT, | |
267 | .expected = 0, | |
b18b396b | 268 | }}, ), |
3a6fa984 BV |
269 | /* |
270 | * MFX_WAIT doesn't fit the way we handle length for most commands. | |
271 | * It has a length field but it uses a non-standard length bias. | |
272 | * It is always 1 dword though, so just treat it as fixed length. | |
273 | */ | |
274 | CMD( MFX_WAIT, SMFX, F, 1, S ), | |
275 | }; | |
276 | ||
277 | static const struct drm_i915_cmd_descriptor vecs_cmds[] = { | |
9c640d1d | 278 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
86ef630d | 279 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
d4d48035 BV |
280 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, |
281 | .bits = {{ | |
282 | .offset = 0, | |
283 | .mask = MI_GLOBAL_GTT, | |
284 | .expected = 0, | |
285 | }}, ), | |
9c640d1d | 286 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
b18b396b BV |
287 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
288 | .bits = {{ | |
289 | .offset = 0, | |
290 | .mask = MI_FLUSH_DW_NOTIFY, | |
291 | .expected = 0, | |
d4d48035 BV |
292 | }, |
293 | { | |
294 | .offset = 1, | |
295 | .mask = MI_FLUSH_DW_USE_GTT, | |
296 | .expected = 0, | |
297 | .condition_offset = 0, | |
298 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
114d4f70 BV |
299 | }, |
300 | { | |
301 | .offset = 0, | |
302 | .mask = MI_FLUSH_DW_STORE_INDEX, | |
303 | .expected = 0, | |
304 | .condition_offset = 0, | |
305 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
d4d48035 BV |
306 | }}, ), |
307 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, | |
308 | .bits = {{ | |
309 | .offset = 0, | |
310 | .mask = MI_GLOBAL_GTT, | |
311 | .expected = 0, | |
b18b396b | 312 | }}, ), |
3a6fa984 BV |
313 | }; |
314 | ||
315 | static const struct drm_i915_cmd_descriptor blt_cmds[] = { | |
9c640d1d | 316 | CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), |
d4d48035 BV |
317 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B, |
318 | .bits = {{ | |
319 | .offset = 0, | |
320 | .mask = MI_GLOBAL_GTT, | |
321 | .expected = 0, | |
322 | }}, ), | |
9c640d1d | 323 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
b18b396b BV |
324 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
325 | .bits = {{ | |
326 | .offset = 0, | |
327 | .mask = MI_FLUSH_DW_NOTIFY, | |
328 | .expected = 0, | |
d4d48035 BV |
329 | }, |
330 | { | |
331 | .offset = 1, | |
332 | .mask = MI_FLUSH_DW_USE_GTT, | |
333 | .expected = 0, | |
334 | .condition_offset = 0, | |
335 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
114d4f70 BV |
336 | }, |
337 | { | |
338 | .offset = 0, | |
339 | .mask = MI_FLUSH_DW_STORE_INDEX, | |
340 | .expected = 0, | |
341 | .condition_offset = 0, | |
342 | .condition_mask = MI_FLUSH_DW_OP_MASK, | |
b18b396b | 343 | }}, ), |
3a6fa984 BV |
344 | CMD( COLOR_BLT, S2D, !F, 0x3F, S ), |
345 | CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ), | |
346 | }; | |
347 | ||
9c640d1d | 348 | static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = { |
17c1eb15 | 349 | CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), |
9c640d1d BV |
350 | CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), |
351 | }; | |
352 | ||
3a6fa984 BV |
353 | #undef CMD |
354 | #undef SMI | |
355 | #undef S3D | |
356 | #undef S2D | |
357 | #undef SMFX | |
358 | #undef F | |
359 | #undef S | |
360 | #undef R | |
361 | #undef W | |
362 | #undef B | |
363 | #undef M | |
364 | ||
365 | static const struct drm_i915_cmd_table gen7_render_cmds[] = { | |
366 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
367 | { render_cmds, ARRAY_SIZE(render_cmds) }, | |
368 | }; | |
369 | ||
370 | static const struct drm_i915_cmd_table hsw_render_ring_cmds[] = { | |
371 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
372 | { render_cmds, ARRAY_SIZE(render_cmds) }, | |
373 | { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) }, | |
374 | }; | |
375 | ||
376 | static const struct drm_i915_cmd_table gen7_video_cmds[] = { | |
377 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
378 | { video_cmds, ARRAY_SIZE(video_cmds) }, | |
379 | }; | |
380 | ||
381 | static const struct drm_i915_cmd_table hsw_vebox_cmds[] = { | |
382 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
383 | { vecs_cmds, ARRAY_SIZE(vecs_cmds) }, | |
384 | }; | |
385 | ||
386 | static const struct drm_i915_cmd_table gen7_blt_cmds[] = { | |
387 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
388 | { blt_cmds, ARRAY_SIZE(blt_cmds) }, | |
389 | }; | |
390 | ||
9c640d1d BV |
391 | static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = { |
392 | { common_cmds, ARRAY_SIZE(common_cmds) }, | |
393 | { blt_cmds, ARRAY_SIZE(blt_cmds) }, | |
394 | { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) }, | |
395 | }; | |
396 | ||
5947de9b BV |
397 | /* |
398 | * Register whitelists, sorted by increasing register offset. | |
4e86f725 FJ |
399 | */ |
400 | ||
401 | /* | |
402 | * An individual whitelist entry granting access to register addr. If | |
403 | * mask is non-zero the argument of immediate register writes will be | |
404 | * AND-ed with mask, and the command will be rejected if the result | |
405 | * doesn't match value. | |
406 | * | |
407 | * Registers with non-zero mask are only allowed to be written using | |
408 | * LRI. | |
409 | */ | |
410 | struct drm_i915_reg_descriptor { | |
f0f59a00 | 411 | i915_reg_t addr; |
4e86f725 FJ |
412 | u32 mask; |
413 | u32 value; | |
414 | }; | |
415 | ||
416 | /* Convenience macro for adding 32-bit registers. */ | |
e597ef40 VS |
417 | #define REG32(_reg, ...) \ |
418 | { .addr = (_reg), __VA_ARGS__ } | |
4e86f725 FJ |
419 | |
420 | /* | |
421 | * Convenience macro for adding 64-bit registers. | |
5947de9b BV |
422 | * |
423 | * Some registers that userspace accesses are 64 bits. The register | |
424 | * access commands only allow 32-bit accesses. Hence, we have to include | |
425 | * entries for both halves of the 64-bit registers. | |
426 | */ | |
e597ef40 VS |
427 | #define REG64(_reg) \ |
428 | { .addr = _reg }, \ | |
429 | { .addr = _reg ## _UDW } | |
430 | ||
431 | #define REG64_IDX(_reg, idx) \ | |
432 | { .addr = _reg(idx) }, \ | |
433 | { .addr = _reg ## _UDW(idx) } | |
5947de9b | 434 | |
4e86f725 | 435 | static const struct drm_i915_reg_descriptor gen7_render_regs[] = { |
c61200c2 | 436 | REG64(GPGPU_THREADS_DISPATCHED), |
5947de9b BV |
437 | REG64(HS_INVOCATION_COUNT), |
438 | REG64(DS_INVOCATION_COUNT), | |
439 | REG64(IA_VERTICES_COUNT), | |
440 | REG64(IA_PRIMITIVES_COUNT), | |
441 | REG64(VS_INVOCATION_COUNT), | |
442 | REG64(GS_INVOCATION_COUNT), | |
443 | REG64(GS_PRIMITIVES_COUNT), | |
444 | REG64(CL_INVOCATION_COUNT), | |
445 | REG64(CL_PRIMITIVES_COUNT), | |
446 | REG64(PS_INVOCATION_COUNT), | |
447 | REG64(PS_DEPTH_COUNT), | |
a6573e1f | 448 | REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), |
4e86f725 | 449 | REG32(OACONTROL), /* Only allowed for LRI and SRM. See below. */ |
f1f55cc0 NR |
450 | REG64(MI_PREDICATE_SRC0), |
451 | REG64(MI_PREDICATE_SRC1), | |
4e86f725 FJ |
452 | REG32(GEN7_3DPRIM_END_OFFSET), |
453 | REG32(GEN7_3DPRIM_START_VERTEX), | |
454 | REG32(GEN7_3DPRIM_VERTEX_COUNT), | |
455 | REG32(GEN7_3DPRIM_INSTANCE_COUNT), | |
456 | REG32(GEN7_3DPRIM_START_INSTANCE), | |
457 | REG32(GEN7_3DPRIM_BASE_VERTEX), | |
7b9748cb JJ |
458 | REG32(GEN7_GPGPU_DISPATCHDIMX), |
459 | REG32(GEN7_GPGPU_DISPATCHDIMY), | |
460 | REG32(GEN7_GPGPU_DISPATCHDIMZ), | |
e597ef40 VS |
461 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0), |
462 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1), | |
463 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2), | |
464 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3), | |
465 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0), | |
466 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1), | |
467 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2), | |
468 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3), | |
4e86f725 FJ |
469 | REG32(GEN7_SO_WRITE_OFFSET(0)), |
470 | REG32(GEN7_SO_WRITE_OFFSET(1)), | |
471 | REG32(GEN7_SO_WRITE_OFFSET(2)), | |
472 | REG32(GEN7_SO_WRITE_OFFSET(3)), | |
473 | REG32(GEN7_L3SQCREG1), | |
474 | REG32(GEN7_L3CNTLREG2), | |
475 | REG32(GEN7_L3CNTLREG3), | |
99c5aeca JJ |
476 | }; |
477 | ||
478 | static const struct drm_i915_reg_descriptor hsw_render_regs[] = { | |
1b85066b JJ |
479 | REG64_IDX(HSW_CS_GPR, 0), |
480 | REG64_IDX(HSW_CS_GPR, 1), | |
481 | REG64_IDX(HSW_CS_GPR, 2), | |
482 | REG64_IDX(HSW_CS_GPR, 3), | |
483 | REG64_IDX(HSW_CS_GPR, 4), | |
484 | REG64_IDX(HSW_CS_GPR, 5), | |
485 | REG64_IDX(HSW_CS_GPR, 6), | |
486 | REG64_IDX(HSW_CS_GPR, 7), | |
487 | REG64_IDX(HSW_CS_GPR, 8), | |
488 | REG64_IDX(HSW_CS_GPR, 9), | |
489 | REG64_IDX(HSW_CS_GPR, 10), | |
490 | REG64_IDX(HSW_CS_GPR, 11), | |
491 | REG64_IDX(HSW_CS_GPR, 12), | |
492 | REG64_IDX(HSW_CS_GPR, 13), | |
493 | REG64_IDX(HSW_CS_GPR, 14), | |
494 | REG64_IDX(HSW_CS_GPR, 15), | |
d351f6d9 FJ |
495 | REG32(HSW_SCRATCH1, |
496 | .mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE, | |
497 | .value = 0), | |
498 | REG32(HSW_ROW_CHICKEN3, | |
499 | .mask = ~(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE << 16 | | |
500 | HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), | |
501 | .value = 0), | |
5947de9b BV |
502 | }; |
503 | ||
4e86f725 FJ |
504 | static const struct drm_i915_reg_descriptor gen7_blt_regs[] = { |
505 | REG32(BCS_SWCTRL), | |
5947de9b BV |
506 | }; |
507 | ||
4e86f725 FJ |
508 | static const struct drm_i915_reg_descriptor ivb_master_regs[] = { |
509 | REG32(FORCEWAKE_MT), | |
510 | REG32(DERRMR), | |
511 | REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_A)), | |
512 | REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_B)), | |
513 | REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_C)), | |
220375aa BV |
514 | }; |
515 | ||
4e86f725 FJ |
516 | static const struct drm_i915_reg_descriptor hsw_master_regs[] = { |
517 | REG32(FORCEWAKE_MT), | |
518 | REG32(DERRMR), | |
220375aa BV |
519 | }; |
520 | ||
5947de9b | 521 | #undef REG64 |
4e86f725 | 522 | #undef REG32 |
5947de9b | 523 | |
361b027b JJ |
524 | struct drm_i915_reg_table { |
525 | const struct drm_i915_reg_descriptor *regs; | |
526 | int num_regs; | |
527 | bool master; | |
528 | }; | |
529 | ||
530 | static const struct drm_i915_reg_table ivb_render_reg_tables[] = { | |
531 | { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, | |
532 | { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true }, | |
533 | }; | |
534 | ||
535 | static const struct drm_i915_reg_table ivb_blt_reg_tables[] = { | |
536 | { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, | |
537 | { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true }, | |
538 | }; | |
539 | ||
540 | static const struct drm_i915_reg_table hsw_render_reg_tables[] = { | |
541 | { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, | |
99c5aeca | 542 | { hsw_render_regs, ARRAY_SIZE(hsw_render_regs), false }, |
361b027b JJ |
543 | { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true }, |
544 | }; | |
545 | ||
546 | static const struct drm_i915_reg_table hsw_blt_reg_tables[] = { | |
547 | { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, | |
548 | { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true }, | |
549 | }; | |
550 | ||
351e3db2 BV |
551 | static u32 gen7_render_get_cmd_length_mask(u32 cmd_header) |
552 | { | |
553 | u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; | |
554 | u32 subclient = | |
555 | (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; | |
556 | ||
557 | if (client == INSTR_MI_CLIENT) | |
558 | return 0x3F; | |
559 | else if (client == INSTR_RC_CLIENT) { | |
560 | if (subclient == INSTR_MEDIA_SUBCLIENT) | |
561 | return 0xFFFF; | |
562 | else | |
563 | return 0xFF; | |
564 | } | |
565 | ||
566 | DRM_DEBUG_DRIVER("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header); | |
567 | return 0; | |
568 | } | |
569 | ||
570 | static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header) | |
571 | { | |
572 | u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; | |
573 | u32 subclient = | |
574 | (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; | |
86ef630d | 575 | u32 op = (cmd_header & INSTR_26_TO_24_MASK) >> INSTR_26_TO_24_SHIFT; |
351e3db2 BV |
576 | |
577 | if (client == INSTR_MI_CLIENT) | |
578 | return 0x3F; | |
579 | else if (client == INSTR_RC_CLIENT) { | |
86ef630d MN |
580 | if (subclient == INSTR_MEDIA_SUBCLIENT) { |
581 | if (op == 6) | |
582 | return 0xFFFF; | |
583 | else | |
584 | return 0xFFF; | |
585 | } else | |
351e3db2 BV |
586 | return 0xFF; |
587 | } | |
588 | ||
589 | DRM_DEBUG_DRIVER("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header); | |
590 | return 0; | |
591 | } | |
592 | ||
593 | static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header) | |
594 | { | |
595 | u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; | |
596 | ||
597 | if (client == INSTR_MI_CLIENT) | |
598 | return 0x3F; | |
599 | else if (client == INSTR_BC_CLIENT) | |
600 | return 0xFF; | |
601 | ||
602 | DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header); | |
603 | return 0; | |
604 | } | |
605 | ||
33a051a5 | 606 | static bool validate_cmds_sorted(const struct intel_engine_cs *engine, |
44e895a8 BV |
607 | const struct drm_i915_cmd_table *cmd_tables, |
608 | int cmd_table_count) | |
351e3db2 BV |
609 | { |
610 | int i; | |
300233ee | 611 | bool ret = true; |
351e3db2 | 612 | |
44e895a8 | 613 | if (!cmd_tables || cmd_table_count == 0) |
300233ee | 614 | return true; |
351e3db2 | 615 | |
44e895a8 BV |
616 | for (i = 0; i < cmd_table_count; i++) { |
617 | const struct drm_i915_cmd_table *table = &cmd_tables[i]; | |
351e3db2 BV |
618 | u32 previous = 0; |
619 | int j; | |
620 | ||
621 | for (j = 0; j < table->count; j++) { | |
622 | const struct drm_i915_cmd_descriptor *desc = | |
8453580c | 623 | &table->table[j]; |
351e3db2 BV |
624 | u32 curr = desc->cmd.value & desc->cmd.mask; |
625 | ||
300233ee | 626 | if (curr < previous) { |
33a051a5 CW |
627 | DRM_ERROR("CMD: %s [%d] command table not sorted: " |
628 | "table=%d entry=%d cmd=0x%08X prev=0x%08X\n", | |
629 | engine->name, engine->id, | |
630 | i, j, curr, previous); | |
300233ee BV |
631 | ret = false; |
632 | } | |
351e3db2 BV |
633 | |
634 | previous = curr; | |
635 | } | |
636 | } | |
300233ee BV |
637 | |
638 | return ret; | |
351e3db2 BV |
639 | } |
640 | ||
33a051a5 | 641 | static bool check_sorted(const struct intel_engine_cs *engine, |
4e86f725 FJ |
642 | const struct drm_i915_reg_descriptor *reg_table, |
643 | int reg_count) | |
351e3db2 BV |
644 | { |
645 | int i; | |
646 | u32 previous = 0; | |
300233ee | 647 | bool ret = true; |
351e3db2 BV |
648 | |
649 | for (i = 0; i < reg_count; i++) { | |
f0f59a00 | 650 | u32 curr = i915_mmio_reg_offset(reg_table[i].addr); |
351e3db2 | 651 | |
300233ee | 652 | if (curr < previous) { |
33a051a5 CW |
653 | DRM_ERROR("CMD: %s [%d] register table not sorted: " |
654 | "entry=%d reg=0x%08X prev=0x%08X\n", | |
655 | engine->name, engine->id, | |
656 | i, curr, previous); | |
300233ee BV |
657 | ret = false; |
658 | } | |
351e3db2 BV |
659 | |
660 | previous = curr; | |
661 | } | |
300233ee BV |
662 | |
663 | return ret; | |
351e3db2 BV |
664 | } |
665 | ||
0bc40be8 | 666 | static bool validate_regs_sorted(struct intel_engine_cs *engine) |
351e3db2 | 667 | { |
361b027b JJ |
668 | int i; |
669 | const struct drm_i915_reg_table *table; | |
670 | ||
671 | for (i = 0; i < engine->reg_table_count; i++) { | |
672 | table = &engine->reg_tables[i]; | |
33a051a5 | 673 | if (!check_sorted(engine, table->regs, table->num_regs)) |
361b027b JJ |
674 | return false; |
675 | } | |
676 | ||
677 | return true; | |
351e3db2 BV |
678 | } |
679 | ||
44e895a8 BV |
680 | struct cmd_node { |
681 | const struct drm_i915_cmd_descriptor *desc; | |
682 | struct hlist_node node; | |
683 | }; | |
684 | ||
685 | /* | |
686 | * Different command ranges have different numbers of bits for the opcode. For | |
687 | * example, MI commands use bits 31:23 while 3D commands use bits 31:16. The | |
688 | * problem is that, for example, MI commands use bits 22:16 for other fields | |
689 | * such as GGTT vs PPGTT bits. If we include those bits in the mask then when | |
690 | * we mask a command from a batch it could hash to the wrong bucket due to | |
691 | * non-opcode bits being set. But if we don't include those bits, some 3D | |
692 | * commands may hash to the same bucket due to not including opcode bits that | |
693 | * make the command unique. For now, we will risk hashing to the same bucket. | |
694 | * | |
695 | * If we attempt to generate a perfect hash, we should be able to look at bits | |
696 | * 31:29 of a command from a batch buffer and use the full mask for that | |
697 | * client. The existing INSTR_CLIENT_MASK/SHIFT defines can be used for this. | |
698 | */ | |
699 | #define CMD_HASH_MASK STD_MI_OPCODE_MASK | |
700 | ||
0bc40be8 | 701 | static int init_hash_table(struct intel_engine_cs *engine, |
44e895a8 BV |
702 | const struct drm_i915_cmd_table *cmd_tables, |
703 | int cmd_table_count) | |
704 | { | |
705 | int i, j; | |
706 | ||
0bc40be8 | 707 | hash_init(engine->cmd_hash); |
44e895a8 BV |
708 | |
709 | for (i = 0; i < cmd_table_count; i++) { | |
710 | const struct drm_i915_cmd_table *table = &cmd_tables[i]; | |
711 | ||
712 | for (j = 0; j < table->count; j++) { | |
713 | const struct drm_i915_cmd_descriptor *desc = | |
714 | &table->table[j]; | |
715 | struct cmd_node *desc_node = | |
716 | kmalloc(sizeof(*desc_node), GFP_KERNEL); | |
717 | ||
718 | if (!desc_node) | |
719 | return -ENOMEM; | |
720 | ||
721 | desc_node->desc = desc; | |
0bc40be8 | 722 | hash_add(engine->cmd_hash, &desc_node->node, |
44e895a8 BV |
723 | desc->cmd.value & CMD_HASH_MASK); |
724 | } | |
725 | } | |
726 | ||
727 | return 0; | |
728 | } | |
729 | ||
0bc40be8 | 730 | static void fini_hash_table(struct intel_engine_cs *engine) |
44e895a8 BV |
731 | { |
732 | struct hlist_node *tmp; | |
733 | struct cmd_node *desc_node; | |
734 | int i; | |
735 | ||
0bc40be8 | 736 | hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) { |
44e895a8 BV |
737 | hash_del(&desc_node->node); |
738 | kfree(desc_node); | |
739 | } | |
740 | } | |
741 | ||
351e3db2 | 742 | /** |
33a051a5 | 743 | * intel_engine_init_cmd_parser() - set cmd parser related fields for an engine |
14bb2c11 | 744 | * @engine: the engine to initialize |
351e3db2 BV |
745 | * |
746 | * Optionally initializes fields related to batch buffer command parsing in the | |
a4872ba6 | 747 | * struct intel_engine_cs based on whether the platform requires software |
351e3db2 | 748 | * command parsing. |
44e895a8 BV |
749 | * |
750 | * Return: non-zero if initialization fails | |
351e3db2 | 751 | */ |
33a051a5 | 752 | int intel_engine_init_cmd_parser(struct intel_engine_cs *engine) |
351e3db2 | 753 | { |
44e895a8 BV |
754 | const struct drm_i915_cmd_table *cmd_tables; |
755 | int cmd_table_count; | |
756 | int ret; | |
757 | ||
c033666a | 758 | if (!IS_GEN7(engine->i915)) |
44e895a8 | 759 | return 0; |
351e3db2 | 760 | |
0bc40be8 | 761 | switch (engine->id) { |
351e3db2 | 762 | case RCS: |
c033666a | 763 | if (IS_HASWELL(engine->i915)) { |
44e895a8 BV |
764 | cmd_tables = hsw_render_ring_cmds; |
765 | cmd_table_count = | |
3a6fa984 BV |
766 | ARRAY_SIZE(hsw_render_ring_cmds); |
767 | } else { | |
44e895a8 BV |
768 | cmd_tables = gen7_render_cmds; |
769 | cmd_table_count = ARRAY_SIZE(gen7_render_cmds); | |
3a6fa984 BV |
770 | } |
771 | ||
c033666a | 772 | if (IS_HASWELL(engine->i915)) { |
361b027b JJ |
773 | engine->reg_tables = hsw_render_reg_tables; |
774 | engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables); | |
220375aa | 775 | } else { |
361b027b JJ |
776 | engine->reg_tables = ivb_render_reg_tables; |
777 | engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables); | |
220375aa BV |
778 | } |
779 | ||
0bc40be8 | 780 | engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask; |
351e3db2 BV |
781 | break; |
782 | case VCS: | |
44e895a8 BV |
783 | cmd_tables = gen7_video_cmds; |
784 | cmd_table_count = ARRAY_SIZE(gen7_video_cmds); | |
0bc40be8 | 785 | engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; |
351e3db2 BV |
786 | break; |
787 | case BCS: | |
c033666a | 788 | if (IS_HASWELL(engine->i915)) { |
44e895a8 BV |
789 | cmd_tables = hsw_blt_ring_cmds; |
790 | cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); | |
9c640d1d | 791 | } else { |
44e895a8 BV |
792 | cmd_tables = gen7_blt_cmds; |
793 | cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); | |
9c640d1d BV |
794 | } |
795 | ||
c033666a | 796 | if (IS_HASWELL(engine->i915)) { |
361b027b JJ |
797 | engine->reg_tables = hsw_blt_reg_tables; |
798 | engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables); | |
220375aa | 799 | } else { |
361b027b JJ |
800 | engine->reg_tables = ivb_blt_reg_tables; |
801 | engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables); | |
220375aa BV |
802 | } |
803 | ||
0bc40be8 | 804 | engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; |
351e3db2 BV |
805 | break; |
806 | case VECS: | |
44e895a8 BV |
807 | cmd_tables = hsw_vebox_cmds; |
808 | cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds); | |
351e3db2 | 809 | /* VECS can use the same length_mask function as VCS */ |
0bc40be8 | 810 | engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; |
351e3db2 BV |
811 | break; |
812 | default: | |
33a051a5 | 813 | MISSING_CASE(engine->id); |
351e3db2 BV |
814 | BUG(); |
815 | } | |
816 | ||
0bc40be8 TU |
817 | BUG_ON(!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)); |
818 | BUG_ON(!validate_regs_sorted(engine)); | |
44e895a8 | 819 | |
0bc40be8 | 820 | WARN_ON(!hash_empty(engine->cmd_hash)); |
bfc882b4 | 821 | |
0bc40be8 | 822 | ret = init_hash_table(engine, cmd_tables, cmd_table_count); |
bfc882b4 DV |
823 | if (ret) { |
824 | DRM_ERROR("CMD: cmd_parser_init failed!\n"); | |
0bc40be8 | 825 | fini_hash_table(engine); |
bfc882b4 | 826 | return ret; |
44e895a8 BV |
827 | } |
828 | ||
0bc40be8 | 829 | engine->needs_cmd_parser = true; |
44e895a8 BV |
830 | |
831 | return 0; | |
832 | } | |
833 | ||
834 | /** | |
33a051a5 | 835 | * intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields |
14bb2c11 | 836 | * @engine: the engine to clean up |
44e895a8 BV |
837 | * |
838 | * Releases any resources related to command parsing that may have been | |
33a051a5 | 839 | * initialized for the specified engine. |
44e895a8 | 840 | */ |
33a051a5 | 841 | void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine) |
44e895a8 | 842 | { |
0bc40be8 | 843 | if (!engine->needs_cmd_parser) |
44e895a8 BV |
844 | return; |
845 | ||
0bc40be8 | 846 | fini_hash_table(engine); |
351e3db2 BV |
847 | } |
848 | ||
849 | static const struct drm_i915_cmd_descriptor* | |
0bc40be8 | 850 | find_cmd_in_table(struct intel_engine_cs *engine, |
351e3db2 BV |
851 | u32 cmd_header) |
852 | { | |
44e895a8 | 853 | struct cmd_node *desc_node; |
351e3db2 | 854 | |
0bc40be8 | 855 | hash_for_each_possible(engine->cmd_hash, desc_node, node, |
44e895a8 BV |
856 | cmd_header & CMD_HASH_MASK) { |
857 | const struct drm_i915_cmd_descriptor *desc = desc_node->desc; | |
351e3db2 BV |
858 | u32 masked_cmd = desc->cmd.mask & cmd_header; |
859 | u32 masked_value = desc->cmd.value & desc->cmd.mask; | |
860 | ||
861 | if (masked_cmd == masked_value) | |
862 | return desc; | |
863 | } | |
864 | ||
865 | return NULL; | |
866 | } | |
867 | ||
868 | /* | |
869 | * Returns a pointer to a descriptor for the command specified by cmd_header. | |
870 | * | |
871 | * The caller must supply space for a default descriptor via the default_desc | |
33a051a5 | 872 | * parameter. If no descriptor for the specified command exists in the engine's |
351e3db2 | 873 | * command parser tables, this function fills in default_desc based on the |
33a051a5 | 874 | * engine's default length encoding and returns default_desc. |
351e3db2 BV |
875 | */ |
876 | static const struct drm_i915_cmd_descriptor* | |
0bc40be8 | 877 | find_cmd(struct intel_engine_cs *engine, |
351e3db2 BV |
878 | u32 cmd_header, |
879 | struct drm_i915_cmd_descriptor *default_desc) | |
880 | { | |
44e895a8 | 881 | const struct drm_i915_cmd_descriptor *desc; |
351e3db2 | 882 | u32 mask; |
351e3db2 | 883 | |
0bc40be8 | 884 | desc = find_cmd_in_table(engine, cmd_header); |
44e895a8 BV |
885 | if (desc) |
886 | return desc; | |
351e3db2 | 887 | |
0bc40be8 | 888 | mask = engine->get_cmd_length_mask(cmd_header); |
351e3db2 BV |
889 | if (!mask) |
890 | return NULL; | |
891 | ||
892 | BUG_ON(!default_desc); | |
893 | default_desc->flags = CMD_DESC_SKIP; | |
894 | default_desc->length.mask = mask; | |
895 | ||
896 | return default_desc; | |
897 | } | |
898 | ||
4e86f725 FJ |
899 | static const struct drm_i915_reg_descriptor * |
900 | find_reg(const struct drm_i915_reg_descriptor *table, | |
901 | int count, u32 addr) | |
351e3db2 | 902 | { |
361b027b | 903 | int i; |
351e3db2 | 904 | |
361b027b JJ |
905 | for (i = 0; i < count; i++) { |
906 | if (i915_mmio_reg_offset(table[i].addr) == addr) | |
907 | return &table[i]; | |
908 | } | |
909 | ||
910 | return NULL; | |
911 | } | |
912 | ||
913 | static const struct drm_i915_reg_descriptor * | |
914 | find_reg_in_tables(const struct drm_i915_reg_table *tables, | |
915 | int count, bool is_master, u32 addr) | |
916 | { | |
917 | int i; | |
918 | const struct drm_i915_reg_table *table; | |
919 | const struct drm_i915_reg_descriptor *reg; | |
920 | ||
921 | for (i = 0; i < count; i++) { | |
922 | table = &tables[i]; | |
923 | if (!table->master || is_master) { | |
924 | reg = find_reg(table->regs, table->num_regs, | |
925 | addr); | |
926 | if (reg != NULL) | |
927 | return reg; | |
351e3db2 BV |
928 | } |
929 | } | |
930 | ||
4e86f725 | 931 | return NULL; |
351e3db2 BV |
932 | } |
933 | ||
17cabf57 CW |
934 | static u32 *vmap_batch(struct drm_i915_gem_object *obj, |
935 | unsigned start, unsigned len) | |
351e3db2 BV |
936 | { |
937 | int i; | |
938 | void *addr = NULL; | |
939 | struct sg_page_iter sg_iter; | |
17cabf57 CW |
940 | int first_page = start >> PAGE_SHIFT; |
941 | int last_page = (len + start + 4095) >> PAGE_SHIFT; | |
942 | int npages = last_page - first_page; | |
351e3db2 BV |
943 | struct page **pages; |
944 | ||
17cabf57 | 945 | pages = drm_malloc_ab(npages, sizeof(*pages)); |
351e3db2 BV |
946 | if (pages == NULL) { |
947 | DRM_DEBUG_DRIVER("Failed to get space for pages\n"); | |
948 | goto finish; | |
949 | } | |
950 | ||
951 | i = 0; | |
72c5ba95 | 952 | for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, first_page) { |
17cabf57 | 953 | pages[i++] = sg_page_iter_page(&sg_iter); |
72c5ba95 MK |
954 | if (i == npages) |
955 | break; | |
956 | } | |
351e3db2 BV |
957 | |
958 | addr = vmap(pages, i, 0, PAGE_KERNEL); | |
959 | if (addr == NULL) { | |
960 | DRM_DEBUG_DRIVER("Failed to vmap pages\n"); | |
961 | goto finish; | |
962 | } | |
963 | ||
964 | finish: | |
965 | if (pages) | |
966 | drm_free_large(pages); | |
967 | return (u32*)addr; | |
968 | } | |
969 | ||
78a42377 BV |
970 | /* Returns a vmap'd pointer to dest_obj, which the caller must unmap */ |
971 | static u32 *copy_batch(struct drm_i915_gem_object *dest_obj, | |
b9ffd80e BV |
972 | struct drm_i915_gem_object *src_obj, |
973 | u32 batch_start_offset, | |
974 | u32 batch_len) | |
78a42377 | 975 | { |
78a42377 | 976 | int needs_clflush = 0; |
17cabf57 CW |
977 | void *src_base, *src; |
978 | void *dst = NULL; | |
979 | int ret; | |
b9ffd80e | 980 | |
17cabf57 CW |
981 | if (batch_len > dest_obj->base.size || |
982 | batch_len + batch_start_offset > src_obj->base.size) | |
b9ffd80e | 983 | return ERR_PTR(-E2BIG); |
78a42377 | 984 | |
de4e783a CW |
985 | if (WARN_ON(dest_obj->pages_pin_count == 0)) |
986 | return ERR_PTR(-ENODEV); | |
987 | ||
78a42377 BV |
988 | ret = i915_gem_obj_prepare_shmem_read(src_obj, &needs_clflush); |
989 | if (ret) { | |
17cabf57 | 990 | DRM_DEBUG_DRIVER("CMD: failed to prepare shadow batch\n"); |
78a42377 BV |
991 | return ERR_PTR(ret); |
992 | } | |
993 | ||
17cabf57 | 994 | src_base = vmap_batch(src_obj, batch_start_offset, batch_len); |
b9ffd80e | 995 | if (!src_base) { |
78a42377 BV |
996 | DRM_DEBUG_DRIVER("CMD: Failed to vmap batch\n"); |
997 | ret = -ENOMEM; | |
998 | goto unpin_src; | |
999 | } | |
1000 | ||
78a42377 BV |
1001 | ret = i915_gem_object_set_to_cpu_domain(dest_obj, true); |
1002 | if (ret) { | |
17cabf57 | 1003 | DRM_DEBUG_DRIVER("CMD: Failed to set shadow batch to CPU\n"); |
78a42377 BV |
1004 | goto unmap_src; |
1005 | } | |
1006 | ||
17cabf57 CW |
1007 | dst = vmap_batch(dest_obj, 0, batch_len); |
1008 | if (!dst) { | |
78a42377 BV |
1009 | DRM_DEBUG_DRIVER("CMD: Failed to vmap shadow batch\n"); |
1010 | ret = -ENOMEM; | |
1011 | goto unmap_src; | |
1012 | } | |
1013 | ||
17cabf57 CW |
1014 | src = src_base + offset_in_page(batch_start_offset); |
1015 | if (needs_clflush) | |
1016 | drm_clflush_virt_range(src, batch_len); | |
b9ffd80e | 1017 | |
17cabf57 | 1018 | memcpy(dst, src, batch_len); |
78a42377 BV |
1019 | |
1020 | unmap_src: | |
b9ffd80e | 1021 | vunmap(src_base); |
78a42377 BV |
1022 | unpin_src: |
1023 | i915_gem_object_unpin_pages(src_obj); | |
1024 | ||
17cabf57 | 1025 | return ret ? ERR_PTR(ret) : dst; |
78a42377 BV |
1026 | } |
1027 | ||
351e3db2 | 1028 | /** |
33a051a5 CW |
1029 | * intel_engine_needs_cmd_parser() - should a given engine use software |
1030 | * command parsing? | |
14bb2c11 | 1031 | * @engine: the engine in question |
351e3db2 BV |
1032 | * |
1033 | * Only certain platforms require software batch buffer command parsing, and | |
32197aab | 1034 | * only when enabled via module parameter. |
351e3db2 | 1035 | * |
33a051a5 | 1036 | * Return: true if the engine requires software command parsing |
351e3db2 | 1037 | */ |
33a051a5 | 1038 | bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine) |
351e3db2 | 1039 | { |
0bc40be8 | 1040 | if (!engine->needs_cmd_parser) |
351e3db2 BV |
1041 | return false; |
1042 | ||
c033666a | 1043 | if (!USES_PPGTT(engine->i915)) |
d4d48035 BV |
1044 | return false; |
1045 | ||
351e3db2 BV |
1046 | return (i915.enable_cmd_parser == 1); |
1047 | } | |
1048 | ||
0bc40be8 | 1049 | static bool check_cmd(const struct intel_engine_cs *engine, |
b651000b | 1050 | const struct drm_i915_cmd_descriptor *desc, |
6a65c5b9 | 1051 | const u32 *cmd, u32 length, |
6e66ea13 BV |
1052 | const bool is_master, |
1053 | bool *oacontrol_set) | |
b651000b BV |
1054 | { |
1055 | if (desc->flags & CMD_DESC_REJECT) { | |
1056 | DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd); | |
1057 | return false; | |
1058 | } | |
1059 | ||
1060 | if ((desc->flags & CMD_DESC_MASTER) && !is_master) { | |
1061 | DRM_DEBUG_DRIVER("CMD: Rejected master-only command: 0x%08X\n", | |
1062 | *cmd); | |
1063 | return false; | |
1064 | } | |
1065 | ||
1066 | if (desc->flags & CMD_DESC_REGISTER) { | |
6e66ea13 | 1067 | /* |
6a65c5b9 FJ |
1068 | * Get the distance between individual register offset |
1069 | * fields if the command can perform more than one | |
1070 | * access at a time. | |
6e66ea13 | 1071 | */ |
6a65c5b9 FJ |
1072 | const u32 step = desc->reg.step ? desc->reg.step : length; |
1073 | u32 offset; | |
1074 | ||
1075 | for (offset = desc->reg.offset; offset < length; | |
1076 | offset += step) { | |
1077 | const u32 reg_addr = cmd[offset] & desc->reg.mask; | |
4e86f725 | 1078 | const struct drm_i915_reg_descriptor *reg = |
361b027b JJ |
1079 | find_reg_in_tables(engine->reg_tables, |
1080 | engine->reg_table_count, | |
1081 | is_master, | |
1082 | reg_addr); | |
4e86f725 FJ |
1083 | |
1084 | if (!reg) { | |
33a051a5 CW |
1085 | DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (exec_id=%d)\n", |
1086 | reg_addr, *cmd, engine->exec_id); | |
4e86f725 FJ |
1087 | return false; |
1088 | } | |
6a65c5b9 FJ |
1089 | |
1090 | /* | |
1091 | * OACONTROL requires some special handling for | |
1092 | * writes. We want to make sure that any batch which | |
1093 | * enables OA also disables it before the end of the | |
1094 | * batch. The goal is to prevent one process from | |
1095 | * snooping on the perf data from another process. To do | |
1096 | * that, we need to check the value that will be written | |
1097 | * to the register. Hence, limit OACONTROL writes to | |
1098 | * only MI_LOAD_REGISTER_IMM commands. | |
1099 | */ | |
f0f59a00 | 1100 | if (reg_addr == i915_mmio_reg_offset(OACONTROL)) { |
f1afe24f | 1101 | if (desc->cmd.value == MI_LOAD_REGISTER_MEM) { |
6a65c5b9 FJ |
1102 | DRM_DEBUG_DRIVER("CMD: Rejected LRM to OACONTROL\n"); |
1103 | return false; | |
1104 | } | |
1105 | ||
6761d0a1 KG |
1106 | if (desc->cmd.value == MI_LOAD_REGISTER_REG) { |
1107 | DRM_DEBUG_DRIVER("CMD: Rejected LRR to OACONTROL\n"); | |
1108 | return false; | |
1109 | } | |
1110 | ||
6a65c5b9 FJ |
1111 | if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1)) |
1112 | *oacontrol_set = (cmd[offset + 1] != 0); | |
00caf019 | 1113 | } |
6e66ea13 | 1114 | |
4e86f725 FJ |
1115 | /* |
1116 | * Check the value written to the register against the | |
1117 | * allowed mask/value pair given in the whitelist entry. | |
1118 | */ | |
1119 | if (reg->mask) { | |
f1afe24f | 1120 | if (desc->cmd.value == MI_LOAD_REGISTER_MEM) { |
4e86f725 FJ |
1121 | DRM_DEBUG_DRIVER("CMD: Rejected LRM to masked register 0x%08X\n", |
1122 | reg_addr); | |
1123 | return false; | |
1124 | } | |
1125 | ||
6761d0a1 KG |
1126 | if (desc->cmd.value == MI_LOAD_REGISTER_REG) { |
1127 | DRM_DEBUG_DRIVER("CMD: Rejected LRR to masked register 0x%08X\n", | |
1128 | reg_addr); | |
1129 | return false; | |
1130 | } | |
1131 | ||
4e86f725 FJ |
1132 | if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) && |
1133 | (offset + 2 > length || | |
1134 | (cmd[offset + 1] & reg->mask) != reg->value)) { | |
1135 | DRM_DEBUG_DRIVER("CMD: Rejected LRI to masked register 0x%08X\n", | |
1136 | reg_addr); | |
6a65c5b9 FJ |
1137 | return false; |
1138 | } | |
b651000b BV |
1139 | } |
1140 | } | |
1141 | } | |
1142 | ||
1143 | if (desc->flags & CMD_DESC_BITMASK) { | |
1144 | int i; | |
1145 | ||
1146 | for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) { | |
1147 | u32 dword; | |
1148 | ||
1149 | if (desc->bits[i].mask == 0) | |
1150 | break; | |
1151 | ||
1152 | if (desc->bits[i].condition_mask != 0) { | |
1153 | u32 offset = | |
1154 | desc->bits[i].condition_offset; | |
1155 | u32 condition = cmd[offset] & | |
1156 | desc->bits[i].condition_mask; | |
1157 | ||
1158 | if (condition == 0) | |
1159 | continue; | |
1160 | } | |
1161 | ||
1162 | dword = cmd[desc->bits[i].offset] & | |
1163 | desc->bits[i].mask; | |
1164 | ||
1165 | if (dword != desc->bits[i].expected) { | |
33a051a5 | 1166 | DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (exec_id=%d)\n", |
b651000b BV |
1167 | *cmd, |
1168 | desc->bits[i].mask, | |
1169 | desc->bits[i].expected, | |
33a051a5 | 1170 | dword, engine->exec_id); |
b651000b BV |
1171 | return false; |
1172 | } | |
1173 | } | |
1174 | } | |
1175 | ||
1176 | return true; | |
1177 | } | |
1178 | ||
351e3db2 BV |
1179 | #define LENGTH_BIAS 2 |
1180 | ||
1181 | /** | |
1182 | * i915_parse_cmds() - parse a submitted batch buffer for privilege violations | |
14bb2c11 | 1183 | * @engine: the engine on which the batch is to execute |
351e3db2 | 1184 | * @batch_obj: the batch buffer in question |
78a42377 | 1185 | * @shadow_batch_obj: copy of the batch buffer in question |
351e3db2 | 1186 | * @batch_start_offset: byte offset in the batch at which execution starts |
b9ffd80e | 1187 | * @batch_len: length of the commands in batch_obj |
351e3db2 BV |
1188 | * @is_master: is the submitting process the drm master? |
1189 | * | |
1190 | * Parses the specified batch buffer looking for privilege violations as | |
1191 | * described in the overview. | |
1192 | * | |
42c7156a BV |
1193 | * Return: non-zero if the parser finds violations or otherwise fails; -EACCES |
1194 | * if the batch appears legal but should use hardware parsing | |
351e3db2 | 1195 | */ |
33a051a5 CW |
1196 | int intel_engine_cmd_parser(struct intel_engine_cs *engine, |
1197 | struct drm_i915_gem_object *batch_obj, | |
1198 | struct drm_i915_gem_object *shadow_batch_obj, | |
1199 | u32 batch_start_offset, | |
1200 | u32 batch_len, | |
1201 | bool is_master) | |
351e3db2 | 1202 | { |
351e3db2 BV |
1203 | u32 *cmd, *batch_base, *batch_end; |
1204 | struct drm_i915_cmd_descriptor default_desc = { 0 }; | |
6e66ea13 | 1205 | bool oacontrol_set = false; /* OACONTROL tracking. See check_cmd() */ |
17cabf57 | 1206 | int ret = 0; |
71745376 | 1207 | |
b9ffd80e BV |
1208 | batch_base = copy_batch(shadow_batch_obj, batch_obj, |
1209 | batch_start_offset, batch_len); | |
78a42377 BV |
1210 | if (IS_ERR(batch_base)) { |
1211 | DRM_DEBUG_DRIVER("CMD: Failed to copy batch\n"); | |
1212 | return PTR_ERR(batch_base); | |
351e3db2 BV |
1213 | } |
1214 | ||
78a42377 | 1215 | /* |
b9ffd80e | 1216 | * We use the batch length as size because the shadow object is as |
78a42377 BV |
1217 | * large or larger and copy_batch() will write MI_NOPs to the extra |
1218 | * space. Parsing should be faster in some cases this way. | |
1219 | */ | |
17cabf57 | 1220 | batch_end = batch_base + (batch_len / sizeof(*batch_end)); |
351e3db2 | 1221 | |
17cabf57 | 1222 | cmd = batch_base; |
351e3db2 BV |
1223 | while (cmd < batch_end) { |
1224 | const struct drm_i915_cmd_descriptor *desc; | |
1225 | u32 length; | |
1226 | ||
1227 | if (*cmd == MI_BATCH_BUFFER_END) | |
1228 | break; | |
1229 | ||
0bc40be8 | 1230 | desc = find_cmd(engine, *cmd, &default_desc); |
351e3db2 BV |
1231 | if (!desc) { |
1232 | DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n", | |
1233 | *cmd); | |
1234 | ret = -EINVAL; | |
1235 | break; | |
1236 | } | |
1237 | ||
42c7156a BV |
1238 | /* |
1239 | * If the batch buffer contains a chained batch, return an | |
1240 | * error that tells the caller to abort and dispatch the | |
1241 | * workload as a non-secure batch. | |
1242 | */ | |
1243 | if (desc->cmd.value == MI_BATCH_BUFFER_START) { | |
1244 | ret = -EACCES; | |
1245 | break; | |
1246 | } | |
1247 | ||
351e3db2 BV |
1248 | if (desc->flags & CMD_DESC_FIXED) |
1249 | length = desc->length.fixed; | |
1250 | else | |
1251 | length = ((*cmd & desc->length.mask) + LENGTH_BIAS); | |
1252 | ||
1253 | if ((batch_end - cmd) < length) { | |
86a25121 | 1254 | DRM_DEBUG_DRIVER("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", |
351e3db2 BV |
1255 | *cmd, |
1256 | length, | |
4b6eab59 | 1257 | batch_end - cmd); |
351e3db2 BV |
1258 | ret = -EINVAL; |
1259 | break; | |
1260 | } | |
1261 | ||
0bc40be8 | 1262 | if (!check_cmd(engine, desc, cmd, length, is_master, |
6a65c5b9 | 1263 | &oacontrol_set)) { |
351e3db2 BV |
1264 | ret = -EINVAL; |
1265 | break; | |
1266 | } | |
1267 | ||
351e3db2 BV |
1268 | cmd += length; |
1269 | } | |
1270 | ||
6e66ea13 BV |
1271 | if (oacontrol_set) { |
1272 | DRM_DEBUG_DRIVER("CMD: batch set OACONTROL but did not clear it\n"); | |
1273 | ret = -EINVAL; | |
1274 | } | |
1275 | ||
351e3db2 BV |
1276 | if (cmd >= batch_end) { |
1277 | DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n"); | |
1278 | ret = -EINVAL; | |
1279 | } | |
1280 | ||
1281 | vunmap(batch_base); | |
1282 | ||
351e3db2 BV |
1283 | return ret; |
1284 | } | |
d728c8ef BV |
1285 | |
1286 | /** | |
1287 | * i915_cmd_parser_get_version() - get the cmd parser version number | |
14bb2c11 | 1288 | * @dev_priv: i915 device private |
d728c8ef BV |
1289 | * |
1290 | * The cmd parser maintains a simple increasing integer version number suitable | |
1291 | * for passing to userspace clients to determine what operations are permitted. | |
1292 | * | |
1293 | * Return: the current version number of the cmd parser | |
1294 | */ | |
1ca3712c | 1295 | int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv) |
d728c8ef | 1296 | { |
1ca3712c CW |
1297 | struct intel_engine_cs *engine; |
1298 | bool active = false; | |
1299 | ||
1300 | /* If the command parser is not enabled, report 0 - unsupported */ | |
1301 | for_each_engine(engine, dev_priv) { | |
33a051a5 | 1302 | if (intel_engine_needs_cmd_parser(engine)) { |
1ca3712c CW |
1303 | active = true; |
1304 | break; | |
1305 | } | |
1306 | } | |
1307 | if (!active) | |
1308 | return 0; | |
1309 | ||
d728c8ef BV |
1310 | /* |
1311 | * Command parser version history | |
1312 | * | |
1313 | * 1. Initial version. Checks batches and reports violations, but leaves | |
1314 | * hardware parsing enabled (so does not allow new use cases). | |
f1f55cc0 NR |
1315 | * 2. Allow access to the MI_PREDICATE_SRC0 and |
1316 | * MI_PREDICATE_SRC1 registers. | |
c61200c2 | 1317 | * 3. Allow access to the GPGPU_THREADS_DISPATCHED register. |
2bbe6bbb | 1318 | * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3. |
7b9748cb | 1319 | * 5. GPGPU dispatch compute indirect registers. |
6cf0716c | 1320 | * 6. TIMESTAMP register and Haswell CS GPR registers |
6761d0a1 | 1321 | * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers. |
d728c8ef | 1322 | */ |
6761d0a1 | 1323 | return 7; |
d728c8ef | 1324 | } |