Commit | Line | Data |
---|---|---|
c8b75bca EA |
1 | /* |
2 | * Copyright (C) 2015 Broadcom | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | #include "drmP.h" | |
10 | #include "drm_gem_cma_helper.h" | |
11 | ||
12 | struct vc4_dev { | |
13 | struct drm_device *dev; | |
14 | ||
15 | struct vc4_hdmi *hdmi; | |
16 | struct vc4_hvs *hvs; | |
17 | struct vc4_crtc *crtc[3]; | |
d3f5168a | 18 | struct vc4_v3d *v3d; |
08302c35 | 19 | struct vc4_dpi *dpi; |
48666d56 DF |
20 | |
21 | struct drm_fbdev_cma *fbdev; | |
c826a6e1 | 22 | |
21461365 EA |
23 | struct vc4_hang_state *hang_state; |
24 | ||
c826a6e1 EA |
25 | /* The kernel-space BO cache. Tracks buffers that have been |
26 | * unreferenced by all other users (refcounts of 0!) but not | |
27 | * yet freed, so we can do cheap allocations. | |
28 | */ | |
29 | struct vc4_bo_cache { | |
30 | /* Array of list heads for entries in the BO cache, | |
31 | * based on number of pages, so we can do O(1) lookups | |
32 | * in the cache when allocating. | |
33 | */ | |
34 | struct list_head *size_list; | |
35 | uint32_t size_list_size; | |
36 | ||
37 | /* List of all BOs in the cache, ordered by age, so we | |
38 | * can do O(1) lookups when trying to free old | |
39 | * buffers. | |
40 | */ | |
41 | struct list_head time_list; | |
42 | struct work_struct time_work; | |
43 | struct timer_list time_timer; | |
44 | } bo_cache; | |
45 | ||
46 | struct vc4_bo_stats { | |
47 | u32 num_allocated; | |
48 | u32 size_allocated; | |
49 | u32 num_cached; | |
50 | u32 size_cached; | |
51 | } bo_stats; | |
52 | ||
53 | /* Protects bo_cache and the BO stats. */ | |
54 | struct mutex bo_lock; | |
d5b1a78a | 55 | |
ca26d28b | 56 | /* Sequence number for the last job queued in bin_job_list. |
d5b1a78a EA |
57 | * Starts at 0 (no jobs emitted). |
58 | */ | |
59 | uint64_t emit_seqno; | |
60 | ||
61 | /* Sequence number for the last completed job on the GPU. | |
62 | * Starts at 0 (no jobs completed). | |
63 | */ | |
64 | uint64_t finished_seqno; | |
65 | ||
ca26d28b VG |
66 | /* List of all struct vc4_exec_info for jobs to be executed in |
67 | * the binner. The first job in the list is the one currently | |
68 | * programmed into ct0ca for execution. | |
d5b1a78a | 69 | */ |
ca26d28b VG |
70 | struct list_head bin_job_list; |
71 | ||
72 | /* List of all struct vc4_exec_info for jobs that have | |
73 | * completed binning and are ready for rendering. The first | |
74 | * job in the list is the one currently programmed into ct1ca | |
75 | * for execution. | |
76 | */ | |
77 | struct list_head render_job_list; | |
78 | ||
d5b1a78a EA |
79 | /* List of the finished vc4_exec_infos waiting to be freed by |
80 | * job_done_work. | |
81 | */ | |
82 | struct list_head job_done_list; | |
83 | /* Spinlock used to synchronize the job_list and seqno | |
84 | * accesses between the IRQ handler and GEM ioctls. | |
85 | */ | |
86 | spinlock_t job_lock; | |
87 | wait_queue_head_t job_wait_queue; | |
88 | struct work_struct job_done_work; | |
89 | ||
b501bacc EA |
90 | /* List of struct vc4_seqno_cb for callbacks to be made from a |
91 | * workqueue when the given seqno is passed. | |
92 | */ | |
93 | struct list_head seqno_cb_list; | |
94 | ||
d5b1a78a EA |
95 | /* The binner overflow memory that's currently set up in |
96 | * BPOA/BPOS registers. When overflow occurs and a new one is | |
97 | * allocated, the previous one will be moved to | |
98 | * vc4->current_exec's free list. | |
99 | */ | |
100 | struct vc4_bo *overflow_mem; | |
101 | struct work_struct overflow_mem_work; | |
102 | ||
36cb6253 EA |
103 | int power_refcount; |
104 | ||
105 | /* Mutex controlling the power refcount. */ | |
106 | struct mutex power_lock; | |
107 | ||
d5b1a78a | 108 | struct { |
d5b1a78a EA |
109 | struct timer_list timer; |
110 | struct work_struct reset_work; | |
111 | } hangcheck; | |
112 | ||
113 | struct semaphore async_modeset; | |
c8b75bca EA |
114 | }; |
115 | ||
116 | static inline struct vc4_dev * | |
117 | to_vc4_dev(struct drm_device *dev) | |
118 | { | |
119 | return (struct vc4_dev *)dev->dev_private; | |
120 | } | |
121 | ||
122 | struct vc4_bo { | |
123 | struct drm_gem_cma_object base; | |
c826a6e1 | 124 | |
d5b1a78a EA |
125 | /* seqno of the last job to render to this BO. */ |
126 | uint64_t seqno; | |
127 | ||
c826a6e1 EA |
128 | /* List entry for the BO's position in either |
129 | * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list | |
130 | */ | |
131 | struct list_head unref_head; | |
132 | ||
133 | /* Time in jiffies when the BO was put in vc4->bo_cache. */ | |
134 | unsigned long free_time; | |
135 | ||
136 | /* List entry for the BO's position in vc4_dev->bo_cache.size_list */ | |
137 | struct list_head size_head; | |
463873d5 EA |
138 | |
139 | /* Struct for shader validation state, if created by | |
140 | * DRM_IOCTL_VC4_CREATE_SHADER_BO. | |
141 | */ | |
142 | struct vc4_validated_shader_info *validated_shader; | |
c8b75bca EA |
143 | }; |
144 | ||
145 | static inline struct vc4_bo * | |
146 | to_vc4_bo(struct drm_gem_object *bo) | |
147 | { | |
148 | return (struct vc4_bo *)bo; | |
149 | } | |
150 | ||
b501bacc EA |
151 | struct vc4_seqno_cb { |
152 | struct work_struct work; | |
153 | uint64_t seqno; | |
154 | void (*func)(struct vc4_seqno_cb *cb); | |
155 | }; | |
156 | ||
d3f5168a | 157 | struct vc4_v3d { |
001bdb55 | 158 | struct vc4_dev *vc4; |
d3f5168a EA |
159 | struct platform_device *pdev; |
160 | void __iomem *regs; | |
161 | }; | |
162 | ||
c8b75bca EA |
163 | struct vc4_hvs { |
164 | struct platform_device *pdev; | |
165 | void __iomem *regs; | |
d8dbf44f EA |
166 | u32 __iomem *dlist; |
167 | ||
168 | /* Memory manager for CRTCs to allocate space in the display | |
169 | * list. Units are dwords. | |
170 | */ | |
171 | struct drm_mm dlist_mm; | |
21af94cf EA |
172 | /* Memory manager for the LBM memory used by HVS scaling. */ |
173 | struct drm_mm lbm_mm; | |
d8dbf44f | 174 | spinlock_t mm_lock; |
21af94cf EA |
175 | |
176 | struct drm_mm_node mitchell_netravali_filter; | |
c8b75bca EA |
177 | }; |
178 | ||
179 | struct vc4_plane { | |
180 | struct drm_plane base; | |
181 | }; | |
182 | ||
183 | static inline struct vc4_plane * | |
184 | to_vc4_plane(struct drm_plane *plane) | |
185 | { | |
186 | return (struct vc4_plane *)plane; | |
187 | } | |
188 | ||
189 | enum vc4_encoder_type { | |
190 | VC4_ENCODER_TYPE_HDMI, | |
191 | VC4_ENCODER_TYPE_VEC, | |
192 | VC4_ENCODER_TYPE_DSI0, | |
193 | VC4_ENCODER_TYPE_DSI1, | |
194 | VC4_ENCODER_TYPE_SMI, | |
195 | VC4_ENCODER_TYPE_DPI, | |
196 | }; | |
197 | ||
198 | struct vc4_encoder { | |
199 | struct drm_encoder base; | |
200 | enum vc4_encoder_type type; | |
201 | u32 clock_select; | |
202 | }; | |
203 | ||
204 | static inline struct vc4_encoder * | |
205 | to_vc4_encoder(struct drm_encoder *encoder) | |
206 | { | |
207 | return container_of(encoder, struct vc4_encoder, base); | |
208 | } | |
209 | ||
d3f5168a EA |
210 | #define V3D_READ(offset) readl(vc4->v3d->regs + offset) |
211 | #define V3D_WRITE(offset, val) writel(val, vc4->v3d->regs + offset) | |
c8b75bca EA |
212 | #define HVS_READ(offset) readl(vc4->hvs->regs + offset) |
213 | #define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset) | |
214 | ||
d5b1a78a EA |
215 | struct vc4_exec_info { |
216 | /* Sequence number for this bin/render job. */ | |
217 | uint64_t seqno; | |
218 | ||
c4ce60dc EA |
219 | /* Last current addresses the hardware was processing when the |
220 | * hangcheck timer checked on us. | |
221 | */ | |
222 | uint32_t last_ct0ca, last_ct1ca; | |
223 | ||
d5b1a78a EA |
224 | /* Kernel-space copy of the ioctl arguments */ |
225 | struct drm_vc4_submit_cl *args; | |
226 | ||
227 | /* This is the array of BOs that were looked up at the start of exec. | |
228 | * Command validation will use indices into this array. | |
229 | */ | |
230 | struct drm_gem_cma_object **bo; | |
231 | uint32_t bo_count; | |
232 | ||
233 | /* Pointers for our position in vc4->job_list */ | |
234 | struct list_head head; | |
235 | ||
236 | /* List of other BOs used in the job that need to be released | |
237 | * once the job is complete. | |
238 | */ | |
239 | struct list_head unref_list; | |
240 | ||
241 | /* Current unvalidated indices into @bo loaded by the non-hardware | |
242 | * VC4_PACKET_GEM_HANDLES. | |
243 | */ | |
244 | uint32_t bo_index[2]; | |
245 | ||
246 | /* This is the BO where we store the validated command lists, shader | |
247 | * records, and uniforms. | |
248 | */ | |
249 | struct drm_gem_cma_object *exec_bo; | |
250 | ||
251 | /** | |
252 | * This tracks the per-shader-record state (packet 64) that | |
253 | * determines the length of the shader record and the offset | |
254 | * it's expected to be found at. It gets read in from the | |
255 | * command lists. | |
256 | */ | |
257 | struct vc4_shader_state { | |
258 | uint32_t addr; | |
259 | /* Maximum vertex index referenced by any primitive using this | |
260 | * shader state. | |
261 | */ | |
262 | uint32_t max_index; | |
263 | } *shader_state; | |
264 | ||
265 | /** How many shader states the user declared they were using. */ | |
266 | uint32_t shader_state_size; | |
267 | /** How many shader state records the validator has seen. */ | |
268 | uint32_t shader_state_count; | |
269 | ||
270 | bool found_tile_binning_mode_config_packet; | |
271 | bool found_start_tile_binning_packet; | |
272 | bool found_increment_semaphore_packet; | |
273 | bool found_flush; | |
274 | uint8_t bin_tiles_x, bin_tiles_y; | |
275 | struct drm_gem_cma_object *tile_bo; | |
276 | uint32_t tile_alloc_offset; | |
277 | ||
278 | /** | |
279 | * Computed addresses pointing into exec_bo where we start the | |
280 | * bin thread (ct0) and render thread (ct1). | |
281 | */ | |
282 | uint32_t ct0ca, ct0ea; | |
283 | uint32_t ct1ca, ct1ea; | |
284 | ||
285 | /* Pointer to the unvalidated bin CL (if present). */ | |
286 | void *bin_u; | |
287 | ||
288 | /* Pointers to the shader recs. These paddr gets incremented as CL | |
289 | * packets are relocated in validate_gl_shader_state, and the vaddrs | |
290 | * (u and v) get incremented and size decremented as the shader recs | |
291 | * themselves are validated. | |
292 | */ | |
293 | void *shader_rec_u; | |
294 | void *shader_rec_v; | |
295 | uint32_t shader_rec_p; | |
296 | uint32_t shader_rec_size; | |
297 | ||
298 | /* Pointers to the uniform data. These pointers are incremented, and | |
299 | * size decremented, as each batch of uniforms is uploaded. | |
300 | */ | |
301 | void *uniforms_u; | |
302 | void *uniforms_v; | |
303 | uint32_t uniforms_p; | |
304 | uint32_t uniforms_size; | |
305 | }; | |
306 | ||
307 | static inline struct vc4_exec_info * | |
ca26d28b VG |
308 | vc4_first_bin_job(struct vc4_dev *vc4) |
309 | { | |
310 | if (list_empty(&vc4->bin_job_list)) | |
311 | return NULL; | |
312 | return list_first_entry(&vc4->bin_job_list, struct vc4_exec_info, head); | |
313 | } | |
314 | ||
315 | static inline struct vc4_exec_info * | |
316 | vc4_first_render_job(struct vc4_dev *vc4) | |
d5b1a78a | 317 | { |
ca26d28b | 318 | if (list_empty(&vc4->render_job_list)) |
d5b1a78a | 319 | return NULL; |
ca26d28b VG |
320 | return list_first_entry(&vc4->render_job_list, |
321 | struct vc4_exec_info, head); | |
d5b1a78a EA |
322 | } |
323 | ||
463873d5 EA |
324 | /** |
325 | * struct vc4_texture_sample_info - saves the offsets into the UBO for texture | |
326 | * setup parameters. | |
327 | * | |
328 | * This will be used at draw time to relocate the reference to the texture | |
329 | * contents in p0, and validate that the offset combined with | |
330 | * width/height/stride/etc. from p1 and p2/p3 doesn't sample outside the BO. | |
331 | * Note that the hardware treats unprovided config parameters as 0, so not all | |
332 | * of them need to be set up for every texure sample, and we'll store ~0 as | |
333 | * the offset to mark the unused ones. | |
334 | * | |
335 | * See the VC4 3D architecture guide page 41 ("Texture and Memory Lookup Unit | |
336 | * Setup") for definitions of the texture parameters. | |
337 | */ | |
338 | struct vc4_texture_sample_info { | |
339 | bool is_direct; | |
340 | uint32_t p_offset[4]; | |
341 | }; | |
342 | ||
343 | /** | |
344 | * struct vc4_validated_shader_info - information about validated shaders that | |
345 | * needs to be used from command list validation. | |
346 | * | |
347 | * For a given shader, each time a shader state record references it, we need | |
348 | * to verify that the shader doesn't read more uniforms than the shader state | |
349 | * record's uniform BO pointer can provide, and we need to apply relocations | |
350 | * and validate the shader state record's uniforms that define the texture | |
351 | * samples. | |
352 | */ | |
353 | struct vc4_validated_shader_info { | |
354 | uint32_t uniforms_size; | |
355 | uint32_t uniforms_src_size; | |
356 | uint32_t num_texture_samples; | |
357 | struct vc4_texture_sample_info *texture_samples; | |
358 | }; | |
359 | ||
c8b75bca EA |
360 | /** |
361 | * _wait_for - magic (register) wait macro | |
362 | * | |
363 | * Does the right thing for modeset paths when run under kdgb or similar atomic | |
364 | * contexts. Note that it's important that we check the condition again after | |
365 | * having timed out, since the timeout could be due to preemption or similar and | |
366 | * we've never had a chance to check the condition before the timeout. | |
367 | */ | |
368 | #define _wait_for(COND, MS, W) ({ \ | |
369 | unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ | |
370 | int ret__ = 0; \ | |
371 | while (!(COND)) { \ | |
372 | if (time_after(jiffies, timeout__)) { \ | |
373 | if (!(COND)) \ | |
374 | ret__ = -ETIMEDOUT; \ | |
375 | break; \ | |
376 | } \ | |
377 | if (W && drm_can_sleep()) { \ | |
378 | msleep(W); \ | |
379 | } else { \ | |
380 | cpu_relax(); \ | |
381 | } \ | |
382 | } \ | |
383 | ret__; \ | |
384 | }) | |
385 | ||
386 | #define wait_for(COND, MS) _wait_for(COND, MS, 1) | |
387 | ||
388 | /* vc4_bo.c */ | |
c826a6e1 | 389 | struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size); |
c8b75bca | 390 | void vc4_free_object(struct drm_gem_object *gem_obj); |
c826a6e1 EA |
391 | struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size, |
392 | bool from_cache); | |
c8b75bca EA |
393 | int vc4_dumb_create(struct drm_file *file_priv, |
394 | struct drm_device *dev, | |
395 | struct drm_mode_create_dumb *args); | |
396 | struct dma_buf *vc4_prime_export(struct drm_device *dev, | |
397 | struct drm_gem_object *obj, int flags); | |
d5bc60f6 EA |
398 | int vc4_create_bo_ioctl(struct drm_device *dev, void *data, |
399 | struct drm_file *file_priv); | |
463873d5 EA |
400 | int vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data, |
401 | struct drm_file *file_priv); | |
d5bc60f6 EA |
402 | int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data, |
403 | struct drm_file *file_priv); | |
21461365 EA |
404 | int vc4_get_hang_state_ioctl(struct drm_device *dev, void *data, |
405 | struct drm_file *file_priv); | |
463873d5 EA |
406 | int vc4_mmap(struct file *filp, struct vm_area_struct *vma); |
407 | int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma); | |
408 | void *vc4_prime_vmap(struct drm_gem_object *obj); | |
c826a6e1 EA |
409 | void vc4_bo_cache_init(struct drm_device *dev); |
410 | void vc4_bo_cache_destroy(struct drm_device *dev); | |
411 | int vc4_bo_stats_debugfs(struct seq_file *m, void *arg); | |
c8b75bca EA |
412 | |
413 | /* vc4_crtc.c */ | |
414 | extern struct platform_driver vc4_crtc_driver; | |
1f43710a DA |
415 | int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id); |
416 | void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id); | |
c8b75bca EA |
417 | int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg); |
418 | ||
419 | /* vc4_debugfs.c */ | |
420 | int vc4_debugfs_init(struct drm_minor *minor); | |
421 | void vc4_debugfs_cleanup(struct drm_minor *minor); | |
422 | ||
423 | /* vc4_drv.c */ | |
424 | void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index); | |
425 | ||
08302c35 EA |
426 | /* vc4_dpi.c */ |
427 | extern struct platform_driver vc4_dpi_driver; | |
428 | int vc4_dpi_debugfs_regs(struct seq_file *m, void *unused); | |
429 | ||
d5b1a78a EA |
430 | /* vc4_gem.c */ |
431 | void vc4_gem_init(struct drm_device *dev); | |
432 | void vc4_gem_destroy(struct drm_device *dev); | |
433 | int vc4_submit_cl_ioctl(struct drm_device *dev, void *data, | |
434 | struct drm_file *file_priv); | |
435 | int vc4_wait_seqno_ioctl(struct drm_device *dev, void *data, | |
436 | struct drm_file *file_priv); | |
437 | int vc4_wait_bo_ioctl(struct drm_device *dev, void *data, | |
438 | struct drm_file *file_priv); | |
ca26d28b VG |
439 | void vc4_submit_next_bin_job(struct drm_device *dev); |
440 | void vc4_submit_next_render_job(struct drm_device *dev); | |
441 | void vc4_move_job_to_render(struct drm_device *dev, struct vc4_exec_info *exec); | |
d5b1a78a EA |
442 | int vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, |
443 | uint64_t timeout_ns, bool interruptible); | |
444 | void vc4_job_handle_completed(struct vc4_dev *vc4); | |
b501bacc EA |
445 | int vc4_queue_seqno_cb(struct drm_device *dev, |
446 | struct vc4_seqno_cb *cb, uint64_t seqno, | |
447 | void (*func)(struct vc4_seqno_cb *cb)); | |
d5b1a78a | 448 | |
c8b75bca EA |
449 | /* vc4_hdmi.c */ |
450 | extern struct platform_driver vc4_hdmi_driver; | |
451 | int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused); | |
452 | ||
d5b1a78a EA |
453 | /* vc4_irq.c */ |
454 | irqreturn_t vc4_irq(int irq, void *arg); | |
455 | void vc4_irq_preinstall(struct drm_device *dev); | |
456 | int vc4_irq_postinstall(struct drm_device *dev); | |
457 | void vc4_irq_uninstall(struct drm_device *dev); | |
458 | void vc4_irq_reset(struct drm_device *dev); | |
459 | ||
c8b75bca EA |
460 | /* vc4_hvs.c */ |
461 | extern struct platform_driver vc4_hvs_driver; | |
462 | void vc4_hvs_dump_state(struct drm_device *dev); | |
463 | int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused); | |
464 | ||
465 | /* vc4_kms.c */ | |
466 | int vc4_kms_load(struct drm_device *dev); | |
467 | ||
468 | /* vc4_plane.c */ | |
469 | struct drm_plane *vc4_plane_init(struct drm_device *dev, | |
470 | enum drm_plane_type type); | |
471 | u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist); | |
472 | u32 vc4_plane_dlist_size(struct drm_plane_state *state); | |
b501bacc EA |
473 | void vc4_plane_async_set_fb(struct drm_plane *plane, |
474 | struct drm_framebuffer *fb); | |
463873d5 | 475 | |
d3f5168a EA |
476 | /* vc4_v3d.c */ |
477 | extern struct platform_driver vc4_v3d_driver; | |
478 | int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused); | |
479 | int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused); | |
d5b1a78a EA |
480 | |
481 | /* vc4_validate.c */ | |
482 | int | |
483 | vc4_validate_bin_cl(struct drm_device *dev, | |
484 | void *validated, | |
485 | void *unvalidated, | |
486 | struct vc4_exec_info *exec); | |
487 | ||
488 | int | |
489 | vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec); | |
490 | ||
491 | struct drm_gem_cma_object *vc4_use_bo(struct vc4_exec_info *exec, | |
492 | uint32_t hindex); | |
493 | ||
494 | int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec); | |
495 | ||
496 | bool vc4_check_tex_size(struct vc4_exec_info *exec, | |
497 | struct drm_gem_cma_object *fbo, | |
498 | uint32_t offset, uint8_t tiling_format, | |
499 | uint32_t width, uint32_t height, uint8_t cpp); | |
d3f5168a | 500 | |
463873d5 EA |
501 | /* vc4_validate_shader.c */ |
502 | struct vc4_validated_shader_info * | |
503 | vc4_validate_shader(struct drm_gem_cma_object *shader_obj); |