2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
23 * Cmdstream submission:
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */
28 #define BO_LOCKED 0x4000
29 #define BO_PINNED 0x2000
31 static struct msm_gem_submit
*submit_create(struct drm_device
*dev
,
32 struct msm_gpu
*gpu
, int nr
)
34 struct msm_gem_submit
*submit
;
35 int sz
= sizeof(*submit
) + (nr
* sizeof(submit
->bos
[0]));
37 submit
= kmalloc(sz
, GFP_TEMPORARY
| __GFP_NOWARN
| __GFP_NORETRY
);
43 submit
->pid
= get_pid(task_pid(current
));
45 /* initially, until copy_from_user() and bo lookup succeeds: */
49 INIT_LIST_HEAD(&submit
->bo_list
);
50 ww_acquire_init(&submit
->ticket
, &reservation_ww_class
);
55 void msm_gem_submit_free(struct msm_gem_submit
*submit
)
57 fence_put(submit
->fence
);
58 list_del(&submit
->node
);
63 static int submit_lookup_objects(struct msm_gem_submit
*submit
,
64 struct drm_msm_gem_submit
*args
, struct drm_file
*file
)
69 spin_lock(&file
->table_lock
);
71 for (i
= 0; i
< args
->nr_bos
; i
++) {
72 struct drm_msm_gem_submit_bo submit_bo
;
73 struct drm_gem_object
*obj
;
74 struct msm_gem_object
*msm_obj
;
75 void __user
*userptr
=
76 u64_to_user_ptr(args
->bos
+ (i
* sizeof(submit_bo
)));
78 ret
= copy_from_user(&submit_bo
, userptr
, sizeof(submit_bo
));
84 if (submit_bo
.flags
& ~MSM_SUBMIT_BO_FLAGS
) {
85 DRM_ERROR("invalid flags: %x\n", submit_bo
.flags
);
90 submit
->bos
[i
].flags
= submit_bo
.flags
;
91 /* in validate_objects() we figure out if this is true: */
92 submit
->bos
[i
].iova
= submit_bo
.presumed
;
94 /* normally use drm_gem_object_lookup(), but for bulk lookup
95 * all under single table_lock just hit object_idr directly:
97 obj
= idr_find(&file
->object_idr
, submit_bo
.handle
);
99 DRM_ERROR("invalid handle %u at index %u\n", submit_bo
.handle
, i
);
104 msm_obj
= to_msm_bo(obj
);
106 if (!list_empty(&msm_obj
->submit_entry
)) {
107 DRM_ERROR("handle %u at index %u already on submit list\n",
108 submit_bo
.handle
, i
);
113 drm_gem_object_reference(obj
);
115 submit
->bos
[i
].obj
= msm_obj
;
117 list_add_tail(&msm_obj
->submit_entry
, &submit
->bo_list
);
122 spin_unlock(&file
->table_lock
);
127 static void submit_unlock_unpin_bo(struct msm_gem_submit
*submit
, int i
)
129 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
131 if (submit
->bos
[i
].flags
& BO_PINNED
)
132 msm_gem_put_iova(&msm_obj
->base
, submit
->gpu
->id
);
134 if (submit
->bos
[i
].flags
& BO_LOCKED
)
135 ww_mutex_unlock(&msm_obj
->resv
->lock
);
137 if (!(submit
->bos
[i
].flags
& BO_VALID
))
138 submit
->bos
[i
].iova
= 0;
140 submit
->bos
[i
].flags
&= ~(BO_LOCKED
| BO_PINNED
);
143 /* This is where we make sure all the bo's are reserved and pin'd: */
144 static int submit_lock_objects(struct msm_gem_submit
*submit
)
146 int contended
, slow_locked
= -1, i
, ret
= 0;
149 for (i
= 0; i
< submit
->nr_bos
; i
++) {
150 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
152 if (slow_locked
== i
)
157 if (!(submit
->bos
[i
].flags
& BO_LOCKED
)) {
158 ret
= ww_mutex_lock_interruptible(&msm_obj
->resv
->lock
,
162 submit
->bos
[i
].flags
|= BO_LOCKED
;
166 ww_acquire_done(&submit
->ticket
);
172 submit_unlock_unpin_bo(submit
, i
);
175 submit_unlock_unpin_bo(submit
, slow_locked
);
177 if (ret
== -EDEADLK
) {
178 struct msm_gem_object
*msm_obj
= submit
->bos
[contended
].obj
;
179 /* we lost out in a seqno race, lock and retry.. */
180 ret
= ww_mutex_lock_slow_interruptible(&msm_obj
->resv
->lock
,
183 submit
->bos
[contended
].flags
|= BO_LOCKED
;
184 slow_locked
= contended
;
192 static int submit_fence_sync(struct msm_gem_submit
*submit
)
196 for (i
= 0; i
< submit
->nr_bos
; i
++) {
197 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
198 bool write
= submit
->bos
[i
].flags
& MSM_SUBMIT_BO_WRITE
;
200 ret
= msm_gem_sync_object(&msm_obj
->base
, submit
->gpu
->fctx
, write
);
208 static int submit_pin_objects(struct msm_gem_submit
*submit
)
212 submit
->valid
= true;
214 for (i
= 0; i
< submit
->nr_bos
; i
++) {
215 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
218 /* if locking succeeded, pin bo: */
219 ret
= msm_gem_get_iova_locked(&msm_obj
->base
,
220 submit
->gpu
->id
, &iova
);
225 submit
->bos
[i
].flags
|= BO_PINNED
;
227 if (iova
== submit
->bos
[i
].iova
) {
228 submit
->bos
[i
].flags
|= BO_VALID
;
230 submit
->bos
[i
].iova
= iova
;
231 /* iova changed, so address in cmdstream is not valid: */
232 submit
->bos
[i
].flags
&= ~BO_VALID
;
233 submit
->valid
= false;
240 static int submit_bo(struct msm_gem_submit
*submit
, uint32_t idx
,
241 struct msm_gem_object
**obj
, uint32_t *iova
, bool *valid
)
243 if (idx
>= submit
->nr_bos
) {
244 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
245 idx
, submit
->nr_bos
);
250 *obj
= submit
->bos
[idx
].obj
;
252 *iova
= submit
->bos
[idx
].iova
;
254 *valid
= !!(submit
->bos
[idx
].flags
& BO_VALID
);
259 /* process the reloc's and patch up the cmdstream as needed: */
260 static int submit_reloc(struct msm_gem_submit
*submit
, struct msm_gem_object
*obj
,
261 uint32_t offset
, uint32_t nr_relocs
, uint64_t relocs
)
263 uint32_t i
, last_offset
= 0;
268 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset
);
272 /* For now, just map the entire thing. Eventually we probably
273 * to do it page-by-page, w/ kmap() if not vmap()d..
275 ptr
= msm_gem_vaddr_locked(&obj
->base
);
279 DBG("failed to map: %d", ret
);
283 for (i
= 0; i
< nr_relocs
; i
++) {
284 struct drm_msm_gem_submit_reloc submit_reloc
;
285 void __user
*userptr
=
286 u64_to_user_ptr(relocs
+ (i
* sizeof(submit_reloc
)));
290 ret
= copy_from_user(&submit_reloc
, userptr
, sizeof(submit_reloc
));
294 if (submit_reloc
.submit_offset
% 4) {
295 DRM_ERROR("non-aligned reloc offset: %u\n",
296 submit_reloc
.submit_offset
);
300 /* offset in dwords: */
301 off
= submit_reloc
.submit_offset
/ 4;
303 if ((off
>= (obj
->base
.size
/ 4)) ||
304 (off
< last_offset
)) {
305 DRM_ERROR("invalid offset %u at reloc %u\n", off
, i
);
309 ret
= submit_bo(submit
, submit_reloc
.reloc_idx
, NULL
, &iova
, &valid
);
316 iova
+= submit_reloc
.reloc_offset
;
318 if (submit_reloc
.shift
< 0)
319 iova
>>= -submit_reloc
.shift
;
321 iova
<<= submit_reloc
.shift
;
323 ptr
[off
] = iova
| submit_reloc
.or;
331 static void submit_cleanup(struct msm_gem_submit
*submit
)
335 for (i
= 0; i
< submit
->nr_bos
; i
++) {
336 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
337 submit_unlock_unpin_bo(submit
, i
);
338 list_del_init(&msm_obj
->submit_entry
);
339 drm_gem_object_unreference(&msm_obj
->base
);
342 ww_acquire_fini(&submit
->ticket
);
345 int msm_ioctl_gem_submit(struct drm_device
*dev
, void *data
,
346 struct drm_file
*file
)
348 struct msm_drm_private
*priv
= dev
->dev_private
;
349 struct drm_msm_gem_submit
*args
= data
;
350 struct msm_file_private
*ctx
= file
->driver_priv
;
351 struct msm_gem_submit
*submit
;
352 struct msm_gpu
*gpu
= priv
->gpu
;
359 /* for now, we just have 3d pipe.. eventually this would need to
360 * be more clever to dispatch to appropriate gpu module:
362 if (args
->pipe
!= MSM_PIPE_3D0
)
365 if (args
->nr_cmds
> MAX_CMDS
)
368 submit
= submit_create(dev
, gpu
, args
->nr_bos
);
372 mutex_lock(&dev
->struct_mutex
);
374 ret
= submit_lookup_objects(submit
, args
, file
);
378 ret
= submit_lock_objects(submit
);
382 ret
= submit_fence_sync(submit
);
386 ret
= submit_pin_objects(submit
);
390 for (i
= 0; i
< args
->nr_cmds
; i
++) {
391 struct drm_msm_gem_submit_cmd submit_cmd
;
392 void __user
*userptr
=
393 u64_to_user_ptr(args
->cmds
+ (i
* sizeof(submit_cmd
)));
394 struct msm_gem_object
*msm_obj
;
397 ret
= copy_from_user(&submit_cmd
, userptr
, sizeof(submit_cmd
));
403 /* validate input from userspace: */
404 switch (submit_cmd
.type
) {
405 case MSM_SUBMIT_CMD_BUF
:
406 case MSM_SUBMIT_CMD_IB_TARGET_BUF
:
407 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF
:
410 DRM_ERROR("invalid type: %08x\n", submit_cmd
.type
);
415 ret
= submit_bo(submit
, submit_cmd
.submit_idx
,
416 &msm_obj
, &iova
, NULL
);
420 if (submit_cmd
.size
% 4) {
421 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
427 if ((submit_cmd
.size
+ submit_cmd
.submit_offset
) >=
428 msm_obj
->base
.size
) {
429 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd
.size
);
434 submit
->cmd
[i
].type
= submit_cmd
.type
;
435 submit
->cmd
[i
].size
= submit_cmd
.size
/ 4;
436 submit
->cmd
[i
].iova
= iova
+ submit_cmd
.submit_offset
;
437 submit
->cmd
[i
].idx
= submit_cmd
.submit_idx
;
442 ret
= submit_reloc(submit
, msm_obj
, submit_cmd
.submit_offset
,
443 submit_cmd
.nr_relocs
, submit_cmd
.relocs
);
450 ret
= msm_gpu_submit(gpu
, submit
, ctx
);
452 args
->fence
= submit
->fence
->seqno
;
455 submit_cleanup(submit
);
457 msm_gem_submit_free(submit
);
458 mutex_unlock(&dev
->struct_mutex
);