Commit | Line | Data |
---|---|---|
6ee73861 BS |
1 | /* |
2 | * Copyright (C) 2008 Ben Skeggs. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining | |
6 | * a copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sublicense, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * The above copyright notice and this permission notice (including the | |
14 | * next paragraph) shall be included in all copies or substantial | |
15 | * portions of the Software. | |
16 | * | |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. | |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE | |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION | |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | |
24 | * | |
25 | */ | |
6ee73861 | 26 | |
ebb945a9 | 27 | #include <linux/dma-buf.h> |
94580299 | 28 | #include <nouveau_drm.h> |
ebb945a9 BS |
29 | |
30 | #include <subdev/fb.h> | |
31 | ||
32 | #include "nouveau_drm.h" | |
6ee73861 | 33 | #include "nouveau_dma.h" |
d375e7d5 | 34 | #include "nouveau_fence.h" |
ebb945a9 | 35 | #include "nouveau_abi16.h" |
6ee73861 | 36 | |
ebb945a9 BS |
37 | #include "nouveau_ttm.h" |
38 | #include "nouveau_gem.h" | |
6ee73861 BS |
39 | |
40 | int | |
41 | nouveau_gem_object_new(struct drm_gem_object *gem) | |
42 | { | |
43 | return 0; | |
44 | } | |
45 | ||
46 | void | |
47 | nouveau_gem_object_del(struct drm_gem_object *gem) | |
48 | { | |
49 | struct nouveau_bo *nvbo = gem->driver_private; | |
50 | struct ttm_buffer_object *bo = &nvbo->bo; | |
51 | ||
52 | if (!nvbo) | |
53 | return; | |
54 | nvbo->gem = NULL; | |
55 | ||
6ee73861 BS |
56 | if (unlikely(nvbo->pin_refcnt)) { |
57 | nvbo->pin_refcnt = 1; | |
58 | nouveau_bo_unpin(nvbo); | |
59 | } | |
60 | ||
22b33e8e DA |
61 | if (gem->import_attach) |
62 | drm_prime_gem_destroy(gem, nvbo->bo.sg); | |
63 | ||
6ee73861 | 64 | ttm_bo_unref(&bo); |
fd632aa3 DV |
65 | |
66 | drm_gem_object_release(gem); | |
67 | kfree(gem); | |
6ee73861 BS |
68 | } |
69 | ||
639212d0 BS |
70 | int |
71 | nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) | |
72 | { | |
ebb945a9 | 73 | struct nouveau_cli *cli = nouveau_cli(file_priv); |
2fd3db6f BS |
74 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
75 | struct nouveau_vma *vma; | |
76 | int ret; | |
639212d0 | 77 | |
ebb945a9 | 78 | if (!cli->base.vm) |
639212d0 BS |
79 | return 0; |
80 | ||
2fd3db6f BS |
81 | ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); |
82 | if (ret) | |
83 | return ret; | |
84 | ||
ebb945a9 | 85 | vma = nouveau_bo_vma_find(nvbo, cli->base.vm); |
2fd3db6f BS |
86 | if (!vma) { |
87 | vma = kzalloc(sizeof(*vma), GFP_KERNEL); | |
88 | if (!vma) { | |
89 | ret = -ENOMEM; | |
90 | goto out; | |
91 | } | |
92 | ||
ebb945a9 | 93 | ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma); |
2fd3db6f BS |
94 | if (ret) { |
95 | kfree(vma); | |
96 | goto out; | |
97 | } | |
98 | } else { | |
99 | vma->refcount++; | |
100 | } | |
101 | ||
102 | out: | |
103 | ttm_bo_unreserve(&nvbo->bo); | |
104 | return ret; | |
639212d0 BS |
105 | } |
106 | ||
107 | void | |
108 | nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) | |
109 | { | |
ebb945a9 | 110 | struct nouveau_cli *cli = nouveau_cli(file_priv); |
2fd3db6f BS |
111 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
112 | struct nouveau_vma *vma; | |
113 | int ret; | |
639212d0 | 114 | |
ebb945a9 | 115 | if (!cli->base.vm) |
639212d0 | 116 | return; |
2fd3db6f BS |
117 | |
118 | ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); | |
119 | if (ret) | |
120 | return; | |
121 | ||
ebb945a9 | 122 | vma = nouveau_bo_vma_find(nvbo, cli->base.vm); |
2fd3db6f | 123 | if (vma) { |
8fe198b2 | 124 | if (--vma->refcount == 0) { |
2fd3db6f | 125 | nouveau_bo_vma_del(nvbo, vma); |
8fe198b2 MS |
126 | kfree(vma); |
127 | } | |
2fd3db6f BS |
128 | } |
129 | ttm_bo_unreserve(&nvbo->bo); | |
639212d0 BS |
130 | } |
131 | ||
6ee73861 | 132 | int |
f6d4e621 BS |
133 | nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, |
134 | uint32_t tile_mode, uint32_t tile_flags, | |
135 | struct nouveau_bo **pnvbo) | |
6ee73861 | 136 | { |
ebb945a9 | 137 | struct nouveau_drm *drm = nouveau_drm(dev); |
6ee73861 | 138 | struct nouveau_bo *nvbo; |
6ba9a683 | 139 | u32 flags = 0; |
6ee73861 BS |
140 | int ret; |
141 | ||
6ba9a683 BS |
142 | if (domain & NOUVEAU_GEM_DOMAIN_VRAM) |
143 | flags |= TTM_PL_FLAG_VRAM; | |
144 | if (domain & NOUVEAU_GEM_DOMAIN_GART) | |
145 | flags |= TTM_PL_FLAG_TT; | |
146 | if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) | |
147 | flags |= TTM_PL_FLAG_SYSTEM; | |
148 | ||
7375c95b | 149 | ret = nouveau_bo_new(dev, size, align, flags, tile_mode, |
22b33e8e | 150 | tile_flags, NULL, pnvbo); |
6ee73861 BS |
151 | if (ret) |
152 | return ret; | |
153 | nvbo = *pnvbo; | |
154 | ||
db5c8e29 BS |
155 | /* we restrict allowed domains on nv50+ to only the types |
156 | * that were requested at creation time. not possibly on | |
157 | * earlier chips without busting the ABI. | |
158 | */ | |
159 | nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | | |
160 | NOUVEAU_GEM_DOMAIN_GART; | |
ebb945a9 | 161 | if (nv_device(drm->device)->card_type >= NV_50) |
db5c8e29 BS |
162 | nvbo->valid_domains &= domain; |
163 | ||
6ee73861 BS |
164 | nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size); |
165 | if (!nvbo->gem) { | |
166 | nouveau_bo_ref(NULL, pnvbo); | |
167 | return -ENOMEM; | |
168 | } | |
169 | ||
5df23979 | 170 | nvbo->bo.persistent_swap_storage = nvbo->gem->filp; |
6ee73861 BS |
171 | nvbo->gem->driver_private = nvbo; |
172 | return 0; | |
173 | } | |
174 | ||
175 | static int | |
e758a311 BS |
176 | nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, |
177 | struct drm_nouveau_gem_info *rep) | |
6ee73861 | 178 | { |
ebb945a9 | 179 | struct nouveau_cli *cli = nouveau_cli(file_priv); |
6ee73861 | 180 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
e758a311 | 181 | struct nouveau_vma *vma; |
6ee73861 BS |
182 | |
183 | if (nvbo->bo.mem.mem_type == TTM_PL_TT) | |
184 | rep->domain = NOUVEAU_GEM_DOMAIN_GART; | |
185 | else | |
186 | rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; | |
187 | ||
e758a311 | 188 | rep->offset = nvbo->bo.offset; |
ebb945a9 BS |
189 | if (cli->base.vm) { |
190 | vma = nouveau_bo_vma_find(nvbo, cli->base.vm); | |
e758a311 BS |
191 | if (!vma) |
192 | return -EINVAL; | |
193 | ||
194 | rep->offset = vma->offset; | |
195 | } | |
196 | ||
6ee73861 | 197 | rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; |
d550c41e | 198 | rep->map_handle = nvbo->bo.addr_space_offset; |
6ee73861 BS |
199 | rep->tile_mode = nvbo->tile_mode; |
200 | rep->tile_flags = nvbo->tile_flags; | |
201 | return 0; | |
202 | } | |
203 | ||
6ee73861 BS |
204 | int |
205 | nouveau_gem_ioctl_new(struct drm_device *dev, void *data, | |
206 | struct drm_file *file_priv) | |
207 | { | |
ebb945a9 BS |
208 | struct nouveau_drm *drm = nouveau_drm(dev); |
209 | struct nouveau_fb *pfb = nouveau_fb(drm->device); | |
6ee73861 BS |
210 | struct drm_nouveau_gem_new *req = data; |
211 | struct nouveau_bo *nvbo = NULL; | |
6ee73861 BS |
212 | int ret = 0; |
213 | ||
ebb945a9 | 214 | drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping; |
6ee73861 | 215 | |
ebb945a9 BS |
216 | if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { |
217 | NV_ERROR(drm, "bad page flags: 0x%08x\n", req->info.tile_flags); | |
6ee73861 | 218 | return -EINVAL; |
60d2a88a | 219 | } |
6ee73861 | 220 | |
f6d4e621 | 221 | ret = nouveau_gem_new(dev, req->info.size, req->align, |
6ba9a683 BS |
222 | req->info.domain, req->info.tile_mode, |
223 | req->info.tile_flags, &nvbo); | |
6ee73861 BS |
224 | if (ret) |
225 | return ret; | |
226 | ||
6ee73861 | 227 | ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle); |
e758a311 BS |
228 | if (ret == 0) { |
229 | ret = nouveau_gem_info(file_priv, nvbo->gem, &req->info); | |
230 | if (ret) | |
231 | drm_gem_handle_delete(file_priv, req->info.handle); | |
232 | } | |
233 | ||
29d08b3e DA |
234 | /* drop reference from allocate - handle holds it now */ |
235 | drm_gem_object_unreference_unlocked(nvbo->gem); | |
6ee73861 BS |
236 | return ret; |
237 | } | |
238 | ||
239 | static int | |
240 | nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, | |
241 | uint32_t write_domains, uint32_t valid_domains) | |
242 | { | |
243 | struct nouveau_bo *nvbo = gem->driver_private; | |
244 | struct ttm_buffer_object *bo = &nvbo->bo; | |
db5c8e29 | 245 | uint32_t domains = valid_domains & nvbo->valid_domains & |
78ad0f7b FJ |
246 | (write_domains ? write_domains : read_domains); |
247 | uint32_t pref_flags = 0, valid_flags = 0; | |
6ee73861 | 248 | |
78ad0f7b | 249 | if (!domains) |
6ee73861 BS |
250 | return -EINVAL; |
251 | ||
78ad0f7b FJ |
252 | if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) |
253 | valid_flags |= TTM_PL_FLAG_VRAM; | |
254 | ||
255 | if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) | |
256 | valid_flags |= TTM_PL_FLAG_TT; | |
257 | ||
258 | if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && | |
259 | bo->mem.mem_type == TTM_PL_VRAM) | |
260 | pref_flags |= TTM_PL_FLAG_VRAM; | |
261 | ||
262 | else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && | |
263 | bo->mem.mem_type == TTM_PL_TT) | |
264 | pref_flags |= TTM_PL_FLAG_TT; | |
265 | ||
266 | else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) | |
267 | pref_flags |= TTM_PL_FLAG_VRAM; | |
268 | ||
269 | else | |
270 | pref_flags |= TTM_PL_FLAG_TT; | |
271 | ||
272 | nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); | |
6ee73861 | 273 | |
6ee73861 BS |
274 | return 0; |
275 | } | |
276 | ||
277 | struct validate_op { | |
6ee73861 BS |
278 | struct list_head vram_list; |
279 | struct list_head gart_list; | |
280 | struct list_head both_list; | |
281 | }; | |
282 | ||
283 | static void | |
284 | validate_fini_list(struct list_head *list, struct nouveau_fence *fence) | |
285 | { | |
286 | struct list_head *entry, *tmp; | |
287 | struct nouveau_bo *nvbo; | |
288 | ||
289 | list_for_each_safe(entry, tmp, list) { | |
290 | nvbo = list_entry(entry, struct nouveau_bo, entry); | |
332b242f FJ |
291 | |
292 | nouveau_bo_fence(nvbo, fence); | |
6ee73861 | 293 | |
a1606a95 BS |
294 | if (unlikely(nvbo->validate_mapped)) { |
295 | ttm_bo_kunmap(&nvbo->kmap); | |
296 | nvbo->validate_mapped = false; | |
297 | } | |
298 | ||
6ee73861 BS |
299 | list_del(&nvbo->entry); |
300 | nvbo->reserved_by = NULL; | |
301 | ttm_bo_unreserve(&nvbo->bo); | |
374c3af8 | 302 | drm_gem_object_unreference_unlocked(nvbo->gem); |
6ee73861 BS |
303 | } |
304 | } | |
305 | ||
306 | static void | |
234896a7 | 307 | validate_fini(struct validate_op *op, struct nouveau_fence* fence) |
6ee73861 | 308 | { |
234896a7 LB |
309 | validate_fini_list(&op->vram_list, fence); |
310 | validate_fini_list(&op->gart_list, fence); | |
311 | validate_fini_list(&op->both_list, fence); | |
6ee73861 BS |
312 | } |
313 | ||
314 | static int | |
315 | validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, | |
316 | struct drm_nouveau_gem_pushbuf_bo *pbbo, | |
317 | int nr_buffers, struct validate_op *op) | |
318 | { | |
ebb945a9 BS |
319 | struct drm_device *dev = chan->drm->dev; |
320 | struct nouveau_drm *drm = nouveau_drm(dev); | |
6ee73861 BS |
321 | uint32_t sequence; |
322 | int trycnt = 0; | |
323 | int ret, i; | |
324 | ||
ebb945a9 | 325 | sequence = atomic_add_return(1, &drm->ttm.validate_sequence); |
6ee73861 BS |
326 | retry: |
327 | if (++trycnt > 100000) { | |
ebb945a9 | 328 | NV_ERROR(drm, "%s failed and gave up.\n", __func__); |
6ee73861 BS |
329 | return -EINVAL; |
330 | } | |
331 | ||
332 | for (i = 0; i < nr_buffers; i++) { | |
333 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; | |
334 | struct drm_gem_object *gem; | |
335 | struct nouveau_bo *nvbo; | |
336 | ||
337 | gem = drm_gem_object_lookup(dev, file_priv, b->handle); | |
338 | if (!gem) { | |
ebb945a9 | 339 | NV_ERROR(drm, "Unknown handle 0x%08x\n", b->handle); |
6ee73861 | 340 | validate_fini(op, NULL); |
bf79cb91 | 341 | return -ENOENT; |
6ee73861 BS |
342 | } |
343 | nvbo = gem->driver_private; | |
344 | ||
345 | if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { | |
ebb945a9 | 346 | NV_ERROR(drm, "multiple instances of buffer %d on " |
6ee73861 | 347 | "validation list\n", b->handle); |
5086f69e | 348 | drm_gem_object_unreference_unlocked(gem); |
6ee73861 BS |
349 | validate_fini(op, NULL); |
350 | return -EINVAL; | |
351 | } | |
352 | ||
938c40ed | 353 | ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence); |
6ee73861 BS |
354 | if (ret) { |
355 | validate_fini(op, NULL); | |
938c40ed BS |
356 | if (unlikely(ret == -EAGAIN)) |
357 | ret = ttm_bo_wait_unreserved(&nvbo->bo, true); | |
374c3af8 | 358 | drm_gem_object_unreference_unlocked(gem); |
938c40ed BS |
359 | if (unlikely(ret)) { |
360 | if (ret != -ERESTARTSYS) | |
ebb945a9 | 361 | NV_ERROR(drm, "fail reserve\n"); |
6ee73861 | 362 | return ret; |
a1606a95 | 363 | } |
6ee73861 BS |
364 | goto retry; |
365 | } | |
366 | ||
a1606a95 | 367 | b->user_priv = (uint64_t)(unsigned long)nvbo; |
6ee73861 BS |
368 | nvbo->reserved_by = file_priv; |
369 | nvbo->pbbo_index = i; | |
370 | if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && | |
371 | (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) | |
372 | list_add_tail(&nvbo->entry, &op->both_list); | |
373 | else | |
374 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) | |
375 | list_add_tail(&nvbo->entry, &op->vram_list); | |
376 | else | |
377 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) | |
378 | list_add_tail(&nvbo->entry, &op->gart_list); | |
379 | else { | |
ebb945a9 | 380 | NV_ERROR(drm, "invalid valid domains: 0x%08x\n", |
6ee73861 | 381 | b->valid_domains); |
0208843d | 382 | list_add_tail(&nvbo->entry, &op->both_list); |
6ee73861 BS |
383 | validate_fini(op, NULL); |
384 | return -EINVAL; | |
385 | } | |
6ee73861 BS |
386 | } |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
525895ba BS |
391 | static int |
392 | validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) | |
393 | { | |
394 | struct nouveau_fence *fence = NULL; | |
395 | int ret = 0; | |
396 | ||
397 | spin_lock(&nvbo->bo.bdev->fence_lock); | |
398 | if (nvbo->bo.sync_obj) | |
399 | fence = nouveau_fence_ref(nvbo->bo.sync_obj); | |
400 | spin_unlock(&nvbo->bo.bdev->fence_lock); | |
401 | ||
402 | if (fence) { | |
403 | ret = nouveau_fence_sync(fence, chan); | |
404 | nouveau_fence_unref(&fence); | |
405 | } | |
406 | ||
407 | return ret; | |
408 | } | |
409 | ||
6ee73861 BS |
410 | static int |
411 | validate_list(struct nouveau_channel *chan, struct list_head *list, | |
412 | struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr) | |
413 | { | |
ebb945a9 | 414 | struct nouveau_drm *drm = chan->drm; |
6ee73861 BS |
415 | struct drm_nouveau_gem_pushbuf_bo __user *upbbo = |
416 | (void __force __user *)(uintptr_t)user_pbbo_ptr; | |
417 | struct nouveau_bo *nvbo; | |
418 | int ret, relocs = 0; | |
419 | ||
420 | list_for_each_entry(nvbo, list, entry) { | |
421 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; | |
6ee73861 | 422 | |
525895ba | 423 | ret = validate_sync(chan, nvbo); |
415e6186 | 424 | if (unlikely(ret)) { |
ebb945a9 | 425 | NV_ERROR(drm, "fail pre-validate sync\n"); |
415e6186 | 426 | return ret; |
6ee73861 BS |
427 | } |
428 | ||
429 | ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains, | |
430 | b->write_domains, | |
431 | b->valid_domains); | |
a1606a95 | 432 | if (unlikely(ret)) { |
ebb945a9 | 433 | NV_ERROR(drm, "fail set_domain\n"); |
6ee73861 | 434 | return ret; |
a1606a95 | 435 | } |
6ee73861 | 436 | |
7a45d764 | 437 | ret = nouveau_bo_validate(nvbo, true, false, false); |
a1606a95 | 438 | if (unlikely(ret)) { |
938c40ed | 439 | if (ret != -ERESTARTSYS) |
ebb945a9 | 440 | NV_ERROR(drm, "fail ttm_validate\n"); |
6ee73861 | 441 | return ret; |
a1606a95 | 442 | } |
6ee73861 | 443 | |
525895ba | 444 | ret = validate_sync(chan, nvbo); |
415e6186 | 445 | if (unlikely(ret)) { |
ebb945a9 | 446 | NV_ERROR(drm, "fail post-validate sync\n"); |
415e6186 BS |
447 | return ret; |
448 | } | |
449 | ||
ebb945a9 | 450 | if (nv_device(drm->device)->card_type < NV_50) { |
a3fcd0a9 BS |
451 | if (nvbo->bo.offset == b->presumed.offset && |
452 | ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && | |
453 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || | |
454 | (nvbo->bo.mem.mem_type == TTM_PL_TT && | |
455 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) | |
456 | continue; | |
457 | ||
458 | if (nvbo->bo.mem.mem_type == TTM_PL_TT) | |
459 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; | |
460 | else | |
461 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; | |
462 | b->presumed.offset = nvbo->bo.offset; | |
463 | b->presumed.valid = 0; | |
464 | relocs++; | |
465 | ||
466 | if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed, | |
467 | &b->presumed, sizeof(b->presumed))) | |
468 | return -EFAULT; | |
469 | } | |
6ee73861 BS |
470 | } |
471 | ||
472 | return relocs; | |
473 | } | |
474 | ||
475 | static int | |
476 | nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, | |
477 | struct drm_file *file_priv, | |
478 | struct drm_nouveau_gem_pushbuf_bo *pbbo, | |
479 | uint64_t user_buffers, int nr_buffers, | |
480 | struct validate_op *op, int *apply_relocs) | |
481 | { | |
ebb945a9 | 482 | struct nouveau_drm *drm = chan->drm; |
6ee73861 BS |
483 | int ret, relocs = 0; |
484 | ||
485 | INIT_LIST_HEAD(&op->vram_list); | |
486 | INIT_LIST_HEAD(&op->gart_list); | |
487 | INIT_LIST_HEAD(&op->both_list); | |
488 | ||
6ee73861 BS |
489 | if (nr_buffers == 0) |
490 | return 0; | |
491 | ||
492 | ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); | |
a1606a95 | 493 | if (unlikely(ret)) { |
938c40ed | 494 | if (ret != -ERESTARTSYS) |
ebb945a9 | 495 | NV_ERROR(drm, "validate_init\n"); |
6ee73861 | 496 | return ret; |
a1606a95 | 497 | } |
6ee73861 BS |
498 | |
499 | ret = validate_list(chan, &op->vram_list, pbbo, user_buffers); | |
500 | if (unlikely(ret < 0)) { | |
938c40ed | 501 | if (ret != -ERESTARTSYS) |
ebb945a9 | 502 | NV_ERROR(drm, "validate vram_list\n"); |
6ee73861 BS |
503 | validate_fini(op, NULL); |
504 | return ret; | |
505 | } | |
506 | relocs += ret; | |
507 | ||
508 | ret = validate_list(chan, &op->gart_list, pbbo, user_buffers); | |
509 | if (unlikely(ret < 0)) { | |
938c40ed | 510 | if (ret != -ERESTARTSYS) |
ebb945a9 | 511 | NV_ERROR(drm, "validate gart_list\n"); |
6ee73861 BS |
512 | validate_fini(op, NULL); |
513 | return ret; | |
514 | } | |
515 | relocs += ret; | |
516 | ||
517 | ret = validate_list(chan, &op->both_list, pbbo, user_buffers); | |
518 | if (unlikely(ret < 0)) { | |
938c40ed | 519 | if (ret != -ERESTARTSYS) |
ebb945a9 | 520 | NV_ERROR(drm, "validate both_list\n"); |
6ee73861 BS |
521 | validate_fini(op, NULL); |
522 | return ret; | |
523 | } | |
524 | relocs += ret; | |
525 | ||
526 | *apply_relocs = relocs; | |
527 | return 0; | |
528 | } | |
529 | ||
530 | static inline void * | |
531 | u_memcpya(uint64_t user, unsigned nmemb, unsigned size) | |
532 | { | |
533 | void *mem; | |
534 | void __user *userptr = (void __force __user *)(uintptr_t)user; | |
535 | ||
536 | mem = kmalloc(nmemb * size, GFP_KERNEL); | |
537 | if (!mem) | |
538 | return ERR_PTR(-ENOMEM); | |
539 | ||
540 | if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) { | |
541 | kfree(mem); | |
542 | return ERR_PTR(-EFAULT); | |
543 | } | |
544 | ||
545 | return mem; | |
546 | } | |
547 | ||
548 | static int | |
a1606a95 BS |
549 | nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev, |
550 | struct drm_nouveau_gem_pushbuf *req, | |
551 | struct drm_nouveau_gem_pushbuf_bo *bo) | |
6ee73861 | 552 | { |
ebb945a9 | 553 | struct nouveau_drm *drm = nouveau_newpriv(dev); |
6ee73861 | 554 | struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; |
12f735b7 LB |
555 | int ret = 0; |
556 | unsigned i; | |
6ee73861 | 557 | |
a1606a95 | 558 | reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); |
6ee73861 BS |
559 | if (IS_ERR(reloc)) |
560 | return PTR_ERR(reloc); | |
561 | ||
a1606a95 | 562 | for (i = 0; i < req->nr_relocs; i++) { |
6ee73861 BS |
563 | struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; |
564 | struct drm_nouveau_gem_pushbuf_bo *b; | |
a1606a95 | 565 | struct nouveau_bo *nvbo; |
6ee73861 BS |
566 | uint32_t data; |
567 | ||
a1606a95 | 568 | if (unlikely(r->bo_index > req->nr_buffers)) { |
ebb945a9 | 569 | NV_ERROR(drm, "reloc bo index invalid\n"); |
6ee73861 BS |
570 | ret = -EINVAL; |
571 | break; | |
572 | } | |
573 | ||
574 | b = &bo[r->bo_index]; | |
a1606a95 | 575 | if (b->presumed.valid) |
6ee73861 BS |
576 | continue; |
577 | ||
a1606a95 | 578 | if (unlikely(r->reloc_bo_index > req->nr_buffers)) { |
ebb945a9 | 579 | NV_ERROR(drm, "reloc container bo index invalid\n"); |
a1606a95 BS |
580 | ret = -EINVAL; |
581 | break; | |
582 | } | |
583 | nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; | |
584 | ||
585 | if (unlikely(r->reloc_bo_offset + 4 > | |
586 | nvbo->bo.mem.num_pages << PAGE_SHIFT)) { | |
ebb945a9 | 587 | NV_ERROR(drm, "reloc outside of bo\n"); |
a1606a95 BS |
588 | ret = -EINVAL; |
589 | break; | |
590 | } | |
591 | ||
592 | if (!nvbo->kmap.virtual) { | |
593 | ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, | |
594 | &nvbo->kmap); | |
595 | if (ret) { | |
ebb945a9 | 596 | NV_ERROR(drm, "failed kmap for reloc\n"); |
a1606a95 BS |
597 | break; |
598 | } | |
599 | nvbo->validate_mapped = true; | |
600 | } | |
601 | ||
6ee73861 | 602 | if (r->flags & NOUVEAU_GEM_RELOC_LOW) |
a1606a95 | 603 | data = b->presumed.offset + r->data; |
6ee73861 BS |
604 | else |
605 | if (r->flags & NOUVEAU_GEM_RELOC_HIGH) | |
a1606a95 | 606 | data = (b->presumed.offset + r->data) >> 32; |
6ee73861 BS |
607 | else |
608 | data = r->data; | |
609 | ||
610 | if (r->flags & NOUVEAU_GEM_RELOC_OR) { | |
a1606a95 | 611 | if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) |
6ee73861 BS |
612 | data |= r->tor; |
613 | else | |
614 | data |= r->vor; | |
615 | } | |
616 | ||
702adba2 | 617 | spin_lock(&nvbo->bo.bdev->fence_lock); |
a1606a95 | 618 | ret = ttm_bo_wait(&nvbo->bo, false, false, false); |
702adba2 | 619 | spin_unlock(&nvbo->bo.bdev->fence_lock); |
a1606a95 | 620 | if (ret) { |
ebb945a9 | 621 | NV_ERROR(drm, "reloc wait_idle failed: %d\n", ret); |
a1606a95 BS |
622 | break; |
623 | } | |
a1606a95 BS |
624 | |
625 | nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); | |
6ee73861 BS |
626 | } |
627 | ||
628 | kfree(reloc); | |
629 | return ret; | |
630 | } | |
631 | ||
632 | int | |
633 | nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, | |
634 | struct drm_file *file_priv) | |
635 | { | |
ebb945a9 BS |
636 | struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); |
637 | struct nouveau_abi16_chan *temp; | |
638 | struct nouveau_drm *drm = nouveau_drm(dev); | |
6ee73861 | 639 | struct drm_nouveau_gem_pushbuf *req = data; |
a1606a95 BS |
640 | struct drm_nouveau_gem_pushbuf_push *push; |
641 | struct drm_nouveau_gem_pushbuf_bo *bo; | |
ebb945a9 | 642 | struct nouveau_channel *chan = NULL; |
6ee73861 | 643 | struct validate_op op; |
6e86e041 | 644 | struct nouveau_fence *fence = NULL; |
a1606a95 | 645 | int i, j, ret = 0, do_reloc = 0; |
6ee73861 | 646 | |
ebb945a9 BS |
647 | if (unlikely(!abi16)) |
648 | return -ENOMEM; | |
6ee73861 | 649 | |
ebb945a9 BS |
650 | list_for_each_entry(temp, &abi16->channels, head) { |
651 | if (temp->chan->handle == (NVDRM_CHAN | req->channel)) { | |
652 | chan = temp->chan; | |
653 | break; | |
654 | } | |
655 | } | |
656 | ||
657 | if (!chan) | |
658 | return nouveau_abi16_put(abi16, -ENOENT); | |
659 | ||
660 | req->vram_available = drm->gem.vram_available; | |
661 | req->gart_available = drm->gem.gart_available; | |
a1606a95 BS |
662 | if (unlikely(req->nr_push == 0)) |
663 | goto out_next; | |
6ee73861 | 664 | |
a1606a95 | 665 | if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { |
ebb945a9 | 666 | NV_ERROR(drm, "pushbuf push count exceeds limit: %d max %d\n", |
a1606a95 | 667 | req->nr_push, NOUVEAU_GEM_MAX_PUSH); |
ebb945a9 | 668 | return nouveau_abi16_put(abi16, -EINVAL); |
6ee73861 BS |
669 | } |
670 | ||
a1606a95 | 671 | if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { |
ebb945a9 | 672 | NV_ERROR(drm, "pushbuf bo count exceeds limit: %d max %d\n", |
a1606a95 | 673 | req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); |
ebb945a9 | 674 | return nouveau_abi16_put(abi16, -EINVAL); |
6ee73861 BS |
675 | } |
676 | ||
a1606a95 | 677 | if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { |
ebb945a9 | 678 | NV_ERROR(drm, "pushbuf reloc count exceeds limit: %d max %d\n", |
a1606a95 | 679 | req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); |
ebb945a9 | 680 | return nouveau_abi16_put(abi16, -EINVAL); |
6ee73861 BS |
681 | } |
682 | ||
a1606a95 | 683 | push = u_memcpya(req->push, req->nr_push, sizeof(*push)); |
ebb945a9 BS |
684 | if (IS_ERR(push)) |
685 | return nouveau_abi16_put(abi16, PTR_ERR(push)); | |
a1606a95 | 686 | |
6ee73861 | 687 | bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); |
a1606a95 BS |
688 | if (IS_ERR(bo)) { |
689 | kfree(push); | |
ebb945a9 | 690 | return nouveau_abi16_put(abi16, PTR_ERR(bo)); |
a1606a95 | 691 | } |
6ee73861 | 692 | |
accf9496 | 693 | /* Ensure all push buffers are on validate list */ |
415e6186 BS |
694 | for (i = 0; i < req->nr_push; i++) { |
695 | if (push[i].bo_index >= req->nr_buffers) { | |
ebb945a9 | 696 | NV_ERROR(drm, "push %d buffer not in list\n", i); |
415e6186 | 697 | ret = -EINVAL; |
7fa0cba2 | 698 | goto out_prevalid; |
415e6186 | 699 | } |
415e6186 BS |
700 | } |
701 | ||
6ee73861 BS |
702 | /* Validate buffer list */ |
703 | ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, | |
704 | req->nr_buffers, &op, &do_reloc); | |
705 | if (ret) { | |
938c40ed | 706 | if (ret != -ERESTARTSYS) |
ebb945a9 | 707 | NV_ERROR(drm, "validate: %d\n", ret); |
7fa0cba2 | 708 | goto out_prevalid; |
6ee73861 BS |
709 | } |
710 | ||
6ee73861 BS |
711 | /* Apply any relocations that are required */ |
712 | if (do_reloc) { | |
a1606a95 | 713 | ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo); |
6ee73861 | 714 | if (ret) { |
ebb945a9 | 715 | NV_ERROR(drm, "reloc apply: %d\n", ret); |
6ee73861 BS |
716 | goto out; |
717 | } | |
6ee73861 | 718 | } |
6ee73861 | 719 | |
9a391ad8 | 720 | if (chan->dma.ib_max) { |
5e120f6e | 721 | ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); |
6ee73861 | 722 | if (ret) { |
ebb945a9 | 723 | NV_ERROR(drm, "nv50cal_space: %d\n", ret); |
6ee73861 BS |
724 | goto out; |
725 | } | |
6ee73861 | 726 | |
a1606a95 BS |
727 | for (i = 0; i < req->nr_push; i++) { |
728 | struct nouveau_bo *nvbo = (void *)(unsigned long) | |
729 | bo[push[i].bo_index].user_priv; | |
730 | ||
731 | nv50_dma_push(chan, nvbo, push[i].offset, | |
732 | push[i].length); | |
733 | } | |
9a391ad8 | 734 | } else |
ebb945a9 | 735 | if (nv_device(drm->device)->chipset >= 0x25) { |
a1606a95 | 736 | ret = RING_SPACE(chan, req->nr_push * 2); |
6ee73861 | 737 | if (ret) { |
ebb945a9 | 738 | NV_ERROR(drm, "cal_space: %d\n", ret); |
6ee73861 BS |
739 | goto out; |
740 | } | |
a1606a95 BS |
741 | |
742 | for (i = 0; i < req->nr_push; i++) { | |
743 | struct nouveau_bo *nvbo = (void *)(unsigned long) | |
744 | bo[push[i].bo_index].user_priv; | |
a1606a95 | 745 | |
3a92d37e | 746 | OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); |
a1606a95 BS |
747 | OUT_RING(chan, 0); |
748 | } | |
6ee73861 | 749 | } else { |
a1606a95 | 750 | ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); |
6ee73861 | 751 | if (ret) { |
ebb945a9 | 752 | NV_ERROR(drm, "jmp_space: %d\n", ret); |
6ee73861 BS |
753 | goto out; |
754 | } | |
6ee73861 | 755 | |
a1606a95 BS |
756 | for (i = 0; i < req->nr_push; i++) { |
757 | struct nouveau_bo *nvbo = (void *)(unsigned long) | |
758 | bo[push[i].bo_index].user_priv; | |
a1606a95 BS |
759 | uint32_t cmd; |
760 | ||
ebb945a9 | 761 | cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); |
a1606a95 BS |
762 | cmd |= 0x20000000; |
763 | if (unlikely(cmd != req->suffix0)) { | |
764 | if (!nvbo->kmap.virtual) { | |
765 | ret = ttm_bo_kmap(&nvbo->bo, 0, | |
766 | nvbo->bo.mem. | |
767 | num_pages, | |
768 | &nvbo->kmap); | |
769 | if (ret) { | |
770 | WIND_RING(chan); | |
771 | goto out; | |
772 | } | |
773 | nvbo->validate_mapped = true; | |
774 | } | |
775 | ||
776 | nouveau_bo_wr32(nvbo, (push[i].offset + | |
777 | push[i].length - 8) / 4, cmd); | |
778 | } | |
779 | ||
3a92d37e BS |
780 | OUT_RING(chan, 0x20000000 | |
781 | (nvbo->bo.offset + push[i].offset)); | |
6ee73861 | 782 | OUT_RING(chan, 0); |
a1606a95 BS |
783 | for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) |
784 | OUT_RING(chan, 0); | |
785 | } | |
6ee73861 BS |
786 | } |
787 | ||
d375e7d5 | 788 | ret = nouveau_fence_new(chan, &fence); |
6ee73861 | 789 | if (ret) { |
ebb945a9 | 790 | NV_ERROR(drm, "error fencing pushbuf: %d\n", ret); |
6ee73861 BS |
791 | WIND_RING(chan); |
792 | goto out; | |
793 | } | |
794 | ||
795 | out: | |
234896a7 | 796 | validate_fini(&op, fence); |
382d62e5 | 797 | nouveau_fence_unref(&fence); |
7fa0cba2 MS |
798 | |
799 | out_prevalid: | |
6ee73861 | 800 | kfree(bo); |
a1606a95 | 801 | kfree(push); |
6ee73861 BS |
802 | |
803 | out_next: | |
9a391ad8 BS |
804 | if (chan->dma.ib_max) { |
805 | req->suffix0 = 0x00000000; | |
806 | req->suffix1 = 0x00000000; | |
807 | } else | |
ebb945a9 | 808 | if (nv_device(drm->device)->chipset >= 0x25) { |
6ee73861 BS |
809 | req->suffix0 = 0x00020000; |
810 | req->suffix1 = 0x00000000; | |
811 | } else { | |
812 | req->suffix0 = 0x20000000 | | |
ebb945a9 | 813 | (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); |
6ee73861 BS |
814 | req->suffix1 = 0x00000000; |
815 | } | |
816 | ||
ebb945a9 | 817 | return nouveau_abi16_put(abi16, ret); |
6ee73861 BS |
818 | } |
819 | ||
6ee73861 BS |
820 | static inline uint32_t |
821 | domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) | |
822 | { | |
823 | uint32_t flags = 0; | |
824 | ||
825 | if (domain & NOUVEAU_GEM_DOMAIN_VRAM) | |
826 | flags |= TTM_PL_FLAG_VRAM; | |
827 | if (domain & NOUVEAU_GEM_DOMAIN_GART) | |
828 | flags |= TTM_PL_FLAG_TT; | |
829 | ||
830 | return flags; | |
831 | } | |
832 | ||
6ee73861 BS |
833 | int |
834 | nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, | |
835 | struct drm_file *file_priv) | |
836 | { | |
837 | struct drm_nouveau_gem_cpu_prep *req = data; | |
838 | struct drm_gem_object *gem; | |
839 | struct nouveau_bo *nvbo; | |
840 | bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); | |
841 | int ret = -EINVAL; | |
842 | ||
6ee73861 BS |
843 | gem = drm_gem_object_lookup(dev, file_priv, req->handle); |
844 | if (!gem) | |
bf79cb91 | 845 | return -ENOENT; |
6ee73861 BS |
846 | nvbo = nouveau_gem_object(gem); |
847 | ||
21e86c1c BS |
848 | spin_lock(&nvbo->bo.bdev->fence_lock); |
849 | ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); | |
850 | spin_unlock(&nvbo->bo.bdev->fence_lock); | |
bc9025bd | 851 | drm_gem_object_unreference_unlocked(gem); |
6ee73861 BS |
852 | return ret; |
853 | } | |
854 | ||
855 | int | |
856 | nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, | |
857 | struct drm_file *file_priv) | |
858 | { | |
21e86c1c | 859 | return 0; |
6ee73861 BS |
860 | } |
861 | ||
862 | int | |
863 | nouveau_gem_ioctl_info(struct drm_device *dev, void *data, | |
864 | struct drm_file *file_priv) | |
865 | { | |
866 | struct drm_nouveau_gem_info *req = data; | |
867 | struct drm_gem_object *gem; | |
868 | int ret; | |
869 | ||
6ee73861 BS |
870 | gem = drm_gem_object_lookup(dev, file_priv, req->handle); |
871 | if (!gem) | |
bf79cb91 | 872 | return -ENOENT; |
6ee73861 | 873 | |
e758a311 | 874 | ret = nouveau_gem_info(file_priv, gem, req); |
bc9025bd | 875 | drm_gem_object_unreference_unlocked(gem); |
6ee73861 BS |
876 | return ret; |
877 | } | |
878 |