Commit | Line | Data |
---|---|---|
771fe6b9 JG |
1 | /* |
2 | * Copyright 2008 Advanced Micro Devices, Inc. | |
3 | * Copyright 2008 Red Hat Inc. | |
4 | * Copyright 2009 Jerome Glisse. | |
5 | * | |
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |
7 | * copy of this software and associated documentation files (the "Software"), | |
8 | * to deal in the Software without restriction, including without limitation | |
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
10 | * and/or sell copies of the Software, and to permit persons to whom the | |
11 | * Software is furnished to do so, subject to the following conditions: | |
12 | * | |
13 | * The above copyright notice and this permission notice shall be included in | |
14 | * all copies or substantial portions of the Software. | |
15 | * | |
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
19 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
22 | * OTHER DEALINGS IN THE SOFTWARE. | |
23 | * | |
24 | * Authors: Dave Airlie | |
25 | * Alex Deucher | |
26 | * Jerome Glisse | |
27 | */ | |
28 | #include "drmP.h" | |
29 | #include "radeon_drm.h" | |
30 | #include "radeon.h" | |
31 | #include "radeon_reg.h" | |
32 | ||
03eec93b AD |
33 | /* |
34 | * GART | |
35 | * The GART (Graphics Aperture Remapping Table) is an aperture | |
36 | * in the GPU's address space. System pages can be mapped into | |
37 | * the aperture and look like contiguous pages from the GPU's | |
38 | * perspective. A page table maps the pages in the aperture | |
39 | * to the actual backing pages in system memory. | |
40 | * | |
41 | * Radeon GPUs support both an internal GART, as described above, | |
42 | * and AGP. AGP works similarly, but the GART table is configured | |
43 | * and maintained by the northbridge rather than the driver. | |
44 | * Radeon hw has a separate AGP aperture that is programmed to | |
45 | * point to the AGP aperture provided by the northbridge and the | |
46 | * requests are passed through to the northbridge aperture. | |
47 | * Both AGP and internal GART can be used at the same time, however | |
48 | * that is not currently supported by the driver. | |
49 | * | |
50 | * This file handles the common internal GART management. | |
51 | */ | |
52 | ||
771fe6b9 JG |
53 | /* |
54 | * Common GART table functions. | |
55 | */ | |
03eec93b AD |
56 | /** |
57 | * radeon_gart_table_ram_alloc - allocate system ram for gart page table | |
58 | * | |
59 | * @rdev: radeon_device pointer | |
60 | * | |
61 | * Allocate system memory for GART page table | |
62 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |
63 | * gart table to be in system memory. | |
64 | * Returns 0 for success, -ENOMEM for failure. | |
65 | */ | |
771fe6b9 JG |
66 | int radeon_gart_table_ram_alloc(struct radeon_device *rdev) |
67 | { | |
68 | void *ptr; | |
69 | ||
70 | ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size, | |
71 | &rdev->gart.table_addr); | |
72 | if (ptr == NULL) { | |
73 | return -ENOMEM; | |
74 | } | |
75 | #ifdef CONFIG_X86 | |
76 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |
77 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |
78 | set_memory_uc((unsigned long)ptr, | |
79 | rdev->gart.table_size >> PAGE_SHIFT); | |
80 | } | |
81 | #endif | |
c9a1be96 JG |
82 | rdev->gart.ptr = ptr; |
83 | memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); | |
771fe6b9 JG |
84 | return 0; |
85 | } | |
86 | ||
03eec93b AD |
87 | /** |
88 | * radeon_gart_table_ram_free - free system ram for gart page table | |
89 | * | |
90 | * @rdev: radeon_device pointer | |
91 | * | |
92 | * Free system memory for GART page table | |
93 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |
94 | * gart table to be in system memory. | |
95 | */ | |
771fe6b9 JG |
96 | void radeon_gart_table_ram_free(struct radeon_device *rdev) |
97 | { | |
c9a1be96 | 98 | if (rdev->gart.ptr == NULL) { |
771fe6b9 JG |
99 | return; |
100 | } | |
101 | #ifdef CONFIG_X86 | |
102 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |
103 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |
c9a1be96 | 104 | set_memory_wb((unsigned long)rdev->gart.ptr, |
771fe6b9 JG |
105 | rdev->gart.table_size >> PAGE_SHIFT); |
106 | } | |
107 | #endif | |
108 | pci_free_consistent(rdev->pdev, rdev->gart.table_size, | |
c9a1be96 | 109 | (void *)rdev->gart.ptr, |
771fe6b9 | 110 | rdev->gart.table_addr); |
c9a1be96 | 111 | rdev->gart.ptr = NULL; |
771fe6b9 JG |
112 | rdev->gart.table_addr = 0; |
113 | } | |
114 | ||
03eec93b AD |
115 | /** |
116 | * radeon_gart_table_vram_alloc - allocate vram for gart page table | |
117 | * | |
118 | * @rdev: radeon_device pointer | |
119 | * | |
120 | * Allocate video memory for GART page table | |
121 | * (pcie r4xx, r5xx+). These asics require the | |
122 | * gart table to be in video memory. | |
123 | * Returns 0 for success, error for failure. | |
124 | */ | |
771fe6b9 JG |
125 | int radeon_gart_table_vram_alloc(struct radeon_device *rdev) |
126 | { | |
771fe6b9 JG |
127 | int r; |
128 | ||
c9a1be96 | 129 | if (rdev->gart.robj == NULL) { |
441921d5 | 130 | r = radeon_bo_create(rdev, rdev->gart.table_size, |
268b2510 | 131 | PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, |
40f5cf99 | 132 | NULL, &rdev->gart.robj); |
771fe6b9 JG |
133 | if (r) { |
134 | return r; | |
135 | } | |
136 | } | |
4aac0473 JG |
137 | return 0; |
138 | } | |
139 | ||
03eec93b AD |
140 | /** |
141 | * radeon_gart_table_vram_pin - pin gart page table in vram | |
142 | * | |
143 | * @rdev: radeon_device pointer | |
144 | * | |
145 | * Pin the GART page table in vram so it will not be moved | |
146 | * by the memory manager (pcie r4xx, r5xx+). These asics require the | |
147 | * gart table to be in video memory. | |
148 | * Returns 0 for success, error for failure. | |
149 | */ | |
4aac0473 JG |
150 | int radeon_gart_table_vram_pin(struct radeon_device *rdev) |
151 | { | |
152 | uint64_t gpu_addr; | |
153 | int r; | |
154 | ||
c9a1be96 | 155 | r = radeon_bo_reserve(rdev->gart.robj, false); |
4c788679 | 156 | if (unlikely(r != 0)) |
771fe6b9 | 157 | return r; |
c9a1be96 | 158 | r = radeon_bo_pin(rdev->gart.robj, |
4c788679 | 159 | RADEON_GEM_DOMAIN_VRAM, &gpu_addr); |
771fe6b9 | 160 | if (r) { |
c9a1be96 | 161 | radeon_bo_unreserve(rdev->gart.robj); |
771fe6b9 JG |
162 | return r; |
163 | } | |
c9a1be96 | 164 | r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); |
4c788679 | 165 | if (r) |
c9a1be96 JG |
166 | radeon_bo_unpin(rdev->gart.robj); |
167 | radeon_bo_unreserve(rdev->gart.robj); | |
771fe6b9 | 168 | rdev->gart.table_addr = gpu_addr; |
4c788679 | 169 | return r; |
771fe6b9 JG |
170 | } |
171 | ||
03eec93b AD |
172 | /** |
173 | * radeon_gart_table_vram_unpin - unpin gart page table in vram | |
174 | * | |
175 | * @rdev: radeon_device pointer | |
176 | * | |
177 | * Unpin the GART page table in vram (pcie r4xx, r5xx+). | |
178 | * These asics require the gart table to be in video memory. | |
179 | */ | |
c9a1be96 | 180 | void radeon_gart_table_vram_unpin(struct radeon_device *rdev) |
771fe6b9 | 181 | { |
4c788679 JG |
182 | int r; |
183 | ||
c9a1be96 | 184 | if (rdev->gart.robj == NULL) { |
771fe6b9 JG |
185 | return; |
186 | } | |
c9a1be96 | 187 | r = radeon_bo_reserve(rdev->gart.robj, false); |
4c788679 | 188 | if (likely(r == 0)) { |
c9a1be96 JG |
189 | radeon_bo_kunmap(rdev->gart.robj); |
190 | radeon_bo_unpin(rdev->gart.robj); | |
191 | radeon_bo_unreserve(rdev->gart.robj); | |
192 | rdev->gart.ptr = NULL; | |
193 | } | |
194 | } | |
195 | ||
03eec93b AD |
196 | /** |
197 | * radeon_gart_table_vram_free - free gart page table vram | |
198 | * | |
199 | * @rdev: radeon_device pointer | |
200 | * | |
201 | * Free the video memory used for the GART page table | |
202 | * (pcie r4xx, r5xx+). These asics require the gart table to | |
203 | * be in video memory. | |
204 | */ | |
c9a1be96 JG |
205 | void radeon_gart_table_vram_free(struct radeon_device *rdev) |
206 | { | |
207 | if (rdev->gart.robj == NULL) { | |
208 | return; | |
4c788679 | 209 | } |
c9a1be96 JG |
210 | radeon_gart_table_vram_unpin(rdev); |
211 | radeon_bo_unref(&rdev->gart.robj); | |
771fe6b9 JG |
212 | } |
213 | ||
771fe6b9 JG |
214 | /* |
215 | * Common gart functions. | |
216 | */ | |
03eec93b AD |
217 | /** |
218 | * radeon_gart_unbind - unbind pages from the gart page table | |
219 | * | |
220 | * @rdev: radeon_device pointer | |
221 | * @offset: offset into the GPU's gart aperture | |
222 | * @pages: number of pages to unbind | |
223 | * | |
224 | * Unbinds the requested pages from the gart page table and | |
225 | * replaces them with the dummy page (all asics). | |
226 | */ | |
771fe6b9 JG |
227 | void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, |
228 | int pages) | |
229 | { | |
230 | unsigned t; | |
231 | unsigned p; | |
232 | int i, j; | |
82568565 | 233 | u64 page_base; |
771fe6b9 JG |
234 | |
235 | if (!rdev->gart.ready) { | |
fcf4de5a | 236 | WARN(1, "trying to unbind memory from uninitialized GART !\n"); |
771fe6b9 JG |
237 | return; |
238 | } | |
a77f1718 MT |
239 | t = offset / RADEON_GPU_PAGE_SIZE; |
240 | p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); | |
771fe6b9 JG |
241 | for (i = 0; i < pages; i++, p++) { |
242 | if (rdev->gart.pages[p]) { | |
771fe6b9 | 243 | rdev->gart.pages[p] = NULL; |
82568565 DA |
244 | rdev->gart.pages_addr[p] = rdev->dummy_page.addr; |
245 | page_base = rdev->gart.pages_addr[p]; | |
a77f1718 | 246 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { |
c9a1be96 JG |
247 | if (rdev->gart.ptr) { |
248 | radeon_gart_set_page(rdev, t, page_base); | |
249 | } | |
82568565 | 250 | page_base += RADEON_GPU_PAGE_SIZE; |
771fe6b9 JG |
251 | } |
252 | } | |
253 | } | |
254 | mb(); | |
255 | radeon_gart_tlb_flush(rdev); | |
256 | } | |
257 | ||
03eec93b AD |
258 | /** |
259 | * radeon_gart_bind - bind pages into the gart page table | |
260 | * | |
261 | * @rdev: radeon_device pointer | |
262 | * @offset: offset into the GPU's gart aperture | |
263 | * @pages: number of pages to bind | |
264 | * @pagelist: pages to bind | |
265 | * @dma_addr: DMA addresses of pages | |
266 | * | |
267 | * Binds the requested pages to the gart page table | |
268 | * (all asics). | |
269 | * Returns 0 for success, -EINVAL for failure. | |
270 | */ | |
771fe6b9 | 271 | int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, |
c39d3516 | 272 | int pages, struct page **pagelist, dma_addr_t *dma_addr) |
771fe6b9 JG |
273 | { |
274 | unsigned t; | |
275 | unsigned p; | |
276 | uint64_t page_base; | |
277 | int i, j; | |
278 | ||
279 | if (!rdev->gart.ready) { | |
fcf4de5a | 280 | WARN(1, "trying to bind memory to uninitialized GART !\n"); |
771fe6b9 JG |
281 | return -EINVAL; |
282 | } | |
a77f1718 MT |
283 | t = offset / RADEON_GPU_PAGE_SIZE; |
284 | p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); | |
771fe6b9 JG |
285 | |
286 | for (i = 0; i < pages; i++, p++) { | |
c52494f6 | 287 | rdev->gart.pages_addr[p] = dma_addr[i]; |
771fe6b9 | 288 | rdev->gart.pages[p] = pagelist[i]; |
c9a1be96 JG |
289 | if (rdev->gart.ptr) { |
290 | page_base = rdev->gart.pages_addr[p]; | |
291 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { | |
292 | radeon_gart_set_page(rdev, t, page_base); | |
293 | page_base += RADEON_GPU_PAGE_SIZE; | |
294 | } | |
771fe6b9 JG |
295 | } |
296 | } | |
297 | mb(); | |
298 | radeon_gart_tlb_flush(rdev); | |
299 | return 0; | |
300 | } | |
301 | ||
03eec93b AD |
302 | /** |
303 | * radeon_gart_restore - bind all pages in the gart page table | |
304 | * | |
305 | * @rdev: radeon_device pointer | |
306 | * | |
307 | * Binds all pages in the gart page table (all asics). | |
308 | * Used to rebuild the gart table on device startup or resume. | |
309 | */ | |
82568565 DA |
310 | void radeon_gart_restore(struct radeon_device *rdev) |
311 | { | |
312 | int i, j, t; | |
313 | u64 page_base; | |
314 | ||
c9a1be96 JG |
315 | if (!rdev->gart.ptr) { |
316 | return; | |
317 | } | |
82568565 DA |
318 | for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { |
319 | page_base = rdev->gart.pages_addr[i]; | |
320 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { | |
321 | radeon_gart_set_page(rdev, t, page_base); | |
322 | page_base += RADEON_GPU_PAGE_SIZE; | |
323 | } | |
324 | } | |
325 | mb(); | |
326 | radeon_gart_tlb_flush(rdev); | |
327 | } | |
328 | ||
03eec93b AD |
329 | /** |
330 | * radeon_gart_init - init the driver info for managing the gart | |
331 | * | |
332 | * @rdev: radeon_device pointer | |
333 | * | |
334 | * Allocate the dummy page and init the gart driver info (all asics). | |
335 | * Returns 0 for success, error for failure. | |
336 | */ | |
771fe6b9 JG |
337 | int radeon_gart_init(struct radeon_device *rdev) |
338 | { | |
82568565 DA |
339 | int r, i; |
340 | ||
771fe6b9 JG |
341 | if (rdev->gart.pages) { |
342 | return 0; | |
343 | } | |
a77f1718 MT |
344 | /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ |
345 | if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { | |
771fe6b9 JG |
346 | DRM_ERROR("Page size is smaller than GPU page size!\n"); |
347 | return -EINVAL; | |
348 | } | |
82568565 DA |
349 | r = radeon_dummy_page_init(rdev); |
350 | if (r) | |
351 | return r; | |
771fe6b9 JG |
352 | /* Compute table size */ |
353 | rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; | |
a77f1718 | 354 | rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; |
771fe6b9 JG |
355 | DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", |
356 | rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); | |
357 | /* Allocate pages table */ | |
358 | rdev->gart.pages = kzalloc(sizeof(void *) * rdev->gart.num_cpu_pages, | |
359 | GFP_KERNEL); | |
360 | if (rdev->gart.pages == NULL) { | |
361 | radeon_gart_fini(rdev); | |
362 | return -ENOMEM; | |
363 | } | |
364 | rdev->gart.pages_addr = kzalloc(sizeof(dma_addr_t) * | |
365 | rdev->gart.num_cpu_pages, GFP_KERNEL); | |
366 | if (rdev->gart.pages_addr == NULL) { | |
367 | radeon_gart_fini(rdev); | |
368 | return -ENOMEM; | |
369 | } | |
82568565 DA |
370 | /* set GART entry to point to the dummy page by default */ |
371 | for (i = 0; i < rdev->gart.num_cpu_pages; i++) { | |
372 | rdev->gart.pages_addr[i] = rdev->dummy_page.addr; | |
373 | } | |
771fe6b9 JG |
374 | return 0; |
375 | } | |
376 | ||
03eec93b AD |
377 | /** |
378 | * radeon_gart_fini - tear down the driver info for managing the gart | |
379 | * | |
380 | * @rdev: radeon_device pointer | |
381 | * | |
382 | * Tear down the gart driver info and free the dummy page (all asics). | |
383 | */ | |
771fe6b9 JG |
384 | void radeon_gart_fini(struct radeon_device *rdev) |
385 | { | |
386 | if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { | |
387 | /* unbind pages */ | |
388 | radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); | |
389 | } | |
390 | rdev->gart.ready = false; | |
391 | kfree(rdev->gart.pages); | |
392 | kfree(rdev->gart.pages_addr); | |
393 | rdev->gart.pages = NULL; | |
394 | rdev->gart.pages_addr = NULL; | |
92656d70 AD |
395 | |
396 | radeon_dummy_page_fini(rdev); | |
771fe6b9 | 397 | } |
721604a1 | 398 | |
09db8644 AD |
399 | /* |
400 | * GPUVM | |
401 | * GPUVM is similar to the legacy gart on older asics, however | |
402 | * rather than there being a single global gart table | |
403 | * for the entire GPU, there are multiple VM page tables active | |
404 | * at any given time. The VM page tables can contain a mix | |
405 | * vram pages and system memory pages and system memory pages | |
406 | * can be mapped as snooped (cached system pages) or unsnooped | |
407 | * (uncached system pages). | |
408 | * Each VM has an ID associated with it and there is a page table | |
409 | * associated with each VMID. When execting a command buffer, | |
410 | * the kernel tells the the ring what VMID to use for that command | |
411 | * buffer. VMIDs are allocated dynamically as commands are submitted. | |
412 | * The userspace drivers maintain their own address space and the kernel | |
413 | * sets up their pages tables accordingly when they submit their | |
414 | * command buffers and a VMID is assigned. | |
415 | * Cayman/Trinity support up to 8 active VMs at any given time; | |
416 | * SI supports 16. | |
417 | */ | |
418 | ||
721604a1 JG |
419 | /* |
420 | * vm helpers | |
421 | * | |
422 | * TODO bind a default page at vm initialization for default address | |
423 | */ | |
c6105f24 | 424 | |
09db8644 AD |
425 | /** |
426 | * radeon_vm_manager_init - init the vm manager | |
427 | * | |
428 | * @rdev: radeon_device pointer | |
429 | * | |
430 | * Init the vm manager (cayman+). | |
431 | * Returns 0 for success, error for failure. | |
432 | */ | |
721604a1 JG |
433 | int radeon_vm_manager_init(struct radeon_device *rdev) |
434 | { | |
c6105f24 CK |
435 | struct radeon_vm *vm; |
436 | struct radeon_bo_va *bo_va; | |
721604a1 JG |
437 | int r; |
438 | ||
c6105f24 | 439 | if (!rdev->vm_manager.enabled) { |
e6b0b6a8 | 440 | /* allocate enough for 2 full VM pts */ |
c6105f24 | 441 | r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, |
e6b0b6a8 | 442 | rdev->vm_manager.max_pfn * 8 * 2, |
c6105f24 CK |
443 | RADEON_GEM_DOMAIN_VRAM); |
444 | if (r) { | |
445 | dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", | |
446 | (rdev->vm_manager.max_pfn * 8) >> 10); | |
447 | return r; | |
448 | } | |
67e915e4 | 449 | |
05b07147 | 450 | r = radeon_asic_vm_init(rdev); |
c6105f24 CK |
451 | if (r) |
452 | return r; | |
089a786e | 453 | |
c6105f24 CK |
454 | rdev->vm_manager.enabled = true; |
455 | ||
456 | r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); | |
457 | if (r) | |
458 | return r; | |
721604a1 | 459 | } |
67e915e4 | 460 | |
c6105f24 CK |
461 | /* restore page table */ |
462 | list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { | |
ee60e29f | 463 | if (vm->sa_bo == NULL) |
c6105f24 | 464 | continue; |
67e915e4 | 465 | |
c6105f24 | 466 | list_for_each_entry(bo_va, &vm->va, vm_list) { |
c6105f24 | 467 | bo_va->valid = false; |
c6105f24 | 468 | } |
c6105f24 CK |
469 | } |
470 | return 0; | |
721604a1 JG |
471 | } |
472 | ||
09db8644 | 473 | /** |
ddf03f5c | 474 | * radeon_vm_free_pt - free the page table for a specific vm |
09db8644 AD |
475 | * |
476 | * @rdev: radeon_device pointer | |
477 | * @vm: vm to unbind | |
478 | * | |
ddf03f5c CK |
479 | * Free the page table of a specific vm (cayman+). |
480 | * | |
481 | * Global and local mutex must be lock! | |
09db8644 | 482 | */ |
ddf03f5c | 483 | static void radeon_vm_free_pt(struct radeon_device *rdev, |
721604a1 JG |
484 | struct radeon_vm *vm) |
485 | { | |
486 | struct radeon_bo_va *bo_va; | |
487 | ||
ddf03f5c CK |
488 | if (!vm->sa_bo) |
489 | return; | |
721604a1 | 490 | |
721604a1 | 491 | list_del_init(&vm->list); |
ddf03f5c | 492 | radeon_sa_bo_free(rdev, &vm->sa_bo, vm->fence); |
721604a1 JG |
493 | vm->pt = NULL; |
494 | ||
495 | list_for_each_entry(bo_va, &vm->va, vm_list) { | |
496 | bo_va->valid = false; | |
497 | } | |
498 | } | |
499 | ||
09db8644 AD |
500 | /** |
501 | * radeon_vm_manager_fini - tear down the vm manager | |
502 | * | |
503 | * @rdev: radeon_device pointer | |
504 | * | |
505 | * Tear down the VM manager (cayman+). | |
506 | */ | |
721604a1 | 507 | void radeon_vm_manager_fini(struct radeon_device *rdev) |
721604a1 JG |
508 | { |
509 | struct radeon_vm *vm, *tmp; | |
ee60e29f | 510 | int i; |
721604a1 | 511 | |
c6105f24 CK |
512 | if (!rdev->vm_manager.enabled) |
513 | return; | |
514 | ||
36ff39c4 | 515 | mutex_lock(&rdev->vm_manager.lock); |
ddf03f5c | 516 | /* free all allocated page tables */ |
721604a1 | 517 | list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { |
ddf03f5c CK |
518 | mutex_lock(&vm->mutex); |
519 | radeon_vm_free_pt(rdev, vm); | |
520 | mutex_unlock(&vm->mutex); | |
721604a1 | 521 | } |
ee60e29f CK |
522 | for (i = 0; i < RADEON_NUM_VM; ++i) { |
523 | radeon_fence_unref(&rdev->vm_manager.active[i]); | |
524 | } | |
05b07147 | 525 | radeon_asic_vm_fini(rdev); |
36ff39c4 | 526 | mutex_unlock(&rdev->vm_manager.lock); |
c6105f24 CK |
527 | |
528 | radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); | |
529 | radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); | |
530 | rdev->vm_manager.enabled = false; | |
721604a1 JG |
531 | } |
532 | ||
09db8644 | 533 | /** |
ddf03f5c | 534 | * radeon_vm_alloc_pt - allocates a page table for a VM |
09db8644 AD |
535 | * |
536 | * @rdev: radeon_device pointer | |
537 | * @vm: vm to bind | |
538 | * | |
ddf03f5c CK |
539 | * Allocate a page table for the requested vm (cayman+). |
540 | * Also starts to populate the page table. | |
09db8644 | 541 | * Returns 0 for success, error for failure. |
ddf03f5c CK |
542 | * |
543 | * Global and local mutex must be locked! | |
09db8644 | 544 | */ |
ddf03f5c | 545 | int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) |
721604a1 JG |
546 | { |
547 | struct radeon_vm *vm_evict; | |
ee60e29f | 548 | int r; |
721604a1 JG |
549 | |
550 | if (vm == NULL) { | |
551 | return -EINVAL; | |
552 | } | |
553 | ||
ee60e29f | 554 | if (vm->sa_bo != NULL) { |
721604a1 JG |
555 | /* update lru */ |
556 | list_del_init(&vm->list); | |
557 | list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); | |
558 | return 0; | |
559 | } | |
560 | ||
561 | retry: | |
562 | r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo, | |
563 | RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8), | |
557017a0 | 564 | RADEON_GPU_PAGE_SIZE, false); |
ddf03f5c | 565 | if (r == -ENOMEM) { |
721604a1 JG |
566 | if (list_empty(&rdev->vm_manager.lru_vm)) { |
567 | return r; | |
568 | } | |
569 | vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, struct radeon_vm, list); | |
ddf03f5c CK |
570 | mutex_lock(&vm_evict->mutex); |
571 | radeon_vm_free_pt(rdev, vm_evict); | |
572 | mutex_unlock(&vm_evict->mutex); | |
721604a1 | 573 | goto retry; |
ddf03f5c CK |
574 | |
575 | } else if (r) { | |
576 | return r; | |
721604a1 | 577 | } |
ddf03f5c | 578 | |
2e0d9910 CK |
579 | vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo); |
580 | vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo); | |
721604a1 JG |
581 | memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8)); |
582 | ||
ee60e29f CK |
583 | list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); |
584 | return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, | |
585 | &rdev->ring_tmp_bo.bo->tbo.mem); | |
586 | } | |
587 | ||
588 | /** | |
589 | * radeon_vm_grab_id - allocate the next free VMID | |
590 | * | |
591 | * @rdev: radeon_device pointer | |
592 | * @vm: vm to allocate id for | |
593 | * @ring: ring we want to submit job to | |
594 | * | |
595 | * Allocate an id for the vm (cayman+). | |
596 | * Returns the fence we need to sync to (if any). | |
597 | * | |
598 | * Global and local mutex must be locked! | |
599 | */ | |
600 | struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, | |
601 | struct radeon_vm *vm, int ring) | |
602 | { | |
603 | struct radeon_fence *best[RADEON_NUM_RINGS] = {}; | |
604 | unsigned choices[2] = {}; | |
605 | unsigned i; | |
606 | ||
607 | /* check if the id is still valid */ | |
608 | if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) | |
609 | return NULL; | |
610 | ||
611 | /* we definately need to flush */ | |
612 | radeon_fence_unref(&vm->last_flush); | |
613 | ||
614 | /* skip over VMID 0, since it is the system VM */ | |
615 | for (i = 1; i < rdev->vm_manager.nvm; ++i) { | |
616 | struct radeon_fence *fence = rdev->vm_manager.active[i]; | |
617 | ||
618 | if (fence == NULL) { | |
619 | /* found a free one */ | |
620 | vm->id = i; | |
621 | return NULL; | |
622 | } | |
623 | ||
624 | if (radeon_fence_is_earlier(fence, best[fence->ring])) { | |
625 | best[fence->ring] = fence; | |
626 | choices[fence->ring == ring ? 0 : 1] = i; | |
721604a1 | 627 | } |
721604a1 JG |
628 | } |
629 | ||
ee60e29f CK |
630 | for (i = 0; i < 2; ++i) { |
631 | if (choices[i]) { | |
632 | vm->id = choices[i]; | |
633 | return rdev->vm_manager.active[choices[i]]; | |
634 | } | |
721604a1 | 635 | } |
ee60e29f CK |
636 | |
637 | /* should never happen */ | |
638 | BUG(); | |
639 | return NULL; | |
640 | } | |
641 | ||
642 | /** | |
643 | * radeon_vm_fence - remember fence for vm | |
644 | * | |
645 | * @rdev: radeon_device pointer | |
646 | * @vm: vm we want to fence | |
647 | * @fence: fence to remember | |
648 | * | |
649 | * Fence the vm (cayman+). | |
650 | * Set the fence used to protect page table and id. | |
651 | * | |
652 | * Global and local mutex must be locked! | |
653 | */ | |
654 | void radeon_vm_fence(struct radeon_device *rdev, | |
655 | struct radeon_vm *vm, | |
656 | struct radeon_fence *fence) | |
657 | { | |
658 | radeon_fence_unref(&rdev->vm_manager.active[vm->id]); | |
659 | rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); | |
660 | ||
661 | radeon_fence_unref(&vm->fence); | |
662 | vm->fence = radeon_fence_ref(fence); | |
721604a1 JG |
663 | } |
664 | ||
665 | /* object have to be reserved */ | |
09db8644 AD |
666 | /** |
667 | * radeon_vm_bo_add - add a bo to a specific vm | |
668 | * | |
669 | * @rdev: radeon_device pointer | |
670 | * @vm: requested vm | |
671 | * @bo: radeon buffer object | |
672 | * @offset: requested offset of the buffer in the VM address space | |
673 | * @flags: attributes of pages (read/write/valid/etc.) | |
674 | * | |
675 | * Add @bo into the requested vm (cayman+). | |
676 | * Add @bo to the list of bos associated with the vm and validate | |
677 | * the offset requested within the vm address space. | |
678 | * Returns 0 for success, error for failure. | |
679 | */ | |
721604a1 JG |
680 | int radeon_vm_bo_add(struct radeon_device *rdev, |
681 | struct radeon_vm *vm, | |
682 | struct radeon_bo *bo, | |
683 | uint64_t offset, | |
684 | uint32_t flags) | |
685 | { | |
686 | struct radeon_bo_va *bo_va, *tmp; | |
687 | struct list_head *head; | |
688 | uint64_t size = radeon_bo_size(bo), last_offset = 0; | |
689 | unsigned last_pfn; | |
690 | ||
691 | bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); | |
692 | if (bo_va == NULL) { | |
693 | return -ENOMEM; | |
694 | } | |
695 | bo_va->vm = vm; | |
696 | bo_va->bo = bo; | |
697 | bo_va->soffset = offset; | |
698 | bo_va->eoffset = offset + size; | |
699 | bo_va->flags = flags; | |
700 | bo_va->valid = false; | |
701 | INIT_LIST_HEAD(&bo_va->bo_list); | |
702 | INIT_LIST_HEAD(&bo_va->vm_list); | |
703 | /* make sure object fit at this offset */ | |
704 | if (bo_va->soffset >= bo_va->eoffset) { | |
705 | kfree(bo_va); | |
706 | return -EINVAL; | |
707 | } | |
708 | ||
709 | last_pfn = bo_va->eoffset / RADEON_GPU_PAGE_SIZE; | |
710 | if (last_pfn > rdev->vm_manager.max_pfn) { | |
711 | kfree(bo_va); | |
712 | dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", | |
713 | last_pfn, rdev->vm_manager.max_pfn); | |
714 | return -EINVAL; | |
715 | } | |
716 | ||
717 | mutex_lock(&vm->mutex); | |
718 | if (last_pfn > vm->last_pfn) { | |
bb409155 CK |
719 | /* release mutex and lock in right order */ |
720 | mutex_unlock(&vm->mutex); | |
36ff39c4 | 721 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 CK |
722 | mutex_lock(&vm->mutex); |
723 | /* and check again */ | |
724 | if (last_pfn > vm->last_pfn) { | |
725 | /* grow va space 32M by 32M */ | |
726 | unsigned align = ((32 << 20) >> 12) - 1; | |
ddf03f5c | 727 | radeon_vm_free_pt(rdev, vm); |
bb409155 CK |
728 | vm->last_pfn = (last_pfn + align) & ~align; |
729 | } | |
36ff39c4 | 730 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 JG |
731 | } |
732 | head = &vm->va; | |
733 | last_offset = 0; | |
734 | list_for_each_entry(tmp, &vm->va, vm_list) { | |
a36e70b2 | 735 | if (bo_va->soffset >= last_offset && bo_va->eoffset <= tmp->soffset) { |
721604a1 JG |
736 | /* bo can be added before this one */ |
737 | break; | |
738 | } | |
96a5844f | 739 | if (bo_va->eoffset > tmp->soffset && bo_va->soffset < tmp->eoffset) { |
721604a1 | 740 | /* bo and tmp overlap, invalid offset */ |
721604a1 JG |
741 | dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", |
742 | bo, (unsigned)bo_va->soffset, tmp->bo, | |
743 | (unsigned)tmp->soffset, (unsigned)tmp->eoffset); | |
55ba70c4 | 744 | kfree(bo_va); |
721604a1 JG |
745 | mutex_unlock(&vm->mutex); |
746 | return -EINVAL; | |
747 | } | |
748 | last_offset = tmp->eoffset; | |
749 | head = &tmp->vm_list; | |
750 | } | |
751 | list_add(&bo_va->vm_list, head); | |
752 | list_add_tail(&bo_va->bo_list, &bo->va); | |
753 | mutex_unlock(&vm->mutex); | |
754 | return 0; | |
755 | } | |
756 | ||
09db8644 AD |
757 | /** |
758 | * radeon_vm_get_addr - get the physical address of the page | |
759 | * | |
760 | * @rdev: radeon_device pointer | |
761 | * @mem: ttm mem | |
762 | * @pfn: pfn | |
763 | * | |
764 | * Look up the physical address of the page that the pte resolves | |
765 | * to (cayman+). | |
766 | * Returns the physical address of the page. | |
767 | */ | |
089a786e CK |
768 | u64 radeon_vm_get_addr(struct radeon_device *rdev, |
769 | struct ttm_mem_reg *mem, | |
770 | unsigned pfn) | |
721604a1 JG |
771 | { |
772 | u64 addr = 0; | |
773 | ||
774 | switch (mem->mem_type) { | |
775 | case TTM_PL_VRAM: | |
776 | addr = (mem->start << PAGE_SHIFT); | |
777 | addr += pfn * RADEON_GPU_PAGE_SIZE; | |
778 | addr += rdev->vm_manager.vram_base_offset; | |
779 | break; | |
780 | case TTM_PL_TT: | |
781 | /* offset inside page table */ | |
782 | addr = mem->start << PAGE_SHIFT; | |
783 | addr += pfn * RADEON_GPU_PAGE_SIZE; | |
784 | addr = addr >> PAGE_SHIFT; | |
785 | /* page table offset */ | |
786 | addr = rdev->gart.pages_addr[addr]; | |
787 | /* in case cpu page size != gpu page size*/ | |
788 | addr += (pfn * RADEON_GPU_PAGE_SIZE) & (~PAGE_MASK); | |
789 | break; | |
790 | default: | |
791 | break; | |
792 | } | |
793 | return addr; | |
794 | } | |
795 | ||
09db8644 AD |
796 | /** |
797 | * radeon_vm_bo_update_pte - map a bo into the vm page table | |
798 | * | |
799 | * @rdev: radeon_device pointer | |
800 | * @vm: requested vm | |
801 | * @bo: radeon buffer object | |
802 | * @mem: ttm mem | |
803 | * | |
804 | * Fill in the page table entries for @bo (cayman+). | |
805 | * Returns 0 for success, -EINVAL for failure. | |
2a6f1abb CK |
806 | * |
807 | * Object have to be reserved & global and local mutex must be locked! | |
09db8644 | 808 | */ |
721604a1 JG |
809 | int radeon_vm_bo_update_pte(struct radeon_device *rdev, |
810 | struct radeon_vm *vm, | |
811 | struct radeon_bo *bo, | |
812 | struct ttm_mem_reg *mem) | |
813 | { | |
2a6f1abb CK |
814 | unsigned ridx = rdev->asic->vm.pt_ring_index; |
815 | struct radeon_ring *ring = &rdev->ring[ridx]; | |
816 | struct radeon_semaphore *sem = NULL; | |
721604a1 | 817 | struct radeon_bo_va *bo_va; |
2a6f1abb | 818 | unsigned ngpu_pages, ndw; |
089a786e | 819 | uint64_t pfn; |
2a6f1abb | 820 | int r; |
721604a1 JG |
821 | |
822 | /* nothing to do if vm isn't bound */ | |
ee60e29f | 823 | if (vm->sa_bo == NULL) |
04bd27ae | 824 | return 0; |
721604a1 JG |
825 | |
826 | bo_va = radeon_bo_va(bo, vm); | |
827 | if (bo_va == NULL) { | |
828 | dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); | |
829 | return -EINVAL; | |
830 | } | |
831 | ||
2a6f1abb | 832 | if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) |
721604a1 JG |
833 | return 0; |
834 | ||
835 | ngpu_pages = radeon_bo_ngpu_pages(bo); | |
836 | bo_va->flags &= ~RADEON_VM_PAGE_VALID; | |
837 | bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; | |
838 | if (mem) { | |
839 | if (mem->mem_type != TTM_PL_SYSTEM) { | |
840 | bo_va->flags |= RADEON_VM_PAGE_VALID; | |
841 | bo_va->valid = true; | |
842 | } | |
843 | if (mem->mem_type == TTM_PL_TT) { | |
844 | bo_va->flags |= RADEON_VM_PAGE_SYSTEM; | |
845 | } | |
2a6f1abb CK |
846 | if (!bo_va->valid) { |
847 | mem = NULL; | |
848 | } | |
849 | } else { | |
850 | bo_va->valid = false; | |
721604a1 | 851 | } |
089a786e | 852 | pfn = bo_va->soffset / RADEON_GPU_PAGE_SIZE; |
2a6f1abb CK |
853 | |
854 | if (vm->fence && radeon_fence_signaled(vm->fence)) { | |
855 | radeon_fence_unref(&vm->fence); | |
856 | } | |
857 | ||
858 | if (vm->fence && vm->fence->ring != ridx) { | |
859 | r = radeon_semaphore_create(rdev, &sem); | |
860 | if (r) { | |
861 | return r; | |
862 | } | |
863 | } | |
864 | ||
865 | /* estimate number of dw needed */ | |
866 | ndw = 32; | |
867 | ndw += (ngpu_pages >> 12) * 3; | |
868 | ndw += ngpu_pages * 2; | |
869 | ||
870 | r = radeon_ring_lock(rdev, ring, ndw); | |
871 | if (r) { | |
872 | return r; | |
873 | } | |
874 | ||
875 | if (sem && radeon_fence_need_sync(vm->fence, ridx)) { | |
876 | radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); | |
877 | radeon_fence_note_sync(vm->fence, ridx); | |
878 | } | |
879 | ||
880 | radeon_asic_vm_set_page(rdev, vm, pfn, mem, ngpu_pages, bo_va->flags); | |
881 | ||
882 | radeon_fence_unref(&vm->fence); | |
883 | r = radeon_fence_emit(rdev, &vm->fence, ridx); | |
884 | if (r) { | |
885 | radeon_ring_unlock_undo(rdev, ring); | |
886 | return r; | |
887 | } | |
888 | radeon_ring_unlock_commit(rdev, ring); | |
889 | radeon_semaphore_free(rdev, &sem, vm->fence); | |
9b40e5d8 | 890 | radeon_fence_unref(&vm->last_flush); |
721604a1 JG |
891 | return 0; |
892 | } | |
893 | ||
09db8644 AD |
894 | /** |
895 | * radeon_vm_bo_rmv - remove a bo to a specific vm | |
896 | * | |
897 | * @rdev: radeon_device pointer | |
898 | * @vm: requested vm | |
899 | * @bo: radeon buffer object | |
900 | * | |
901 | * Remove @bo from the requested vm (cayman+). | |
902 | * Remove @bo from the list of bos associated with the vm and | |
903 | * remove the ptes for @bo in the page table. | |
904 | * Returns 0 for success. | |
ddf03f5c CK |
905 | * |
906 | * Object have to be reserved! | |
09db8644 | 907 | */ |
721604a1 JG |
908 | int radeon_vm_bo_rmv(struct radeon_device *rdev, |
909 | struct radeon_vm *vm, | |
910 | struct radeon_bo *bo) | |
911 | { | |
912 | struct radeon_bo_va *bo_va; | |
2a6f1abb | 913 | int r; |
721604a1 JG |
914 | |
915 | bo_va = radeon_bo_va(bo, vm); | |
916 | if (bo_va == NULL) | |
917 | return 0; | |
918 | ||
36ff39c4 | 919 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 | 920 | mutex_lock(&vm->mutex); |
2a6f1abb | 921 | r = radeon_vm_bo_update_pte(rdev, vm, bo, NULL); |
36ff39c4 | 922 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 | 923 | list_del(&bo_va->vm_list); |
a7eef882 | 924 | mutex_unlock(&vm->mutex); |
108b0d34 | 925 | list_del(&bo_va->bo_list); |
721604a1 JG |
926 | |
927 | kfree(bo_va); | |
2a6f1abb | 928 | return r; |
721604a1 JG |
929 | } |
930 | ||
09db8644 AD |
931 | /** |
932 | * radeon_vm_bo_invalidate - mark the bo as invalid | |
933 | * | |
934 | * @rdev: radeon_device pointer | |
935 | * @vm: requested vm | |
936 | * @bo: radeon buffer object | |
937 | * | |
938 | * Mark @bo as invalid (cayman+). | |
939 | */ | |
721604a1 JG |
940 | void radeon_vm_bo_invalidate(struct radeon_device *rdev, |
941 | struct radeon_bo *bo) | |
942 | { | |
943 | struct radeon_bo_va *bo_va; | |
944 | ||
945 | BUG_ON(!atomic_read(&bo->tbo.reserved)); | |
946 | list_for_each_entry(bo_va, &bo->va, bo_list) { | |
947 | bo_va->valid = false; | |
948 | } | |
949 | } | |
950 | ||
09db8644 AD |
951 | /** |
952 | * radeon_vm_init - initialize a vm instance | |
953 | * | |
954 | * @rdev: radeon_device pointer | |
955 | * @vm: requested vm | |
956 | * | |
957 | * Init @vm (cayman+). | |
958 | * Map the IB pool and any other shared objects into the VM | |
959 | * by default as it's used by all VMs. | |
960 | * Returns 0 for success, error for failure. | |
961 | */ | |
721604a1 JG |
962 | int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) |
963 | { | |
964 | int r; | |
965 | ||
ee60e29f | 966 | vm->id = 0; |
721604a1 JG |
967 | vm->fence = NULL; |
968 | mutex_init(&vm->mutex); | |
969 | INIT_LIST_HEAD(&vm->list); | |
970 | INIT_LIST_HEAD(&vm->va); | |
c21b328e AD |
971 | /* SI requires equal sized PTs for all VMs, so always set |
972 | * last_pfn to max_pfn. cayman allows variable sized | |
973 | * pts so we can grow then as needed. Once we switch | |
974 | * to two level pts we can unify this again. | |
975 | */ | |
976 | if (rdev->family >= CHIP_TAHITI) | |
977 | vm->last_pfn = rdev->vm_manager.max_pfn; | |
978 | else | |
979 | vm->last_pfn = 0; | |
721604a1 JG |
980 | /* map the ib pool buffer at 0 in virtual address space, set |
981 | * read only | |
982 | */ | |
ca19f21e | 983 | r = radeon_vm_bo_add(rdev, vm, rdev->ring_tmp_bo.bo, RADEON_VA_IB_OFFSET, |
721604a1 JG |
984 | RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED); |
985 | return r; | |
986 | } | |
987 | ||
09db8644 | 988 | /** |
f59abbf2 | 989 | * radeon_vm_fini - tear down a vm instance |
09db8644 AD |
990 | * |
991 | * @rdev: radeon_device pointer | |
992 | * @vm: requested vm | |
993 | * | |
994 | * Tear down @vm (cayman+). | |
995 | * Unbind the VM and remove all bos from the vm bo list | |
996 | */ | |
721604a1 JG |
997 | void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) |
998 | { | |
999 | struct radeon_bo_va *bo_va, *tmp; | |
1000 | int r; | |
1001 | ||
36ff39c4 | 1002 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 | 1003 | mutex_lock(&vm->mutex); |
ddf03f5c | 1004 | radeon_vm_free_pt(rdev, vm); |
36ff39c4 | 1005 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 | 1006 | |
e43b5ec0 JG |
1007 | /* remove all bo at this point non are busy any more because unbind |
1008 | * waited for the last vm fence to signal | |
1009 | */ | |
c507f7ef | 1010 | r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false); |
721604a1 | 1011 | if (!r) { |
c507f7ef | 1012 | bo_va = radeon_bo_va(rdev->ring_tmp_bo.bo, vm); |
721604a1 JG |
1013 | list_del_init(&bo_va->bo_list); |
1014 | list_del_init(&bo_va->vm_list); | |
c507f7ef | 1015 | radeon_bo_unreserve(rdev->ring_tmp_bo.bo); |
721604a1 JG |
1016 | kfree(bo_va); |
1017 | } | |
1018 | if (!list_empty(&vm->va)) { | |
1019 | dev_err(rdev->dev, "still active bo inside vm\n"); | |
1020 | } | |
1021 | list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { | |
1022 | list_del_init(&bo_va->vm_list); | |
1023 | r = radeon_bo_reserve(bo_va->bo, false); | |
1024 | if (!r) { | |
1025 | list_del_init(&bo_va->bo_list); | |
1026 | radeon_bo_unreserve(bo_va->bo); | |
1027 | kfree(bo_va); | |
1028 | } | |
1029 | } | |
ddf03f5c CK |
1030 | radeon_fence_unref(&vm->fence); |
1031 | radeon_fence_unref(&vm->last_flush); | |
721604a1 JG |
1032 | mutex_unlock(&vm->mutex); |
1033 | } |