2 * Copyright (C) 2007 Ben Skeggs.
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include "nouveau_drv.h"
32 struct nv50_instmem_priv
{
33 uint32_t save1700
[5]; /* 0x1700->0x1710 */
35 struct nouveau_gpuobj_ref
*pramin_pt
;
36 struct nouveau_gpuobj_ref
*pramin_bar
;
37 struct nouveau_gpuobj_ref
*fb_bar
;
40 #define NV50_INSTMEM_PAGE_SHIFT 12
41 #define NV50_INSTMEM_PAGE_SIZE (1 << NV50_INSTMEM_PAGE_SHIFT)
42 #define NV50_INSTMEM_PT_SIZE(a) (((a) >> 12) << 3)
44 /*NOTE: - Assumes 0x1700 already covers the correct MiB of PRAMIN
46 #define BAR0_WI32(g, o, v) do { \
48 if ((g)->im_backing) { \
49 offset = (g)->im_backing_start; \
51 offset = chan->ramin->gpuobj->im_backing_start; \
52 offset += (g)->im_pramin->start; \
55 nv_wr32(dev, NV_RAMIN + (offset & 0xfffff), (v)); \
59 nv50_instmem_init(struct drm_device
*dev
)
61 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
62 struct nouveau_channel
*chan
;
63 uint32_t c_offset
, c_size
, c_ramfc
, c_vmpd
, c_base
, pt_size
;
64 uint32_t save_nv001700
;
66 struct nv50_instmem_priv
*priv
;
69 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
72 dev_priv
->engine
.instmem
.priv
= priv
;
74 /* Save state, will restore at takedown. */
75 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
76 priv
->save1700
[(i
-0x1700)/4] = nv_rd32(dev
, i
);
78 /* Reserve the last MiB of VRAM, we should probably try to avoid
79 * setting up the below tables over the top of the VBIOS image at
82 dev_priv
->ramin_rsvd_vram
= 1 << 20;
83 c_offset
= dev_priv
->vram_size
- dev_priv
->ramin_rsvd_vram
;
85 c_vmpd
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x1400 : 0x200;
86 c_ramfc
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x0 : 0x20;
87 c_base
= c_vmpd
+ 0x4000;
88 pt_size
= NV50_INSTMEM_PT_SIZE(dev_priv
->ramin_size
);
90 NV_DEBUG(dev
, " Rsvd VRAM base: 0x%08x\n", c_offset
);
91 NV_DEBUG(dev
, " VBIOS image: 0x%08x\n",
92 (nv_rd32(dev
, 0x619f04) & ~0xff) << 8);
93 NV_DEBUG(dev
, " Aperture size: %d MiB\n", dev_priv
->ramin_size
>> 20);
94 NV_DEBUG(dev
, " PT size: %d KiB\n", pt_size
>> 10);
96 /* Determine VM layout, we need to do this first to make sure
97 * we allocate enough memory for all the page tables.
99 dev_priv
->vm_gart_base
= roundup(NV50_VM_BLOCK
, NV50_VM_BLOCK
);
100 dev_priv
->vm_gart_size
= NV50_VM_BLOCK
;
102 dev_priv
->vm_vram_base
= dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
;
103 dev_priv
->vm_vram_size
= dev_priv
->vram_size
;
104 if (dev_priv
->vm_vram_size
> NV50_VM_MAX_VRAM
)
105 dev_priv
->vm_vram_size
= NV50_VM_MAX_VRAM
;
106 dev_priv
->vm_vram_size
= roundup(dev_priv
->vm_vram_size
, NV50_VM_BLOCK
);
107 dev_priv
->vm_vram_pt_nr
= dev_priv
->vm_vram_size
/ NV50_VM_BLOCK
;
109 dev_priv
->vm_end
= dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
;
111 NV_DEBUG(dev
, "NV50VM: GART 0x%016llx-0x%016llx\n",
112 dev_priv
->vm_gart_base
,
113 dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
- 1);
114 NV_DEBUG(dev
, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
115 dev_priv
->vm_vram_base
,
116 dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
- 1);
118 c_size
+= dev_priv
->vm_vram_pt_nr
* (NV50_VM_BLOCK
/ 65536 * 8);
120 /* Map BAR0 PRAMIN aperture over the memory we want to use */
121 save_nv001700
= nv_rd32(dev
, NV50_PUNK_BAR0_PRAMIN
);
122 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (c_offset
>> 16));
124 /* Create a fake channel, and use it as our "dummy" channels 0/127.
125 * The main reason for creating a channel is so we can use the gpuobj
126 * code. However, it's probably worth noting that NVIDIA also setup
127 * their channels 0/127 with the same values they configure here.
128 * So, there may be some other reason for doing this.
130 * Have to create the entire channel manually, as the real channel
131 * creation code assumes we have PRAMIN access, and we don't until
134 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
139 chan
->file_priv
= (struct drm_file
*)-2;
140 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = chan
;
142 /* Channel's PRAMIN object + heap */
143 ret
= nouveau_gpuobj_new_fake(dev
, 0, c_offset
, c_size
, 0,
148 if (drm_mm_init(&chan
->ramin_heap
, c_base
, c_size
- c_base
))
151 /* RAMFC + zero channel's PRAMIN up to start of VM pagedir */
152 ret
= nouveau_gpuobj_new_fake(dev
, c_ramfc
, c_offset
+ c_ramfc
,
153 0x4000, 0, NULL
, &chan
->ramfc
);
157 for (i
= 0; i
< c_vmpd
; i
+= 4)
158 BAR0_WI32(chan
->ramin
->gpuobj
, i
, 0);
160 /* VM page directory */
161 ret
= nouveau_gpuobj_new_fake(dev
, c_vmpd
, c_offset
+ c_vmpd
,
162 0x4000, 0, &chan
->vm_pd
, NULL
);
165 for (i
= 0; i
< 0x4000; i
+= 8) {
166 BAR0_WI32(chan
->vm_pd
, i
+ 0x00, 0x00000000);
167 BAR0_WI32(chan
->vm_pd
, i
+ 0x04, 0x00000000);
170 /* PRAMIN page table, cheat and map into VM at 0x0000000000.
171 * We map the entire fake channel into the start of the PRAMIN BAR
173 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0, pt_size
, 0x1000,
174 0, &priv
->pramin_pt
);
179 if (dev_priv
->vram_sys_base
) {
180 v
+= dev_priv
->vram_sys_base
;
185 while (v
< dev_priv
->vram_sys_base
+ c_offset
+ c_size
) {
186 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, lower_32_bits(v
));
187 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, upper_32_bits(v
));
192 while (i
< pt_size
) {
193 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, 0x00000000);
194 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, 0x00000000);
198 BAR0_WI32(chan
->vm_pd
, 0x00, priv
->pramin_pt
->instance
| 0x63);
199 BAR0_WI32(chan
->vm_pd
, 0x04, 0x00000000);
201 /* VRAM page table(s), mapped into VM at +1GiB */
202 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
203 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0,
204 NV50_VM_BLOCK
/65536*8, 0, 0,
205 &chan
->vm_vram_pt
[i
]);
207 NV_ERROR(dev
, "Error creating VRAM page tables: %d\n",
209 dev_priv
->vm_vram_pt_nr
= i
;
212 dev_priv
->vm_vram_pt
[i
] = chan
->vm_vram_pt
[i
]->gpuobj
;
214 for (v
= 0; v
< dev_priv
->vm_vram_pt
[i
]->im_pramin
->size
;
216 BAR0_WI32(dev_priv
->vm_vram_pt
[i
], v
, 0);
218 BAR0_WI32(chan
->vm_pd
, 0x10 + (i
*8),
219 chan
->vm_vram_pt
[i
]->instance
| 0x61);
220 BAR0_WI32(chan
->vm_pd
, 0x14 + (i
*8), 0);
223 /* DMA object for PRAMIN BAR */
224 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
228 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x00, 0x7fc00000);
229 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x04, dev_priv
->ramin_size
- 1);
230 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x08, 0x00000000);
231 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x0c, 0x00000000);
232 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x10, 0x00000000);
233 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x14, 0x00000000);
235 /* DMA object for FB BAR */
236 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
240 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x00, 0x7fc00000);
241 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x04, 0x40000000 +
242 pci_resource_len(dev
->pdev
, 1) - 1);
243 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x08, 0x40000000);
244 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x0c, 0x00000000);
245 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x10, 0x00000000);
246 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x14, 0x00000000);
248 /* Poke the relevant regs, and pray it works :) */
249 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
250 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
251 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
252 NV50_PUNK_BAR_CFG_BASE_VALID
);
253 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
254 NV50_PUNK_BAR1_CTXDMA_VALID
);
255 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
256 NV50_PUNK_BAR3_CTXDMA_VALID
);
258 for (i
= 0; i
< 8; i
++)
259 nv_wr32(dev
, 0x1900 + (i
*4), 0);
261 /* Assume that praying isn't enough, check that we can re-read the
262 * entire fake channel back from the PRAMIN BAR */
263 for (i
= 0; i
< c_size
; i
+= 4) {
264 if (nv_rd32(dev
, NV_RAMIN
+ i
) != nv_ri32(dev
, i
)) {
265 NV_ERROR(dev
, "Error reading back PRAMIN at 0x%08x\n",
271 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, save_nv001700
);
273 /* Global PRAMIN heap */
274 if (drm_mm_init(&dev_priv
->ramin_heap
, c_size
, dev_priv
->ramin_size
- c_size
)) {
275 NV_ERROR(dev
, "Failed to init RAMIN heap\n");
278 /*XXX: incorrect, but needed to make hash func "work" */
279 dev_priv
->ramht_offset
= 0x10000;
280 dev_priv
->ramht_bits
= 9;
281 dev_priv
->ramht_size
= (1 << dev_priv
->ramht_bits
);
286 nv50_instmem_takedown(struct drm_device
*dev
)
288 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
289 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
290 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
298 /* Restore state from before init */
299 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
300 nv_wr32(dev
, i
, priv
->save1700
[(i
- 0x1700) / 4]);
302 nouveau_gpuobj_ref_del(dev
, &priv
->fb_bar
);
303 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_bar
);
304 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_pt
);
306 /* Destroy dummy channel */
308 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
309 nouveau_gpuobj_ref_del(dev
, &chan
->vm_vram_pt
[i
]);
310 dev_priv
->vm_vram_pt
[i
] = NULL
;
312 dev_priv
->vm_vram_pt_nr
= 0;
314 nouveau_gpuobj_del(dev
, &chan
->vm_pd
);
315 nouveau_gpuobj_ref_del(dev
, &chan
->ramfc
);
316 nouveau_gpuobj_ref_del(dev
, &chan
->ramin
);
317 drm_mm_takedown(&chan
->ramin_heap
);
319 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = NULL
;
323 dev_priv
->engine
.instmem
.priv
= NULL
;
328 nv50_instmem_suspend(struct drm_device
*dev
)
330 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
331 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
332 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
335 ramin
->im_backing_suspend
= vmalloc(ramin
->im_pramin
->size
);
336 if (!ramin
->im_backing_suspend
)
339 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
340 ramin
->im_backing_suspend
[i
/4] = nv_ri32(dev
, i
);
345 nv50_instmem_resume(struct drm_device
*dev
)
347 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
348 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
349 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
350 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
353 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (ramin
->im_backing_start
>> 16));
354 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
355 BAR0_WI32(ramin
, i
, ramin
->im_backing_suspend
[i
/4]);
356 vfree(ramin
->im_backing_suspend
);
357 ramin
->im_backing_suspend
= NULL
;
359 /* Poke the relevant regs, and pray it works :) */
360 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
361 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
362 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
363 NV50_PUNK_BAR_CFG_BASE_VALID
);
364 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
365 NV50_PUNK_BAR1_CTXDMA_VALID
);
366 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
367 NV50_PUNK_BAR3_CTXDMA_VALID
);
369 for (i
= 0; i
< 8; i
++)
370 nv_wr32(dev
, 0x1900 + (i
*4), 0);
374 nv50_instmem_populate(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
,
379 if (gpuobj
->im_backing
)
382 *sz
= ALIGN(*sz
, NV50_INSTMEM_PAGE_SIZE
);
386 ret
= nouveau_bo_new(dev
, NULL
, *sz
, 0, TTM_PL_FLAG_VRAM
, 0, 0x0000,
387 true, false, &gpuobj
->im_backing
);
389 NV_ERROR(dev
, "error getting PRAMIN backing pages: %d\n", ret
);
393 ret
= nouveau_bo_pin(gpuobj
->im_backing
, TTM_PL_FLAG_VRAM
);
395 NV_ERROR(dev
, "error pinning PRAMIN backing VRAM: %d\n", ret
);
396 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
400 gpuobj
->im_backing_start
= gpuobj
->im_backing
->bo
.mem
.mm_node
->start
;
401 gpuobj
->im_backing_start
<<= PAGE_SHIFT
;
407 nv50_instmem_clear(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
409 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
411 if (gpuobj
&& gpuobj
->im_backing
) {
412 if (gpuobj
->im_bound
)
413 dev_priv
->engine
.instmem
.unbind(dev
, gpuobj
);
414 nouveau_bo_unpin(gpuobj
->im_backing
);
415 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
416 gpuobj
->im_backing
= NULL
;
421 nv50_instmem_bind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
423 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
424 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
425 struct nouveau_gpuobj
*pramin_pt
= priv
->pramin_pt
->gpuobj
;
426 uint32_t pte
, pte_end
;
429 if (!gpuobj
->im_backing
|| !gpuobj
->im_pramin
|| gpuobj
->im_bound
)
432 NV_DEBUG(dev
, "st=0x%lx sz=0x%lx\n",
433 gpuobj
->im_pramin
->start
, gpuobj
->im_pramin
->size
);
435 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
436 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
437 vram
= gpuobj
->im_backing_start
;
439 NV_DEBUG(dev
, "pramin=0x%lx, pte=%d, pte_end=%d\n",
440 gpuobj
->im_pramin
->start
, pte
, pte_end
);
441 NV_DEBUG(dev
, "first vram page: 0x%08x\n", gpuobj
->im_backing_start
);
444 if (dev_priv
->vram_sys_base
) {
445 vram
+= dev_priv
->vram_sys_base
;
449 while (pte
< pte_end
) {
450 nv_wo32(dev
, pramin_pt
, pte
++, lower_32_bits(vram
));
451 nv_wo32(dev
, pramin_pt
, pte
++, upper_32_bits(vram
));
452 vram
+= NV50_INSTMEM_PAGE_SIZE
;
454 dev_priv
->engine
.instmem
.flush(dev
);
456 nv50_vm_flush(dev
, 4);
457 nv50_vm_flush(dev
, 6);
459 gpuobj
->im_bound
= 1;
464 nv50_instmem_unbind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
466 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
467 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
468 uint32_t pte
, pte_end
;
470 if (gpuobj
->im_bound
== 0)
473 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
474 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
476 while (pte
< pte_end
) {
477 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
478 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
480 dev_priv
->engine
.instmem
.flush(dev
);
482 gpuobj
->im_bound
= 0;
487 nv50_instmem_flush(struct drm_device
*dev
)
489 nv_wr32(dev
, 0x00330c, 0x00000001);
490 if (!nv_wait(0x00330c, 0x00000002, 0x00000000))
491 NV_ERROR(dev
, "PRAMIN flush timeout\n");
495 nv84_instmem_flush(struct drm_device
*dev
)
497 nv_wr32(dev
, 0x070000, 0x00000001);
498 if (!nv_wait(0x070000, 0x00000002, 0x00000000))
499 NV_ERROR(dev
, "PRAMIN flush timeout\n");
503 nv50_vm_flush(struct drm_device
*dev
, int engine
)
505 nv_wr32(dev
, 0x100c80, (engine
<< 16) | 1);
506 if (!nv_wait(0x100c80, 0x00000001, 0x00000000))
507 NV_ERROR(dev
, "vm flush timeout: engine %d\n", engine
);