2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include <core/client.h>
26 #include <core/handle.h>
27 #include <core/namedb.h>
28 #include <core/gpuobj.h>
29 #include <core/engctx.h>
30 #include <core/class.h>
31 #include <core/math.h>
32 #include <core/enum.h>
34 #include <subdev/timer.h>
35 #include <subdev/bar.h>
36 #include <subdev/vm.h>
38 #include <engine/dmaobj.h>
39 #include <engine/fifo.h>
41 struct nvc0_fifo_priv
{
42 struct nouveau_fifo base
;
43 struct nouveau_gpuobj
*playlist
[2];
46 struct nouveau_gpuobj
*mem
;
47 struct nouveau_vma bar
;
52 struct nvc0_fifo_base
{
53 struct nouveau_fifo_base base
;
54 struct nouveau_gpuobj
*pgd
;
55 struct nouveau_vm
*vm
;
58 struct nvc0_fifo_chan
{
59 struct nouveau_fifo_chan base
;
62 /*******************************************************************************
63 * FIFO channel objects
64 ******************************************************************************/
67 nvc0_fifo_playlist_update(struct nvc0_fifo_priv
*priv
)
69 struct nouveau_bar
*bar
= nouveau_bar(priv
);
70 struct nouveau_gpuobj
*cur
;
73 cur
= priv
->playlist
[priv
->cur_playlist
];
74 priv
->cur_playlist
= !priv
->cur_playlist
;
76 for (i
= 0, p
= 0; i
< 128; i
++) {
77 if (!(nv_rd32(priv
, 0x003004 + (i
* 8)) & 1))
79 nv_wo32(cur
, p
+ 0, i
);
80 nv_wo32(cur
, p
+ 4, 0x00000004);
85 nv_wr32(priv
, 0x002270, cur
->addr
>> 12);
86 nv_wr32(priv
, 0x002274, 0x01f00000 | (p
>> 3));
87 if (!nv_wait(priv
, 0x00227c, 0x00100000, 0x00000000))
88 nv_error(priv
, "playlist update failed\n");
92 nvc0_fifo_context_attach(struct nouveau_object
*parent
,
93 struct nouveau_object
*object
)
95 struct nouveau_bar
*bar
= nouveau_bar(parent
);
96 struct nvc0_fifo_base
*base
= (void *)parent
->parent
;
97 struct nouveau_engctx
*ectx
= (void *)object
;
101 switch (nv_engidx(object
->engine
)) {
102 case NVDEV_ENGINE_SW
: return 0;
103 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
104 case NVDEV_ENGINE_COPY0
: addr
= 0x0230; break;
105 case NVDEV_ENGINE_COPY1
: addr
= 0x0240; break;
110 if (!ectx
->vma
.node
) {
111 ret
= nouveau_gpuobj_map_vm(nv_gpuobj(ectx
), base
->vm
,
112 NV_MEM_ACCESS_RW
, &ectx
->vma
);
117 nv_wo32(base
, addr
+ 0x00, lower_32_bits(ectx
->vma
.offset
) | 4);
118 nv_wo32(base
, addr
+ 0x04, upper_32_bits(ectx
->vma
.offset
));
124 nvc0_fifo_context_detach(struct nouveau_object
*parent
, bool suspend
,
125 struct nouveau_object
*object
)
127 struct nouveau_bar
*bar
= nouveau_bar(parent
);
128 struct nvc0_fifo_priv
*priv
= (void *)parent
->engine
;
129 struct nvc0_fifo_base
*base
= (void *)parent
->parent
;
130 struct nvc0_fifo_chan
*chan
= (void *)parent
;
133 switch (nv_engidx(object
->engine
)) {
134 case NVDEV_ENGINE_SW
: return 0;
135 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
136 case NVDEV_ENGINE_COPY0
: addr
= 0x0230; break;
137 case NVDEV_ENGINE_COPY1
: addr
= 0x0240; break;
142 nv_wo32(base
, addr
+ 0x00, 0x00000000);
143 nv_wo32(base
, addr
+ 0x04, 0x00000000);
146 nv_wr32(priv
, 0x002634, chan
->base
.chid
);
147 if (!nv_wait(priv
, 0x002634, 0xffffffff, chan
->base
.chid
)) {
148 nv_error(priv
, "channel %d kick timeout\n", chan
->base
.chid
);
157 nvc0_fifo_chan_ctor(struct nouveau_object
*parent
,
158 struct nouveau_object
*engine
,
159 struct nouveau_oclass
*oclass
, void *data
, u32 size
,
160 struct nouveau_object
**pobject
)
162 struct nouveau_bar
*bar
= nouveau_bar(parent
);
163 struct nvc0_fifo_priv
*priv
= (void *)engine
;
164 struct nvc0_fifo_base
*base
= (void *)parent
;
165 struct nvc0_fifo_chan
*chan
;
166 struct nv_channel_ind_class
*args
= data
;
167 u64 usermem
, ioffset
, ilength
;
170 if (size
< sizeof(*args
))
173 ret
= nouveau_fifo_channel_create(parent
, engine
, oclass
, 1,
174 priv
->user
.bar
.offset
, 0x1000,
176 (1 << NVDEV_ENGINE_SW
) |
177 (1 << NVDEV_ENGINE_GR
) |
178 (1 << NVDEV_ENGINE_COPY0
) |
179 (1 << NVDEV_ENGINE_COPY1
), &chan
);
180 *pobject
= nv_object(chan
);
184 nv_parent(chan
)->context_attach
= nvc0_fifo_context_attach
;
185 nv_parent(chan
)->context_detach
= nvc0_fifo_context_detach
;
187 usermem
= chan
->base
.chid
* 0x1000;
188 ioffset
= args
->ioffset
;
189 ilength
= log2i(args
->ilength
/ 8);
191 for (i
= 0; i
< 0x1000; i
+= 4)
192 nv_wo32(priv
->user
.mem
, usermem
+ i
, 0x00000000);
194 nv_wo32(base
, 0x08, lower_32_bits(priv
->user
.mem
->addr
+ usermem
));
195 nv_wo32(base
, 0x0c, upper_32_bits(priv
->user
.mem
->addr
+ usermem
));
196 nv_wo32(base
, 0x10, 0x0000face);
197 nv_wo32(base
, 0x30, 0xfffff902);
198 nv_wo32(base
, 0x48, lower_32_bits(ioffset
));
199 nv_wo32(base
, 0x4c, upper_32_bits(ioffset
) | (ilength
<< 16));
200 nv_wo32(base
, 0x54, 0x00000002);
201 nv_wo32(base
, 0x84, 0x20400000);
202 nv_wo32(base
, 0x94, 0x30000001);
203 nv_wo32(base
, 0x9c, 0x00000100);
204 nv_wo32(base
, 0xa4, 0x1f1f1f1f);
205 nv_wo32(base
, 0xa8, 0x1f1f1f1f);
206 nv_wo32(base
, 0xac, 0x0000001f);
207 nv_wo32(base
, 0xb8, 0xf8000000);
208 nv_wo32(base
, 0xf8, 0x10003080); /* 0x002310 */
209 nv_wo32(base
, 0xfc, 0x10000010); /* 0x002350 */
215 nvc0_fifo_chan_init(struct nouveau_object
*object
)
217 struct nouveau_gpuobj
*base
= nv_gpuobj(object
->parent
);
218 struct nvc0_fifo_priv
*priv
= (void *)object
->engine
;
219 struct nvc0_fifo_chan
*chan
= (void *)object
;
220 u32 chid
= chan
->base
.chid
;
223 ret
= nouveau_fifo_channel_init(&chan
->base
);
227 nv_wr32(priv
, 0x003000 + (chid
* 8), 0xc0000000 | base
->addr
>> 12);
228 nv_wr32(priv
, 0x003004 + (chid
* 8), 0x001f0001);
229 nvc0_fifo_playlist_update(priv
);
234 nvc0_fifo_chan_fini(struct nouveau_object
*object
, bool suspend
)
236 struct nvc0_fifo_priv
*priv
= (void *)object
->engine
;
237 struct nvc0_fifo_chan
*chan
= (void *)object
;
238 u32 chid
= chan
->base
.chid
;
240 nv_mask(priv
, 0x003004 + (chid
* 8), 0x00000001, 0x00000000);
241 nvc0_fifo_playlist_update(priv
);
242 nv_wr32(priv
, 0x003000 + (chid
* 8), 0x00000000);
244 return nouveau_fifo_channel_fini(&chan
->base
, suspend
);
247 static struct nouveau_ofuncs
249 .ctor
= nvc0_fifo_chan_ctor
,
250 .dtor
= _nouveau_fifo_channel_dtor
,
251 .init
= nvc0_fifo_chan_init
,
252 .fini
= nvc0_fifo_chan_fini
,
253 .rd32
= _nouveau_fifo_channel_rd32
,
254 .wr32
= _nouveau_fifo_channel_wr32
,
257 static struct nouveau_oclass
258 nvc0_fifo_sclass
[] = {
259 { 0x906f, &nvc0_fifo_ofuncs
},
263 /*******************************************************************************
264 * FIFO context - instmem heap and vm setup
265 ******************************************************************************/
268 nvc0_fifo_context_ctor(struct nouveau_object
*parent
,
269 struct nouveau_object
*engine
,
270 struct nouveau_oclass
*oclass
, void *data
, u32 size
,
271 struct nouveau_object
**pobject
)
273 struct nvc0_fifo_base
*base
;
276 ret
= nouveau_fifo_context_create(parent
, engine
, oclass
, NULL
, 0x1000,
277 0x1000, NVOBJ_FLAG_ZERO_ALLOC
|
278 NVOBJ_FLAG_HEAP
, &base
);
279 *pobject
= nv_object(base
);
283 ret
= nouveau_gpuobj_new(parent
, NULL
, 0x10000, 0x1000, 0, &base
->pgd
);
287 nv_wo32(base
, 0x0200, lower_32_bits(base
->pgd
->addr
));
288 nv_wo32(base
, 0x0204, upper_32_bits(base
->pgd
->addr
));
289 nv_wo32(base
, 0x0208, 0xffffffff);
290 nv_wo32(base
, 0x020c, 0x000000ff);
292 ret
= nouveau_vm_ref(nouveau_client(parent
)->vm
, &base
->vm
, base
->pgd
);
300 nvc0_fifo_context_dtor(struct nouveau_object
*object
)
302 struct nvc0_fifo_base
*base
= (void *)object
;
303 nouveau_vm_ref(NULL
, &base
->vm
, base
->pgd
);
304 nouveau_gpuobj_ref(NULL
, &base
->pgd
);
305 nouveau_fifo_context_destroy(&base
->base
);
308 static struct nouveau_oclass
310 .handle
= NV_ENGCTX(FIFO
, 0xc0),
311 .ofuncs
= &(struct nouveau_ofuncs
) {
312 .ctor
= nvc0_fifo_context_ctor
,
313 .dtor
= nvc0_fifo_context_dtor
,
314 .init
= _nouveau_fifo_context_init
,
315 .fini
= _nouveau_fifo_context_fini
,
316 .rd32
= _nouveau_fifo_context_rd32
,
317 .wr32
= _nouveau_fifo_context_wr32
,
321 /*******************************************************************************
323 ******************************************************************************/
325 struct nouveau_enum nvc0_fifo_fault_unit
[] = {
327 { 0x03, "PEEPHOLE" },
333 { 0x13, "PCOUNTER" },
341 struct nouveau_enum nvc0_fifo_fault_reason
[] = {
342 { 0x00, "PT_NOT_PRESENT" },
343 { 0x01, "PT_TOO_SHORT" },
344 { 0x02, "PAGE_NOT_PRESENT" },
345 { 0x03, "VM_LIMIT_EXCEEDED" },
346 { 0x04, "NO_CHANNEL" },
347 { 0x05, "PAGE_SYSTEM_ONLY" },
348 { 0x06, "PAGE_READ_ONLY" },
349 { 0x0a, "COMPRESSED_SYSRAM" },
350 { 0x0c, "INVALID_STORAGE_TYPE" },
354 struct nouveau_enum nvc0_fifo_fault_hubclient
[] = {
357 { 0x04, "DISPATCH" },
360 { 0x07, "BAR_READ" },
361 { 0x08, "BAR_WRITE" },
365 { 0x11, "PCOUNTER" },
368 { 0x15, "CCACHE_POST" },
372 struct nouveau_enum nvc0_fifo_fault_gpcclient
[] = {
380 struct nouveau_bitfield nvc0_fifo_subfifo_intr
[] = {
381 /* { 0x00008000, "" } seen with null ib push */
382 { 0x00200000, "ILLEGAL_MTHD" },
383 { 0x00800000, "EMPTY_SUBC" },
388 nvc0_fifo_isr_vm_fault(struct nvc0_fifo_priv
*priv
, int unit
)
390 u32 inst
= nv_rd32(priv
, 0x2800 + (unit
* 0x10));
391 u32 valo
= nv_rd32(priv
, 0x2804 + (unit
* 0x10));
392 u32 vahi
= nv_rd32(priv
, 0x2808 + (unit
* 0x10));
393 u32 stat
= nv_rd32(priv
, 0x280c + (unit
* 0x10));
394 u32 client
= (stat
& 0x00001f00) >> 8;
396 nv_error(priv
, "%s fault at 0x%010llx [", (stat
& 0x00000080) ?
397 "write" : "read", (u64
)vahi
<< 32 | valo
);
398 nouveau_enum_print(nvc0_fifo_fault_reason
, stat
& 0x0000000f);
400 nouveau_enum_print(nvc0_fifo_fault_unit
, unit
);
401 if (stat
& 0x00000040) {
403 nouveau_enum_print(nvc0_fifo_fault_hubclient
, client
);
405 printk("/GPC%d/", (stat
& 0x1f000000) >> 24);
406 nouveau_enum_print(nvc0_fifo_fault_gpcclient
, client
);
408 printk(" on channel 0x%010llx\n", (u64
)inst
<< 12);
412 nvc0_fifo_swmthd(struct nvc0_fifo_priv
*priv
, u32 chid
, u32 mthd
, u32 data
)
414 struct nvc0_fifo_chan
*chan
= NULL
;
415 struct nouveau_handle
*bind
;
419 spin_lock_irqsave(&priv
->base
.lock
, flags
);
420 if (likely(chid
>= priv
->base
.min
&& chid
<= priv
->base
.max
))
421 chan
= (void *)priv
->base
.channel
[chid
];
425 bind
= nouveau_namedb_get_class(nv_namedb(chan
), 0x906e);
427 if (!mthd
|| !nv_call(bind
->object
, mthd
, data
))
429 nouveau_namedb_put(bind
);
433 spin_unlock_irqrestore(&priv
->base
.lock
, flags
);
438 nvc0_fifo_isr_subfifo_intr(struct nvc0_fifo_priv
*priv
, int unit
)
440 u32 stat
= nv_rd32(priv
, 0x040108 + (unit
* 0x2000));
441 u32 addr
= nv_rd32(priv
, 0x0400c0 + (unit
* 0x2000));
442 u32 data
= nv_rd32(priv
, 0x0400c4 + (unit
* 0x2000));
443 u32 chid
= nv_rd32(priv
, 0x040120 + (unit
* 0x2000)) & 0x7f;
444 u32 subc
= (addr
& 0x00070000) >> 16;
445 u32 mthd
= (addr
& 0x00003ffc);
448 if (stat
& 0x00200000) {
449 if (mthd
== 0x0054) {
450 if (!nvc0_fifo_swmthd(priv
, chid
, 0x0500, 0x00000000))
455 if (stat
& 0x00800000) {
456 if (!nvc0_fifo_swmthd(priv
, chid
, mthd
, data
))
461 nv_error(priv
, "SUBFIFO%d:", unit
);
462 nouveau_bitfield_print(nvc0_fifo_subfifo_intr
, show
);
464 nv_error(priv
, "SUBFIFO%d: ch %d subc %d mthd 0x%04x "
466 unit
, chid
, subc
, mthd
, data
);
469 nv_wr32(priv
, 0x0400c0 + (unit
* 0x2000), 0x80600008);
470 nv_wr32(priv
, 0x040108 + (unit
* 0x2000), stat
);
474 nvc0_fifo_intr(struct nouveau_subdev
*subdev
)
476 struct nvc0_fifo_priv
*priv
= (void *)subdev
;
477 u32 mask
= nv_rd32(priv
, 0x002140);
478 u32 stat
= nv_rd32(priv
, 0x002100) & mask
;
480 if (stat
& 0x00000100) {
481 nv_info(priv
, "unknown status 0x00000100\n");
482 nv_wr32(priv
, 0x002100, 0x00000100);
486 if (stat
& 0x10000000) {
487 u32 units
= nv_rd32(priv
, 0x00259c);
492 nvc0_fifo_isr_vm_fault(priv
, i
);
496 nv_wr32(priv
, 0x00259c, units
);
500 if (stat
& 0x20000000) {
501 u32 units
= nv_rd32(priv
, 0x0025a0);
506 nvc0_fifo_isr_subfifo_intr(priv
, i
);
510 nv_wr32(priv
, 0x0025a0, units
);
514 if (stat
& 0x40000000) {
515 nv_warn(priv
, "unknown status 0x40000000\n");
516 nv_mask(priv
, 0x002a00, 0x00000000, 0x00000000);
521 nv_fatal(priv
, "unhandled status 0x%08x\n", stat
);
522 nv_wr32(priv
, 0x002100, stat
);
523 nv_wr32(priv
, 0x002140, 0);
528 nvc0_fifo_ctor(struct nouveau_object
*parent
, struct nouveau_object
*engine
,
529 struct nouveau_oclass
*oclass
, void *data
, u32 size
,
530 struct nouveau_object
**pobject
)
532 struct nvc0_fifo_priv
*priv
;
535 ret
= nouveau_fifo_create(parent
, engine
, oclass
, 0, 127, &priv
);
536 *pobject
= nv_object(priv
);
540 ret
= nouveau_gpuobj_new(parent
, NULL
, 0x1000, 0x1000, 0,
545 ret
= nouveau_gpuobj_new(parent
, NULL
, 0x1000, 0x1000, 0,
550 ret
= nouveau_gpuobj_new(parent
, NULL
, 128 * 0x1000, 0x1000, 0,
555 ret
= nouveau_gpuobj_map(priv
->user
.mem
, NV_MEM_ACCESS_RW
,
560 nv_subdev(priv
)->unit
= 0x00000100;
561 nv_subdev(priv
)->intr
= nvc0_fifo_intr
;
562 nv_engine(priv
)->cclass
= &nvc0_fifo_cclass
;
563 nv_engine(priv
)->sclass
= nvc0_fifo_sclass
;
568 nvc0_fifo_dtor(struct nouveau_object
*object
)
570 struct nvc0_fifo_priv
*priv
= (void *)object
;
572 nouveau_gpuobj_unmap(&priv
->user
.bar
);
573 nouveau_gpuobj_ref(NULL
, &priv
->user
.mem
);
574 nouveau_gpuobj_ref(NULL
, &priv
->playlist
[1]);
575 nouveau_gpuobj_ref(NULL
, &priv
->playlist
[0]);
577 nouveau_fifo_destroy(&priv
->base
);
581 nvc0_fifo_init(struct nouveau_object
*object
)
583 struct nvc0_fifo_priv
*priv
= (void *)object
;
586 ret
= nouveau_fifo_init(&priv
->base
);
590 nv_wr32(priv
, 0x000204, 0xffffffff);
591 nv_wr32(priv
, 0x002204, 0xffffffff);
593 priv
->spoon_nr
= hweight32(nv_rd32(priv
, 0x002204));
594 nv_debug(priv
, "%d subfifo(s)\n", priv
->spoon_nr
);
596 /* assign engines to subfifos */
597 if (priv
->spoon_nr
>= 3) {
598 nv_wr32(priv
, 0x002208, ~(1 << 0)); /* PGRAPH */
599 nv_wr32(priv
, 0x00220c, ~(1 << 1)); /* PVP */
600 nv_wr32(priv
, 0x002210, ~(1 << 1)); /* PPP */
601 nv_wr32(priv
, 0x002214, ~(1 << 1)); /* PBSP */
602 nv_wr32(priv
, 0x002218, ~(1 << 2)); /* PCE0 */
603 nv_wr32(priv
, 0x00221c, ~(1 << 1)); /* PCE1 */
607 for (i
= 0; i
< priv
->spoon_nr
; i
++) {
608 nv_mask(priv
, 0x04013c + (i
* 0x2000), 0x10000100, 0x00000000);
609 nv_wr32(priv
, 0x040108 + (i
* 0x2000), 0xffffffff); /* INTR */
610 nv_wr32(priv
, 0x04010c + (i
* 0x2000), 0xfffffeff); /* INTREN */
613 nv_mask(priv
, 0x002200, 0x00000001, 0x00000001);
614 nv_wr32(priv
, 0x002254, 0x10000000 | priv
->user
.bar
.offset
>> 12);
616 nv_wr32(priv
, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
617 nv_wr32(priv
, 0x002100, 0xffffffff);
618 nv_wr32(priv
, 0x002140, 0xbfffffff);
622 struct nouveau_oclass
624 .handle
= NV_ENGINE(FIFO
, 0xc0),
625 .ofuncs
= &(struct nouveau_ofuncs
) {
626 .ctor
= nvc0_fifo_ctor
,
627 .dtor
= nvc0_fifo_dtor
,
628 .init
= nvc0_fifo_init
,
629 .fini
= _nouveau_fifo_fini
,