2 * Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
25 #include <nvif/class.h>
26 #include <subdev/timer.h>
28 static struct nvkm_oclass
30 { FERMI_TWOD_A
, &nvkm_object_ofuncs
},
31 { KEPLER_INLINE_TO_MEMORY_A
, &nvkm_object_ofuncs
},
32 { KEPLER_C
, &gf100_fermi_ofuncs
, gf100_gr_9097_omthds
},
33 { KEPLER_COMPUTE_A
, &nvkm_object_ofuncs
, gf100_gr_90c0_omthds
},
38 gk20a_gr_init_dtor(struct gf100_gr_pack
*pack
)
49 static struct gf100_gr_pack
*
50 gk20a_gr_av_to_init(struct gf100_gr_fuc
*fuc
)
52 struct gf100_gr_init
*init
;
53 struct gf100_gr_pack
*pack
;
54 const int nent
= (fuc
->size
/ sizeof(struct gk20a_fw_av
));
57 pack
= vzalloc((sizeof(*pack
) * 2) + (sizeof(*init
) * (nent
+ 1)));
59 return ERR_PTR(-ENOMEM
);
61 init
= (void *)(pack
+ 2);
65 for (i
= 0; i
< nent
; i
++) {
66 struct gf100_gr_init
*ent
= &init
[i
];
67 struct gk20a_fw_av
*av
= &((struct gk20a_fw_av
*)fuc
->data
)[i
];
85 static struct gf100_gr_pack
*
86 gk20a_gr_aiv_to_init(struct gf100_gr_fuc
*fuc
)
88 struct gf100_gr_init
*init
;
89 struct gf100_gr_pack
*pack
;
90 const int nent
= (fuc
->size
/ sizeof(struct gk20a_fw_aiv
));
93 pack
= vzalloc((sizeof(*pack
) * 2) + (sizeof(*init
) * (nent
+ 1)));
95 return ERR_PTR(-ENOMEM
);
97 init
= (void *)(pack
+ 2);
101 for (i
= 0; i
< nent
; i
++) {
102 struct gf100_gr_init
*ent
= &init
[i
];
103 struct gk20a_fw_aiv
*av
= &((struct gk20a_fw_aiv
*)fuc
->data
)[i
];
105 ent
->addr
= av
->addr
;
106 ent
->data
= av
->data
;
114 static struct gf100_gr_pack
*
115 gk20a_gr_av_to_method(struct gf100_gr_fuc
*fuc
)
117 struct gf100_gr_init
*init
;
118 struct gf100_gr_pack
*pack
;
119 /* We don't suppose we will initialize more than 16 classes here... */
120 static const unsigned int max_classes
= 16;
121 const int nent
= (fuc
->size
/ sizeof(struct gk20a_fw_av
));
125 pack
= vzalloc((sizeof(*pack
) * max_classes
) +
126 (sizeof(*init
) * (nent
+ 1)));
128 return ERR_PTR(-ENOMEM
);
130 init
= (void *)(pack
+ max_classes
);
132 for (i
= 0; i
< nent
; i
++) {
133 struct gf100_gr_init
*ent
= &init
[i
];
134 struct gk20a_fw_av
*av
= &((struct gk20a_fw_av
*)fuc
->data
)[i
];
135 u32
class = av
->addr
& 0xffff;
136 u32 addr
= (av
->addr
& 0xffff0000) >> 14;
138 if (prevclass
!= class) {
139 pack
[classidx
].init
= ent
;
140 pack
[classidx
].type
= class;
142 if (++classidx
>= max_classes
) {
144 return ERR_PTR(-ENOSPC
);
149 ent
->data
= av
->data
;
158 gk20a_gr_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
159 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
160 struct nvkm_object
**pobject
)
163 struct gf100_gr_priv
*priv
;
164 struct gf100_gr_fuc fuc
;
166 err
= gf100_gr_ctor(parent
, engine
, oclass
, data
, size
, pobject
);
170 priv
= (void *)*pobject
;
172 err
= gf100_gr_ctor_fw(priv
, "sw_nonctx", &fuc
);
175 priv
->fuc_sw_nonctx
= gk20a_gr_av_to_init(&fuc
);
176 gf100_gr_dtor_fw(&fuc
);
177 if (IS_ERR(priv
->fuc_sw_nonctx
))
178 return PTR_ERR(priv
->fuc_sw_nonctx
);
180 err
= gf100_gr_ctor_fw(priv
, "sw_ctx", &fuc
);
183 priv
->fuc_sw_ctx
= gk20a_gr_aiv_to_init(&fuc
);
184 gf100_gr_dtor_fw(&fuc
);
185 if (IS_ERR(priv
->fuc_sw_ctx
))
186 return PTR_ERR(priv
->fuc_sw_ctx
);
188 err
= gf100_gr_ctor_fw(priv
, "sw_bundle_init", &fuc
);
191 priv
->fuc_bundle
= gk20a_gr_av_to_init(&fuc
);
192 gf100_gr_dtor_fw(&fuc
);
193 if (IS_ERR(priv
->fuc_bundle
))
194 return PTR_ERR(priv
->fuc_bundle
);
196 err
= gf100_gr_ctor_fw(priv
, "sw_method_init", &fuc
);
199 priv
->fuc_method
= gk20a_gr_av_to_method(&fuc
);
200 gf100_gr_dtor_fw(&fuc
);
201 if (IS_ERR(priv
->fuc_method
))
202 return PTR_ERR(priv
->fuc_method
);
208 gk20a_gr_dtor(struct nvkm_object
*object
)
210 struct gf100_gr_priv
*priv
= (void *)object
;
212 gk20a_gr_init_dtor(priv
->fuc_method
);
213 gk20a_gr_init_dtor(priv
->fuc_bundle
);
214 gk20a_gr_init_dtor(priv
->fuc_sw_ctx
);
215 gk20a_gr_init_dtor(priv
->fuc_sw_nonctx
);
217 gf100_gr_dtor(object
);
221 gk20a_gr_wait_mem_scrubbing(struct gf100_gr_priv
*priv
)
223 if (!nv_wait(priv
, 0x40910c, 0x6, 0x0)) {
224 nv_error(priv
, "FECS mem scrubbing timeout\n");
228 if (!nv_wait(priv
, 0x41a10c, 0x6, 0x0)) {
229 nv_error(priv
, "GPCCS mem scrubbing timeout\n");
237 gk20a_gr_set_hww_esr_report_mask(struct gf100_gr_priv
*priv
)
239 nv_wr32(priv
, 0x419e44, 0x1ffffe);
240 nv_wr32(priv
, 0x419e4c, 0x7f);
244 gk20a_gr_init(struct nvkm_object
*object
)
246 struct gk20a_gr_oclass
*oclass
= (void *)object
->oclass
;
247 struct gf100_gr_priv
*priv
= (void *)object
;
248 const u32 magicgpc918
= DIV_ROUND_UP(0x00800000, priv
->tpc_total
);
249 u32 data
[TPC_MAX
/ 8] = {};
254 ret
= nvkm_gr_init(&priv
->base
);
259 nv_wr32(priv
, 0x40802c, 0x1);
261 gf100_gr_mmio(priv
, priv
->fuc_sw_nonctx
);
263 ret
= gk20a_gr_wait_mem_scrubbing(priv
);
267 ret
= gf100_gr_wait_idle(priv
);
271 /* MMU debug buffer */
272 nv_wr32(priv
, 0x100cc8, priv
->unk4188b4
->addr
>> 8);
273 nv_wr32(priv
, 0x100ccc, priv
->unk4188b8
->addr
>> 8);
275 if (oclass
->init_gpc_mmu
)
276 oclass
->init_gpc_mmu(priv
);
278 /* Set the PE as stream master */
279 nv_mask(priv
, 0x503018, 0x1, 0x1);
282 memset(data
, 0x00, sizeof(data
));
283 memcpy(tpcnr
, priv
->tpc_nr
, sizeof(priv
->tpc_nr
));
284 for (i
= 0, gpc
= -1; i
< priv
->tpc_total
; i
++) {
286 gpc
= (gpc
+ 1) % priv
->gpc_nr
;
287 } while (!tpcnr
[gpc
]);
288 tpc
= priv
->tpc_nr
[gpc
] - tpcnr
[gpc
]--;
290 data
[i
/ 8] |= tpc
<< ((i
% 8) * 4);
293 nv_wr32(priv
, GPC_BCAST(0x0980), data
[0]);
294 nv_wr32(priv
, GPC_BCAST(0x0984), data
[1]);
295 nv_wr32(priv
, GPC_BCAST(0x0988), data
[2]);
296 nv_wr32(priv
, GPC_BCAST(0x098c), data
[3]);
298 for (gpc
= 0; gpc
< priv
->gpc_nr
; gpc
++) {
299 nv_wr32(priv
, GPC_UNIT(gpc
, 0x0914),
300 priv
->magic_not_rop_nr
<< 8 | priv
->tpc_nr
[gpc
]);
301 nv_wr32(priv
, GPC_UNIT(gpc
, 0x0910), 0x00040000 |
303 nv_wr32(priv
, GPC_UNIT(gpc
, 0x0918), magicgpc918
);
306 nv_wr32(priv
, GPC_BCAST(0x3fd4), magicgpc918
);
308 /* Enable FIFO access */
309 nv_wr32(priv
, 0x400500, 0x00010001);
311 /* Enable interrupts */
312 nv_wr32(priv
, 0x400100, 0xffffffff);
313 nv_wr32(priv
, 0x40013c, 0xffffffff);
315 /* Enable FECS error interrupts */
316 nv_wr32(priv
, 0x409c24, 0x000f0000);
318 /* Enable hardware warning exceptions */
319 nv_wr32(priv
, 0x404000, 0xc0000000);
320 nv_wr32(priv
, 0x404600, 0xc0000000);
322 if (oclass
->set_hww_esr_report_mask
)
323 oclass
->set_hww_esr_report_mask(priv
);
325 /* Enable TPC exceptions per GPC */
326 nv_wr32(priv
, 0x419d0c, 0x2);
327 nv_wr32(priv
, 0x41ac94, (((1 << priv
->tpc_total
) - 1) & 0xff) << 16);
329 /* Reset and enable all exceptions */
330 nv_wr32(priv
, 0x400108, 0xffffffff);
331 nv_wr32(priv
, 0x400138, 0xffffffff);
332 nv_wr32(priv
, 0x400118, 0xffffffff);
333 nv_wr32(priv
, 0x400130, 0xffffffff);
334 nv_wr32(priv
, 0x40011c, 0xffffffff);
335 nv_wr32(priv
, 0x400134, 0xffffffff);
337 gf100_gr_zbc_init(priv
);
339 return gf100_gr_init_ctxctl(priv
);
343 gk20a_gr_oclass
= &(struct gk20a_gr_oclass
) {
345 .base
.handle
= NV_ENGINE(GR
, 0xea),
346 .base
.ofuncs
= &(struct nvkm_ofuncs
) {
347 .ctor
= gk20a_gr_ctor
,
348 .dtor
= gk20a_gr_dtor
,
349 .init
= gk20a_gr_init
,
350 .fini
= _nvkm_gr_fini
,
352 .cclass
= &gk20a_grctx_oclass
,
353 .sclass
= gk20a_gr_sclass
,
356 .set_hww_esr_report_mask
= gk20a_gr_set_hww_esr_report_mask
,