projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/radeon: add support for newer mc ucode on CI (v2)
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
35
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
66
67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
72 extern void sumo_rlc_fini(struct radeon_device *rdev);
73 extern int sumo_rlc_init(struct radeon_device *rdev);
74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
75 extern void si_rlc_reset(struct radeon_device *rdev);
76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
77 extern int cik_sdma_resume(struct radeon_device *rdev);
78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
79 extern void cik_sdma_fini(struct radeon_device *rdev);
80 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
81 static void cik_rlc_stop(struct radeon_device *rdev);
82 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
83 static void cik_program_aspm(struct radeon_device *rdev);
84 static void cik_init_pg(struct radeon_device *rdev);
85 static void cik_init_cg(struct radeon_device *rdev);
86 static void cik_fini_pg(struct radeon_device *rdev);
87 static void cik_fini_cg(struct radeon_device *rdev);
88 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
89 bool enable);
90
91 /* get temperature in millidegrees */
92 int ci_get_temp(struct radeon_device *rdev)
93 {
94 u32 temp;
95 int actual_temp = 0;
96
97 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
98 CTF_TEMP_SHIFT;
99
100 if (temp & 0x200)
101 actual_temp = 255;
102 else
103 actual_temp = temp & 0x1ff;
104
105 actual_temp = actual_temp * 1000;
106
107 return actual_temp;
108 }
109
110 /* get temperature in millidegrees */
111 int kv_get_temp(struct radeon_device *rdev)
112 {
113 u32 temp;
114 int actual_temp = 0;
115
116 temp = RREG32_SMC(0xC0300E0C);
117
118 if (temp)
119 actual_temp = (temp / 8) - 49;
120 else
121 actual_temp = 0;
122
123 actual_temp = actual_temp * 1000;
124
125 return actual_temp;
126 }
127
128 /*
129 * Indirect registers accessor
130 */
131 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
132 {
133 unsigned long flags;
134 u32 r;
135
136 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
137 WREG32(PCIE_INDEX, reg);
138 (void)RREG32(PCIE_INDEX);
139 r = RREG32(PCIE_DATA);
140 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
141 return r;
142 }
143
144 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
145 {
146 unsigned long flags;
147
148 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
149 WREG32(PCIE_INDEX, reg);
150 (void)RREG32(PCIE_INDEX);
151 WREG32(PCIE_DATA, v);
152 (void)RREG32(PCIE_DATA);
153 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
154 }
155
156 static const u32 spectre_rlc_save_restore_register_list[] =
157 {
158 (0x0e00 << 16) | (0xc12c >> 2),
159 0x00000000,
160 (0x0e00 << 16) | (0xc140 >> 2),
161 0x00000000,
162 (0x0e00 << 16) | (0xc150 >> 2),
163 0x00000000,
164 (0x0e00 << 16) | (0xc15c >> 2),
165 0x00000000,
166 (0x0e00 << 16) | (0xc168 >> 2),
167 0x00000000,
168 (0x0e00 << 16) | (0xc170 >> 2),
169 0x00000000,
170 (0x0e00 << 16) | (0xc178 >> 2),
171 0x00000000,
172 (0x0e00 << 16) | (0xc204 >> 2),
173 0x00000000,
174 (0x0e00 << 16) | (0xc2b4 >> 2),
175 0x00000000,
176 (0x0e00 << 16) | (0xc2b8 >> 2),
177 0x00000000,
178 (0x0e00 << 16) | (0xc2bc >> 2),
179 0x00000000,
180 (0x0e00 << 16) | (0xc2c0 >> 2),
181 0x00000000,
182 (0x0e00 << 16) | (0x8228 >> 2),
183 0x00000000,
184 (0x0e00 << 16) | (0x829c >> 2),
185 0x00000000,
186 (0x0e00 << 16) | (0x869c >> 2),
187 0x00000000,
188 (0x0600 << 16) | (0x98f4 >> 2),
189 0x00000000,
190 (0x0e00 << 16) | (0x98f8 >> 2),
191 0x00000000,
192 (0x0e00 << 16) | (0x9900 >> 2),
193 0x00000000,
194 (0x0e00 << 16) | (0xc260 >> 2),
195 0x00000000,
196 (0x0e00 << 16) | (0x90e8 >> 2),
197 0x00000000,
198 (0x0e00 << 16) | (0x3c000 >> 2),
199 0x00000000,
200 (0x0e00 << 16) | (0x3c00c >> 2),
201 0x00000000,
202 (0x0e00 << 16) | (0x8c1c >> 2),
203 0x00000000,
204 (0x0e00 << 16) | (0x9700 >> 2),
205 0x00000000,
206 (0x0e00 << 16) | (0xcd20 >> 2),
207 0x00000000,
208 (0x4e00 << 16) | (0xcd20 >> 2),
209 0x00000000,
210 (0x5e00 << 16) | (0xcd20 >> 2),
211 0x00000000,
212 (0x6e00 << 16) | (0xcd20 >> 2),
213 0x00000000,
214 (0x7e00 << 16) | (0xcd20 >> 2),
215 0x00000000,
216 (0x8e00 << 16) | (0xcd20 >> 2),
217 0x00000000,
218 (0x9e00 << 16) | (0xcd20 >> 2),
219 0x00000000,
220 (0xae00 << 16) | (0xcd20 >> 2),
221 0x00000000,
222 (0xbe00 << 16) | (0xcd20 >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0x89bc >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0x8900 >> 2),
227 0x00000000,
228 0x3,
229 (0x0e00 << 16) | (0xc130 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc134 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc1fc >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0xc208 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0xc264 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xc268 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0xc26c >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0xc270 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc274 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc278 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0xc27c >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0xc280 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0xc284 >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0xc288 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xc28c >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xc290 >> 2),
260 0x00000000,
261 (0x0e00 << 16) | (0xc294 >> 2),
262 0x00000000,
263 (0x0e00 << 16) | (0xc298 >> 2),
264 0x00000000,
265 (0x0e00 << 16) | (0xc29c >> 2),
266 0x00000000,
267 (0x0e00 << 16) | (0xc2a0 >> 2),
268 0x00000000,
269 (0x0e00 << 16) | (0xc2a4 >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0xc2a8 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc2ac >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc2b0 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x301d0 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x30238 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0x30250 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0x30254 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0x30258 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0x3025c >> 2),
288 0x00000000,
289 (0x4e00 << 16) | (0xc900 >> 2),
290 0x00000000,
291 (0x5e00 << 16) | (0xc900 >> 2),
292 0x00000000,
293 (0x6e00 << 16) | (0xc900 >> 2),
294 0x00000000,
295 (0x7e00 << 16) | (0xc900 >> 2),
296 0x00000000,
297 (0x8e00 << 16) | (0xc900 >> 2),
298 0x00000000,
299 (0x9e00 << 16) | (0xc900 >> 2),
300 0x00000000,
301 (0xae00 << 16) | (0xc900 >> 2),
302 0x00000000,
303 (0xbe00 << 16) | (0xc900 >> 2),
304 0x00000000,
305 (0x4e00 << 16) | (0xc904 >> 2),
306 0x00000000,
307 (0x5e00 << 16) | (0xc904 >> 2),
308 0x00000000,
309 (0x6e00 << 16) | (0xc904 >> 2),
310 0x00000000,
311 (0x7e00 << 16) | (0xc904 >> 2),
312 0x00000000,
313 (0x8e00 << 16) | (0xc904 >> 2),
314 0x00000000,
315 (0x9e00 << 16) | (0xc904 >> 2),
316 0x00000000,
317 (0xae00 << 16) | (0xc904 >> 2),
318 0x00000000,
319 (0xbe00 << 16) | (0xc904 >> 2),
320 0x00000000,
321 (0x4e00 << 16) | (0xc908 >> 2),
322 0x00000000,
323 (0x5e00 << 16) | (0xc908 >> 2),
324 0x00000000,
325 (0x6e00 << 16) | (0xc908 >> 2),
326 0x00000000,
327 (0x7e00 << 16) | (0xc908 >> 2),
328 0x00000000,
329 (0x8e00 << 16) | (0xc908 >> 2),
330 0x00000000,
331 (0x9e00 << 16) | (0xc908 >> 2),
332 0x00000000,
333 (0xae00 << 16) | (0xc908 >> 2),
334 0x00000000,
335 (0xbe00 << 16) | (0xc908 >> 2),
336 0x00000000,
337 (0x4e00 << 16) | (0xc90c >> 2),
338 0x00000000,
339 (0x5e00 << 16) | (0xc90c >> 2),
340 0x00000000,
341 (0x6e00 << 16) | (0xc90c >> 2),
342 0x00000000,
343 (0x7e00 << 16) | (0xc90c >> 2),
344 0x00000000,
345 (0x8e00 << 16) | (0xc90c >> 2),
346 0x00000000,
347 (0x9e00 << 16) | (0xc90c >> 2),
348 0x00000000,
349 (0xae00 << 16) | (0xc90c >> 2),
350 0x00000000,
351 (0xbe00 << 16) | (0xc90c >> 2),
352 0x00000000,
353 (0x4e00 << 16) | (0xc910 >> 2),
354 0x00000000,
355 (0x5e00 << 16) | (0xc910 >> 2),
356 0x00000000,
357 (0x6e00 << 16) | (0xc910 >> 2),
358 0x00000000,
359 (0x7e00 << 16) | (0xc910 >> 2),
360 0x00000000,
361 (0x8e00 << 16) | (0xc910 >> 2),
362 0x00000000,
363 (0x9e00 << 16) | (0xc910 >> 2),
364 0x00000000,
365 (0xae00 << 16) | (0xc910 >> 2),
366 0x00000000,
367 (0xbe00 << 16) | (0xc910 >> 2),
368 0x00000000,
369 (0x0e00 << 16) | (0xc99c >> 2),
370 0x00000000,
371 (0x0e00 << 16) | (0x9834 >> 2),
372 0x00000000,
373 (0x0000 << 16) | (0x30f00 >> 2),
374 0x00000000,
375 (0x0001 << 16) | (0x30f00 >> 2),
376 0x00000000,
377 (0x0000 << 16) | (0x30f04 >> 2),
378 0x00000000,
379 (0x0001 << 16) | (0x30f04 >> 2),
380 0x00000000,
381 (0x0000 << 16) | (0x30f08 >> 2),
382 0x00000000,
383 (0x0001 << 16) | (0x30f08 >> 2),
384 0x00000000,
385 (0x0000 << 16) | (0x30f0c >> 2),
386 0x00000000,
387 (0x0001 << 16) | (0x30f0c >> 2),
388 0x00000000,
389 (0x0600 << 16) | (0x9b7c >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0x8a14 >> 2),
392 0x00000000,
393 (0x0e00 << 16) | (0x8a18 >> 2),
394 0x00000000,
395 (0x0600 << 16) | (0x30a00 >> 2),
396 0x00000000,
397 (0x0e00 << 16) | (0x8bf0 >> 2),
398 0x00000000,
399 (0x0e00 << 16) | (0x8bcc >> 2),
400 0x00000000,
401 (0x0e00 << 16) | (0x8b24 >> 2),
402 0x00000000,
403 (0x0e00 << 16) | (0x30a04 >> 2),
404 0x00000000,
405 (0x0600 << 16) | (0x30a10 >> 2),
406 0x00000000,
407 (0x0600 << 16) | (0x30a14 >> 2),
408 0x00000000,
409 (0x0600 << 16) | (0x30a18 >> 2),
410 0x00000000,
411 (0x0600 << 16) | (0x30a2c >> 2),
412 0x00000000,
413 (0x0e00 << 16) | (0xc700 >> 2),
414 0x00000000,
415 (0x0e00 << 16) | (0xc704 >> 2),
416 0x00000000,
417 (0x0e00 << 16) | (0xc708 >> 2),
418 0x00000000,
419 (0x0e00 << 16) | (0xc768 >> 2),
420 0x00000000,
421 (0x0400 << 16) | (0xc770 >> 2),
422 0x00000000,
423 (0x0400 << 16) | (0xc774 >> 2),
424 0x00000000,
425 (0x0400 << 16) | (0xc778 >> 2),
426 0x00000000,
427 (0x0400 << 16) | (0xc77c >> 2),
428 0x00000000,
429 (0x0400 << 16) | (0xc780 >> 2),
430 0x00000000,
431 (0x0400 << 16) | (0xc784 >> 2),
432 0x00000000,
433 (0x0400 << 16) | (0xc788 >> 2),
434 0x00000000,
435 (0x0400 << 16) | (0xc78c >> 2),
436 0x00000000,
437 (0x0400 << 16) | (0xc798 >> 2),
438 0x00000000,
439 (0x0400 << 16) | (0xc79c >> 2),
440 0x00000000,
441 (0x0400 << 16) | (0xc7a0 >> 2),
442 0x00000000,
443 (0x0400 << 16) | (0xc7a4 >> 2),
444 0x00000000,
445 (0x0400 << 16) | (0xc7a8 >> 2),
446 0x00000000,
447 (0x0400 << 16) | (0xc7ac >> 2),
448 0x00000000,
449 (0x0400 << 16) | (0xc7b0 >> 2),
450 0x00000000,
451 (0x0400 << 16) | (0xc7b4 >> 2),
452 0x00000000,
453 (0x0e00 << 16) | (0x9100 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0x3c010 >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x92a8 >> 2),
458 0x00000000,
459 (0x0e00 << 16) | (0x92ac >> 2),
460 0x00000000,
461 (0x0e00 << 16) | (0x92b4 >> 2),
462 0x00000000,
463 (0x0e00 << 16) | (0x92b8 >> 2),
464 0x00000000,
465 (0x0e00 << 16) | (0x92bc >> 2),
466 0x00000000,
467 (0x0e00 << 16) | (0x92c0 >> 2),
468 0x00000000,
469 (0x0e00 << 16) | (0x92c4 >> 2),
470 0x00000000,
471 (0x0e00 << 16) | (0x92c8 >> 2),
472 0x00000000,
473 (0x0e00 << 16) | (0x92cc >> 2),
474 0x00000000,
475 (0x0e00 << 16) | (0x92d0 >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8c00 >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0x8c04 >> 2),
480 0x00000000,
481 (0x0e00 << 16) | (0x8c20 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0x8c38 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0x8c3c >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0xae00 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0x9604 >> 2),
490 0x00000000,
491 (0x0e00 << 16) | (0xac08 >> 2),
492 0x00000000,
493 (0x0e00 << 16) | (0xac0c >> 2),
494 0x00000000,
495 (0x0e00 << 16) | (0xac10 >> 2),
496 0x00000000,
497 (0x0e00 << 16) | (0xac14 >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xac58 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xac68 >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xac6c >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xac70 >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0xac74 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0xac78 >> 2),
510 0x00000000,
511 (0x0e00 << 16) | (0xac7c >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0xac80 >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0xac84 >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0xac88 >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0xac8c >> 2),
520 0x00000000,
521 (0x0e00 << 16) | (0x970c >> 2),
522 0x00000000,
523 (0x0e00 << 16) | (0x9714 >> 2),
524 0x00000000,
525 (0x0e00 << 16) | (0x9718 >> 2),
526 0x00000000,
527 (0x0e00 << 16) | (0x971c >> 2),
528 0x00000000,
529 (0x0e00 << 16) | (0x31068 >> 2),
530 0x00000000,
531 (0x4e00 << 16) | (0x31068 >> 2),
532 0x00000000,
533 (0x5e00 << 16) | (0x31068 >> 2),
534 0x00000000,
535 (0x6e00 << 16) | (0x31068 >> 2),
536 0x00000000,
537 (0x7e00 << 16) | (0x31068 >> 2),
538 0x00000000,
539 (0x8e00 << 16) | (0x31068 >> 2),
540 0x00000000,
541 (0x9e00 << 16) | (0x31068 >> 2),
542 0x00000000,
543 (0xae00 << 16) | (0x31068 >> 2),
544 0x00000000,
545 (0xbe00 << 16) | (0x31068 >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0xcd10 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0xcd14 >> 2),
550 0x00000000,
551 (0x0e00 << 16) | (0x88b0 >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x88b4 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x88b8 >> 2),
556 0x00000000,
557 (0x0e00 << 16) | (0x88bc >> 2),
558 0x00000000,
559 (0x0400 << 16) | (0x89c0 >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x88c4 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x88c8 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x88d0 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x88d4 >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x88d8 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x8980 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x30938 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x3093c >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x30940 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x89a0 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x30900 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x30904 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x89b4 >> 2),
586 0x00000000,
587 (0x0e00 << 16) | (0x3c210 >> 2),
588 0x00000000,
589 (0x0e00 << 16) | (0x3c214 >> 2),
590 0x00000000,
591 (0x0e00 << 16) | (0x3c218 >> 2),
592 0x00000000,
593 (0x0e00 << 16) | (0x8904 >> 2),
594 0x00000000,
595 0x5,
596 (0x0e00 << 16) | (0x8c28 >> 2),
597 (0x0e00 << 16) | (0x8c2c >> 2),
598 (0x0e00 << 16) | (0x8c30 >> 2),
599 (0x0e00 << 16) | (0x8c34 >> 2),
600 (0x0e00 << 16) | (0x9600 >> 2),
601 };
602
603 static const u32 kalindi_rlc_save_restore_register_list[] =
604 {
605 (0x0e00 << 16) | (0xc12c >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xc140 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xc150 >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xc15c >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xc168 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xc170 >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xc204 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0xc2b4 >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0xc2b8 >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0xc2bc >> 2),
624 0x00000000,
625 (0x0e00 << 16) | (0xc2c0 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0x8228 >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0x829c >> 2),
630 0x00000000,
631 (0x0e00 << 16) | (0x869c >> 2),
632 0x00000000,
633 (0x0600 << 16) | (0x98f4 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0x98f8 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x9900 >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0xc260 >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x90e8 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0x3c000 >> 2),
644 0x00000000,
645 (0x0e00 << 16) | (0x3c00c >> 2),
646 0x00000000,
647 (0x0e00 << 16) | (0x8c1c >> 2),
648 0x00000000,
649 (0x0e00 << 16) | (0x9700 >> 2),
650 0x00000000,
651 (0x0e00 << 16) | (0xcd20 >> 2),
652 0x00000000,
653 (0x4e00 << 16) | (0xcd20 >> 2),
654 0x00000000,
655 (0x5e00 << 16) | (0xcd20 >> 2),
656 0x00000000,
657 (0x6e00 << 16) | (0xcd20 >> 2),
658 0x00000000,
659 (0x7e00 << 16) | (0xcd20 >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0x89bc >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0x8900 >> 2),
664 0x00000000,
665 0x3,
666 (0x0e00 << 16) | (0xc130 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc134 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc1fc >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc208 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc264 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc268 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc26c >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0xc270 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0xc274 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xc28c >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0xc290 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0xc294 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc298 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0xc2a0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0xc2a4 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0xc2a8 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0xc2ac >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x301d0 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x30238 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x30250 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x30254 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x30258 >> 2),
709 0x00000000,
710 (0x0e00 << 16) | (0x3025c >> 2),
711 0x00000000,
712 (0x4e00 << 16) | (0xc900 >> 2),
713 0x00000000,
714 (0x5e00 << 16) | (0xc900 >> 2),
715 0x00000000,
716 (0x6e00 << 16) | (0xc900 >> 2),
717 0x00000000,
718 (0x7e00 << 16) | (0xc900 >> 2),
719 0x00000000,
720 (0x4e00 << 16) | (0xc904 >> 2),
721 0x00000000,
722 (0x5e00 << 16) | (0xc904 >> 2),
723 0x00000000,
724 (0x6e00 << 16) | (0xc904 >> 2),
725 0x00000000,
726 (0x7e00 << 16) | (0xc904 >> 2),
727 0x00000000,
728 (0x4e00 << 16) | (0xc908 >> 2),
729 0x00000000,
730 (0x5e00 << 16) | (0xc908 >> 2),
731 0x00000000,
732 (0x6e00 << 16) | (0xc908 >> 2),
733 0x00000000,
734 (0x7e00 << 16) | (0xc908 >> 2),
735 0x00000000,
736 (0x4e00 << 16) | (0xc90c >> 2),
737 0x00000000,
738 (0x5e00 << 16) | (0xc90c >> 2),
739 0x00000000,
740 (0x6e00 << 16) | (0xc90c >> 2),
741 0x00000000,
742 (0x7e00 << 16) | (0xc90c >> 2),
743 0x00000000,
744 (0x4e00 << 16) | (0xc910 >> 2),
745 0x00000000,
746 (0x5e00 << 16) | (0xc910 >> 2),
747 0x00000000,
748 (0x6e00 << 16) | (0xc910 >> 2),
749 0x00000000,
750 (0x7e00 << 16) | (0xc910 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0xc99c >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x9834 >> 2),
755 0x00000000,
756 (0x0000 << 16) | (0x30f00 >> 2),
757 0x00000000,
758 (0x0000 << 16) | (0x30f04 >> 2),
759 0x00000000,
760 (0x0000 << 16) | (0x30f08 >> 2),
761 0x00000000,
762 (0x0000 << 16) | (0x30f0c >> 2),
763 0x00000000,
764 (0x0600 << 16) | (0x9b7c >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0x8a14 >> 2),
767 0x00000000,
768 (0x0e00 << 16) | (0x8a18 >> 2),
769 0x00000000,
770 (0x0600 << 16) | (0x30a00 >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0x8bf0 >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x8bcc >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x8b24 >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0x30a04 >> 2),
779 0x00000000,
780 (0x0600 << 16) | (0x30a10 >> 2),
781 0x00000000,
782 (0x0600 << 16) | (0x30a14 >> 2),
783 0x00000000,
784 (0x0600 << 16) | (0x30a18 >> 2),
785 0x00000000,
786 (0x0600 << 16) | (0x30a2c >> 2),
787 0x00000000,
788 (0x0e00 << 16) | (0xc700 >> 2),
789 0x00000000,
790 (0x0e00 << 16) | (0xc704 >> 2),
791 0x00000000,
792 (0x0e00 << 16) | (0xc708 >> 2),
793 0x00000000,
794 (0x0e00 << 16) | (0xc768 >> 2),
795 0x00000000,
796 (0x0400 << 16) | (0xc770 >> 2),
797 0x00000000,
798 (0x0400 << 16) | (0xc774 >> 2),
799 0x00000000,
800 (0x0400 << 16) | (0xc798 >> 2),
801 0x00000000,
802 (0x0400 << 16) | (0xc79c >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0x9100 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0x3c010 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0x8c00 >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0x8c04 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0x8c20 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0x8c38 >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0x8c3c >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0xae00 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0x9604 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0xac08 >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0xac0c >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0xac10 >> 2),
827 0x00000000,
828 (0x0e00 << 16) | (0xac14 >> 2),
829 0x00000000,
830 (0x0e00 << 16) | (0xac58 >> 2),
831 0x00000000,
832 (0x0e00 << 16) | (0xac68 >> 2),
833 0x00000000,
834 (0x0e00 << 16) | (0xac6c >> 2),
835 0x00000000,
836 (0x0e00 << 16) | (0xac70 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xac74 >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0xac78 >> 2),
841 0x00000000,
842 (0x0e00 << 16) | (0xac7c >> 2),
843 0x00000000,
844 (0x0e00 << 16) | (0xac80 >> 2),
845 0x00000000,
846 (0x0e00 << 16) | (0xac84 >> 2),
847 0x00000000,
848 (0x0e00 << 16) | (0xac88 >> 2),
849 0x00000000,
850 (0x0e00 << 16) | (0xac8c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x970c >> 2),
853 0x00000000,
854 (0x0e00 << 16) | (0x9714 >> 2),
855 0x00000000,
856 (0x0e00 << 16) | (0x9718 >> 2),
857 0x00000000,
858 (0x0e00 << 16) | (0x971c >> 2),
859 0x00000000,
860 (0x0e00 << 16) | (0x31068 >> 2),
861 0x00000000,
862 (0x4e00 << 16) | (0x31068 >> 2),
863 0x00000000,
864 (0x5e00 << 16) | (0x31068 >> 2),
865 0x00000000,
866 (0x6e00 << 16) | (0x31068 >> 2),
867 0x00000000,
868 (0x7e00 << 16) | (0x31068 >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0xcd10 >> 2),
871 0x00000000,
872 (0x0e00 << 16) | (0xcd14 >> 2),
873 0x00000000,
874 (0x0e00 << 16) | (0x88b0 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0x88b4 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0x88b8 >> 2),
879 0x00000000,
880 (0x0e00 << 16) | (0x88bc >> 2),
881 0x00000000,
882 (0x0400 << 16) | (0x89c0 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x88c4 >> 2),
885 0x00000000,
886 (0x0e00 << 16) | (0x88c8 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x88d0 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x88d4 >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x88d8 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x8980 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x30938 >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x3093c >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x30940 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x89a0 >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0x30900 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x30904 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0x89b4 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0x3e1fc >> 2),
911 0x00000000,
912 (0x0e00 << 16) | (0x3c210 >> 2),
913 0x00000000,
914 (0x0e00 << 16) | (0x3c214 >> 2),
915 0x00000000,
916 (0x0e00 << 16) | (0x3c218 >> 2),
917 0x00000000,
918 (0x0e00 << 16) | (0x8904 >> 2),
919 0x00000000,
920 0x5,
921 (0x0e00 << 16) | (0x8c28 >> 2),
922 (0x0e00 << 16) | (0x8c2c >> 2),
923 (0x0e00 << 16) | (0x8c30 >> 2),
924 (0x0e00 << 16) | (0x8c34 >> 2),
925 (0x0e00 << 16) | (0x9600 >> 2),
926 };
927
928 static const u32 bonaire_golden_spm_registers[] =
929 {
930 0x30800, 0xe0ffffff, 0xe0000000
931 };
932
933 static const u32 bonaire_golden_common_registers[] =
934 {
935 0xc770, 0xffffffff, 0x00000800,
936 0xc774, 0xffffffff, 0x00000800,
937 0xc798, 0xffffffff, 0x00007fbf,
938 0xc79c, 0xffffffff, 0x00007faf
939 };
940
941 static const u32 bonaire_golden_registers[] =
942 {
943 0x3354, 0x00000333, 0x00000333,
944 0x3350, 0x000c0fc0, 0x00040200,
945 0x9a10, 0x00010000, 0x00058208,
946 0x3c000, 0xffff1fff, 0x00140000,
947 0x3c200, 0xfdfc0fff, 0x00000100,
948 0x3c234, 0x40000000, 0x40000200,
949 0x9830, 0xffffffff, 0x00000000,
950 0x9834, 0xf00fffff, 0x00000400,
951 0x9838, 0x0002021c, 0x00020200,
952 0xc78, 0x00000080, 0x00000000,
953 0x5bb0, 0x000000f0, 0x00000070,
954 0x5bc0, 0xf0311fff, 0x80300000,
955 0x98f8, 0x73773777, 0x12010001,
956 0x350c, 0x00810000, 0x408af000,
957 0x7030, 0x31000111, 0x00000011,
958 0x2f48, 0x73773777, 0x12010001,
959 0x220c, 0x00007fb6, 0x0021a1b1,
960 0x2210, 0x00007fb6, 0x002021b1,
961 0x2180, 0x00007fb6, 0x00002191,
962 0x2218, 0x00007fb6, 0x002121b1,
963 0x221c, 0x00007fb6, 0x002021b1,
964 0x21dc, 0x00007fb6, 0x00002191,
965 0x21e0, 0x00007fb6, 0x00002191,
966 0x3628, 0x0000003f, 0x0000000a,
967 0x362c, 0x0000003f, 0x0000000a,
968 0x2ae4, 0x00073ffe, 0x000022a2,
969 0x240c, 0x000007ff, 0x00000000,
970 0x8a14, 0xf000003f, 0x00000007,
971 0x8bf0, 0x00002001, 0x00000001,
972 0x8b24, 0xffffffff, 0x00ffffff,
973 0x30a04, 0x0000ff0f, 0x00000000,
974 0x28a4c, 0x07ffffff, 0x06000000,
975 0x4d8, 0x00000fff, 0x00000100,
976 0x3e78, 0x00000001, 0x00000002,
977 0x9100, 0x03000000, 0x0362c688,
978 0x8c00, 0x000000ff, 0x00000001,
979 0xe40, 0x00001fff, 0x00001fff,
980 0x9060, 0x0000007f, 0x00000020,
981 0x9508, 0x00010000, 0x00010000,
982 0xac14, 0x000003ff, 0x000000f3,
983 0xac0c, 0xffffffff, 0x00001032
984 };
985
986 static const u32 bonaire_mgcg_cgcg_init[] =
987 {
988 0xc420, 0xffffffff, 0xfffffffc,
989 0x30800, 0xffffffff, 0xe0000000,
990 0x3c2a0, 0xffffffff, 0x00000100,
991 0x3c208, 0xffffffff, 0x00000100,
992 0x3c2c0, 0xffffffff, 0xc0000100,
993 0x3c2c8, 0xffffffff, 0xc0000100,
994 0x3c2c4, 0xffffffff, 0xc0000100,
995 0x55e4, 0xffffffff, 0x00600100,
996 0x3c280, 0xffffffff, 0x00000100,
997 0x3c214, 0xffffffff, 0x06000100,
998 0x3c220, 0xffffffff, 0x00000100,
999 0x3c218, 0xffffffff, 0x06000100,
1000 0x3c204, 0xffffffff, 0x00000100,
1001 0x3c2e0, 0xffffffff, 0x00000100,
1002 0x3c224, 0xffffffff, 0x00000100,
1003 0x3c200, 0xffffffff, 0x00000100,
1004 0x3c230, 0xffffffff, 0x00000100,
1005 0x3c234, 0xffffffff, 0x00000100,
1006 0x3c250, 0xffffffff, 0x00000100,
1007 0x3c254, 0xffffffff, 0x00000100,
1008 0x3c258, 0xffffffff, 0x00000100,
1009 0x3c25c, 0xffffffff, 0x00000100,
1010 0x3c260, 0xffffffff, 0x00000100,
1011 0x3c27c, 0xffffffff, 0x00000100,
1012 0x3c278, 0xffffffff, 0x00000100,
1013 0x3c210, 0xffffffff, 0x06000100,
1014 0x3c290, 0xffffffff, 0x00000100,
1015 0x3c274, 0xffffffff, 0x00000100,
1016 0x3c2b4, 0xffffffff, 0x00000100,
1017 0x3c2b0, 0xffffffff, 0x00000100,
1018 0x3c270, 0xffffffff, 0x00000100,
1019 0x30800, 0xffffffff, 0xe0000000,
1020 0x3c020, 0xffffffff, 0x00010000,
1021 0x3c024, 0xffffffff, 0x00030002,
1022 0x3c028, 0xffffffff, 0x00040007,
1023 0x3c02c, 0xffffffff, 0x00060005,
1024 0x3c030, 0xffffffff, 0x00090008,
1025 0x3c034, 0xffffffff, 0x00010000,
1026 0x3c038, 0xffffffff, 0x00030002,
1027 0x3c03c, 0xffffffff, 0x00040007,
1028 0x3c040, 0xffffffff, 0x00060005,
1029 0x3c044, 0xffffffff, 0x00090008,
1030 0x3c048, 0xffffffff, 0x00010000,
1031 0x3c04c, 0xffffffff, 0x00030002,
1032 0x3c050, 0xffffffff, 0x00040007,
1033 0x3c054, 0xffffffff, 0x00060005,
1034 0x3c058, 0xffffffff, 0x00090008,
1035 0x3c05c, 0xffffffff, 0x00010000,
1036 0x3c060, 0xffffffff, 0x00030002,
1037 0x3c064, 0xffffffff, 0x00040007,
1038 0x3c068, 0xffffffff, 0x00060005,
1039 0x3c06c, 0xffffffff, 0x00090008,
1040 0x3c070, 0xffffffff, 0x00010000,
1041 0x3c074, 0xffffffff, 0x00030002,
1042 0x3c078, 0xffffffff, 0x00040007,
1043 0x3c07c, 0xffffffff, 0x00060005,
1044 0x3c080, 0xffffffff, 0x00090008,
1045 0x3c084, 0xffffffff, 0x00010000,
1046 0x3c088, 0xffffffff, 0x00030002,
1047 0x3c08c, 0xffffffff, 0x00040007,
1048 0x3c090, 0xffffffff, 0x00060005,
1049 0x3c094, 0xffffffff, 0x00090008,
1050 0x3c098, 0xffffffff, 0x00010000,
1051 0x3c09c, 0xffffffff, 0x00030002,
1052 0x3c0a0, 0xffffffff, 0x00040007,
1053 0x3c0a4, 0xffffffff, 0x00060005,
1054 0x3c0a8, 0xffffffff, 0x00090008,
1055 0x3c000, 0xffffffff, 0x96e00200,
1056 0x8708, 0xffffffff, 0x00900100,
1057 0xc424, 0xffffffff, 0x0020003f,
1058 0x38, 0xffffffff, 0x0140001c,
1059 0x3c, 0x000f0000, 0x000f0000,
1060 0x220, 0xffffffff, 0xC060000C,
1061 0x224, 0xc0000fff, 0x00000100,
1062 0xf90, 0xffffffff, 0x00000100,
1063 0xf98, 0x00000101, 0x00000000,
1064 0x20a8, 0xffffffff, 0x00000104,
1065 0x55e4, 0xff000fff, 0x00000100,
1066 0x30cc, 0xc0000fff, 0x00000104,
1067 0xc1e4, 0x00000001, 0x00000001,
1068 0xd00c, 0xff000ff0, 0x00000100,
1069 0xd80c, 0xff000ff0, 0x00000100
1070 };
1071
1072 static const u32 spectre_golden_spm_registers[] =
1073 {
1074 0x30800, 0xe0ffffff, 0xe0000000
1075 };
1076
1077 static const u32 spectre_golden_common_registers[] =
1078 {
1079 0xc770, 0xffffffff, 0x00000800,
1080 0xc774, 0xffffffff, 0x00000800,
1081 0xc798, 0xffffffff, 0x00007fbf,
1082 0xc79c, 0xffffffff, 0x00007faf
1083 };
1084
1085 static const u32 spectre_golden_registers[] =
1086 {
1087 0x3c000, 0xffff1fff, 0x96940200,
1088 0x3c00c, 0xffff0001, 0xff000000,
1089 0x3c200, 0xfffc0fff, 0x00000100,
1090 0x6ed8, 0x00010101, 0x00010000,
1091 0x9834, 0xf00fffff, 0x00000400,
1092 0x9838, 0xfffffffc, 0x00020200,
1093 0x5bb0, 0x000000f0, 0x00000070,
1094 0x5bc0, 0xf0311fff, 0x80300000,
1095 0x98f8, 0x73773777, 0x12010001,
1096 0x9b7c, 0x00ff0000, 0x00fc0000,
1097 0x2f48, 0x73773777, 0x12010001,
1098 0x8a14, 0xf000003f, 0x00000007,
1099 0x8b24, 0xffffffff, 0x00ffffff,
1100 0x28350, 0x3f3f3fff, 0x00000082,
1101 0x28354, 0x0000003f, 0x00000000,
1102 0x3e78, 0x00000001, 0x00000002,
1103 0x913c, 0xffff03df, 0x00000004,
1104 0xc768, 0x00000008, 0x00000008,
1105 0x8c00, 0x000008ff, 0x00000800,
1106 0x9508, 0x00010000, 0x00010000,
1107 0xac0c, 0xffffffff, 0x54763210,
1108 0x214f8, 0x01ff01ff, 0x00000002,
1109 0x21498, 0x007ff800, 0x00200000,
1110 0x2015c, 0xffffffff, 0x00000f40,
1111 0x30934, 0xffffffff, 0x00000001
1112 };
1113
1114 static const u32 spectre_mgcg_cgcg_init[] =
1115 {
1116 0xc420, 0xffffffff, 0xfffffffc,
1117 0x30800, 0xffffffff, 0xe0000000,
1118 0x3c2a0, 0xffffffff, 0x00000100,
1119 0x3c208, 0xffffffff, 0x00000100,
1120 0x3c2c0, 0xffffffff, 0x00000100,
1121 0x3c2c8, 0xffffffff, 0x00000100,
1122 0x3c2c4, 0xffffffff, 0x00000100,
1123 0x55e4, 0xffffffff, 0x00600100,
1124 0x3c280, 0xffffffff, 0x00000100,
1125 0x3c214, 0xffffffff, 0x06000100,
1126 0x3c220, 0xffffffff, 0x00000100,
1127 0x3c218, 0xffffffff, 0x06000100,
1128 0x3c204, 0xffffffff, 0x00000100,
1129 0x3c2e0, 0xffffffff, 0x00000100,
1130 0x3c224, 0xffffffff, 0x00000100,
1131 0x3c200, 0xffffffff, 0x00000100,
1132 0x3c230, 0xffffffff, 0x00000100,
1133 0x3c234, 0xffffffff, 0x00000100,
1134 0x3c250, 0xffffffff, 0x00000100,
1135 0x3c254, 0xffffffff, 0x00000100,
1136 0x3c258, 0xffffffff, 0x00000100,
1137 0x3c25c, 0xffffffff, 0x00000100,
1138 0x3c260, 0xffffffff, 0x00000100,
1139 0x3c27c, 0xffffffff, 0x00000100,
1140 0x3c278, 0xffffffff, 0x00000100,
1141 0x3c210, 0xffffffff, 0x06000100,
1142 0x3c290, 0xffffffff, 0x00000100,
1143 0x3c274, 0xffffffff, 0x00000100,
1144 0x3c2b4, 0xffffffff, 0x00000100,
1145 0x3c2b0, 0xffffffff, 0x00000100,
1146 0x3c270, 0xffffffff, 0x00000100,
1147 0x30800, 0xffffffff, 0xe0000000,
1148 0x3c020, 0xffffffff, 0x00010000,
1149 0x3c024, 0xffffffff, 0x00030002,
1150 0x3c028, 0xffffffff, 0x00040007,
1151 0x3c02c, 0xffffffff, 0x00060005,
1152 0x3c030, 0xffffffff, 0x00090008,
1153 0x3c034, 0xffffffff, 0x00010000,
1154 0x3c038, 0xffffffff, 0x00030002,
1155 0x3c03c, 0xffffffff, 0x00040007,
1156 0x3c040, 0xffffffff, 0x00060005,
1157 0x3c044, 0xffffffff, 0x00090008,
1158 0x3c048, 0xffffffff, 0x00010000,
1159 0x3c04c, 0xffffffff, 0x00030002,
1160 0x3c050, 0xffffffff, 0x00040007,
1161 0x3c054, 0xffffffff, 0x00060005,
1162 0x3c058, 0xffffffff, 0x00090008,
1163 0x3c05c, 0xffffffff, 0x00010000,
1164 0x3c060, 0xffffffff, 0x00030002,
1165 0x3c064, 0xffffffff, 0x00040007,
1166 0x3c068, 0xffffffff, 0x00060005,
1167 0x3c06c, 0xffffffff, 0x00090008,
1168 0x3c070, 0xffffffff, 0x00010000,
1169 0x3c074, 0xffffffff, 0x00030002,
1170 0x3c078, 0xffffffff, 0x00040007,
1171 0x3c07c, 0xffffffff, 0x00060005,
1172 0x3c080, 0xffffffff, 0x00090008,
1173 0x3c084, 0xffffffff, 0x00010000,
1174 0x3c088, 0xffffffff, 0x00030002,
1175 0x3c08c, 0xffffffff, 0x00040007,
1176 0x3c090, 0xffffffff, 0x00060005,
1177 0x3c094, 0xffffffff, 0x00090008,
1178 0x3c098, 0xffffffff, 0x00010000,
1179 0x3c09c, 0xffffffff, 0x00030002,
1180 0x3c0a0, 0xffffffff, 0x00040007,
1181 0x3c0a4, 0xffffffff, 0x00060005,
1182 0x3c0a8, 0xffffffff, 0x00090008,
1183 0x3c0ac, 0xffffffff, 0x00010000,
1184 0x3c0b0, 0xffffffff, 0x00030002,
1185 0x3c0b4, 0xffffffff, 0x00040007,
1186 0x3c0b8, 0xffffffff, 0x00060005,
1187 0x3c0bc, 0xffffffff, 0x00090008,
1188 0x3c000, 0xffffffff, 0x96e00200,
1189 0x8708, 0xffffffff, 0x00900100,
1190 0xc424, 0xffffffff, 0x0020003f,
1191 0x38, 0xffffffff, 0x0140001c,
1192 0x3c, 0x000f0000, 0x000f0000,
1193 0x220, 0xffffffff, 0xC060000C,
1194 0x224, 0xc0000fff, 0x00000100,
1195 0xf90, 0xffffffff, 0x00000100,
1196 0xf98, 0x00000101, 0x00000000,
1197 0x20a8, 0xffffffff, 0x00000104,
1198 0x55e4, 0xff000fff, 0x00000100,
1199 0x30cc, 0xc0000fff, 0x00000104,
1200 0xc1e4, 0x00000001, 0x00000001,
1201 0xd00c, 0xff000ff0, 0x00000100,
1202 0xd80c, 0xff000ff0, 0x00000100
1203 };
1204
1205 static const u32 kalindi_golden_spm_registers[] =
1206 {
1207 0x30800, 0xe0ffffff, 0xe0000000
1208 };
1209
1210 static const u32 kalindi_golden_common_registers[] =
1211 {
1212 0xc770, 0xffffffff, 0x00000800,
1213 0xc774, 0xffffffff, 0x00000800,
1214 0xc798, 0xffffffff, 0x00007fbf,
1215 0xc79c, 0xffffffff, 0x00007faf
1216 };
1217
1218 static const u32 kalindi_golden_registers[] =
1219 {
1220 0x3c000, 0xffffdfff, 0x6e944040,
1221 0x55e4, 0xff607fff, 0xfc000100,
1222 0x3c220, 0xff000fff, 0x00000100,
1223 0x3c224, 0xff000fff, 0x00000100,
1224 0x3c200, 0xfffc0fff, 0x00000100,
1225 0x6ed8, 0x00010101, 0x00010000,
1226 0x9830, 0xffffffff, 0x00000000,
1227 0x9834, 0xf00fffff, 0x00000400,
1228 0x5bb0, 0x000000f0, 0x00000070,
1229 0x5bc0, 0xf0311fff, 0x80300000,
1230 0x98f8, 0x73773777, 0x12010001,
1231 0x98fc, 0xffffffff, 0x00000010,
1232 0x9b7c, 0x00ff0000, 0x00fc0000,
1233 0x8030, 0x00001f0f, 0x0000100a,
1234 0x2f48, 0x73773777, 0x12010001,
1235 0x2408, 0x000fffff, 0x000c007f,
1236 0x8a14, 0xf000003f, 0x00000007,
1237 0x8b24, 0x3fff3fff, 0x00ffcfff,
1238 0x30a04, 0x0000ff0f, 0x00000000,
1239 0x28a4c, 0x07ffffff, 0x06000000,
1240 0x4d8, 0x00000fff, 0x00000100,
1241 0x3e78, 0x00000001, 0x00000002,
1242 0xc768, 0x00000008, 0x00000008,
1243 0x8c00, 0x000000ff, 0x00000003,
1244 0x214f8, 0x01ff01ff, 0x00000002,
1245 0x21498, 0x007ff800, 0x00200000,
1246 0x2015c, 0xffffffff, 0x00000f40,
1247 0x88c4, 0x001f3ae3, 0x00000082,
1248 0x88d4, 0x0000001f, 0x00000010,
1249 0x30934, 0xffffffff, 0x00000000
1250 };
1251
1252 static const u32 kalindi_mgcg_cgcg_init[] =
1253 {
1254 0xc420, 0xffffffff, 0xfffffffc,
1255 0x30800, 0xffffffff, 0xe0000000,
1256 0x3c2a0, 0xffffffff, 0x00000100,
1257 0x3c208, 0xffffffff, 0x00000100,
1258 0x3c2c0, 0xffffffff, 0x00000100,
1259 0x3c2c8, 0xffffffff, 0x00000100,
1260 0x3c2c4, 0xffffffff, 0x00000100,
1261 0x55e4, 0xffffffff, 0x00600100,
1262 0x3c280, 0xffffffff, 0x00000100,
1263 0x3c214, 0xffffffff, 0x06000100,
1264 0x3c220, 0xffffffff, 0x00000100,
1265 0x3c218, 0xffffffff, 0x06000100,
1266 0x3c204, 0xffffffff, 0x00000100,
1267 0x3c2e0, 0xffffffff, 0x00000100,
1268 0x3c224, 0xffffffff, 0x00000100,
1269 0x3c200, 0xffffffff, 0x00000100,
1270 0x3c230, 0xffffffff, 0x00000100,
1271 0x3c234, 0xffffffff, 0x00000100,
1272 0x3c250, 0xffffffff, 0x00000100,
1273 0x3c254, 0xffffffff, 0x00000100,
1274 0x3c258, 0xffffffff, 0x00000100,
1275 0x3c25c, 0xffffffff, 0x00000100,
1276 0x3c260, 0xffffffff, 0x00000100,
1277 0x3c27c, 0xffffffff, 0x00000100,
1278 0x3c278, 0xffffffff, 0x00000100,
1279 0x3c210, 0xffffffff, 0x06000100,
1280 0x3c290, 0xffffffff, 0x00000100,
1281 0x3c274, 0xffffffff, 0x00000100,
1282 0x3c2b4, 0xffffffff, 0x00000100,
1283 0x3c2b0, 0xffffffff, 0x00000100,
1284 0x3c270, 0xffffffff, 0x00000100,
1285 0x30800, 0xffffffff, 0xe0000000,
1286 0x3c020, 0xffffffff, 0x00010000,
1287 0x3c024, 0xffffffff, 0x00030002,
1288 0x3c028, 0xffffffff, 0x00040007,
1289 0x3c02c, 0xffffffff, 0x00060005,
1290 0x3c030, 0xffffffff, 0x00090008,
1291 0x3c034, 0xffffffff, 0x00010000,
1292 0x3c038, 0xffffffff, 0x00030002,
1293 0x3c03c, 0xffffffff, 0x00040007,
1294 0x3c040, 0xffffffff, 0x00060005,
1295 0x3c044, 0xffffffff, 0x00090008,
1296 0x3c000, 0xffffffff, 0x96e00200,
1297 0x8708, 0xffffffff, 0x00900100,
1298 0xc424, 0xffffffff, 0x0020003f,
1299 0x38, 0xffffffff, 0x0140001c,
1300 0x3c, 0x000f0000, 0x000f0000,
1301 0x220, 0xffffffff, 0xC060000C,
1302 0x224, 0xc0000fff, 0x00000100,
1303 0x20a8, 0xffffffff, 0x00000104,
1304 0x55e4, 0xff000fff, 0x00000100,
1305 0x30cc, 0xc0000fff, 0x00000104,
1306 0xc1e4, 0x00000001, 0x00000001,
1307 0xd00c, 0xff000ff0, 0x00000100,
1308 0xd80c, 0xff000ff0, 0x00000100
1309 };
1310
1311 static const u32 hawaii_golden_spm_registers[] =
1312 {
1313 0x30800, 0xe0ffffff, 0xe0000000
1314 };
1315
1316 static const u32 hawaii_golden_common_registers[] =
1317 {
1318 0x30800, 0xffffffff, 0xe0000000,
1319 0x28350, 0xffffffff, 0x3a00161a,
1320 0x28354, 0xffffffff, 0x0000002e,
1321 0x9a10, 0xffffffff, 0x00018208,
1322 0x98f8, 0xffffffff, 0x12011003
1323 };
1324
1325 static const u32 hawaii_golden_registers[] =
1326 {
1327 0x3354, 0x00000333, 0x00000333,
1328 0x9a10, 0x00010000, 0x00058208,
1329 0x9830, 0xffffffff, 0x00000000,
1330 0x9834, 0xf00fffff, 0x00000400,
1331 0x9838, 0x0002021c, 0x00020200,
1332 0xc78, 0x00000080, 0x00000000,
1333 0x5bb0, 0x000000f0, 0x00000070,
1334 0x5bc0, 0xf0311fff, 0x80300000,
1335 0x350c, 0x00810000, 0x408af000,
1336 0x7030, 0x31000111, 0x00000011,
1337 0x2f48, 0x73773777, 0x12010001,
1338 0x2120, 0x0000007f, 0x0000001b,
1339 0x21dc, 0x00007fb6, 0x00002191,
1340 0x3628, 0x0000003f, 0x0000000a,
1341 0x362c, 0x0000003f, 0x0000000a,
1342 0x2ae4, 0x00073ffe, 0x000022a2,
1343 0x240c, 0x000007ff, 0x00000000,
1344 0x8bf0, 0x00002001, 0x00000001,
1345 0x8b24, 0xffffffff, 0x00ffffff,
1346 0x30a04, 0x0000ff0f, 0x00000000,
1347 0x28a4c, 0x07ffffff, 0x06000000,
1348 0x3e78, 0x00000001, 0x00000002,
1349 0xc768, 0x00000008, 0x00000008,
1350 0xc770, 0x00000f00, 0x00000800,
1351 0xc774, 0x00000f00, 0x00000800,
1352 0xc798, 0x00ffffff, 0x00ff7fbf,
1353 0xc79c, 0x00ffffff, 0x00ff7faf,
1354 0x8c00, 0x000000ff, 0x00000800,
1355 0xe40, 0x00001fff, 0x00001fff,
1356 0x9060, 0x0000007f, 0x00000020,
1357 0x9508, 0x00010000, 0x00010000,
1358 0xae00, 0x00100000, 0x000ff07c,
1359 0xac14, 0x000003ff, 0x0000000f,
1360 0xac10, 0xffffffff, 0x7564fdec,
1361 0xac0c, 0xffffffff, 0x3120b9a8,
1362 0xac08, 0x20000000, 0x0f9c0000
1363 };
1364
1365 static const u32 hawaii_mgcg_cgcg_init[] =
1366 {
1367 0xc420, 0xffffffff, 0xfffffffd,
1368 0x30800, 0xffffffff, 0xe0000000,
1369 0x3c2a0, 0xffffffff, 0x00000100,
1370 0x3c208, 0xffffffff, 0x00000100,
1371 0x3c2c0, 0xffffffff, 0x00000100,
1372 0x3c2c8, 0xffffffff, 0x00000100,
1373 0x3c2c4, 0xffffffff, 0x00000100,
1374 0x55e4, 0xffffffff, 0x00200100,
1375 0x3c280, 0xffffffff, 0x00000100,
1376 0x3c214, 0xffffffff, 0x06000100,
1377 0x3c220, 0xffffffff, 0x00000100,
1378 0x3c218, 0xffffffff, 0x06000100,
1379 0x3c204, 0xffffffff, 0x00000100,
1380 0x3c2e0, 0xffffffff, 0x00000100,
1381 0x3c224, 0xffffffff, 0x00000100,
1382 0x3c200, 0xffffffff, 0x00000100,
1383 0x3c230, 0xffffffff, 0x00000100,
1384 0x3c234, 0xffffffff, 0x00000100,
1385 0x3c250, 0xffffffff, 0x00000100,
1386 0x3c254, 0xffffffff, 0x00000100,
1387 0x3c258, 0xffffffff, 0x00000100,
1388 0x3c25c, 0xffffffff, 0x00000100,
1389 0x3c260, 0xffffffff, 0x00000100,
1390 0x3c27c, 0xffffffff, 0x00000100,
1391 0x3c278, 0xffffffff, 0x00000100,
1392 0x3c210, 0xffffffff, 0x06000100,
1393 0x3c290, 0xffffffff, 0x00000100,
1394 0x3c274, 0xffffffff, 0x00000100,
1395 0x3c2b4, 0xffffffff, 0x00000100,
1396 0x3c2b0, 0xffffffff, 0x00000100,
1397 0x3c270, 0xffffffff, 0x00000100,
1398 0x30800, 0xffffffff, 0xe0000000,
1399 0x3c020, 0xffffffff, 0x00010000,
1400 0x3c024, 0xffffffff, 0x00030002,
1401 0x3c028, 0xffffffff, 0x00040007,
1402 0x3c02c, 0xffffffff, 0x00060005,
1403 0x3c030, 0xffffffff, 0x00090008,
1404 0x3c034, 0xffffffff, 0x00010000,
1405 0x3c038, 0xffffffff, 0x00030002,
1406 0x3c03c, 0xffffffff, 0x00040007,
1407 0x3c040, 0xffffffff, 0x00060005,
1408 0x3c044, 0xffffffff, 0x00090008,
1409 0x3c048, 0xffffffff, 0x00010000,
1410 0x3c04c, 0xffffffff, 0x00030002,
1411 0x3c050, 0xffffffff, 0x00040007,
1412 0x3c054, 0xffffffff, 0x00060005,
1413 0x3c058, 0xffffffff, 0x00090008,
1414 0x3c05c, 0xffffffff, 0x00010000,
1415 0x3c060, 0xffffffff, 0x00030002,
1416 0x3c064, 0xffffffff, 0x00040007,
1417 0x3c068, 0xffffffff, 0x00060005,
1418 0x3c06c, 0xffffffff, 0x00090008,
1419 0x3c070, 0xffffffff, 0x00010000,
1420 0x3c074, 0xffffffff, 0x00030002,
1421 0x3c078, 0xffffffff, 0x00040007,
1422 0x3c07c, 0xffffffff, 0x00060005,
1423 0x3c080, 0xffffffff, 0x00090008,
1424 0x3c084, 0xffffffff, 0x00010000,
1425 0x3c088, 0xffffffff, 0x00030002,
1426 0x3c08c, 0xffffffff, 0x00040007,
1427 0x3c090, 0xffffffff, 0x00060005,
1428 0x3c094, 0xffffffff, 0x00090008,
1429 0x3c098, 0xffffffff, 0x00010000,
1430 0x3c09c, 0xffffffff, 0x00030002,
1431 0x3c0a0, 0xffffffff, 0x00040007,
1432 0x3c0a4, 0xffffffff, 0x00060005,
1433 0x3c0a8, 0xffffffff, 0x00090008,
1434 0x3c0ac, 0xffffffff, 0x00010000,
1435 0x3c0b0, 0xffffffff, 0x00030002,
1436 0x3c0b4, 0xffffffff, 0x00040007,
1437 0x3c0b8, 0xffffffff, 0x00060005,
1438 0x3c0bc, 0xffffffff, 0x00090008,
1439 0x3c0c0, 0xffffffff, 0x00010000,
1440 0x3c0c4, 0xffffffff, 0x00030002,
1441 0x3c0c8, 0xffffffff, 0x00040007,
1442 0x3c0cc, 0xffffffff, 0x00060005,
1443 0x3c0d0, 0xffffffff, 0x00090008,
1444 0x3c0d4, 0xffffffff, 0x00010000,
1445 0x3c0d8, 0xffffffff, 0x00030002,
1446 0x3c0dc, 0xffffffff, 0x00040007,
1447 0x3c0e0, 0xffffffff, 0x00060005,
1448 0x3c0e4, 0xffffffff, 0x00090008,
1449 0x3c0e8, 0xffffffff, 0x00010000,
1450 0x3c0ec, 0xffffffff, 0x00030002,
1451 0x3c0f0, 0xffffffff, 0x00040007,
1452 0x3c0f4, 0xffffffff, 0x00060005,
1453 0x3c0f8, 0xffffffff, 0x00090008,
1454 0xc318, 0xffffffff, 0x00020200,
1455 0x3350, 0xffffffff, 0x00000200,
1456 0x15c0, 0xffffffff, 0x00000400,
1457 0x55e8, 0xffffffff, 0x00000000,
1458 0x2f50, 0xffffffff, 0x00000902,
1459 0x3c000, 0xffffffff, 0x96940200,
1460 0x8708, 0xffffffff, 0x00900100,
1461 0xc424, 0xffffffff, 0x0020003f,
1462 0x38, 0xffffffff, 0x0140001c,
1463 0x3c, 0x000f0000, 0x000f0000,
1464 0x220, 0xffffffff, 0xc060000c,
1465 0x224, 0xc0000fff, 0x00000100,
1466 0xf90, 0xffffffff, 0x00000100,
1467 0xf98, 0x00000101, 0x00000000,
1468 0x20a8, 0xffffffff, 0x00000104,
1469 0x55e4, 0xff000fff, 0x00000100,
1470 0x30cc, 0xc0000fff, 0x00000104,
1471 0xc1e4, 0x00000001, 0x00000001,
1472 0xd00c, 0xff000ff0, 0x00000100,
1473 0xd80c, 0xff000ff0, 0x00000100
1474 };
1475
1476 static void cik_init_golden_registers(struct radeon_device *rdev)
1477 {
1478 switch (rdev->family) {
1479 case CHIP_BONAIRE:
1480 radeon_program_register_sequence(rdev,
1481 bonaire_mgcg_cgcg_init,
1482 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_common_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1489 radeon_program_register_sequence(rdev,
1490 bonaire_golden_spm_registers,
1491 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1492 break;
1493 case CHIP_KABINI:
1494 radeon_program_register_sequence(rdev,
1495 kalindi_mgcg_cgcg_init,
1496 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_common_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1503 radeon_program_register_sequence(rdev,
1504 kalindi_golden_spm_registers,
1505 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1506 break;
1507 case CHIP_KAVERI:
1508 radeon_program_register_sequence(rdev,
1509 spectre_mgcg_cgcg_init,
1510 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_common_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1517 radeon_program_register_sequence(rdev,
1518 spectre_golden_spm_registers,
1519 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1520 break;
1521 case CHIP_HAWAII:
1522 radeon_program_register_sequence(rdev,
1523 hawaii_mgcg_cgcg_init,
1524 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_common_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1531 radeon_program_register_sequence(rdev,
1532 hawaii_golden_spm_registers,
1533 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1534 break;
1535 default:
1536 break;
1537 }
1538 }
1539
1540 /**
1541 * cik_get_xclk - get the xclk
1542 *
1543 * @rdev: radeon_device pointer
1544 *
1545 * Returns the reference clock used by the gfx engine
1546 * (CIK).
1547 */
1548 u32 cik_get_xclk(struct radeon_device *rdev)
1549 {
1550 u32 reference_clock = rdev->clock.spll.reference_freq;
1551
1552 if (rdev->flags & RADEON_IS_IGP) {
1553 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1554 return reference_clock / 2;
1555 } else {
1556 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1557 return reference_clock / 4;
1558 }
1559 return reference_clock;
1560 }
1561
1562 /**
1563 * cik_mm_rdoorbell - read a doorbell dword
1564 *
1565 * @rdev: radeon_device pointer
1566 * @index: doorbell index
1567 *
1568 * Returns the value in the doorbell aperture at the
1569 * requested doorbell index (CIK).
1570 */
1571 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1572 {
1573 if (index < rdev->doorbell.num_doorbells) {
1574 return readl(rdev->doorbell.ptr + index);
1575 } else {
1576 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1577 return 0;
1578 }
1579 }
1580
1581 /**
1582 * cik_mm_wdoorbell - write a doorbell dword
1583 *
1584 * @rdev: radeon_device pointer
1585 * @index: doorbell index
1586 * @v: value to write
1587 *
1588 * Writes @v to the doorbell aperture at the
1589 * requested doorbell index (CIK).
1590 */
1591 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1592 {
1593 if (index < rdev->doorbell.num_doorbells) {
1594 writel(v, rdev->doorbell.ptr + index);
1595 } else {
1596 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1597 }
1598 }
1599
1600 #define BONAIRE_IO_MC_REGS_SIZE 36
1601
1602 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1603 {
1604 {0x00000070, 0x04400000},
1605 {0x00000071, 0x80c01803},
1606 {0x00000072, 0x00004004},
1607 {0x00000073, 0x00000100},
1608 {0x00000074, 0x00ff0000},
1609 {0x00000075, 0x34000000},
1610 {0x00000076, 0x08000014},
1611 {0x00000077, 0x00cc08ec},
1612 {0x00000078, 0x00000400},
1613 {0x00000079, 0x00000000},
1614 {0x0000007a, 0x04090000},
1615 {0x0000007c, 0x00000000},
1616 {0x0000007e, 0x4408a8e8},
1617 {0x0000007f, 0x00000304},
1618 {0x00000080, 0x00000000},
1619 {0x00000082, 0x00000001},
1620 {0x00000083, 0x00000002},
1621 {0x00000084, 0xf3e4f400},
1622 {0x00000085, 0x052024e3},
1623 {0x00000087, 0x00000000},
1624 {0x00000088, 0x01000000},
1625 {0x0000008a, 0x1c0a0000},
1626 {0x0000008b, 0xff010000},
1627 {0x0000008d, 0xffffefff},
1628 {0x0000008e, 0xfff3efff},
1629 {0x0000008f, 0xfff3efbf},
1630 {0x00000092, 0xf7ffffff},
1631 {0x00000093, 0xffffff7f},
1632 {0x00000095, 0x00101101},
1633 {0x00000096, 0x00000fff},
1634 {0x00000097, 0x00116fff},
1635 {0x00000098, 0x60010000},
1636 {0x00000099, 0x10010000},
1637 {0x0000009a, 0x00006000},
1638 {0x0000009b, 0x00001000},
1639 {0x0000009f, 0x00b48000}
1640 };
1641
1642 #define HAWAII_IO_MC_REGS_SIZE 22
1643
1644 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1645 {
1646 {0x0000007d, 0x40000000},
1647 {0x0000007e, 0x40180304},
1648 {0x0000007f, 0x0000ff00},
1649 {0x00000081, 0x00000000},
1650 {0x00000083, 0x00000800},
1651 {0x00000086, 0x00000000},
1652 {0x00000087, 0x00000100},
1653 {0x00000088, 0x00020100},
1654 {0x00000089, 0x00000000},
1655 {0x0000008b, 0x00040000},
1656 {0x0000008c, 0x00000100},
1657 {0x0000008e, 0xff010000},
1658 {0x00000090, 0xffffefff},
1659 {0x00000091, 0xfff3efff},
1660 {0x00000092, 0xfff3efbf},
1661 {0x00000093, 0xf7ffffff},
1662 {0x00000094, 0xffffff7f},
1663 {0x00000095, 0x00000fff},
1664 {0x00000096, 0x00116fff},
1665 {0x00000097, 0x60010000},
1666 {0x00000098, 0x10010000},
1667 {0x0000009f, 0x00c79000}
1668 };
1669
1670
1671 /**
1672 * cik_srbm_select - select specific register instances
1673 *
1674 * @rdev: radeon_device pointer
1675 * @me: selected ME (micro engine)
1676 * @pipe: pipe
1677 * @queue: queue
1678 * @vmid: VMID
1679 *
1680 * Switches the currently active registers instances. Some
1681 * registers are instanced per VMID, others are instanced per
1682 * me/pipe/queue combination.
1683 */
1684 static void cik_srbm_select(struct radeon_device *rdev,
1685 u32 me, u32 pipe, u32 queue, u32 vmid)
1686 {
1687 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1688 MEID(me & 0x3) |
1689 VMID(vmid & 0xf) |
1690 QUEUEID(queue & 0x7));
1691 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1692 }
1693
1694 /* ucode loading */
1695 /**
1696 * ci_mc_load_microcode - load MC ucode into the hw
1697 *
1698 * @rdev: radeon_device pointer
1699 *
1700 * Load the GDDR MC ucode into the hw (CIK).
1701 * Returns 0 on success, error on failure.
1702 */
1703 int ci_mc_load_microcode(struct radeon_device *rdev)
1704 {
1705 const __be32 *fw_data;
1706 u32 running, blackout = 0;
1707 u32 *io_mc_regs;
1708 int i, regs_size, ucode_size = rdev->mc_fw->size / 4;
1709
1710 if (!rdev->mc_fw)
1711 return -EINVAL;
1712
1713 switch (rdev->family) {
1714 case CHIP_BONAIRE:
1715 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1716 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1717 break;
1718 case CHIP_HAWAII:
1719 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1720 regs_size = HAWAII_IO_MC_REGS_SIZE;
1721 break;
1722 default:
1723 return -EINVAL;
1724 }
1725
1726 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1727
1728 if (running == 0) {
1729 if (running) {
1730 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1731 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1732 }
1733
1734 /* reset the engine and set to writable */
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1736 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1737
1738 /* load mc io regs */
1739 for (i = 0; i < regs_size; i++) {
1740 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1741 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1742 }
1743 /* load the MC ucode */
1744 fw_data = (const __be32 *)rdev->mc_fw->data;
1745 for (i = 0; i < ucode_size; i++)
1746 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1747
1748 /* put the engine back into the active state */
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1752
1753 /* wait for training to complete */
1754 for (i = 0; i < rdev->usec_timeout; i++) {
1755 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1756 break;
1757 udelay(1);
1758 }
1759 for (i = 0; i < rdev->usec_timeout; i++) {
1760 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1761 break;
1762 udelay(1);
1763 }
1764
1765 if (running)
1766 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1767 }
1768
1769 return 0;
1770 }
1771
1772 /**
1773 * cik_init_microcode - load ucode images from disk
1774 *
1775 * @rdev: radeon_device pointer
1776 *
1777 * Use the firmware interface to load the ucode images into
1778 * the driver (not loaded into hw).
1779 * Returns 0 on success, error on failure.
1780 */
1781 static int cik_init_microcode(struct radeon_device *rdev)
1782 {
1783 const char *chip_name;
1784 size_t pfp_req_size, me_req_size, ce_req_size,
1785 mec_req_size, rlc_req_size, mc_req_size = 0,
1786 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1787 char fw_name[30];
1788 int err;
1789
1790 DRM_DEBUG("\n");
1791
1792 switch (rdev->family) {
1793 case CHIP_BONAIRE:
1794 chip_name = "BONAIRE";
1795 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1796 me_req_size = CIK_ME_UCODE_SIZE * 4;
1797 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1798 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1799 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1800 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1801 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1802 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1803 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1804 break;
1805 case CHIP_HAWAII:
1806 chip_name = "HAWAII";
1807 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1808 me_req_size = CIK_ME_UCODE_SIZE * 4;
1809 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1810 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1811 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1812 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1813 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1814 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1815 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1816 break;
1817 case CHIP_KAVERI:
1818 chip_name = "KAVERI";
1819 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1820 me_req_size = CIK_ME_UCODE_SIZE * 4;
1821 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1822 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1823 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1824 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1825 break;
1826 case CHIP_KABINI:
1827 chip_name = "KABINI";
1828 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1829 me_req_size = CIK_ME_UCODE_SIZE * 4;
1830 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1831 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1832 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1833 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1834 break;
1835 default: BUG();
1836 }
1837
1838 DRM_INFO("Loading %s Microcode\n", chip_name);
1839
1840 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1841 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1842 if (err)
1843 goto out;
1844 if (rdev->pfp_fw->size != pfp_req_size) {
1845 printk(KERN_ERR
1846 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1847 rdev->pfp_fw->size, fw_name);
1848 err = -EINVAL;
1849 goto out;
1850 }
1851
1852 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1853 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1854 if (err)
1855 goto out;
1856 if (rdev->me_fw->size != me_req_size) {
1857 printk(KERN_ERR
1858 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1859 rdev->me_fw->size, fw_name);
1860 err = -EINVAL;
1861 }
1862
1863 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1864 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1865 if (err)
1866 goto out;
1867 if (rdev->ce_fw->size != ce_req_size) {
1868 printk(KERN_ERR
1869 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1870 rdev->ce_fw->size, fw_name);
1871 err = -EINVAL;
1872 }
1873
1874 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1875 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1876 if (err)
1877 goto out;
1878 if (rdev->mec_fw->size != mec_req_size) {
1879 printk(KERN_ERR
1880 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1881 rdev->mec_fw->size, fw_name);
1882 err = -EINVAL;
1883 }
1884
1885 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1886 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1887 if (err)
1888 goto out;
1889 if (rdev->rlc_fw->size != rlc_req_size) {
1890 printk(KERN_ERR
1891 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1892 rdev->rlc_fw->size, fw_name);
1893 err = -EINVAL;
1894 }
1895
1896 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1897 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1898 if (err)
1899 goto out;
1900 if (rdev->sdma_fw->size != sdma_req_size) {
1901 printk(KERN_ERR
1902 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1903 rdev->sdma_fw->size, fw_name);
1904 err = -EINVAL;
1905 }
1906
1907 /* No SMC, MC ucode on APUs */
1908 if (!(rdev->flags & RADEON_IS_IGP)) {
1909 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1910 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1911 if (err) {
1912 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1913 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1914 if (err)
1915 goto out;
1916 }
1917 if ((rdev->mc_fw->size != mc_req_size) &&
1918 (rdev->mc_fw->size != mc2_req_size)){
1919 printk(KERN_ERR
1920 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1921 rdev->mc_fw->size, fw_name);
1922 err = -EINVAL;
1923 }
1924 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1925
1926 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1927 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1928 if (err) {
1929 printk(KERN_ERR
1930 "smc: error loading firmware \"%s\"\n",
1931 fw_name);
1932 release_firmware(rdev->smc_fw);
1933 rdev->smc_fw = NULL;
1934 err = 0;
1935 } else if (rdev->smc_fw->size != smc_req_size) {
1936 printk(KERN_ERR
1937 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1938 rdev->smc_fw->size, fw_name);
1939 err = -EINVAL;
1940 }
1941 }
1942
1943 out:
1944 if (err) {
1945 if (err != -EINVAL)
1946 printk(KERN_ERR
1947 "cik_cp: Failed to load firmware \"%s\"\n",
1948 fw_name);
1949 release_firmware(rdev->pfp_fw);
1950 rdev->pfp_fw = NULL;
1951 release_firmware(rdev->me_fw);
1952 rdev->me_fw = NULL;
1953 release_firmware(rdev->ce_fw);
1954 rdev->ce_fw = NULL;
1955 release_firmware(rdev->rlc_fw);
1956 rdev->rlc_fw = NULL;
1957 release_firmware(rdev->mc_fw);
1958 rdev->mc_fw = NULL;
1959 release_firmware(rdev->smc_fw);
1960 rdev->smc_fw = NULL;
1961 }
1962 return err;
1963 }
1964
1965 /*
1966 * Core functions
1967 */
1968 /**
1969 * cik_tiling_mode_table_init - init the hw tiling table
1970 *
1971 * @rdev: radeon_device pointer
1972 *
1973 * Starting with SI, the tiling setup is done globally in a
1974 * set of 32 tiling modes. Rather than selecting each set of
1975 * parameters per surface as on older asics, we just select
1976 * which index in the tiling table we want to use, and the
1977 * surface uses those parameters (CIK).
1978 */
1979 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1980 {
1981 const u32 num_tile_mode_states = 32;
1982 const u32 num_secondary_tile_mode_states = 16;
1983 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1984 u32 num_pipe_configs;
1985 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1986 rdev->config.cik.max_shader_engines;
1987
1988 switch (rdev->config.cik.mem_row_size_in_kb) {
1989 case 1:
1990 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1991 break;
1992 case 2:
1993 default:
1994 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1995 break;
1996 case 4:
1997 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1998 break;
1999 }
2000
2001 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2002 if (num_pipe_configs > 8)
2003 num_pipe_configs = 16;
2004
2005 if (num_pipe_configs == 16) {
2006 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2007 switch (reg_offset) {
2008 case 0:
2009 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2010 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2011 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2012 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2013 break;
2014 case 1:
2015 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2016 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2017 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2018 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2019 break;
2020 case 2:
2021 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2022 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2023 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2024 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2025 break;
2026 case 3:
2027 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2028 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2029 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2030 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2031 break;
2032 case 4:
2033 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2034 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2035 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2036 TILE_SPLIT(split_equal_to_row_size));
2037 break;
2038 case 5:
2039 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2040 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2042 break;
2043 case 6:
2044 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2045 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2046 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2047 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2048 break;
2049 case 7:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2052 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2053 TILE_SPLIT(split_equal_to_row_size));
2054 break;
2055 case 8:
2056 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2057 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2058 break;
2059 case 9:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2061 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2062 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2063 break;
2064 case 10:
2065 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2066 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2067 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2068 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2069 break;
2070 case 11:
2071 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2072 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2073 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2074 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2075 break;
2076 case 12:
2077 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2078 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2079 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2080 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2081 break;
2082 case 13:
2083 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2085 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2086 break;
2087 case 14:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 16:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2096 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2097 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2098 break;
2099 case 17:
2100 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2101 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2102 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2103 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2104 break;
2105 case 27:
2106 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2107 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2108 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2109 break;
2110 case 28:
2111 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2112 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2113 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2115 break;
2116 case 29:
2117 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2118 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2119 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2120 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2121 break;
2122 case 30:
2123 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2124 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2125 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2126 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2127 break;
2128 default:
2129 gb_tile_moden = 0;
2130 break;
2131 }
2132 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2133 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2134 }
2135 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2136 switch (reg_offset) {
2137 case 0:
2138 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2139 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2140 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2141 NUM_BANKS(ADDR_SURF_16_BANK));
2142 break;
2143 case 1:
2144 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2147 NUM_BANKS(ADDR_SURF_16_BANK));
2148 break;
2149 case 2:
2150 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2151 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2152 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2153 NUM_BANKS(ADDR_SURF_16_BANK));
2154 break;
2155 case 3:
2156 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2159 NUM_BANKS(ADDR_SURF_16_BANK));
2160 break;
2161 case 4:
2162 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2163 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2164 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2165 NUM_BANKS(ADDR_SURF_8_BANK));
2166 break;
2167 case 5:
2168 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2169 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2170 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2171 NUM_BANKS(ADDR_SURF_4_BANK));
2172 break;
2173 case 6:
2174 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2177 NUM_BANKS(ADDR_SURF_2_BANK));
2178 break;
2179 case 8:
2180 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2183 NUM_BANKS(ADDR_SURF_16_BANK));
2184 break;
2185 case 9:
2186 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2187 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2188 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2189 NUM_BANKS(ADDR_SURF_16_BANK));
2190 break;
2191 case 10:
2192 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2193 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2194 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2195 NUM_BANKS(ADDR_SURF_16_BANK));
2196 break;
2197 case 11:
2198 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2199 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2200 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2201 NUM_BANKS(ADDR_SURF_8_BANK));
2202 break;
2203 case 12:
2204 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2207 NUM_BANKS(ADDR_SURF_4_BANK));
2208 break;
2209 case 13:
2210 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2211 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2212 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2213 NUM_BANKS(ADDR_SURF_2_BANK));
2214 break;
2215 case 14:
2216 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2217 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2218 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2219 NUM_BANKS(ADDR_SURF_2_BANK));
2220 break;
2221 default:
2222 gb_tile_moden = 0;
2223 break;
2224 }
2225 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2226 }
2227 } else if (num_pipe_configs == 8) {
2228 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2229 switch (reg_offset) {
2230 case 0:
2231 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2232 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2233 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2235 break;
2236 case 1:
2237 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2238 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2239 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2241 break;
2242 case 2:
2243 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2244 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2245 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2246 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2247 break;
2248 case 3:
2249 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2250 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2251 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2252 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2253 break;
2254 case 4:
2255 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2256 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2257 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2258 TILE_SPLIT(split_equal_to_row_size));
2259 break;
2260 case 5:
2261 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2262 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2263 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2264 break;
2265 case 6:
2266 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2267 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2268 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2269 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2270 break;
2271 case 7:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 TILE_SPLIT(split_equal_to_row_size));
2276 break;
2277 case 8:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2279 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2280 break;
2281 case 9:
2282 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2283 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2284 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2285 break;
2286 case 10:
2287 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2288 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2289 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2290 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2291 break;
2292 case 11:
2293 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2294 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2295 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2296 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2297 break;
2298 case 12:
2299 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2300 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2301 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2302 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2303 break;
2304 case 13:
2305 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2306 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2308 break;
2309 case 14:
2310 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2311 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2312 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2313 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2314 break;
2315 case 16:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 17:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 27:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2329 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2331 break;
2332 case 28:
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2336 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2337 break;
2338 case 29:
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2340 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2343 break;
2344 case 30:
2345 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2347 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2348 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2349 break;
2350 default:
2351 gb_tile_moden = 0;
2352 break;
2353 }
2354 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2355 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2356 }
2357 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2358 switch (reg_offset) {
2359 case 0:
2360 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2361 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2362 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2363 NUM_BANKS(ADDR_SURF_16_BANK));
2364 break;
2365 case 1:
2366 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2369 NUM_BANKS(ADDR_SURF_16_BANK));
2370 break;
2371 case 2:
2372 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2375 NUM_BANKS(ADDR_SURF_16_BANK));
2376 break;
2377 case 3:
2378 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2381 NUM_BANKS(ADDR_SURF_16_BANK));
2382 break;
2383 case 4:
2384 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2387 NUM_BANKS(ADDR_SURF_8_BANK));
2388 break;
2389 case 5:
2390 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2391 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2392 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2393 NUM_BANKS(ADDR_SURF_4_BANK));
2394 break;
2395 case 6:
2396 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2399 NUM_BANKS(ADDR_SURF_2_BANK));
2400 break;
2401 case 8:
2402 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2403 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2404 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2405 NUM_BANKS(ADDR_SURF_16_BANK));
2406 break;
2407 case 9:
2408 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2411 NUM_BANKS(ADDR_SURF_16_BANK));
2412 break;
2413 case 10:
2414 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2416 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2417 NUM_BANKS(ADDR_SURF_16_BANK));
2418 break;
2419 case 11:
2420 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2421 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2422 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2423 NUM_BANKS(ADDR_SURF_16_BANK));
2424 break;
2425 case 12:
2426 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2429 NUM_BANKS(ADDR_SURF_8_BANK));
2430 break;
2431 case 13:
2432 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2435 NUM_BANKS(ADDR_SURF_4_BANK));
2436 break;
2437 case 14:
2438 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2441 NUM_BANKS(ADDR_SURF_2_BANK));
2442 break;
2443 default:
2444 gb_tile_moden = 0;
2445 break;
2446 }
2447 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2448 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2449 }
2450 } else if (num_pipe_configs == 4) {
2451 if (num_rbs == 4) {
2452 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2453 switch (reg_offset) {
2454 case 0:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2459 break;
2460 case 1:
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2465 break;
2466 case 2:
2467 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2469 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2470 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2471 break;
2472 case 3:
2473 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2475 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2476 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2477 break;
2478 case 4:
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2481 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2482 TILE_SPLIT(split_equal_to_row_size));
2483 break;
2484 case 5:
2485 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2486 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2487 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2488 break;
2489 case 6:
2490 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2491 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2492 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2493 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2494 break;
2495 case 7:
2496 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2499 TILE_SPLIT(split_equal_to_row_size));
2500 break;
2501 case 8:
2502 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2503 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2504 break;
2505 case 9:
2506 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2507 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2508 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2509 break;
2510 case 10:
2511 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2512 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2513 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2514 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2515 break;
2516 case 11:
2517 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2518 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2519 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2520 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2521 break;
2522 case 12:
2523 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2526 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2527 break;
2528 case 13:
2529 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2530 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2531 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2532 break;
2533 case 14:
2534 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2535 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2536 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2537 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2538 break;
2539 case 16:
2540 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2541 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2543 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2544 break;
2545 case 17:
2546 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2547 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2548 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2549 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2550 break;
2551 case 27:
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2553 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2554 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2555 break;
2556 case 28:
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2560 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2561 break;
2562 case 29:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2567 break;
2568 case 30:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2572 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2573 break;
2574 default:
2575 gb_tile_moden = 0;
2576 break;
2577 }
2578 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2579 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2580 }
2581 } else if (num_rbs < 4) {
2582 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2583 switch (reg_offset) {
2584 case 0:
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2589 break;
2590 case 1:
2591 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2595 break;
2596 case 2:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2601 break;
2602 case 3:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2607 break;
2608 case 4:
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2612 TILE_SPLIT(split_equal_to_row_size));
2613 break;
2614 case 5:
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2616 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2618 break;
2619 case 6:
2620 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2621 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2622 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2624 break;
2625 case 7:
2626 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2627 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2628 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2629 TILE_SPLIT(split_equal_to_row_size));
2630 break;
2631 case 8:
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2633 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2634 break;
2635 case 9:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2639 break;
2640 case 10:
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2645 break;
2646 case 11:
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2651 break;
2652 case 12:
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 break;
2658 case 13:
2659 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2660 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2661 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2662 break;
2663 case 14:
2664 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2665 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2666 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2667 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2668 break;
2669 case 16:
2670 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2671 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2672 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2673 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2674 break;
2675 case 17:
2676 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2679 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2680 break;
2681 case 27:
2682 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2683 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2684 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2685 break;
2686 case 28:
2687 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2688 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2689 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2690 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2691 break;
2692 case 29:
2693 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2694 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2696 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2697 break;
2698 case 30:
2699 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2700 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2701 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2702 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703 break;
2704 default:
2705 gb_tile_moden = 0;
2706 break;
2707 }
2708 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2709 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2710 }
2711 }
2712 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2713 switch (reg_offset) {
2714 case 0:
2715 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2718 NUM_BANKS(ADDR_SURF_16_BANK));
2719 break;
2720 case 1:
2721 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2722 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2723 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2724 NUM_BANKS(ADDR_SURF_16_BANK));
2725 break;
2726 case 2:
2727 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2730 NUM_BANKS(ADDR_SURF_16_BANK));
2731 break;
2732 case 3:
2733 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2734 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2735 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2736 NUM_BANKS(ADDR_SURF_16_BANK));
2737 break;
2738 case 4:
2739 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2740 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2741 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2742 NUM_BANKS(ADDR_SURF_16_BANK));
2743 break;
2744 case 5:
2745 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2746 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2747 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2748 NUM_BANKS(ADDR_SURF_8_BANK));
2749 break;
2750 case 6:
2751 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2752 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2753 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2754 NUM_BANKS(ADDR_SURF_4_BANK));
2755 break;
2756 case 8:
2757 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2758 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2759 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2760 NUM_BANKS(ADDR_SURF_16_BANK));
2761 break;
2762 case 9:
2763 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2764 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2765 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2766 NUM_BANKS(ADDR_SURF_16_BANK));
2767 break;
2768 case 10:
2769 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2770 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2771 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2772 NUM_BANKS(ADDR_SURF_16_BANK));
2773 break;
2774 case 11:
2775 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2776 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2777 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2778 NUM_BANKS(ADDR_SURF_16_BANK));
2779 break;
2780 case 12:
2781 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2782 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2783 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2784 NUM_BANKS(ADDR_SURF_16_BANK));
2785 break;
2786 case 13:
2787 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2788 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2789 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2790 NUM_BANKS(ADDR_SURF_8_BANK));
2791 break;
2792 case 14:
2793 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2794 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2795 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2796 NUM_BANKS(ADDR_SURF_4_BANK));
2797 break;
2798 default:
2799 gb_tile_moden = 0;
2800 break;
2801 }
2802 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2803 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2804 }
2805 } else if (num_pipe_configs == 2) {
2806 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2807 switch (reg_offset) {
2808 case 0:
2809 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2811 PIPE_CONFIG(ADDR_SURF_P2) |
2812 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2813 break;
2814 case 1:
2815 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2817 PIPE_CONFIG(ADDR_SURF_P2) |
2818 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2819 break;
2820 case 2:
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2823 PIPE_CONFIG(ADDR_SURF_P2) |
2824 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2825 break;
2826 case 3:
2827 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2828 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2829 PIPE_CONFIG(ADDR_SURF_P2) |
2830 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2831 break;
2832 case 4:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2835 PIPE_CONFIG(ADDR_SURF_P2) |
2836 TILE_SPLIT(split_equal_to_row_size));
2837 break;
2838 case 5:
2839 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2840 PIPE_CONFIG(ADDR_SURF_P2) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2842 break;
2843 case 6:
2844 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2845 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2846 PIPE_CONFIG(ADDR_SURF_P2) |
2847 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2848 break;
2849 case 7:
2850 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2851 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2852 PIPE_CONFIG(ADDR_SURF_P2) |
2853 TILE_SPLIT(split_equal_to_row_size));
2854 break;
2855 case 8:
2856 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2857 PIPE_CONFIG(ADDR_SURF_P2);
2858 break;
2859 case 9:
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2861 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2862 PIPE_CONFIG(ADDR_SURF_P2));
2863 break;
2864 case 10:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 11:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 12:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P2) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2881 break;
2882 case 13:
2883 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2884 PIPE_CONFIG(ADDR_SURF_P2) |
2885 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2886 break;
2887 case 14:
2888 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2890 PIPE_CONFIG(ADDR_SURF_P2) |
2891 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2892 break;
2893 case 16:
2894 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2896 PIPE_CONFIG(ADDR_SURF_P2) |
2897 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2898 break;
2899 case 17:
2900 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P2) |
2903 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2904 break;
2905 case 27:
2906 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2908 PIPE_CONFIG(ADDR_SURF_P2));
2909 break;
2910 case 28:
2911 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P2) |
2914 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2915 break;
2916 case 29:
2917 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P2) |
2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2921 break;
2922 case 30:
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2924 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2925 PIPE_CONFIG(ADDR_SURF_P2) |
2926 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2927 break;
2928 default:
2929 gb_tile_moden = 0;
2930 break;
2931 }
2932 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2933 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2934 }
2935 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2936 switch (reg_offset) {
2937 case 0:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 1:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2947 NUM_BANKS(ADDR_SURF_16_BANK));
2948 break;
2949 case 2:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 3:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 4:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 5:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 6:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2977 NUM_BANKS(ADDR_SURF_8_BANK));
2978 break;
2979 case 8:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 9:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2989 NUM_BANKS(ADDR_SURF_16_BANK));
2990 break;
2991 case 10:
2992 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2995 NUM_BANKS(ADDR_SURF_16_BANK));
2996 break;
2997 case 11:
2998 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2999 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3000 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3001 NUM_BANKS(ADDR_SURF_16_BANK));
3002 break;
3003 case 12:
3004 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3007 NUM_BANKS(ADDR_SURF_16_BANK));
3008 break;
3009 case 13:
3010 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3011 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3012 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3013 NUM_BANKS(ADDR_SURF_16_BANK));
3014 break;
3015 case 14:
3016 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3017 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3018 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3019 NUM_BANKS(ADDR_SURF_8_BANK));
3020 break;
3021 default:
3022 gb_tile_moden = 0;
3023 break;
3024 }
3025 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3026 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3027 }
3028 } else
3029 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3030 }
3031
3032 /**
3033 * cik_select_se_sh - select which SE, SH to address
3034 *
3035 * @rdev: radeon_device pointer
3036 * @se_num: shader engine to address
3037 * @sh_num: sh block to address
3038 *
3039 * Select which SE, SH combinations to address. Certain
3040 * registers are instanced per SE or SH. 0xffffffff means
3041 * broadcast to all SEs or SHs (CIK).
3042 */
3043 static void cik_select_se_sh(struct radeon_device *rdev,
3044 u32 se_num, u32 sh_num)
3045 {
3046 u32 data = INSTANCE_BROADCAST_WRITES;
3047
3048 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3049 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3050 else if (se_num == 0xffffffff)
3051 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3052 else if (sh_num == 0xffffffff)
3053 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3054 else
3055 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3056 WREG32(GRBM_GFX_INDEX, data);
3057 }
3058
3059 /**
3060 * cik_create_bitmask - create a bitmask
3061 *
3062 * @bit_width: length of the mask
3063 *
3064 * create a variable length bit mask (CIK).
3065 * Returns the bitmask.
3066 */
3067 static u32 cik_create_bitmask(u32 bit_width)
3068 {
3069 u32 i, mask = 0;
3070
3071 for (i = 0; i < bit_width; i++) {
3072 mask <<= 1;
3073 mask |= 1;
3074 }
3075 return mask;
3076 }
3077
3078 /**
3079 * cik_get_rb_disabled - computes the mask of disabled RBs
3080 *
3081 * @rdev: radeon_device pointer
3082 * @max_rb_num: max RBs (render backends) for the asic
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 *
3086 * Calculates the bitmask of disabled RBs (CIK).
3087 * Returns the disabled RB bitmask.
3088 */
3089 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3090 u32 max_rb_num_per_se,
3091 u32 sh_per_se)
3092 {
3093 u32 data, mask;
3094
3095 data = RREG32(CC_RB_BACKEND_DISABLE);
3096 if (data & 1)
3097 data &= BACKEND_DISABLE_MASK;
3098 else
3099 data = 0;
3100 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3101
3102 data >>= BACKEND_DISABLE_SHIFT;
3103
3104 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3105
3106 return data & mask;
3107 }
3108
3109 /**
3110 * cik_setup_rb - setup the RBs on the asic
3111 *
3112 * @rdev: radeon_device pointer
3113 * @se_num: number of SEs (shader engines) for the asic
3114 * @sh_per_se: number of SH blocks per SE for the asic
3115 * @max_rb_num: max RBs (render backends) for the asic
3116 *
3117 * Configures per-SE/SH RB registers (CIK).
3118 */
3119 static void cik_setup_rb(struct radeon_device *rdev,
3120 u32 se_num, u32 sh_per_se,
3121 u32 max_rb_num_per_se)
3122 {
3123 int i, j;
3124 u32 data, mask;
3125 u32 disabled_rbs = 0;
3126 u32 enabled_rbs = 0;
3127
3128 for (i = 0; i < se_num; i++) {
3129 for (j = 0; j < sh_per_se; j++) {
3130 cik_select_se_sh(rdev, i, j);
3131 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3132 if (rdev->family == CHIP_HAWAII)
3133 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3134 else
3135 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3136 }
3137 }
3138 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3139
3140 mask = 1;
3141 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3142 if (!(disabled_rbs & mask))
3143 enabled_rbs |= mask;
3144 mask <<= 1;
3145 }
3146
3147 rdev->config.cik.backend_enable_mask = enabled_rbs;
3148
3149 for (i = 0; i < se_num; i++) {
3150 cik_select_se_sh(rdev, i, 0xffffffff);
3151 data = 0;
3152 for (j = 0; j < sh_per_se; j++) {
3153 switch (enabled_rbs & 3) {
3154 case 0:
3155 if (j == 0)
3156 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3157 else
3158 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3159 break;
3160 case 1:
3161 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3162 break;
3163 case 2:
3164 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3165 break;
3166 case 3:
3167 default:
3168 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3169 break;
3170 }
3171 enabled_rbs >>= 2;
3172 }
3173 WREG32(PA_SC_RASTER_CONFIG, data);
3174 }
3175 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3176 }
3177
3178 /**
3179 * cik_gpu_init - setup the 3D engine
3180 *
3181 * @rdev: radeon_device pointer
3182 *
3183 * Configures the 3D engine and tiling configuration
3184 * registers so that the 3D engine is usable.
3185 */
3186 static void cik_gpu_init(struct radeon_device *rdev)
3187 {
3188 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3189 u32 mc_shared_chmap, mc_arb_ramcfg;
3190 u32 hdp_host_path_cntl;
3191 u32 tmp;
3192 int i, j;
3193
3194 switch (rdev->family) {
3195 case CHIP_BONAIRE:
3196 rdev->config.cik.max_shader_engines = 2;
3197 rdev->config.cik.max_tile_pipes = 4;
3198 rdev->config.cik.max_cu_per_sh = 7;
3199 rdev->config.cik.max_sh_per_se = 1;
3200 rdev->config.cik.max_backends_per_se = 2;
3201 rdev->config.cik.max_texture_channel_caches = 4;
3202 rdev->config.cik.max_gprs = 256;
3203 rdev->config.cik.max_gs_threads = 32;
3204 rdev->config.cik.max_hw_contexts = 8;
3205
3206 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3207 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3208 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3209 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3210 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3211 break;
3212 case CHIP_HAWAII:
3213 rdev->config.cik.max_shader_engines = 4;
3214 rdev->config.cik.max_tile_pipes = 16;
3215 rdev->config.cik.max_cu_per_sh = 11;
3216 rdev->config.cik.max_sh_per_se = 1;
3217 rdev->config.cik.max_backends_per_se = 4;
3218 rdev->config.cik.max_texture_channel_caches = 16;
3219 rdev->config.cik.max_gprs = 256;
3220 rdev->config.cik.max_gs_threads = 32;
3221 rdev->config.cik.max_hw_contexts = 8;
3222
3223 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3224 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3225 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3226 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3227 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3228 break;
3229 case CHIP_KAVERI:
3230 rdev->config.cik.max_shader_engines = 1;
3231 rdev->config.cik.max_tile_pipes = 4;
3232 if ((rdev->pdev->device == 0x1304) ||
3233 (rdev->pdev->device == 0x1305) ||
3234 (rdev->pdev->device == 0x130C) ||
3235 (rdev->pdev->device == 0x130F) ||
3236 (rdev->pdev->device == 0x1310) ||
3237 (rdev->pdev->device == 0x1311) ||
3238 (rdev->pdev->device == 0x131C)) {
3239 rdev->config.cik.max_cu_per_sh = 8;
3240 rdev->config.cik.max_backends_per_se = 2;
3241 } else if ((rdev->pdev->device == 0x1309) ||
3242 (rdev->pdev->device == 0x130A) ||
3243 (rdev->pdev->device == 0x130D) ||
3244 (rdev->pdev->device == 0x1313) ||
3245 (rdev->pdev->device == 0x131D)) {
3246 rdev->config.cik.max_cu_per_sh = 6;
3247 rdev->config.cik.max_backends_per_se = 2;
3248 } else if ((rdev->pdev->device == 0x1306) ||
3249 (rdev->pdev->device == 0x1307) ||
3250 (rdev->pdev->device == 0x130B) ||
3251 (rdev->pdev->device == 0x130E) ||
3252 (rdev->pdev->device == 0x1315) ||
3253 (rdev->pdev->device == 0x131B)) {
3254 rdev->config.cik.max_cu_per_sh = 4;
3255 rdev->config.cik.max_backends_per_se = 1;
3256 } else {
3257 rdev->config.cik.max_cu_per_sh = 3;
3258 rdev->config.cik.max_backends_per_se = 1;
3259 }
3260 rdev->config.cik.max_sh_per_se = 1;
3261 rdev->config.cik.max_texture_channel_caches = 4;
3262 rdev->config.cik.max_gprs = 256;
3263 rdev->config.cik.max_gs_threads = 16;
3264 rdev->config.cik.max_hw_contexts = 8;
3265
3266 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3267 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3268 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3269 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3270 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3271 break;
3272 case CHIP_KABINI:
3273 default:
3274 rdev->config.cik.max_shader_engines = 1;
3275 rdev->config.cik.max_tile_pipes = 2;
3276 rdev->config.cik.max_cu_per_sh = 2;
3277 rdev->config.cik.max_sh_per_se = 1;
3278 rdev->config.cik.max_backends_per_se = 1;
3279 rdev->config.cik.max_texture_channel_caches = 2;
3280 rdev->config.cik.max_gprs = 256;
3281 rdev->config.cik.max_gs_threads = 16;
3282 rdev->config.cik.max_hw_contexts = 8;
3283
3284 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3285 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3286 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3287 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3288 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3289 break;
3290 }
3291
3292 /* Initialize HDP */
3293 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3294 WREG32((0x2c14 + j), 0x00000000);
3295 WREG32((0x2c18 + j), 0x00000000);
3296 WREG32((0x2c1c + j), 0x00000000);
3297 WREG32((0x2c20 + j), 0x00000000);
3298 WREG32((0x2c24 + j), 0x00000000);
3299 }
3300
3301 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3302
3303 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3304
3305 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3306 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3307
3308 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3309 rdev->config.cik.mem_max_burst_length_bytes = 256;
3310 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3311 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3312 if (rdev->config.cik.mem_row_size_in_kb > 4)
3313 rdev->config.cik.mem_row_size_in_kb = 4;
3314 /* XXX use MC settings? */
3315 rdev->config.cik.shader_engine_tile_size = 32;
3316 rdev->config.cik.num_gpus = 1;
3317 rdev->config.cik.multi_gpu_tile_size = 64;
3318
3319 /* fix up row size */
3320 gb_addr_config &= ~ROW_SIZE_MASK;
3321 switch (rdev->config.cik.mem_row_size_in_kb) {
3322 case 1:
3323 default:
3324 gb_addr_config |= ROW_SIZE(0);
3325 break;
3326 case 2:
3327 gb_addr_config |= ROW_SIZE(1);
3328 break;
3329 case 4:
3330 gb_addr_config |= ROW_SIZE(2);
3331 break;
3332 }
3333
3334 /* setup tiling info dword. gb_addr_config is not adequate since it does
3335 * not have bank info, so create a custom tiling dword.
3336 * bits 3:0 num_pipes
3337 * bits 7:4 num_banks
3338 * bits 11:8 group_size
3339 * bits 15:12 row_size
3340 */
3341 rdev->config.cik.tile_config = 0;
3342 switch (rdev->config.cik.num_tile_pipes) {
3343 case 1:
3344 rdev->config.cik.tile_config |= (0 << 0);
3345 break;
3346 case 2:
3347 rdev->config.cik.tile_config |= (1 << 0);
3348 break;
3349 case 4:
3350 rdev->config.cik.tile_config |= (2 << 0);
3351 break;
3352 case 8:
3353 default:
3354 /* XXX what about 12? */
3355 rdev->config.cik.tile_config |= (3 << 0);
3356 break;
3357 }
3358 rdev->config.cik.tile_config |=
3359 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3360 rdev->config.cik.tile_config |=
3361 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3362 rdev->config.cik.tile_config |=
3363 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3364
3365 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3366 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3367 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3368 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3369 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3370 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3371 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3372 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3373
3374 cik_tiling_mode_table_init(rdev);
3375
3376 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3377 rdev->config.cik.max_sh_per_se,
3378 rdev->config.cik.max_backends_per_se);
3379
3380 /* set HW defaults for 3D engine */
3381 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3382
3383 WREG32(SX_DEBUG_1, 0x20);
3384
3385 WREG32(TA_CNTL_AUX, 0x00010000);
3386
3387 tmp = RREG32(SPI_CONFIG_CNTL);
3388 tmp |= 0x03000000;
3389 WREG32(SPI_CONFIG_CNTL, tmp);
3390
3391 WREG32(SQ_CONFIG, 1);
3392
3393 WREG32(DB_DEBUG, 0);
3394
3395 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3396 tmp |= 0x00000400;
3397 WREG32(DB_DEBUG2, tmp);
3398
3399 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3400 tmp |= 0x00020200;
3401 WREG32(DB_DEBUG3, tmp);
3402
3403 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3404 tmp |= 0x00018208;
3405 WREG32(CB_HW_CONTROL, tmp);
3406
3407 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3408
3409 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3410 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3411 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3412 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3413
3414 WREG32(VGT_NUM_INSTANCES, 1);
3415
3416 WREG32(CP_PERFMON_CNTL, 0);
3417
3418 WREG32(SQ_CONFIG, 0);
3419
3420 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3421 FORCE_EOV_MAX_REZ_CNT(255)));
3422
3423 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3424 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3425
3426 WREG32(VGT_GS_VERTEX_REUSE, 16);
3427 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3428
3429 tmp = RREG32(HDP_MISC_CNTL);
3430 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3431 WREG32(HDP_MISC_CNTL, tmp);
3432
3433 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3434 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3435
3436 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3437 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3438
3439 udelay(50);
3440 }
3441
3442 /*
3443 * GPU scratch registers helpers function.
3444 */
3445 /**
3446 * cik_scratch_init - setup driver info for CP scratch regs
3447 *
3448 * @rdev: radeon_device pointer
3449 *
3450 * Set up the number and offset of the CP scratch registers.
3451 * NOTE: use of CP scratch registers is a legacy inferface and
3452 * is not used by default on newer asics (r6xx+). On newer asics,
3453 * memory buffers are used for fences rather than scratch regs.
3454 */
3455 static void cik_scratch_init(struct radeon_device *rdev)
3456 {
3457 int i;
3458
3459 rdev->scratch.num_reg = 7;
3460 rdev->scratch.reg_base = SCRATCH_REG0;
3461 for (i = 0; i < rdev->scratch.num_reg; i++) {
3462 rdev->scratch.free[i] = true;
3463 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3464 }
3465 }
3466
3467 /**
3468 * cik_ring_test - basic gfx ring test
3469 *
3470 * @rdev: radeon_device pointer
3471 * @ring: radeon_ring structure holding ring information
3472 *
3473 * Allocate a scratch register and write to it using the gfx ring (CIK).
3474 * Provides a basic gfx ring test to verify that the ring is working.
3475 * Used by cik_cp_gfx_resume();
3476 * Returns 0 on success, error on failure.
3477 */
3478 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3479 {
3480 uint32_t scratch;
3481 uint32_t tmp = 0;
3482 unsigned i;
3483 int r;
3484
3485 r = radeon_scratch_get(rdev, &scratch);
3486 if (r) {
3487 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3488 return r;
3489 }
3490 WREG32(scratch, 0xCAFEDEAD);
3491 r = radeon_ring_lock(rdev, ring, 3);
3492 if (r) {
3493 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3494 radeon_scratch_free(rdev, scratch);
3495 return r;
3496 }
3497 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3498 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3499 radeon_ring_write(ring, 0xDEADBEEF);
3500 radeon_ring_unlock_commit(rdev, ring);
3501
3502 for (i = 0; i < rdev->usec_timeout; i++) {
3503 tmp = RREG32(scratch);
3504 if (tmp == 0xDEADBEEF)
3505 break;
3506 DRM_UDELAY(1);
3507 }
3508 if (i < rdev->usec_timeout) {
3509 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3510 } else {
3511 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3512 ring->idx, scratch, tmp);
3513 r = -EINVAL;
3514 }
3515 radeon_scratch_free(rdev, scratch);
3516 return r;
3517 }
3518
3519 /**
3520 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3521 *
3522 * @rdev: radeon_device pointer
3523 * @ridx: radeon ring index
3524 *
3525 * Emits an hdp flush on the cp.
3526 */
3527 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3528 int ridx)
3529 {
3530 struct radeon_ring *ring = &rdev->ring[ridx];
3531 u32 ref_and_mask;
3532
3533 switch (ring->idx) {
3534 case CAYMAN_RING_TYPE_CP1_INDEX:
3535 case CAYMAN_RING_TYPE_CP2_INDEX:
3536 default:
3537 switch (ring->me) {
3538 case 0:
3539 ref_and_mask = CP2 << ring->pipe;
3540 break;
3541 case 1:
3542 ref_and_mask = CP6 << ring->pipe;
3543 break;
3544 default:
3545 return;
3546 }
3547 break;
3548 case RADEON_RING_TYPE_GFX_INDEX:
3549 ref_and_mask = CP0;
3550 break;
3551 }
3552
3553 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3554 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3555 WAIT_REG_MEM_FUNCTION(3) | /* == */
3556 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3557 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3558 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3559 radeon_ring_write(ring, ref_and_mask);
3560 radeon_ring_write(ring, ref_and_mask);
3561 radeon_ring_write(ring, 0x20); /* poll interval */
3562 }
3563
3564 /**
3565 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3566 *
3567 * @rdev: radeon_device pointer
3568 * @fence: radeon fence object
3569 *
3570 * Emits a fence sequnce number on the gfx ring and flushes
3571 * GPU caches.
3572 */
3573 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3574 struct radeon_fence *fence)
3575 {
3576 struct radeon_ring *ring = &rdev->ring[fence->ring];
3577 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3578
3579 /* EVENT_WRITE_EOP - flush caches, send int */
3580 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3581 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3582 EOP_TC_ACTION_EN |
3583 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3584 EVENT_INDEX(5)));
3585 radeon_ring_write(ring, addr & 0xfffffffc);
3586 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3587 radeon_ring_write(ring, fence->seq);
3588 radeon_ring_write(ring, 0);
3589 /* HDP flush */
3590 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3591 }
3592
3593 /**
3594 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3595 *
3596 * @rdev: radeon_device pointer
3597 * @fence: radeon fence object
3598 *
3599 * Emits a fence sequnce number on the compute ring and flushes
3600 * GPU caches.
3601 */
3602 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3603 struct radeon_fence *fence)
3604 {
3605 struct radeon_ring *ring = &rdev->ring[fence->ring];
3606 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3607
3608 /* RELEASE_MEM - flush caches, send int */
3609 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3610 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3611 EOP_TC_ACTION_EN |
3612 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3613 EVENT_INDEX(5)));
3614 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3615 radeon_ring_write(ring, addr & 0xfffffffc);
3616 radeon_ring_write(ring, upper_32_bits(addr));
3617 radeon_ring_write(ring, fence->seq);
3618 radeon_ring_write(ring, 0);
3619 /* HDP flush */
3620 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3621 }
3622
3623 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3624 struct radeon_ring *ring,
3625 struct radeon_semaphore *semaphore,
3626 bool emit_wait)
3627 {
3628 uint64_t addr = semaphore->gpu_addr;
3629 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3630
3631 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3632 radeon_ring_write(ring, addr & 0xffffffff);
3633 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3634
3635 return true;
3636 }
3637
3638 /**
3639 * cik_copy_cpdma - copy pages using the CP DMA engine
3640 *
3641 * @rdev: radeon_device pointer
3642 * @src_offset: src GPU address
3643 * @dst_offset: dst GPU address
3644 * @num_gpu_pages: number of GPU pages to xfer
3645 * @fence: radeon fence object
3646 *
3647 * Copy GPU paging using the CP DMA engine (CIK+).
3648 * Used by the radeon ttm implementation to move pages if
3649 * registered as the asic copy callback.
3650 */
3651 int cik_copy_cpdma(struct radeon_device *rdev,
3652 uint64_t src_offset, uint64_t dst_offset,
3653 unsigned num_gpu_pages,
3654 struct radeon_fence **fence)
3655 {
3656 struct radeon_semaphore *sem = NULL;
3657 int ring_index = rdev->asic->copy.blit_ring_index;
3658 struct radeon_ring *ring = &rdev->ring[ring_index];
3659 u32 size_in_bytes, cur_size_in_bytes, control;
3660 int i, num_loops;
3661 int r = 0;
3662
3663 r = radeon_semaphore_create(rdev, &sem);
3664 if (r) {
3665 DRM_ERROR("radeon: moving bo (%d).\n", r);
3666 return r;
3667 }
3668
3669 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3670 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3671 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3672 if (r) {
3673 DRM_ERROR("radeon: moving bo (%d).\n", r);
3674 radeon_semaphore_free(rdev, &sem, NULL);
3675 return r;
3676 }
3677
3678 radeon_semaphore_sync_to(sem, *fence);
3679 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3680
3681 for (i = 0; i < num_loops; i++) {
3682 cur_size_in_bytes = size_in_bytes;
3683 if (cur_size_in_bytes > 0x1fffff)
3684 cur_size_in_bytes = 0x1fffff;
3685 size_in_bytes -= cur_size_in_bytes;
3686 control = 0;
3687 if (size_in_bytes == 0)
3688 control |= PACKET3_DMA_DATA_CP_SYNC;
3689 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3690 radeon_ring_write(ring, control);
3691 radeon_ring_write(ring, lower_32_bits(src_offset));
3692 radeon_ring_write(ring, upper_32_bits(src_offset));
3693 radeon_ring_write(ring, lower_32_bits(dst_offset));
3694 radeon_ring_write(ring, upper_32_bits(dst_offset));
3695 radeon_ring_write(ring, cur_size_in_bytes);
3696 src_offset += cur_size_in_bytes;
3697 dst_offset += cur_size_in_bytes;
3698 }
3699
3700 r = radeon_fence_emit(rdev, fence, ring->idx);
3701 if (r) {
3702 radeon_ring_unlock_undo(rdev, ring);
3703 return r;
3704 }
3705
3706 radeon_ring_unlock_commit(rdev, ring);
3707 radeon_semaphore_free(rdev, &sem, *fence);
3708
3709 return r;
3710 }
3711
3712 /*
3713 * IB stuff
3714 */
3715 /**
3716 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3717 *
3718 * @rdev: radeon_device pointer
3719 * @ib: radeon indirect buffer object
3720 *
3721 * Emits an DE (drawing engine) or CE (constant engine) IB
3722 * on the gfx ring. IBs are usually generated by userspace
3723 * acceleration drivers and submitted to the kernel for
3724 * sheduling on the ring. This function schedules the IB
3725 * on the gfx ring for execution by the GPU.
3726 */
3727 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3728 {
3729 struct radeon_ring *ring = &rdev->ring[ib->ring];
3730 u32 header, control = INDIRECT_BUFFER_VALID;
3731
3732 if (ib->is_const_ib) {
3733 /* set switch buffer packet before const IB */
3734 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3735 radeon_ring_write(ring, 0);
3736
3737 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3738 } else {
3739 u32 next_rptr;
3740 if (ring->rptr_save_reg) {
3741 next_rptr = ring->wptr + 3 + 4;
3742 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3743 radeon_ring_write(ring, ((ring->rptr_save_reg -
3744 PACKET3_SET_UCONFIG_REG_START) >> 2));
3745 radeon_ring_write(ring, next_rptr);
3746 } else if (rdev->wb.enabled) {
3747 next_rptr = ring->wptr + 5 + 4;
3748 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3749 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3750 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3751 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3752 radeon_ring_write(ring, next_rptr);
3753 }
3754
3755 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3756 }
3757
3758 control |= ib->length_dw |
3759 (ib->vm ? (ib->vm->id << 24) : 0);
3760
3761 radeon_ring_write(ring, header);
3762 radeon_ring_write(ring,
3763 #ifdef __BIG_ENDIAN
3764 (2 << 0) |
3765 #endif
3766 (ib->gpu_addr & 0xFFFFFFFC));
3767 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3768 radeon_ring_write(ring, control);
3769 }
3770
3771 /**
3772 * cik_ib_test - basic gfx ring IB test
3773 *
3774 * @rdev: radeon_device pointer
3775 * @ring: radeon_ring structure holding ring information
3776 *
3777 * Allocate an IB and execute it on the gfx ring (CIK).
3778 * Provides a basic gfx ring test to verify that IBs are working.
3779 * Returns 0 on success, error on failure.
3780 */
3781 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3782 {
3783 struct radeon_ib ib;
3784 uint32_t scratch;
3785 uint32_t tmp = 0;
3786 unsigned i;
3787 int r;
3788
3789 r = radeon_scratch_get(rdev, &scratch);
3790 if (r) {
3791 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3792 return r;
3793 }
3794 WREG32(scratch, 0xCAFEDEAD);
3795 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3796 if (r) {
3797 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3798 radeon_scratch_free(rdev, scratch);
3799 return r;
3800 }
3801 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3802 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3803 ib.ptr[2] = 0xDEADBEEF;
3804 ib.length_dw = 3;
3805 r = radeon_ib_schedule(rdev, &ib, NULL);
3806 if (r) {
3807 radeon_scratch_free(rdev, scratch);
3808 radeon_ib_free(rdev, &ib);
3809 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3810 return r;
3811 }
3812 r = radeon_fence_wait(ib.fence, false);
3813 if (r) {
3814 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3815 radeon_scratch_free(rdev, scratch);
3816 radeon_ib_free(rdev, &ib);
3817 return r;
3818 }
3819 for (i = 0; i < rdev->usec_timeout; i++) {
3820 tmp = RREG32(scratch);
3821 if (tmp == 0xDEADBEEF)
3822 break;
3823 DRM_UDELAY(1);
3824 }
3825 if (i < rdev->usec_timeout) {
3826 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3827 } else {
3828 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3829 scratch, tmp);
3830 r = -EINVAL;
3831 }
3832 radeon_scratch_free(rdev, scratch);
3833 radeon_ib_free(rdev, &ib);
3834 return r;
3835 }
3836
3837 /*
3838 * CP.
3839 * On CIK, gfx and compute now have independant command processors.
3840 *
3841 * GFX
3842 * Gfx consists of a single ring and can process both gfx jobs and
3843 * compute jobs. The gfx CP consists of three microengines (ME):
3844 * PFP - Pre-Fetch Parser
3845 * ME - Micro Engine
3846 * CE - Constant Engine
3847 * The PFP and ME make up what is considered the Drawing Engine (DE).
3848 * The CE is an asynchronous engine used for updating buffer desciptors
3849 * used by the DE so that they can be loaded into cache in parallel
3850 * while the DE is processing state update packets.
3851 *
3852 * Compute
3853 * The compute CP consists of two microengines (ME):
3854 * MEC1 - Compute MicroEngine 1
3855 * MEC2 - Compute MicroEngine 2
3856 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3857 * The queues are exposed to userspace and are programmed directly
3858 * by the compute runtime.
3859 */
3860 /**
3861 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3862 *
3863 * @rdev: radeon_device pointer
3864 * @enable: enable or disable the MEs
3865 *
3866 * Halts or unhalts the gfx MEs.
3867 */
3868 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3869 {
3870 if (enable)
3871 WREG32(CP_ME_CNTL, 0);
3872 else {
3873 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3874 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3875 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3876 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3877 }
3878 udelay(50);
3879 }
3880
3881 /**
3882 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3883 *
3884 * @rdev: radeon_device pointer
3885 *
3886 * Loads the gfx PFP, ME, and CE ucode.
3887 * Returns 0 for success, -EINVAL if the ucode is not available.
3888 */
3889 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3890 {
3891 const __be32 *fw_data;
3892 int i;
3893
3894 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3895 return -EINVAL;
3896
3897 cik_cp_gfx_enable(rdev, false);
3898
3899 /* PFP */
3900 fw_data = (const __be32 *)rdev->pfp_fw->data;
3901 WREG32(CP_PFP_UCODE_ADDR, 0);
3902 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3903 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3904 WREG32(CP_PFP_UCODE_ADDR, 0);
3905
3906 /* CE */
3907 fw_data = (const __be32 *)rdev->ce_fw->data;
3908 WREG32(CP_CE_UCODE_ADDR, 0);
3909 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3910 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3911 WREG32(CP_CE_UCODE_ADDR, 0);
3912
3913 /* ME */
3914 fw_data = (const __be32 *)rdev->me_fw->data;
3915 WREG32(CP_ME_RAM_WADDR, 0);
3916 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3917 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3918 WREG32(CP_ME_RAM_WADDR, 0);
3919
3920 WREG32(CP_PFP_UCODE_ADDR, 0);
3921 WREG32(CP_CE_UCODE_ADDR, 0);
3922 WREG32(CP_ME_RAM_WADDR, 0);
3923 WREG32(CP_ME_RAM_RADDR, 0);
3924 return 0;
3925 }
3926
3927 /**
3928 * cik_cp_gfx_start - start the gfx ring
3929 *
3930 * @rdev: radeon_device pointer
3931 *
3932 * Enables the ring and loads the clear state context and other
3933 * packets required to init the ring.
3934 * Returns 0 for success, error for failure.
3935 */
3936 static int cik_cp_gfx_start(struct radeon_device *rdev)
3937 {
3938 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3939 int r, i;
3940
3941 /* init the CP */
3942 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3943 WREG32(CP_ENDIAN_SWAP, 0);
3944 WREG32(CP_DEVICE_ID, 1);
3945
3946 cik_cp_gfx_enable(rdev, true);
3947
3948 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3949 if (r) {
3950 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3951 return r;
3952 }
3953
3954 /* init the CE partitions. CE only used for gfx on CIK */
3955 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3956 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3957 radeon_ring_write(ring, 0xc000);
3958 radeon_ring_write(ring, 0xc000);
3959
3960 /* setup clear context state */
3961 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3962 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3963
3964 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3965 radeon_ring_write(ring, 0x80000000);
3966 radeon_ring_write(ring, 0x80000000);
3967
3968 for (i = 0; i < cik_default_size; i++)
3969 radeon_ring_write(ring, cik_default_state[i]);
3970
3971 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3972 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3973
3974 /* set clear context state */
3975 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3976 radeon_ring_write(ring, 0);
3977
3978 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3979 radeon_ring_write(ring, 0x00000316);
3980 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3981 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3982
3983 radeon_ring_unlock_commit(rdev, ring);
3984
3985 return 0;
3986 }
3987
3988 /**
3989 * cik_cp_gfx_fini - stop the gfx ring
3990 *
3991 * @rdev: radeon_device pointer
3992 *
3993 * Stop the gfx ring and tear down the driver ring
3994 * info.
3995 */
3996 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3997 {
3998 cik_cp_gfx_enable(rdev, false);
3999 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4000 }
4001
4002 /**
4003 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4004 *
4005 * @rdev: radeon_device pointer
4006 *
4007 * Program the location and size of the gfx ring buffer
4008 * and test it to make sure it's working.
4009 * Returns 0 for success, error for failure.
4010 */
4011 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4012 {
4013 struct radeon_ring *ring;
4014 u32 tmp;
4015 u32 rb_bufsz;
4016 u64 rb_addr;
4017 int r;
4018
4019 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4020 if (rdev->family != CHIP_HAWAII)
4021 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4022
4023 /* Set the write pointer delay */
4024 WREG32(CP_RB_WPTR_DELAY, 0);
4025
4026 /* set the RB to use vmid 0 */
4027 WREG32(CP_RB_VMID, 0);
4028
4029 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4030
4031 /* ring 0 - compute and gfx */
4032 /* Set ring buffer size */
4033 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4034 rb_bufsz = order_base_2(ring->ring_size / 8);
4035 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4036 #ifdef __BIG_ENDIAN
4037 tmp |= BUF_SWAP_32BIT;
4038 #endif
4039 WREG32(CP_RB0_CNTL, tmp);
4040
4041 /* Initialize the ring buffer's read and write pointers */
4042 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4043 ring->wptr = 0;
4044 WREG32(CP_RB0_WPTR, ring->wptr);
4045
4046 /* set the wb address wether it's enabled or not */
4047 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4048 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4049
4050 /* scratch register shadowing is no longer supported */
4051 WREG32(SCRATCH_UMSK, 0);
4052
4053 if (!rdev->wb.enabled)
4054 tmp |= RB_NO_UPDATE;
4055
4056 mdelay(1);
4057 WREG32(CP_RB0_CNTL, tmp);
4058
4059 rb_addr = ring->gpu_addr >> 8;
4060 WREG32(CP_RB0_BASE, rb_addr);
4061 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4062
4063 /* start the ring */
4064 cik_cp_gfx_start(rdev);
4065 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4066 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4067 if (r) {
4068 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4069 return r;
4070 }
4071
4072 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4073 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4074
4075 return 0;
4076 }
4077
4078 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4079 struct radeon_ring *ring)
4080 {
4081 u32 rptr;
4082
4083 if (rdev->wb.enabled)
4084 rptr = rdev->wb.wb[ring->rptr_offs/4];
4085 else
4086 rptr = RREG32(CP_RB0_RPTR);
4087
4088 return rptr;
4089 }
4090
4091 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4092 struct radeon_ring *ring)
4093 {
4094 u32 wptr;
4095
4096 wptr = RREG32(CP_RB0_WPTR);
4097
4098 return wptr;
4099 }
4100
4101 void cik_gfx_set_wptr(struct radeon_device *rdev,
4102 struct radeon_ring *ring)
4103 {
4104 WREG32(CP_RB0_WPTR, ring->wptr);
4105 (void)RREG32(CP_RB0_WPTR);
4106 }
4107
4108 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4109 struct radeon_ring *ring)
4110 {
4111 u32 rptr;
4112
4113 if (rdev->wb.enabled) {
4114 rptr = rdev->wb.wb[ring->rptr_offs/4];
4115 } else {
4116 mutex_lock(&rdev->srbm_mutex);
4117 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4118 rptr = RREG32(CP_HQD_PQ_RPTR);
4119 cik_srbm_select(rdev, 0, 0, 0, 0);
4120 mutex_unlock(&rdev->srbm_mutex);
4121 }
4122
4123 return rptr;
4124 }
4125
4126 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4127 struct radeon_ring *ring)
4128 {
4129 u32 wptr;
4130
4131 if (rdev->wb.enabled) {
4132 /* XXX check if swapping is necessary on BE */
4133 wptr = rdev->wb.wb[ring->wptr_offs/4];
4134 } else {
4135 mutex_lock(&rdev->srbm_mutex);
4136 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4137 wptr = RREG32(CP_HQD_PQ_WPTR);
4138 cik_srbm_select(rdev, 0, 0, 0, 0);
4139 mutex_unlock(&rdev->srbm_mutex);
4140 }
4141
4142 return wptr;
4143 }
4144
4145 void cik_compute_set_wptr(struct radeon_device *rdev,
4146 struct radeon_ring *ring)
4147 {
4148 /* XXX check if swapping is necessary on BE */
4149 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4150 WDOORBELL32(ring->doorbell_index, ring->wptr);
4151 }
4152
4153 /**
4154 * cik_cp_compute_enable - enable/disable the compute CP MEs
4155 *
4156 * @rdev: radeon_device pointer
4157 * @enable: enable or disable the MEs
4158 *
4159 * Halts or unhalts the compute MEs.
4160 */
4161 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4162 {
4163 if (enable)
4164 WREG32(CP_MEC_CNTL, 0);
4165 else {
4166 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4167 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4168 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4169 }
4170 udelay(50);
4171 }
4172
4173 /**
4174 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4175 *
4176 * @rdev: radeon_device pointer
4177 *
4178 * Loads the compute MEC1&2 ucode.
4179 * Returns 0 for success, -EINVAL if the ucode is not available.
4180 */
4181 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4182 {
4183 const __be32 *fw_data;
4184 int i;
4185
4186 if (!rdev->mec_fw)
4187 return -EINVAL;
4188
4189 cik_cp_compute_enable(rdev, false);
4190
4191 /* MEC1 */
4192 fw_data = (const __be32 *)rdev->mec_fw->data;
4193 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4194 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4195 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4196 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4197
4198 if (rdev->family == CHIP_KAVERI) {
4199 /* MEC2 */
4200 fw_data = (const __be32 *)rdev->mec_fw->data;
4201 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4202 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4203 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4204 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4205 }
4206
4207 return 0;
4208 }
4209
4210 /**
4211 * cik_cp_compute_start - start the compute queues
4212 *
4213 * @rdev: radeon_device pointer
4214 *
4215 * Enable the compute queues.
4216 * Returns 0 for success, error for failure.
4217 */
4218 static int cik_cp_compute_start(struct radeon_device *rdev)
4219 {
4220 cik_cp_compute_enable(rdev, true);
4221
4222 return 0;
4223 }
4224
4225 /**
4226 * cik_cp_compute_fini - stop the compute queues
4227 *
4228 * @rdev: radeon_device pointer
4229 *
4230 * Stop the compute queues and tear down the driver queue
4231 * info.
4232 */
4233 static void cik_cp_compute_fini(struct radeon_device *rdev)
4234 {
4235 int i, idx, r;
4236
4237 cik_cp_compute_enable(rdev, false);
4238
4239 for (i = 0; i < 2; i++) {
4240 if (i == 0)
4241 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4242 else
4243 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4244
4245 if (rdev->ring[idx].mqd_obj) {
4246 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4247 if (unlikely(r != 0))
4248 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4249
4250 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4251 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4252
4253 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4254 rdev->ring[idx].mqd_obj = NULL;
4255 }
4256 }
4257 }
4258
4259 static void cik_mec_fini(struct radeon_device *rdev)
4260 {
4261 int r;
4262
4263 if (rdev->mec.hpd_eop_obj) {
4264 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4265 if (unlikely(r != 0))
4266 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4267 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4268 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4269
4270 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4271 rdev->mec.hpd_eop_obj = NULL;
4272 }
4273 }
4274
4275 #define MEC_HPD_SIZE 2048
4276
4277 static int cik_mec_init(struct radeon_device *rdev)
4278 {
4279 int r;
4280 u32 *hpd;
4281
4282 /*
4283 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4284 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4285 */
4286 if (rdev->family == CHIP_KAVERI)
4287 rdev->mec.num_mec = 2;
4288 else
4289 rdev->mec.num_mec = 1;
4290 rdev->mec.num_pipe = 4;
4291 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4292
4293 if (rdev->mec.hpd_eop_obj == NULL) {
4294 r = radeon_bo_create(rdev,
4295 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4296 PAGE_SIZE, true,
4297 RADEON_GEM_DOMAIN_GTT, NULL,
4298 &rdev->mec.hpd_eop_obj);
4299 if (r) {
4300 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4301 return r;
4302 }
4303 }
4304
4305 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4306 if (unlikely(r != 0)) {
4307 cik_mec_fini(rdev);
4308 return r;
4309 }
4310 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4311 &rdev->mec.hpd_eop_gpu_addr);
4312 if (r) {
4313 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4314 cik_mec_fini(rdev);
4315 return r;
4316 }
4317 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4318 if (r) {
4319 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4320 cik_mec_fini(rdev);
4321 return r;
4322 }
4323
4324 /* clear memory. Not sure if this is required or not */
4325 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4326
4327 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4328 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4329
4330 return 0;
4331 }
4332
4333 struct hqd_registers
4334 {
4335 u32 cp_mqd_base_addr;
4336 u32 cp_mqd_base_addr_hi;
4337 u32 cp_hqd_active;
4338 u32 cp_hqd_vmid;
4339 u32 cp_hqd_persistent_state;
4340 u32 cp_hqd_pipe_priority;
4341 u32 cp_hqd_queue_priority;
4342 u32 cp_hqd_quantum;
4343 u32 cp_hqd_pq_base;
4344 u32 cp_hqd_pq_base_hi;
4345 u32 cp_hqd_pq_rptr;
4346 u32 cp_hqd_pq_rptr_report_addr;
4347 u32 cp_hqd_pq_rptr_report_addr_hi;
4348 u32 cp_hqd_pq_wptr_poll_addr;
4349 u32 cp_hqd_pq_wptr_poll_addr_hi;
4350 u32 cp_hqd_pq_doorbell_control;
4351 u32 cp_hqd_pq_wptr;
4352 u32 cp_hqd_pq_control;
4353 u32 cp_hqd_ib_base_addr;
4354 u32 cp_hqd_ib_base_addr_hi;
4355 u32 cp_hqd_ib_rptr;
4356 u32 cp_hqd_ib_control;
4357 u32 cp_hqd_iq_timer;
4358 u32 cp_hqd_iq_rptr;
4359 u32 cp_hqd_dequeue_request;
4360 u32 cp_hqd_dma_offload;
4361 u32 cp_hqd_sema_cmd;
4362 u32 cp_hqd_msg_type;
4363 u32 cp_hqd_atomic0_preop_lo;
4364 u32 cp_hqd_atomic0_preop_hi;
4365 u32 cp_hqd_atomic1_preop_lo;
4366 u32 cp_hqd_atomic1_preop_hi;
4367 u32 cp_hqd_hq_scheduler0;
4368 u32 cp_hqd_hq_scheduler1;
4369 u32 cp_mqd_control;
4370 };
4371
4372 struct bonaire_mqd
4373 {
4374 u32 header;
4375 u32 dispatch_initiator;
4376 u32 dimensions[3];
4377 u32 start_idx[3];
4378 u32 num_threads[3];
4379 u32 pipeline_stat_enable;
4380 u32 perf_counter_enable;
4381 u32 pgm[2];
4382 u32 tba[2];
4383 u32 tma[2];
4384 u32 pgm_rsrc[2];
4385 u32 vmid;
4386 u32 resource_limits;
4387 u32 static_thread_mgmt01[2];
4388 u32 tmp_ring_size;
4389 u32 static_thread_mgmt23[2];
4390 u32 restart[3];
4391 u32 thread_trace_enable;
4392 u32 reserved1;
4393 u32 user_data[16];
4394 u32 vgtcs_invoke_count[2];
4395 struct hqd_registers queue_state;
4396 u32 dequeue_cntr;
4397 u32 interrupt_queue[64];
4398 };
4399
4400 /**
4401 * cik_cp_compute_resume - setup the compute queue registers
4402 *
4403 * @rdev: radeon_device pointer
4404 *
4405 * Program the compute queues and test them to make sure they
4406 * are working.
4407 * Returns 0 for success, error for failure.
4408 */
4409 static int cik_cp_compute_resume(struct radeon_device *rdev)
4410 {
4411 int r, i, idx;
4412 u32 tmp;
4413 bool use_doorbell = true;
4414 u64 hqd_gpu_addr;
4415 u64 mqd_gpu_addr;
4416 u64 eop_gpu_addr;
4417 u64 wb_gpu_addr;
4418 u32 *buf;
4419 struct bonaire_mqd *mqd;
4420
4421 r = cik_cp_compute_start(rdev);
4422 if (r)
4423 return r;
4424
4425 /* fix up chicken bits */
4426 tmp = RREG32(CP_CPF_DEBUG);
4427 tmp |= (1 << 23);
4428 WREG32(CP_CPF_DEBUG, tmp);
4429
4430 /* init the pipes */
4431 mutex_lock(&rdev->srbm_mutex);
4432 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4433 int me = (i < 4) ? 1 : 2;
4434 int pipe = (i < 4) ? i : (i - 4);
4435
4436 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4437
4438 cik_srbm_select(rdev, me, pipe, 0, 0);
4439
4440 /* write the EOP addr */
4441 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4442 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4443
4444 /* set the VMID assigned */
4445 WREG32(CP_HPD_EOP_VMID, 0);
4446
4447 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4448 tmp = RREG32(CP_HPD_EOP_CONTROL);
4449 tmp &= ~EOP_SIZE_MASK;
4450 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4451 WREG32(CP_HPD_EOP_CONTROL, tmp);
4452 }
4453 cik_srbm_select(rdev, 0, 0, 0, 0);
4454 mutex_unlock(&rdev->srbm_mutex);
4455
4456 /* init the queues. Just two for now. */
4457 for (i = 0; i < 2; i++) {
4458 if (i == 0)
4459 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4460 else
4461 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4462
4463 if (rdev->ring[idx].mqd_obj == NULL) {
4464 r = radeon_bo_create(rdev,
4465 sizeof(struct bonaire_mqd),
4466 PAGE_SIZE, true,
4467 RADEON_GEM_DOMAIN_GTT, NULL,
4468 &rdev->ring[idx].mqd_obj);
4469 if (r) {
4470 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4471 return r;
4472 }
4473 }
4474
4475 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4476 if (unlikely(r != 0)) {
4477 cik_cp_compute_fini(rdev);
4478 return r;
4479 }
4480 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4481 &mqd_gpu_addr);
4482 if (r) {
4483 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4484 cik_cp_compute_fini(rdev);
4485 return r;
4486 }
4487 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4488 if (r) {
4489 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4490 cik_cp_compute_fini(rdev);
4491 return r;
4492 }
4493
4494 /* init the mqd struct */
4495 memset(buf, 0, sizeof(struct bonaire_mqd));
4496
4497 mqd = (struct bonaire_mqd *)buf;
4498 mqd->header = 0xC0310800;
4499 mqd->static_thread_mgmt01[0] = 0xffffffff;
4500 mqd->static_thread_mgmt01[1] = 0xffffffff;
4501 mqd->static_thread_mgmt23[0] = 0xffffffff;
4502 mqd->static_thread_mgmt23[1] = 0xffffffff;
4503
4504 mutex_lock(&rdev->srbm_mutex);
4505 cik_srbm_select(rdev, rdev->ring[idx].me,
4506 rdev->ring[idx].pipe,
4507 rdev->ring[idx].queue, 0);
4508
4509 /* disable wptr polling */
4510 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4511 tmp &= ~WPTR_POLL_EN;
4512 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4513
4514 /* enable doorbell? */
4515 mqd->queue_state.cp_hqd_pq_doorbell_control =
4516 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4517 if (use_doorbell)
4518 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4519 else
4520 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4521 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4522 mqd->queue_state.cp_hqd_pq_doorbell_control);
4523
4524 /* disable the queue if it's active */
4525 mqd->queue_state.cp_hqd_dequeue_request = 0;
4526 mqd->queue_state.cp_hqd_pq_rptr = 0;
4527 mqd->queue_state.cp_hqd_pq_wptr= 0;
4528 if (RREG32(CP_HQD_ACTIVE) & 1) {
4529 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4530 for (i = 0; i < rdev->usec_timeout; i++) {
4531 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4532 break;
4533 udelay(1);
4534 }
4535 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4536 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4537 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4538 }
4539
4540 /* set the pointer to the MQD */
4541 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4542 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4543 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4544 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4545 /* set MQD vmid to 0 */
4546 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4547 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4548 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4549
4550 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4551 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4552 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4553 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4554 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4555 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4556
4557 /* set up the HQD, this is similar to CP_RB0_CNTL */
4558 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4559 mqd->queue_state.cp_hqd_pq_control &=
4560 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4561
4562 mqd->queue_state.cp_hqd_pq_control |=
4563 order_base_2(rdev->ring[idx].ring_size / 8);
4564 mqd->queue_state.cp_hqd_pq_control |=
4565 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4566 #ifdef __BIG_ENDIAN
4567 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4568 #endif
4569 mqd->queue_state.cp_hqd_pq_control &=
4570 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4571 mqd->queue_state.cp_hqd_pq_control |=
4572 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4573 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4574
4575 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4576 if (i == 0)
4577 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4578 else
4579 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4580 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4581 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4582 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4583 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4584 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4585
4586 /* set the wb address wether it's enabled or not */
4587 if (i == 0)
4588 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4589 else
4590 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4591 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4592 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4593 upper_32_bits(wb_gpu_addr) & 0xffff;
4594 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4595 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4596 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4597 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4598
4599 /* enable the doorbell if requested */
4600 if (use_doorbell) {
4601 mqd->queue_state.cp_hqd_pq_doorbell_control =
4602 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4603 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4604 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4605 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4606 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4607 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4608 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4609
4610 } else {
4611 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4612 }
4613 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4614 mqd->queue_state.cp_hqd_pq_doorbell_control);
4615
4616 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4617 rdev->ring[idx].wptr = 0;
4618 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4619 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4620 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4621
4622 /* set the vmid for the queue */
4623 mqd->queue_state.cp_hqd_vmid = 0;
4624 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4625
4626 /* activate the queue */
4627 mqd->queue_state.cp_hqd_active = 1;
4628 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4629
4630 cik_srbm_select(rdev, 0, 0, 0, 0);
4631 mutex_unlock(&rdev->srbm_mutex);
4632
4633 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4634 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4635
4636 rdev->ring[idx].ready = true;
4637 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4638 if (r)
4639 rdev->ring[idx].ready = false;
4640 }
4641
4642 return 0;
4643 }
4644
4645 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4646 {
4647 cik_cp_gfx_enable(rdev, enable);
4648 cik_cp_compute_enable(rdev, enable);
4649 }
4650
4651 static int cik_cp_load_microcode(struct radeon_device *rdev)
4652 {
4653 int r;
4654
4655 r = cik_cp_gfx_load_microcode(rdev);
4656 if (r)
4657 return r;
4658 r = cik_cp_compute_load_microcode(rdev);
4659 if (r)
4660 return r;
4661
4662 return 0;
4663 }
4664
4665 static void cik_cp_fini(struct radeon_device *rdev)
4666 {
4667 cik_cp_gfx_fini(rdev);
4668 cik_cp_compute_fini(rdev);
4669 }
4670
4671 static int cik_cp_resume(struct radeon_device *rdev)
4672 {
4673 int r;
4674
4675 cik_enable_gui_idle_interrupt(rdev, false);
4676
4677 r = cik_cp_load_microcode(rdev);
4678 if (r)
4679 return r;
4680
4681 r = cik_cp_gfx_resume(rdev);
4682 if (r)
4683 return r;
4684 r = cik_cp_compute_resume(rdev);
4685 if (r)
4686 return r;
4687
4688 cik_enable_gui_idle_interrupt(rdev, true);
4689
4690 return 0;
4691 }
4692
4693 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4694 {
4695 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4696 RREG32(GRBM_STATUS));
4697 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4698 RREG32(GRBM_STATUS2));
4699 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4700 RREG32(GRBM_STATUS_SE0));
4701 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4702 RREG32(GRBM_STATUS_SE1));
4703 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4704 RREG32(GRBM_STATUS_SE2));
4705 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4706 RREG32(GRBM_STATUS_SE3));
4707 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4708 RREG32(SRBM_STATUS));
4709 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4710 RREG32(SRBM_STATUS2));
4711 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4712 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4713 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4714 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4715 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4716 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4717 RREG32(CP_STALLED_STAT1));
4718 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4719 RREG32(CP_STALLED_STAT2));
4720 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4721 RREG32(CP_STALLED_STAT3));
4722 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4723 RREG32(CP_CPF_BUSY_STAT));
4724 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4725 RREG32(CP_CPF_STALLED_STAT1));
4726 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4727 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4728 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4729 RREG32(CP_CPC_STALLED_STAT1));
4730 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4731 }
4732
4733 /**
4734 * cik_gpu_check_soft_reset - check which blocks are busy
4735 *
4736 * @rdev: radeon_device pointer
4737 *
4738 * Check which blocks are busy and return the relevant reset
4739 * mask to be used by cik_gpu_soft_reset().
4740 * Returns a mask of the blocks to be reset.
4741 */
4742 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4743 {
4744 u32 reset_mask = 0;
4745 u32 tmp;
4746
4747 /* GRBM_STATUS */
4748 tmp = RREG32(GRBM_STATUS);
4749 if (tmp & (PA_BUSY | SC_BUSY |
4750 BCI_BUSY | SX_BUSY |
4751 TA_BUSY | VGT_BUSY |
4752 DB_BUSY | CB_BUSY |
4753 GDS_BUSY | SPI_BUSY |
4754 IA_BUSY | IA_BUSY_NO_DMA))
4755 reset_mask |= RADEON_RESET_GFX;
4756
4757 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4758 reset_mask |= RADEON_RESET_CP;
4759
4760 /* GRBM_STATUS2 */
4761 tmp = RREG32(GRBM_STATUS2);
4762 if (tmp & RLC_BUSY)
4763 reset_mask |= RADEON_RESET_RLC;
4764
4765 /* SDMA0_STATUS_REG */
4766 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4767 if (!(tmp & SDMA_IDLE))
4768 reset_mask |= RADEON_RESET_DMA;
4769
4770 /* SDMA1_STATUS_REG */
4771 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4772 if (!(tmp & SDMA_IDLE))
4773 reset_mask |= RADEON_RESET_DMA1;
4774
4775 /* SRBM_STATUS2 */
4776 tmp = RREG32(SRBM_STATUS2);
4777 if (tmp & SDMA_BUSY)
4778 reset_mask |= RADEON_RESET_DMA;
4779
4780 if (tmp & SDMA1_BUSY)
4781 reset_mask |= RADEON_RESET_DMA1;
4782
4783 /* SRBM_STATUS */
4784 tmp = RREG32(SRBM_STATUS);
4785
4786 if (tmp & IH_BUSY)
4787 reset_mask |= RADEON_RESET_IH;
4788
4789 if (tmp & SEM_BUSY)
4790 reset_mask |= RADEON_RESET_SEM;
4791
4792 if (tmp & GRBM_RQ_PENDING)
4793 reset_mask |= RADEON_RESET_GRBM;
4794
4795 if (tmp & VMC_BUSY)
4796 reset_mask |= RADEON_RESET_VMC;
4797
4798 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4799 MCC_BUSY | MCD_BUSY))
4800 reset_mask |= RADEON_RESET_MC;
4801
4802 if (evergreen_is_display_hung(rdev))
4803 reset_mask |= RADEON_RESET_DISPLAY;
4804
4805 /* Skip MC reset as it's mostly likely not hung, just busy */
4806 if (reset_mask & RADEON_RESET_MC) {
4807 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4808 reset_mask &= ~RADEON_RESET_MC;
4809 }
4810
4811 return reset_mask;
4812 }
4813
4814 /**
4815 * cik_gpu_soft_reset - soft reset GPU
4816 *
4817 * @rdev: radeon_device pointer
4818 * @reset_mask: mask of which blocks to reset
4819 *
4820 * Soft reset the blocks specified in @reset_mask.
4821 */
4822 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4823 {
4824 struct evergreen_mc_save save;
4825 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4826 u32 tmp;
4827
4828 if (reset_mask == 0)
4829 return;
4830
4831 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4832
4833 cik_print_gpu_status_regs(rdev);
4834 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4835 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4836 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4837 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4838
4839 /* disable CG/PG */
4840 cik_fini_pg(rdev);
4841 cik_fini_cg(rdev);
4842
4843 /* stop the rlc */
4844 cik_rlc_stop(rdev);
4845
4846 /* Disable GFX parsing/prefetching */
4847 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4848
4849 /* Disable MEC parsing/prefetching */
4850 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4851
4852 if (reset_mask & RADEON_RESET_DMA) {
4853 /* sdma0 */
4854 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4855 tmp |= SDMA_HALT;
4856 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4857 }
4858 if (reset_mask & RADEON_RESET_DMA1) {
4859 /* sdma1 */
4860 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4861 tmp |= SDMA_HALT;
4862 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4863 }
4864
4865 evergreen_mc_stop(rdev, &save);
4866 if (evergreen_mc_wait_for_idle(rdev)) {
4867 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4868 }
4869
4870 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4871 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4872
4873 if (reset_mask & RADEON_RESET_CP) {
4874 grbm_soft_reset |= SOFT_RESET_CP;
4875
4876 srbm_soft_reset |= SOFT_RESET_GRBM;
4877 }
4878
4879 if (reset_mask & RADEON_RESET_DMA)
4880 srbm_soft_reset |= SOFT_RESET_SDMA;
4881
4882 if (reset_mask & RADEON_RESET_DMA1)
4883 srbm_soft_reset |= SOFT_RESET_SDMA1;
4884
4885 if (reset_mask & RADEON_RESET_DISPLAY)
4886 srbm_soft_reset |= SOFT_RESET_DC;
4887
4888 if (reset_mask & RADEON_RESET_RLC)
4889 grbm_soft_reset |= SOFT_RESET_RLC;
4890
4891 if (reset_mask & RADEON_RESET_SEM)
4892 srbm_soft_reset |= SOFT_RESET_SEM;
4893
4894 if (reset_mask & RADEON_RESET_IH)
4895 srbm_soft_reset |= SOFT_RESET_IH;
4896
4897 if (reset_mask & RADEON_RESET_GRBM)
4898 srbm_soft_reset |= SOFT_RESET_GRBM;
4899
4900 if (reset_mask & RADEON_RESET_VMC)
4901 srbm_soft_reset |= SOFT_RESET_VMC;
4902
4903 if (!(rdev->flags & RADEON_IS_IGP)) {
4904 if (reset_mask & RADEON_RESET_MC)
4905 srbm_soft_reset |= SOFT_RESET_MC;
4906 }
4907
4908 if (grbm_soft_reset) {
4909 tmp = RREG32(GRBM_SOFT_RESET);
4910 tmp |= grbm_soft_reset;
4911 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4912 WREG32(GRBM_SOFT_RESET, tmp);
4913 tmp = RREG32(GRBM_SOFT_RESET);
4914
4915 udelay(50);
4916
4917 tmp &= ~grbm_soft_reset;
4918 WREG32(GRBM_SOFT_RESET, tmp);
4919 tmp = RREG32(GRBM_SOFT_RESET);
4920 }
4921
4922 if (srbm_soft_reset) {
4923 tmp = RREG32(SRBM_SOFT_RESET);
4924 tmp |= srbm_soft_reset;
4925 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4926 WREG32(SRBM_SOFT_RESET, tmp);
4927 tmp = RREG32(SRBM_SOFT_RESET);
4928
4929 udelay(50);
4930
4931 tmp &= ~srbm_soft_reset;
4932 WREG32(SRBM_SOFT_RESET, tmp);
4933 tmp = RREG32(SRBM_SOFT_RESET);
4934 }
4935
4936 /* Wait a little for things to settle down */
4937 udelay(50);
4938
4939 evergreen_mc_resume(rdev, &save);
4940 udelay(50);
4941
4942 cik_print_gpu_status_regs(rdev);
4943 }
4944
4945 struct kv_reset_save_regs {
4946 u32 gmcon_reng_execute;
4947 u32 gmcon_misc;
4948 u32 gmcon_misc3;
4949 };
4950
4951 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4952 struct kv_reset_save_regs *save)
4953 {
4954 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4955 save->gmcon_misc = RREG32(GMCON_MISC);
4956 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4957
4958 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4959 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4960 STCTRL_STUTTER_EN));
4961 }
4962
4963 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4964 struct kv_reset_save_regs *save)
4965 {
4966 int i;
4967
4968 WREG32(GMCON_PGFSM_WRITE, 0);
4969 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4970
4971 for (i = 0; i < 5; i++)
4972 WREG32(GMCON_PGFSM_WRITE, 0);
4973
4974 WREG32(GMCON_PGFSM_WRITE, 0);
4975 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4976
4977 for (i = 0; i < 5; i++)
4978 WREG32(GMCON_PGFSM_WRITE, 0);
4979
4980 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4981 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4982
4983 for (i = 0; i < 5; i++)
4984 WREG32(GMCON_PGFSM_WRITE, 0);
4985
4986 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4987 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4988
4989 for (i = 0; i < 5; i++)
4990 WREG32(GMCON_PGFSM_WRITE, 0);
4991
4992 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4993 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4994
4995 for (i = 0; i < 5; i++)
4996 WREG32(GMCON_PGFSM_WRITE, 0);
4997
4998 WREG32(GMCON_PGFSM_WRITE, 0);
4999 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5000
5001 for (i = 0; i < 5; i++)
5002 WREG32(GMCON_PGFSM_WRITE, 0);
5003
5004 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5005 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5006
5007 for (i = 0; i < 5; i++)
5008 WREG32(GMCON_PGFSM_WRITE, 0);
5009
5010 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5011 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5012
5013 for (i = 0; i < 5; i++)
5014 WREG32(GMCON_PGFSM_WRITE, 0);
5015
5016 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5017 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5018
5019 for (i = 0; i < 5; i++)
5020 WREG32(GMCON_PGFSM_WRITE, 0);
5021
5022 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5023 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5024
5025 for (i = 0; i < 5; i++)
5026 WREG32(GMCON_PGFSM_WRITE, 0);
5027
5028 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5029 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5030
5031 WREG32(GMCON_MISC3, save->gmcon_misc3);
5032 WREG32(GMCON_MISC, save->gmcon_misc);
5033 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5034 }
5035
5036 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5037 {
5038 struct evergreen_mc_save save;
5039 struct kv_reset_save_regs kv_save = { 0 };
5040 u32 tmp, i;
5041
5042 dev_info(rdev->dev, "GPU pci config reset\n");
5043
5044 /* disable dpm? */
5045
5046 /* disable cg/pg */
5047 cik_fini_pg(rdev);
5048 cik_fini_cg(rdev);
5049
5050 /* Disable GFX parsing/prefetching */
5051 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5052
5053 /* Disable MEC parsing/prefetching */
5054 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5055
5056 /* sdma0 */
5057 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5058 tmp |= SDMA_HALT;
5059 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5060 /* sdma1 */
5061 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5062 tmp |= SDMA_HALT;
5063 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5064 /* XXX other engines? */
5065
5066 /* halt the rlc, disable cp internal ints */
5067 cik_rlc_stop(rdev);
5068
5069 udelay(50);
5070
5071 /* disable mem access */
5072 evergreen_mc_stop(rdev, &save);
5073 if (evergreen_mc_wait_for_idle(rdev)) {
5074 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5075 }
5076
5077 if (rdev->flags & RADEON_IS_IGP)
5078 kv_save_regs_for_reset(rdev, &kv_save);
5079
5080 /* disable BM */
5081 pci_clear_master(rdev->pdev);
5082 /* reset */
5083 radeon_pci_config_reset(rdev);
5084
5085 udelay(100);
5086
5087 /* wait for asic to come out of reset */
5088 for (i = 0; i < rdev->usec_timeout; i++) {
5089 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5090 break;
5091 udelay(1);
5092 }
5093
5094 /* does asic init need to be run first??? */
5095 if (rdev->flags & RADEON_IS_IGP)
5096 kv_restore_regs_for_reset(rdev, &kv_save);
5097 }
5098
5099 /**
5100 * cik_asic_reset - soft reset GPU
5101 *
5102 * @rdev: radeon_device pointer
5103 *
5104 * Look up which blocks are hung and attempt
5105 * to reset them.
5106 * Returns 0 for success.
5107 */
5108 int cik_asic_reset(struct radeon_device *rdev)
5109 {
5110 u32 reset_mask;
5111
5112 reset_mask = cik_gpu_check_soft_reset(rdev);
5113
5114 if (reset_mask)
5115 r600_set_bios_scratch_engine_hung(rdev, true);
5116
5117 /* try soft reset */
5118 cik_gpu_soft_reset(rdev, reset_mask);
5119
5120 reset_mask = cik_gpu_check_soft_reset(rdev);
5121
5122 /* try pci config reset */
5123 if (reset_mask && radeon_hard_reset)
5124 cik_gpu_pci_config_reset(rdev);
5125
5126 reset_mask = cik_gpu_check_soft_reset(rdev);
5127
5128 if (!reset_mask)
5129 r600_set_bios_scratch_engine_hung(rdev, false);
5130
5131 return 0;
5132 }
5133
5134 /**
5135 * cik_gfx_is_lockup - check if the 3D engine is locked up
5136 *
5137 * @rdev: radeon_device pointer
5138 * @ring: radeon_ring structure holding ring information
5139 *
5140 * Check if the 3D engine is locked up (CIK).
5141 * Returns true if the engine is locked, false if not.
5142 */
5143 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5144 {
5145 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5146
5147 if (!(reset_mask & (RADEON_RESET_GFX |
5148 RADEON_RESET_COMPUTE |
5149 RADEON_RESET_CP))) {
5150 radeon_ring_lockup_update(rdev, ring);
5151 return false;
5152 }
5153 return radeon_ring_test_lockup(rdev, ring);
5154 }
5155
5156 /* MC */
5157 /**
5158 * cik_mc_program - program the GPU memory controller
5159 *
5160 * @rdev: radeon_device pointer
5161 *
5162 * Set the location of vram, gart, and AGP in the GPU's
5163 * physical address space (CIK).
5164 */
5165 static void cik_mc_program(struct radeon_device *rdev)
5166 {
5167 struct evergreen_mc_save save;
5168 u32 tmp;
5169 int i, j;
5170
5171 /* Initialize HDP */
5172 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5173 WREG32((0x2c14 + j), 0x00000000);
5174 WREG32((0x2c18 + j), 0x00000000);
5175 WREG32((0x2c1c + j), 0x00000000);
5176 WREG32((0x2c20 + j), 0x00000000);
5177 WREG32((0x2c24 + j), 0x00000000);
5178 }
5179 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5180
5181 evergreen_mc_stop(rdev, &save);
5182 if (radeon_mc_wait_for_idle(rdev)) {
5183 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5184 }
5185 /* Lockout access through VGA aperture*/
5186 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5187 /* Update configuration */
5188 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5189 rdev->mc.vram_start >> 12);
5190 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5191 rdev->mc.vram_end >> 12);
5192 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5193 rdev->vram_scratch.gpu_addr >> 12);
5194 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5195 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5196 WREG32(MC_VM_FB_LOCATION, tmp);
5197 /* XXX double check these! */
5198 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5199 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5200 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5201 WREG32(MC_VM_AGP_BASE, 0);
5202 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5203 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5204 if (radeon_mc_wait_for_idle(rdev)) {
5205 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5206 }
5207 evergreen_mc_resume(rdev, &save);
5208 /* we need to own VRAM, so turn off the VGA renderer here
5209 * to stop it overwriting our objects */
5210 rv515_vga_render_disable(rdev);
5211 }
5212
5213 /**
5214 * cik_mc_init - initialize the memory controller driver params
5215 *
5216 * @rdev: radeon_device pointer
5217 *
5218 * Look up the amount of vram, vram width, and decide how to place
5219 * vram and gart within the GPU's physical address space (CIK).
5220 * Returns 0 for success.
5221 */
5222 static int cik_mc_init(struct radeon_device *rdev)
5223 {
5224 u32 tmp;
5225 int chansize, numchan;
5226
5227 /* Get VRAM informations */
5228 rdev->mc.vram_is_ddr = true;
5229 tmp = RREG32(MC_ARB_RAMCFG);
5230 if (tmp & CHANSIZE_MASK) {
5231 chansize = 64;
5232 } else {
5233 chansize = 32;
5234 }
5235 tmp = RREG32(MC_SHARED_CHMAP);
5236 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5237 case 0:
5238 default:
5239 numchan = 1;
5240 break;
5241 case 1:
5242 numchan = 2;
5243 break;
5244 case 2:
5245 numchan = 4;
5246 break;
5247 case 3:
5248 numchan = 8;
5249 break;
5250 case 4:
5251 numchan = 3;
5252 break;
5253 case 5:
5254 numchan = 6;
5255 break;
5256 case 6:
5257 numchan = 10;
5258 break;
5259 case 7:
5260 numchan = 12;
5261 break;
5262 case 8:
5263 numchan = 16;
5264 break;
5265 }
5266 rdev->mc.vram_width = numchan * chansize;
5267 /* Could aper size report 0 ? */
5268 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5269 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5270 /* size in MB on si */
5271 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5272 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5273 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5274 si_vram_gtt_location(rdev, &rdev->mc);
5275 radeon_update_bandwidth_info(rdev);
5276
5277 return 0;
5278 }
5279
5280 /*
5281 * GART
5282 * VMID 0 is the physical GPU addresses as used by the kernel.
5283 * VMIDs 1-15 are used for userspace clients and are handled
5284 * by the radeon vm/hsa code.
5285 */
5286 /**
5287 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5288 *
5289 * @rdev: radeon_device pointer
5290 *
5291 * Flush the TLB for the VMID 0 page table (CIK).
5292 */
5293 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5294 {
5295 /* flush hdp cache */
5296 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5297
5298 /* bits 0-15 are the VM contexts0-15 */
5299 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5300 }
5301
5302 /**
5303 * cik_pcie_gart_enable - gart enable
5304 *
5305 * @rdev: radeon_device pointer
5306 *
5307 * This sets up the TLBs, programs the page tables for VMID0,
5308 * sets up the hw for VMIDs 1-15 which are allocated on
5309 * demand, and sets up the global locations for the LDS, GDS,
5310 * and GPUVM for FSA64 clients (CIK).
5311 * Returns 0 for success, errors for failure.
5312 */
5313 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5314 {
5315 int r, i;
5316
5317 if (rdev->gart.robj == NULL) {
5318 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5319 return -EINVAL;
5320 }
5321 r = radeon_gart_table_vram_pin(rdev);
5322 if (r)
5323 return r;
5324 radeon_gart_restore(rdev);
5325 /* Setup TLB control */
5326 WREG32(MC_VM_MX_L1_TLB_CNTL,
5327 (0xA << 7) |
5328 ENABLE_L1_TLB |
5329 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5330 ENABLE_ADVANCED_DRIVER_MODEL |
5331 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5332 /* Setup L2 cache */
5333 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5334 ENABLE_L2_FRAGMENT_PROCESSING |
5335 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5336 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5337 EFFECTIVE_L2_QUEUE_SIZE(7) |
5338 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5339 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5340 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5341 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5342 /* setup context0 */
5343 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5344 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5345 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5346 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5347 (u32)(rdev->dummy_page.addr >> 12));
5348 WREG32(VM_CONTEXT0_CNTL2, 0);
5349 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5350 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5351
5352 WREG32(0x15D4, 0);
5353 WREG32(0x15D8, 0);
5354 WREG32(0x15DC, 0);
5355
5356 /* empty context1-15 */
5357 /* FIXME start with 4G, once using 2 level pt switch to full
5358 * vm size space
5359 */
5360 /* set vm size, must be a multiple of 4 */
5361 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5362 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5363 for (i = 1; i < 16; i++) {
5364 if (i < 8)
5365 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5366 rdev->gart.table_addr >> 12);
5367 else
5368 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5369 rdev->gart.table_addr >> 12);
5370 }
5371
5372 /* enable context1-15 */
5373 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5374 (u32)(rdev->dummy_page.addr >> 12));
5375 WREG32(VM_CONTEXT1_CNTL2, 4);
5376 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5377 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5378 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5379 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5380 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5381 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5382 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5383 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5384 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5385 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5386 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5387 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5388 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5389
5390 if (rdev->family == CHIP_KAVERI) {
5391 u32 tmp = RREG32(CHUB_CONTROL);
5392 tmp &= ~BYPASS_VM;
5393 WREG32(CHUB_CONTROL, tmp);
5394 }
5395
5396 /* XXX SH_MEM regs */
5397 /* where to put LDS, scratch, GPUVM in FSA64 space */
5398 mutex_lock(&rdev->srbm_mutex);
5399 for (i = 0; i < 16; i++) {
5400 cik_srbm_select(rdev, 0, 0, 0, i);
5401 /* CP and shaders */
5402 WREG32(SH_MEM_CONFIG, 0);
5403 WREG32(SH_MEM_APE1_BASE, 1);
5404 WREG32(SH_MEM_APE1_LIMIT, 0);
5405 WREG32(SH_MEM_BASES, 0);
5406 /* SDMA GFX */
5407 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5408 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5409 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5410 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5411 /* XXX SDMA RLC - todo */
5412 }
5413 cik_srbm_select(rdev, 0, 0, 0, 0);
5414 mutex_unlock(&rdev->srbm_mutex);
5415
5416 cik_pcie_gart_tlb_flush(rdev);
5417 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5418 (unsigned)(rdev->mc.gtt_size >> 20),
5419 (unsigned long long)rdev->gart.table_addr);
5420 rdev->gart.ready = true;
5421 return 0;
5422 }
5423
5424 /**
5425 * cik_pcie_gart_disable - gart disable
5426 *
5427 * @rdev: radeon_device pointer
5428 *
5429 * This disables all VM page table (CIK).
5430 */
5431 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5432 {
5433 /* Disable all tables */
5434 WREG32(VM_CONTEXT0_CNTL, 0);
5435 WREG32(VM_CONTEXT1_CNTL, 0);
5436 /* Setup TLB control */
5437 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5438 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5439 /* Setup L2 cache */
5440 WREG32(VM_L2_CNTL,
5441 ENABLE_L2_FRAGMENT_PROCESSING |
5442 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5443 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5444 EFFECTIVE_L2_QUEUE_SIZE(7) |
5445 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5446 WREG32(VM_L2_CNTL2, 0);
5447 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5448 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5449 radeon_gart_table_vram_unpin(rdev);
5450 }
5451
5452 /**
5453 * cik_pcie_gart_fini - vm fini callback
5454 *
5455 * @rdev: radeon_device pointer
5456 *
5457 * Tears down the driver GART/VM setup (CIK).
5458 */
5459 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5460 {
5461 cik_pcie_gart_disable(rdev);
5462 radeon_gart_table_vram_free(rdev);
5463 radeon_gart_fini(rdev);
5464 }
5465
5466 /* vm parser */
5467 /**
5468 * cik_ib_parse - vm ib_parse callback
5469 *
5470 * @rdev: radeon_device pointer
5471 * @ib: indirect buffer pointer
5472 *
5473 * CIK uses hw IB checking so this is a nop (CIK).
5474 */
5475 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5476 {
5477 return 0;
5478 }
5479
5480 /*
5481 * vm
5482 * VMID 0 is the physical GPU addresses as used by the kernel.
5483 * VMIDs 1-15 are used for userspace clients and are handled
5484 * by the radeon vm/hsa code.
5485 */
5486 /**
5487 * cik_vm_init - cik vm init callback
5488 *
5489 * @rdev: radeon_device pointer
5490 *
5491 * Inits cik specific vm parameters (number of VMs, base of vram for
5492 * VMIDs 1-15) (CIK).
5493 * Returns 0 for success.
5494 */
5495 int cik_vm_init(struct radeon_device *rdev)
5496 {
5497 /* number of VMs */
5498 rdev->vm_manager.nvm = 16;
5499 /* base offset of vram pages */
5500 if (rdev->flags & RADEON_IS_IGP) {
5501 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5502 tmp <<= 22;
5503 rdev->vm_manager.vram_base_offset = tmp;
5504 } else
5505 rdev->vm_manager.vram_base_offset = 0;
5506
5507 return 0;
5508 }
5509
5510 /**
5511 * cik_vm_fini - cik vm fini callback
5512 *
5513 * @rdev: radeon_device pointer
5514 *
5515 * Tear down any asic specific VM setup (CIK).
5516 */
5517 void cik_vm_fini(struct radeon_device *rdev)
5518 {
5519 }
5520
5521 /**
5522 * cik_vm_decode_fault - print human readable fault info
5523 *
5524 * @rdev: radeon_device pointer
5525 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5526 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5527 *
5528 * Print human readable fault information (CIK).
5529 */
5530 static void cik_vm_decode_fault(struct radeon_device *rdev,
5531 u32 status, u32 addr, u32 mc_client)
5532 {
5533 u32 mc_id;
5534 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5535 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5536 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5537 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5538
5539 if (rdev->family == CHIP_HAWAII)
5540 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5541 else
5542 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5543
5544 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5545 protections, vmid, addr,
5546 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5547 block, mc_client, mc_id);
5548 }
5549
5550 /**
5551 * cik_vm_flush - cik vm flush using the CP
5552 *
5553 * @rdev: radeon_device pointer
5554 *
5555 * Update the page table base and flush the VM TLB
5556 * using the CP (CIK).
5557 */
5558 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5559 {
5560 struct radeon_ring *ring = &rdev->ring[ridx];
5561
5562 if (vm == NULL)
5563 return;
5564
5565 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5566 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5567 WRITE_DATA_DST_SEL(0)));
5568 if (vm->id < 8) {
5569 radeon_ring_write(ring,
5570 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5571 } else {
5572 radeon_ring_write(ring,
5573 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5574 }
5575 radeon_ring_write(ring, 0);
5576 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5577
5578 /* update SH_MEM_* regs */
5579 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5580 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5581 WRITE_DATA_DST_SEL(0)));
5582 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5583 radeon_ring_write(ring, 0);
5584 radeon_ring_write(ring, VMID(vm->id));
5585
5586 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5587 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5588 WRITE_DATA_DST_SEL(0)));
5589 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5590 radeon_ring_write(ring, 0);
5591
5592 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5593 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5594 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5595 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5596
5597 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5598 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5599 WRITE_DATA_DST_SEL(0)));
5600 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5601 radeon_ring_write(ring, 0);
5602 radeon_ring_write(ring, VMID(0));
5603
5604 /* HDP flush */
5605 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5606
5607 /* bits 0-15 are the VM contexts0-15 */
5608 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5609 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5610 WRITE_DATA_DST_SEL(0)));
5611 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5612 radeon_ring_write(ring, 0);
5613 radeon_ring_write(ring, 1 << vm->id);
5614
5615 /* compute doesn't have PFP */
5616 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5617 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5618 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5619 radeon_ring_write(ring, 0x0);
5620 }
5621 }
5622
5623 /*
5624 * RLC
5625 * The RLC is a multi-purpose microengine that handles a
5626 * variety of functions, the most important of which is
5627 * the interrupt controller.
5628 */
5629 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5630 bool enable)
5631 {
5632 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5633
5634 if (enable)
5635 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5636 else
5637 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5638 WREG32(CP_INT_CNTL_RING0, tmp);
5639 }
5640
5641 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5642 {
5643 u32 tmp;
5644
5645 tmp = RREG32(RLC_LB_CNTL);
5646 if (enable)
5647 tmp |= LOAD_BALANCE_ENABLE;
5648 else
5649 tmp &= ~LOAD_BALANCE_ENABLE;
5650 WREG32(RLC_LB_CNTL, tmp);
5651 }
5652
5653 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5654 {
5655 u32 i, j, k;
5656 u32 mask;
5657
5658 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5659 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5660 cik_select_se_sh(rdev, i, j);
5661 for (k = 0; k < rdev->usec_timeout; k++) {
5662 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5663 break;
5664 udelay(1);
5665 }
5666 }
5667 }
5668 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5669
5670 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5671 for (k = 0; k < rdev->usec_timeout; k++) {
5672 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5673 break;
5674 udelay(1);
5675 }
5676 }
5677
5678 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5679 {
5680 u32 tmp;
5681
5682 tmp = RREG32(RLC_CNTL);
5683 if (tmp != rlc)
5684 WREG32(RLC_CNTL, rlc);
5685 }
5686
5687 static u32 cik_halt_rlc(struct radeon_device *rdev)
5688 {
5689 u32 data, orig;
5690
5691 orig = data = RREG32(RLC_CNTL);
5692
5693 if (data & RLC_ENABLE) {
5694 u32 i;
5695
5696 data &= ~RLC_ENABLE;
5697 WREG32(RLC_CNTL, data);
5698
5699 for (i = 0; i < rdev->usec_timeout; i++) {
5700 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5701 break;
5702 udelay(1);
5703 }
5704
5705 cik_wait_for_rlc_serdes(rdev);
5706 }
5707
5708 return orig;
5709 }
5710
5711 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5712 {
5713 u32 tmp, i, mask;
5714
5715 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5716 WREG32(RLC_GPR_REG2, tmp);
5717
5718 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5719 for (i = 0; i < rdev->usec_timeout; i++) {
5720 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5721 break;
5722 udelay(1);
5723 }
5724
5725 for (i = 0; i < rdev->usec_timeout; i++) {
5726 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5727 break;
5728 udelay(1);
5729 }
5730 }
5731
5732 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5733 {
5734 u32 tmp;
5735
5736 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5737 WREG32(RLC_GPR_REG2, tmp);
5738 }
5739
5740 /**
5741 * cik_rlc_stop - stop the RLC ME
5742 *
5743 * @rdev: radeon_device pointer
5744 *
5745 * Halt the RLC ME (MicroEngine) (CIK).
5746 */
5747 static void cik_rlc_stop(struct radeon_device *rdev)
5748 {
5749 WREG32(RLC_CNTL, 0);
5750
5751 cik_enable_gui_idle_interrupt(rdev, false);
5752
5753 cik_wait_for_rlc_serdes(rdev);
5754 }
5755
5756 /**
5757 * cik_rlc_start - start the RLC ME
5758 *
5759 * @rdev: radeon_device pointer
5760 *
5761 * Unhalt the RLC ME (MicroEngine) (CIK).
5762 */
5763 static void cik_rlc_start(struct radeon_device *rdev)
5764 {
5765 WREG32(RLC_CNTL, RLC_ENABLE);
5766
5767 cik_enable_gui_idle_interrupt(rdev, true);
5768
5769 udelay(50);
5770 }
5771
5772 /**
5773 * cik_rlc_resume - setup the RLC hw
5774 *
5775 * @rdev: radeon_device pointer
5776 *
5777 * Initialize the RLC registers, load the ucode,
5778 * and start the RLC (CIK).
5779 * Returns 0 for success, -EINVAL if the ucode is not available.
5780 */
5781 static int cik_rlc_resume(struct radeon_device *rdev)
5782 {
5783 u32 i, size, tmp;
5784 const __be32 *fw_data;
5785
5786 if (!rdev->rlc_fw)
5787 return -EINVAL;
5788
5789 switch (rdev->family) {
5790 case CHIP_BONAIRE:
5791 case CHIP_HAWAII:
5792 default:
5793 size = BONAIRE_RLC_UCODE_SIZE;
5794 break;
5795 case CHIP_KAVERI:
5796 size = KV_RLC_UCODE_SIZE;
5797 break;
5798 case CHIP_KABINI:
5799 size = KB_RLC_UCODE_SIZE;
5800 break;
5801 }
5802
5803 cik_rlc_stop(rdev);
5804
5805 /* disable CG */
5806 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5807 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5808
5809 si_rlc_reset(rdev);
5810
5811 cik_init_pg(rdev);
5812
5813 cik_init_cg(rdev);
5814
5815 WREG32(RLC_LB_CNTR_INIT, 0);
5816 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5817
5818 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5819 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5820 WREG32(RLC_LB_PARAMS, 0x00600408);
5821 WREG32(RLC_LB_CNTL, 0x80000004);
5822
5823 WREG32(RLC_MC_CNTL, 0);
5824 WREG32(RLC_UCODE_CNTL, 0);
5825
5826 fw_data = (const __be32 *)rdev->rlc_fw->data;
5827 WREG32(RLC_GPM_UCODE_ADDR, 0);
5828 for (i = 0; i < size; i++)
5829 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5830 WREG32(RLC_GPM_UCODE_ADDR, 0);
5831
5832 /* XXX - find out what chips support lbpw */
5833 cik_enable_lbpw(rdev, false);
5834
5835 if (rdev->family == CHIP_BONAIRE)
5836 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5837
5838 cik_rlc_start(rdev);
5839
5840 return 0;
5841 }
5842
5843 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5844 {
5845 u32 data, orig, tmp, tmp2;
5846
5847 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5848
5849 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5850 cik_enable_gui_idle_interrupt(rdev, true);
5851
5852 tmp = cik_halt_rlc(rdev);
5853
5854 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5855 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5856 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5857 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5858 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5859
5860 cik_update_rlc(rdev, tmp);
5861
5862 data |= CGCG_EN | CGLS_EN;
5863 } else {
5864 cik_enable_gui_idle_interrupt(rdev, false);
5865
5866 RREG32(CB_CGTT_SCLK_CTRL);
5867 RREG32(CB_CGTT_SCLK_CTRL);
5868 RREG32(CB_CGTT_SCLK_CTRL);
5869 RREG32(CB_CGTT_SCLK_CTRL);
5870
5871 data &= ~(CGCG_EN | CGLS_EN);
5872 }
5873
5874 if (orig != data)
5875 WREG32(RLC_CGCG_CGLS_CTRL, data);
5876
5877 }
5878
5879 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5880 {
5881 u32 data, orig, tmp = 0;
5882
5883 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5884 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5885 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5886 orig = data = RREG32(CP_MEM_SLP_CNTL);
5887 data |= CP_MEM_LS_EN;
5888 if (orig != data)
5889 WREG32(CP_MEM_SLP_CNTL, data);
5890 }
5891 }
5892
5893 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5894 data &= 0xfffffffd;
5895 if (orig != data)
5896 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5897
5898 tmp = cik_halt_rlc(rdev);
5899
5900 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5901 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5902 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5903 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5904 WREG32(RLC_SERDES_WR_CTRL, data);
5905
5906 cik_update_rlc(rdev, tmp);
5907
5908 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5909 orig = data = RREG32(CGTS_SM_CTRL_REG);
5910 data &= ~SM_MODE_MASK;
5911 data |= SM_MODE(0x2);
5912 data |= SM_MODE_ENABLE;
5913 data &= ~CGTS_OVERRIDE;
5914 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5915 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5916 data &= ~CGTS_LS_OVERRIDE;
5917 data &= ~ON_MONITOR_ADD_MASK;
5918 data |= ON_MONITOR_ADD_EN;
5919 data |= ON_MONITOR_ADD(0x96);
5920 if (orig != data)
5921 WREG32(CGTS_SM_CTRL_REG, data);
5922 }
5923 } else {
5924 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5925 data |= 0x00000002;
5926 if (orig != data)
5927 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5928
5929 data = RREG32(RLC_MEM_SLP_CNTL);
5930 if (data & RLC_MEM_LS_EN) {
5931 data &= ~RLC_MEM_LS_EN;
5932 WREG32(RLC_MEM_SLP_CNTL, data);
5933 }
5934
5935 data = RREG32(CP_MEM_SLP_CNTL);
5936 if (data & CP_MEM_LS_EN) {
5937 data &= ~CP_MEM_LS_EN;
5938 WREG32(CP_MEM_SLP_CNTL, data);
5939 }
5940
5941 orig = data = RREG32(CGTS_SM_CTRL_REG);
5942 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5943 if (orig != data)
5944 WREG32(CGTS_SM_CTRL_REG, data);
5945
5946 tmp = cik_halt_rlc(rdev);
5947
5948 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5949 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5950 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5951 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5952 WREG32(RLC_SERDES_WR_CTRL, data);
5953
5954 cik_update_rlc(rdev, tmp);
5955 }
5956 }
5957
5958 static const u32 mc_cg_registers[] =
5959 {
5960 MC_HUB_MISC_HUB_CG,
5961 MC_HUB_MISC_SIP_CG,
5962 MC_HUB_MISC_VM_CG,
5963 MC_XPB_CLK_GAT,
5964 ATC_MISC_CG,
5965 MC_CITF_MISC_WR_CG,
5966 MC_CITF_MISC_RD_CG,
5967 MC_CITF_MISC_VM_CG,
5968 VM_L2_CG,
5969 };
5970
5971 static void cik_enable_mc_ls(struct radeon_device *rdev,
5972 bool enable)
5973 {
5974 int i;
5975 u32 orig, data;
5976
5977 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5978 orig = data = RREG32(mc_cg_registers[i]);
5979 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5980 data |= MC_LS_ENABLE;
5981 else
5982 data &= ~MC_LS_ENABLE;
5983 if (data != orig)
5984 WREG32(mc_cg_registers[i], data);
5985 }
5986 }
5987
5988 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5989 bool enable)
5990 {
5991 int i;
5992 u32 orig, data;
5993
5994 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5995 orig = data = RREG32(mc_cg_registers[i]);
5996 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5997 data |= MC_CG_ENABLE;
5998 else
5999 data &= ~MC_CG_ENABLE;
6000 if (data != orig)
6001 WREG32(mc_cg_registers[i], data);
6002 }
6003 }
6004
6005 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6006 bool enable)
6007 {
6008 u32 orig, data;
6009
6010 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6011 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6012 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6013 } else {
6014 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6015 data |= 0xff000000;
6016 if (data != orig)
6017 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6018
6019 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6020 data |= 0xff000000;
6021 if (data != orig)
6022 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6023 }
6024 }
6025
6026 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6027 bool enable)
6028 {
6029 u32 orig, data;
6030
6031 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6032 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6033 data |= 0x100;
6034 if (orig != data)
6035 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6036
6037 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6038 data |= 0x100;
6039 if (orig != data)
6040 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6041 } else {
6042 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6043 data &= ~0x100;
6044 if (orig != data)
6045 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6046
6047 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6048 data &= ~0x100;
6049 if (orig != data)
6050 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6051 }
6052 }
6053
6054 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6055 bool enable)
6056 {
6057 u32 orig, data;
6058
6059 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6060 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6061 data = 0xfff;
6062 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6063
6064 orig = data = RREG32(UVD_CGC_CTRL);
6065 data |= DCM;
6066 if (orig != data)
6067 WREG32(UVD_CGC_CTRL, data);
6068 } else {
6069 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6070 data &= ~0xfff;
6071 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6072
6073 orig = data = RREG32(UVD_CGC_CTRL);
6074 data &= ~DCM;
6075 if (orig != data)
6076 WREG32(UVD_CGC_CTRL, data);
6077 }
6078 }
6079
6080 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6081 bool enable)
6082 {
6083 u32 orig, data;
6084
6085 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6086
6087 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6088 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6089 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6090 else
6091 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6092 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6093
6094 if (orig != data)
6095 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6096 }
6097
6098 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6099 bool enable)
6100 {
6101 u32 orig, data;
6102
6103 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6104
6105 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6106 data &= ~CLOCK_GATING_DIS;
6107 else
6108 data |= CLOCK_GATING_DIS;
6109
6110 if (orig != data)
6111 WREG32(HDP_HOST_PATH_CNTL, data);
6112 }
6113
6114 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6115 bool enable)
6116 {
6117 u32 orig, data;
6118
6119 orig = data = RREG32(HDP_MEM_POWER_LS);
6120
6121 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6122 data |= HDP_LS_ENABLE;
6123 else
6124 data &= ~HDP_LS_ENABLE;
6125
6126 if (orig != data)
6127 WREG32(HDP_MEM_POWER_LS, data);
6128 }
6129
6130 void cik_update_cg(struct radeon_device *rdev,
6131 u32 block, bool enable)
6132 {
6133
6134 if (block & RADEON_CG_BLOCK_GFX) {
6135 cik_enable_gui_idle_interrupt(rdev, false);
6136 /* order matters! */
6137 if (enable) {
6138 cik_enable_mgcg(rdev, true);
6139 cik_enable_cgcg(rdev, true);
6140 } else {
6141 cik_enable_cgcg(rdev, false);
6142 cik_enable_mgcg(rdev, false);
6143 }
6144 cik_enable_gui_idle_interrupt(rdev, true);
6145 }
6146
6147 if (block & RADEON_CG_BLOCK_MC) {
6148 if (!(rdev->flags & RADEON_IS_IGP)) {
6149 cik_enable_mc_mgcg(rdev, enable);
6150 cik_enable_mc_ls(rdev, enable);
6151 }
6152 }
6153
6154 if (block & RADEON_CG_BLOCK_SDMA) {
6155 cik_enable_sdma_mgcg(rdev, enable);
6156 cik_enable_sdma_mgls(rdev, enable);
6157 }
6158
6159 if (block & RADEON_CG_BLOCK_BIF) {
6160 cik_enable_bif_mgls(rdev, enable);
6161 }
6162
6163 if (block & RADEON_CG_BLOCK_UVD) {
6164 if (rdev->has_uvd)
6165 cik_enable_uvd_mgcg(rdev, enable);
6166 }
6167
6168 if (block & RADEON_CG_BLOCK_HDP) {
6169 cik_enable_hdp_mgcg(rdev, enable);
6170 cik_enable_hdp_ls(rdev, enable);
6171 }
6172
6173 if (block & RADEON_CG_BLOCK_VCE) {
6174 vce_v2_0_enable_mgcg(rdev, enable);
6175 }
6176 }
6177
6178 static void cik_init_cg(struct radeon_device *rdev)
6179 {
6180
6181 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6182
6183 if (rdev->has_uvd)
6184 si_init_uvd_internal_cg(rdev);
6185
6186 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6187 RADEON_CG_BLOCK_SDMA |
6188 RADEON_CG_BLOCK_BIF |
6189 RADEON_CG_BLOCK_UVD |
6190 RADEON_CG_BLOCK_HDP), true);
6191 }
6192
6193 static void cik_fini_cg(struct radeon_device *rdev)
6194 {
6195 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6196 RADEON_CG_BLOCK_SDMA |
6197 RADEON_CG_BLOCK_BIF |
6198 RADEON_CG_BLOCK_UVD |
6199 RADEON_CG_BLOCK_HDP), false);
6200
6201 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6202 }
6203
6204 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6205 bool enable)
6206 {
6207 u32 data, orig;
6208
6209 orig = data = RREG32(RLC_PG_CNTL);
6210 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6211 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6212 else
6213 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6214 if (orig != data)
6215 WREG32(RLC_PG_CNTL, data);
6216 }
6217
6218 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6219 bool enable)
6220 {
6221 u32 data, orig;
6222
6223 orig = data = RREG32(RLC_PG_CNTL);
6224 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6225 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6226 else
6227 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6228 if (orig != data)
6229 WREG32(RLC_PG_CNTL, data);
6230 }
6231
6232 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6233 {
6234 u32 data, orig;
6235
6236 orig = data = RREG32(RLC_PG_CNTL);
6237 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6238 data &= ~DISABLE_CP_PG;
6239 else
6240 data |= DISABLE_CP_PG;
6241 if (orig != data)
6242 WREG32(RLC_PG_CNTL, data);
6243 }
6244
6245 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6246 {
6247 u32 data, orig;
6248
6249 orig = data = RREG32(RLC_PG_CNTL);
6250 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6251 data &= ~DISABLE_GDS_PG;
6252 else
6253 data |= DISABLE_GDS_PG;
6254 if (orig != data)
6255 WREG32(RLC_PG_CNTL, data);
6256 }
6257
6258 #define CP_ME_TABLE_SIZE 96
6259 #define CP_ME_TABLE_OFFSET 2048
6260 #define CP_MEC_TABLE_OFFSET 4096
6261
6262 void cik_init_cp_pg_table(struct radeon_device *rdev)
6263 {
6264 const __be32 *fw_data;
6265 volatile u32 *dst_ptr;
6266 int me, i, max_me = 4;
6267 u32 bo_offset = 0;
6268 u32 table_offset;
6269
6270 if (rdev->family == CHIP_KAVERI)
6271 max_me = 5;
6272
6273 if (rdev->rlc.cp_table_ptr == NULL)
6274 return;
6275
6276 /* write the cp table buffer */
6277 dst_ptr = rdev->rlc.cp_table_ptr;
6278 for (me = 0; me < max_me; me++) {
6279 if (me == 0) {
6280 fw_data = (const __be32 *)rdev->ce_fw->data;
6281 table_offset = CP_ME_TABLE_OFFSET;
6282 } else if (me == 1) {
6283 fw_data = (const __be32 *)rdev->pfp_fw->data;
6284 table_offset = CP_ME_TABLE_OFFSET;
6285 } else if (me == 2) {
6286 fw_data = (const __be32 *)rdev->me_fw->data;
6287 table_offset = CP_ME_TABLE_OFFSET;
6288 } else {
6289 fw_data = (const __be32 *)rdev->mec_fw->data;
6290 table_offset = CP_MEC_TABLE_OFFSET;
6291 }
6292
6293 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6294 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6295 }
6296 bo_offset += CP_ME_TABLE_SIZE;
6297 }
6298 }
6299
6300 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6301 bool enable)
6302 {
6303 u32 data, orig;
6304
6305 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6306 orig = data = RREG32(RLC_PG_CNTL);
6307 data |= GFX_PG_ENABLE;
6308 if (orig != data)
6309 WREG32(RLC_PG_CNTL, data);
6310
6311 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6312 data |= AUTO_PG_EN;
6313 if (orig != data)
6314 WREG32(RLC_AUTO_PG_CTRL, data);
6315 } else {
6316 orig = data = RREG32(RLC_PG_CNTL);
6317 data &= ~GFX_PG_ENABLE;
6318 if (orig != data)
6319 WREG32(RLC_PG_CNTL, data);
6320
6321 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6322 data &= ~AUTO_PG_EN;
6323 if (orig != data)
6324 WREG32(RLC_AUTO_PG_CTRL, data);
6325
6326 data = RREG32(DB_RENDER_CONTROL);
6327 }
6328 }
6329
6330 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6331 {
6332 u32 mask = 0, tmp, tmp1;
6333 int i;
6334
6335 cik_select_se_sh(rdev, se, sh);
6336 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6337 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6338 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6339
6340 tmp &= 0xffff0000;
6341
6342 tmp |= tmp1;
6343 tmp >>= 16;
6344
6345 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6346 mask <<= 1;
6347 mask |= 1;
6348 }
6349
6350 return (~tmp) & mask;
6351 }
6352
6353 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6354 {
6355 u32 i, j, k, active_cu_number = 0;
6356 u32 mask, counter, cu_bitmap;
6357 u32 tmp = 0;
6358
6359 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6360 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6361 mask = 1;
6362 cu_bitmap = 0;
6363 counter = 0;
6364 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6365 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6366 if (counter < 2)
6367 cu_bitmap |= mask;
6368 counter ++;
6369 }
6370 mask <<= 1;
6371 }
6372
6373 active_cu_number += counter;
6374 tmp |= (cu_bitmap << (i * 16 + j * 8));
6375 }
6376 }
6377
6378 WREG32(RLC_PG_AO_CU_MASK, tmp);
6379
6380 tmp = RREG32(RLC_MAX_PG_CU);
6381 tmp &= ~MAX_PU_CU_MASK;
6382 tmp |= MAX_PU_CU(active_cu_number);
6383 WREG32(RLC_MAX_PG_CU, tmp);
6384 }
6385
6386 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6387 bool enable)
6388 {
6389 u32 data, orig;
6390
6391 orig = data = RREG32(RLC_PG_CNTL);
6392 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6393 data |= STATIC_PER_CU_PG_ENABLE;
6394 else
6395 data &= ~STATIC_PER_CU_PG_ENABLE;
6396 if (orig != data)
6397 WREG32(RLC_PG_CNTL, data);
6398 }
6399
6400 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6401 bool enable)
6402 {
6403 u32 data, orig;
6404
6405 orig = data = RREG32(RLC_PG_CNTL);
6406 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6407 data |= DYN_PER_CU_PG_ENABLE;
6408 else
6409 data &= ~DYN_PER_CU_PG_ENABLE;
6410 if (orig != data)
6411 WREG32(RLC_PG_CNTL, data);
6412 }
6413
6414 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6415 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6416
6417 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6418 {
6419 u32 data, orig;
6420 u32 i;
6421
6422 if (rdev->rlc.cs_data) {
6423 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6424 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6425 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6426 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6427 } else {
6428 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6429 for (i = 0; i < 3; i++)
6430 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6431 }
6432 if (rdev->rlc.reg_list) {
6433 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6434 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6435 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6436 }
6437
6438 orig = data = RREG32(RLC_PG_CNTL);
6439 data |= GFX_PG_SRC;
6440 if (orig != data)
6441 WREG32(RLC_PG_CNTL, data);
6442
6443 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6444 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6445
6446 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6447 data &= ~IDLE_POLL_COUNT_MASK;
6448 data |= IDLE_POLL_COUNT(0x60);
6449 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6450
6451 data = 0x10101010;
6452 WREG32(RLC_PG_DELAY, data);
6453
6454 data = RREG32(RLC_PG_DELAY_2);
6455 data &= ~0xff;
6456 data |= 0x3;
6457 WREG32(RLC_PG_DELAY_2, data);
6458
6459 data = RREG32(RLC_AUTO_PG_CTRL);
6460 data &= ~GRBM_REG_SGIT_MASK;
6461 data |= GRBM_REG_SGIT(0x700);
6462 WREG32(RLC_AUTO_PG_CTRL, data);
6463
6464 }
6465
6466 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6467 {
6468 cik_enable_gfx_cgpg(rdev, enable);
6469 cik_enable_gfx_static_mgpg(rdev, enable);
6470 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6471 }
6472
6473 u32 cik_get_csb_size(struct radeon_device *rdev)
6474 {
6475 u32 count = 0;
6476 const struct cs_section_def *sect = NULL;
6477 const struct cs_extent_def *ext = NULL;
6478
6479 if (rdev->rlc.cs_data == NULL)
6480 return 0;
6481
6482 /* begin clear state */
6483 count += 2;
6484 /* context control state */
6485 count += 3;
6486
6487 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6488 for (ext = sect->section; ext->extent != NULL; ++ext) {
6489 if (sect->id == SECT_CONTEXT)
6490 count += 2 + ext->reg_count;
6491 else
6492 return 0;
6493 }
6494 }
6495 /* pa_sc_raster_config/pa_sc_raster_config1 */
6496 count += 4;
6497 /* end clear state */
6498 count += 2;
6499 /* clear state */
6500 count += 2;
6501
6502 return count;
6503 }
6504
6505 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6506 {
6507 u32 count = 0, i;
6508 const struct cs_section_def *sect = NULL;
6509 const struct cs_extent_def *ext = NULL;
6510
6511 if (rdev->rlc.cs_data == NULL)
6512 return;
6513 if (buffer == NULL)
6514 return;
6515
6516 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6517 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6518
6519 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6520 buffer[count++] = cpu_to_le32(0x80000000);
6521 buffer[count++] = cpu_to_le32(0x80000000);
6522
6523 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6524 for (ext = sect->section; ext->extent != NULL; ++ext) {
6525 if (sect->id == SECT_CONTEXT) {
6526 buffer[count++] =
6527 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6528 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6529 for (i = 0; i < ext->reg_count; i++)
6530 buffer[count++] = cpu_to_le32(ext->extent[i]);
6531 } else {
6532 return;
6533 }
6534 }
6535 }
6536
6537 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6538 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6539 switch (rdev->family) {
6540 case CHIP_BONAIRE:
6541 buffer[count++] = cpu_to_le32(0x16000012);
6542 buffer[count++] = cpu_to_le32(0x00000000);
6543 break;
6544 case CHIP_KAVERI:
6545 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6546 buffer[count++] = cpu_to_le32(0x00000000);
6547 break;
6548 case CHIP_KABINI:
6549 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6550 buffer[count++] = cpu_to_le32(0x00000000);
6551 break;
6552 case CHIP_HAWAII:
6553 buffer[count++] = cpu_to_le32(0x3a00161a);
6554 buffer[count++] = cpu_to_le32(0x0000002e);
6555 break;
6556 default:
6557 buffer[count++] = cpu_to_le32(0x00000000);
6558 buffer[count++] = cpu_to_le32(0x00000000);
6559 break;
6560 }
6561
6562 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6563 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6564
6565 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6566 buffer[count++] = cpu_to_le32(0);
6567 }
6568
6569 static void cik_init_pg(struct radeon_device *rdev)
6570 {
6571 if (rdev->pg_flags) {
6572 cik_enable_sck_slowdown_on_pu(rdev, true);
6573 cik_enable_sck_slowdown_on_pd(rdev, true);
6574 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6575 cik_init_gfx_cgpg(rdev);
6576 cik_enable_cp_pg(rdev, true);
6577 cik_enable_gds_pg(rdev, true);
6578 }
6579 cik_init_ao_cu_mask(rdev);
6580 cik_update_gfx_pg(rdev, true);
6581 }
6582 }
6583
6584 static void cik_fini_pg(struct radeon_device *rdev)
6585 {
6586 if (rdev->pg_flags) {
6587 cik_update_gfx_pg(rdev, false);
6588 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6589 cik_enable_cp_pg(rdev, false);
6590 cik_enable_gds_pg(rdev, false);
6591 }
6592 }
6593 }
6594
6595 /*
6596 * Interrupts
6597 * Starting with r6xx, interrupts are handled via a ring buffer.
6598 * Ring buffers are areas of GPU accessible memory that the GPU
6599 * writes interrupt vectors into and the host reads vectors out of.
6600 * There is a rptr (read pointer) that determines where the
6601 * host is currently reading, and a wptr (write pointer)
6602 * which determines where the GPU has written. When the
6603 * pointers are equal, the ring is idle. When the GPU
6604 * writes vectors to the ring buffer, it increments the
6605 * wptr. When there is an interrupt, the host then starts
6606 * fetching commands and processing them until the pointers are
6607 * equal again at which point it updates the rptr.
6608 */
6609
6610 /**
6611 * cik_enable_interrupts - Enable the interrupt ring buffer
6612 *
6613 * @rdev: radeon_device pointer
6614 *
6615 * Enable the interrupt ring buffer (CIK).
6616 */
6617 static void cik_enable_interrupts(struct radeon_device *rdev)
6618 {
6619 u32 ih_cntl = RREG32(IH_CNTL);
6620 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6621
6622 ih_cntl |= ENABLE_INTR;
6623 ih_rb_cntl |= IH_RB_ENABLE;
6624 WREG32(IH_CNTL, ih_cntl);
6625 WREG32(IH_RB_CNTL, ih_rb_cntl);
6626 rdev->ih.enabled = true;
6627 }
6628
6629 /**
6630 * cik_disable_interrupts - Disable the interrupt ring buffer
6631 *
6632 * @rdev: radeon_device pointer
6633 *
6634 * Disable the interrupt ring buffer (CIK).
6635 */
6636 static void cik_disable_interrupts(struct radeon_device *rdev)
6637 {
6638 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6639 u32 ih_cntl = RREG32(IH_CNTL);
6640
6641 ih_rb_cntl &= ~IH_RB_ENABLE;
6642 ih_cntl &= ~ENABLE_INTR;
6643 WREG32(IH_RB_CNTL, ih_rb_cntl);
6644 WREG32(IH_CNTL, ih_cntl);
6645 /* set rptr, wptr to 0 */
6646 WREG32(IH_RB_RPTR, 0);
6647 WREG32(IH_RB_WPTR, 0);
6648 rdev->ih.enabled = false;
6649 rdev->ih.rptr = 0;
6650 }
6651
6652 /**
6653 * cik_disable_interrupt_state - Disable all interrupt sources
6654 *
6655 * @rdev: radeon_device pointer
6656 *
6657 * Clear all interrupt enable bits used by the driver (CIK).
6658 */
6659 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6660 {
6661 u32 tmp;
6662
6663 /* gfx ring */
6664 tmp = RREG32(CP_INT_CNTL_RING0) &
6665 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6666 WREG32(CP_INT_CNTL_RING0, tmp);
6667 /* sdma */
6668 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6669 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6670 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6671 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6672 /* compute queues */
6673 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6674 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6675 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6676 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6677 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6678 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6679 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6680 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6681 /* grbm */
6682 WREG32(GRBM_INT_CNTL, 0);
6683 /* vline/vblank, etc. */
6684 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6685 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6686 if (rdev->num_crtc >= 4) {
6687 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6688 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6689 }
6690 if (rdev->num_crtc >= 6) {
6691 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6692 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6693 }
6694
6695 /* dac hotplug */
6696 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6697
6698 /* digital hotplug */
6699 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6700 WREG32(DC_HPD1_INT_CONTROL, tmp);
6701 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6702 WREG32(DC_HPD2_INT_CONTROL, tmp);
6703 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6704 WREG32(DC_HPD3_INT_CONTROL, tmp);
6705 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6706 WREG32(DC_HPD4_INT_CONTROL, tmp);
6707 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6708 WREG32(DC_HPD5_INT_CONTROL, tmp);
6709 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6710 WREG32(DC_HPD6_INT_CONTROL, tmp);
6711
6712 }
6713
6714 /**
6715 * cik_irq_init - init and enable the interrupt ring
6716 *
6717 * @rdev: radeon_device pointer
6718 *
6719 * Allocate a ring buffer for the interrupt controller,
6720 * enable the RLC, disable interrupts, enable the IH
6721 * ring buffer and enable it (CIK).
6722 * Called at device load and reume.
6723 * Returns 0 for success, errors for failure.
6724 */
6725 static int cik_irq_init(struct radeon_device *rdev)
6726 {
6727 int ret = 0;
6728 int rb_bufsz;
6729 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6730
6731 /* allocate ring */
6732 ret = r600_ih_ring_alloc(rdev);
6733 if (ret)
6734 return ret;
6735
6736 /* disable irqs */
6737 cik_disable_interrupts(rdev);
6738
6739 /* init rlc */
6740 ret = cik_rlc_resume(rdev);
6741 if (ret) {
6742 r600_ih_ring_fini(rdev);
6743 return ret;
6744 }
6745
6746 /* setup interrupt control */
6747 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6748 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6749 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6750 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6751 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6752 */
6753 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6754 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6755 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6756 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6757
6758 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6759 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6760
6761 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6762 IH_WPTR_OVERFLOW_CLEAR |
6763 (rb_bufsz << 1));
6764
6765 if (rdev->wb.enabled)
6766 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6767
6768 /* set the writeback address whether it's enabled or not */
6769 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6770 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6771
6772 WREG32(IH_RB_CNTL, ih_rb_cntl);
6773
6774 /* set rptr, wptr to 0 */
6775 WREG32(IH_RB_RPTR, 0);
6776 WREG32(IH_RB_WPTR, 0);
6777
6778 /* Default settings for IH_CNTL (disabled at first) */
6779 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6780 /* RPTR_REARM only works if msi's are enabled */
6781 if (rdev->msi_enabled)
6782 ih_cntl |= RPTR_REARM;
6783 WREG32(IH_CNTL, ih_cntl);
6784
6785 /* force the active interrupt state to all disabled */
6786 cik_disable_interrupt_state(rdev);
6787
6788 pci_set_master(rdev->pdev);
6789
6790 /* enable irqs */
6791 cik_enable_interrupts(rdev);
6792
6793 return ret;
6794 }
6795
6796 /**
6797 * cik_irq_set - enable/disable interrupt sources
6798 *
6799 * @rdev: radeon_device pointer
6800 *
6801 * Enable interrupt sources on the GPU (vblanks, hpd,
6802 * etc.) (CIK).
6803 * Returns 0 for success, errors for failure.
6804 */
6805 int cik_irq_set(struct radeon_device *rdev)
6806 {
6807 u32 cp_int_cntl;
6808 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6809 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6810 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6811 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6812 u32 grbm_int_cntl = 0;
6813 u32 dma_cntl, dma_cntl1;
6814 u32 thermal_int;
6815
6816 if (!rdev->irq.installed) {
6817 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6818 return -EINVAL;
6819 }
6820 /* don't enable anything if the ih is disabled */
6821 if (!rdev->ih.enabled) {
6822 cik_disable_interrupts(rdev);
6823 /* force the active interrupt state to all disabled */
6824 cik_disable_interrupt_state(rdev);
6825 return 0;
6826 }
6827
6828 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6829 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6830 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6831
6832 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6833 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6834 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6835 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6836 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6837 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6838
6839 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6840 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6841
6842 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6843 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6844 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6845 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6846 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6847 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6848 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6849 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6850
6851 if (rdev->flags & RADEON_IS_IGP)
6852 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6853 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6854 else
6855 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6856 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6857
6858 /* enable CP interrupts on all rings */
6859 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6860 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6861 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6862 }
6863 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6864 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6865 DRM_DEBUG("si_irq_set: sw int cp1\n");
6866 if (ring->me == 1) {
6867 switch (ring->pipe) {
6868 case 0:
6869 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6870 break;
6871 case 1:
6872 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6873 break;
6874 case 2:
6875 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6876 break;
6877 case 3:
6878 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6879 break;
6880 default:
6881 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6882 break;
6883 }
6884 } else if (ring->me == 2) {
6885 switch (ring->pipe) {
6886 case 0:
6887 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6888 break;
6889 case 1:
6890 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6891 break;
6892 case 2:
6893 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6894 break;
6895 case 3:
6896 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6897 break;
6898 default:
6899 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6900 break;
6901 }
6902 } else {
6903 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6904 }
6905 }
6906 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6907 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6908 DRM_DEBUG("si_irq_set: sw int cp2\n");
6909 if (ring->me == 1) {
6910 switch (ring->pipe) {
6911 case 0:
6912 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6913 break;
6914 case 1:
6915 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6916 break;
6917 case 2:
6918 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6919 break;
6920 case 3:
6921 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6922 break;
6923 default:
6924 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6925 break;
6926 }
6927 } else if (ring->me == 2) {
6928 switch (ring->pipe) {
6929 case 0:
6930 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6931 break;
6932 case 1:
6933 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6934 break;
6935 case 2:
6936 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6937 break;
6938 case 3:
6939 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6940 break;
6941 default:
6942 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6943 break;
6944 }
6945 } else {
6946 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6947 }
6948 }
6949
6950 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6951 DRM_DEBUG("cik_irq_set: sw int dma\n");
6952 dma_cntl |= TRAP_ENABLE;
6953 }
6954
6955 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6956 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6957 dma_cntl1 |= TRAP_ENABLE;
6958 }
6959
6960 if (rdev->irq.crtc_vblank_int[0] ||
6961 atomic_read(&rdev->irq.pflip[0])) {
6962 DRM_DEBUG("cik_irq_set: vblank 0\n");
6963 crtc1 |= VBLANK_INTERRUPT_MASK;
6964 }
6965 if (rdev->irq.crtc_vblank_int[1] ||
6966 atomic_read(&rdev->irq.pflip[1])) {
6967 DRM_DEBUG("cik_irq_set: vblank 1\n");
6968 crtc2 |= VBLANK_INTERRUPT_MASK;
6969 }
6970 if (rdev->irq.crtc_vblank_int[2] ||
6971 atomic_read(&rdev->irq.pflip[2])) {
6972 DRM_DEBUG("cik_irq_set: vblank 2\n");
6973 crtc3 |= VBLANK_INTERRUPT_MASK;
6974 }
6975 if (rdev->irq.crtc_vblank_int[3] ||
6976 atomic_read(&rdev->irq.pflip[3])) {
6977 DRM_DEBUG("cik_irq_set: vblank 3\n");
6978 crtc4 |= VBLANK_INTERRUPT_MASK;
6979 }
6980 if (rdev->irq.crtc_vblank_int[4] ||
6981 atomic_read(&rdev->irq.pflip[4])) {
6982 DRM_DEBUG("cik_irq_set: vblank 4\n");
6983 crtc5 |= VBLANK_INTERRUPT_MASK;
6984 }
6985 if (rdev->irq.crtc_vblank_int[5] ||
6986 atomic_read(&rdev->irq.pflip[5])) {
6987 DRM_DEBUG("cik_irq_set: vblank 5\n");
6988 crtc6 |= VBLANK_INTERRUPT_MASK;
6989 }
6990 if (rdev->irq.hpd[0]) {
6991 DRM_DEBUG("cik_irq_set: hpd 1\n");
6992 hpd1 |= DC_HPDx_INT_EN;
6993 }
6994 if (rdev->irq.hpd[1]) {
6995 DRM_DEBUG("cik_irq_set: hpd 2\n");
6996 hpd2 |= DC_HPDx_INT_EN;
6997 }
6998 if (rdev->irq.hpd[2]) {
6999 DRM_DEBUG("cik_irq_set: hpd 3\n");
7000 hpd3 |= DC_HPDx_INT_EN;
7001 }
7002 if (rdev->irq.hpd[3]) {
7003 DRM_DEBUG("cik_irq_set: hpd 4\n");
7004 hpd4 |= DC_HPDx_INT_EN;
7005 }
7006 if (rdev->irq.hpd[4]) {
7007 DRM_DEBUG("cik_irq_set: hpd 5\n");
7008 hpd5 |= DC_HPDx_INT_EN;
7009 }
7010 if (rdev->irq.hpd[5]) {
7011 DRM_DEBUG("cik_irq_set: hpd 6\n");
7012 hpd6 |= DC_HPDx_INT_EN;
7013 }
7014
7015 if (rdev->irq.dpm_thermal) {
7016 DRM_DEBUG("dpm thermal\n");
7017 if (rdev->flags & RADEON_IS_IGP)
7018 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7019 else
7020 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7021 }
7022
7023 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7024
7025 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7026 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7027
7028 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7029 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7030 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7031 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7032 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7033 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7034 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7035 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7036
7037 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7038
7039 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7040 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7041 if (rdev->num_crtc >= 4) {
7042 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7043 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7044 }
7045 if (rdev->num_crtc >= 6) {
7046 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7047 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7048 }
7049
7050 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7051 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7052 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7053 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7054 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7055 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7056
7057 if (rdev->flags & RADEON_IS_IGP)
7058 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7059 else
7060 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7061
7062 return 0;
7063 }
7064
7065 /**
7066 * cik_irq_ack - ack interrupt sources
7067 *
7068 * @rdev: radeon_device pointer
7069 *
7070 * Ack interrupt sources on the GPU (vblanks, hpd,
7071 * etc.) (CIK). Certain interrupts sources are sw
7072 * generated and do not require an explicit ack.
7073 */
7074 static inline void cik_irq_ack(struct radeon_device *rdev)
7075 {
7076 u32 tmp;
7077
7078 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7079 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7080 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7081 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7082 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7083 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7084 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7085
7086 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7087 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7088 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7089 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7090 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7091 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7092 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7093 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7094
7095 if (rdev->num_crtc >= 4) {
7096 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7097 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7098 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7099 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7100 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7101 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7102 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7103 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7104 }
7105
7106 if (rdev->num_crtc >= 6) {
7107 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7108 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7109 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7110 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7111 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7112 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7113 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7114 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7115 }
7116
7117 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7118 tmp = RREG32(DC_HPD1_INT_CONTROL);
7119 tmp |= DC_HPDx_INT_ACK;
7120 WREG32(DC_HPD1_INT_CONTROL, tmp);
7121 }
7122 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7123 tmp = RREG32(DC_HPD2_INT_CONTROL);
7124 tmp |= DC_HPDx_INT_ACK;
7125 WREG32(DC_HPD2_INT_CONTROL, tmp);
7126 }
7127 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7128 tmp = RREG32(DC_HPD3_INT_CONTROL);
7129 tmp |= DC_HPDx_INT_ACK;
7130 WREG32(DC_HPD3_INT_CONTROL, tmp);
7131 }
7132 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7133 tmp = RREG32(DC_HPD4_INT_CONTROL);
7134 tmp |= DC_HPDx_INT_ACK;
7135 WREG32(DC_HPD4_INT_CONTROL, tmp);
7136 }
7137 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7138 tmp = RREG32(DC_HPD5_INT_CONTROL);
7139 tmp |= DC_HPDx_INT_ACK;
7140 WREG32(DC_HPD5_INT_CONTROL, tmp);
7141 }
7142 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7143 tmp = RREG32(DC_HPD5_INT_CONTROL);
7144 tmp |= DC_HPDx_INT_ACK;
7145 WREG32(DC_HPD6_INT_CONTROL, tmp);
7146 }
7147 }
7148
7149 /**
7150 * cik_irq_disable - disable interrupts
7151 *
7152 * @rdev: radeon_device pointer
7153 *
7154 * Disable interrupts on the hw (CIK).
7155 */
7156 static void cik_irq_disable(struct radeon_device *rdev)
7157 {
7158 cik_disable_interrupts(rdev);
7159 /* Wait and acknowledge irq */
7160 mdelay(1);
7161 cik_irq_ack(rdev);
7162 cik_disable_interrupt_state(rdev);
7163 }
7164
7165 /**
7166 * cik_irq_disable - disable interrupts for suspend
7167 *
7168 * @rdev: radeon_device pointer
7169 *
7170 * Disable interrupts and stop the RLC (CIK).
7171 * Used for suspend.
7172 */
7173 static void cik_irq_suspend(struct radeon_device *rdev)
7174 {
7175 cik_irq_disable(rdev);
7176 cik_rlc_stop(rdev);
7177 }
7178
7179 /**
7180 * cik_irq_fini - tear down interrupt support
7181 *
7182 * @rdev: radeon_device pointer
7183 *
7184 * Disable interrupts on the hw and free the IH ring
7185 * buffer (CIK).
7186 * Used for driver unload.
7187 */
7188 static void cik_irq_fini(struct radeon_device *rdev)
7189 {
7190 cik_irq_suspend(rdev);
7191 r600_ih_ring_fini(rdev);
7192 }
7193
7194 /**
7195 * cik_get_ih_wptr - get the IH ring buffer wptr
7196 *
7197 * @rdev: radeon_device pointer
7198 *
7199 * Get the IH ring buffer wptr from either the register
7200 * or the writeback memory buffer (CIK). Also check for
7201 * ring buffer overflow and deal with it.
7202 * Used by cik_irq_process().
7203 * Returns the value of the wptr.
7204 */
7205 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7206 {
7207 u32 wptr, tmp;
7208
7209 if (rdev->wb.enabled)
7210 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7211 else
7212 wptr = RREG32(IH_RB_WPTR);
7213
7214 if (wptr & RB_OVERFLOW) {
7215 /* When a ring buffer overflow happen start parsing interrupt
7216 * from the last not overwritten vector (wptr + 16). Hopefully
7217 * this should allow us to catchup.
7218 */
7219 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7220 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7221 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7222 tmp = RREG32(IH_RB_CNTL);
7223 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7224 WREG32(IH_RB_CNTL, tmp);
7225 }
7226 return (wptr & rdev->ih.ptr_mask);
7227 }
7228
7229 /* CIK IV Ring
7230 * Each IV ring entry is 128 bits:
7231 * [7:0] - interrupt source id
7232 * [31:8] - reserved
7233 * [59:32] - interrupt source data
7234 * [63:60] - reserved
7235 * [71:64] - RINGID
7236 * CP:
7237 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7238 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7239 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7240 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7241 * PIPE_ID - ME0 0=3D
7242 * - ME1&2 compute dispatcher (4 pipes each)
7243 * SDMA:
7244 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7245 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7246 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7247 * [79:72] - VMID
7248 * [95:80] - PASID
7249 * [127:96] - reserved
7250 */
7251 /**
7252 * cik_irq_process - interrupt handler
7253 *
7254 * @rdev: radeon_device pointer
7255 *
7256 * Interrupt hander (CIK). Walk the IH ring,
7257 * ack interrupts and schedule work to handle
7258 * interrupt events.
7259 * Returns irq process return code.
7260 */
7261 int cik_irq_process(struct radeon_device *rdev)
7262 {
7263 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7264 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7265 u32 wptr;
7266 u32 rptr;
7267 u32 src_id, src_data, ring_id;
7268 u8 me_id, pipe_id, queue_id;
7269 u32 ring_index;
7270 bool queue_hotplug = false;
7271 bool queue_reset = false;
7272 u32 addr, status, mc_client;
7273 bool queue_thermal = false;
7274
7275 if (!rdev->ih.enabled || rdev->shutdown)
7276 return IRQ_NONE;
7277
7278 wptr = cik_get_ih_wptr(rdev);
7279
7280 restart_ih:
7281 /* is somebody else already processing irqs? */
7282 if (atomic_xchg(&rdev->ih.lock, 1))
7283 return IRQ_NONE;
7284
7285 rptr = rdev->ih.rptr;
7286 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7287
7288 /* Order reading of wptr vs. reading of IH ring data */
7289 rmb();
7290
7291 /* display interrupts */
7292 cik_irq_ack(rdev);
7293
7294 while (rptr != wptr) {
7295 /* wptr/rptr are in bytes! */
7296 ring_index = rptr / 4;
7297 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7298 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7299 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7300
7301 switch (src_id) {
7302 case 1: /* D1 vblank/vline */
7303 switch (src_data) {
7304 case 0: /* D1 vblank */
7305 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7306 if (rdev->irq.crtc_vblank_int[0]) {
7307 drm_handle_vblank(rdev->ddev, 0);
7308 rdev->pm.vblank_sync = true;
7309 wake_up(&rdev->irq.vblank_queue);
7310 }
7311 if (atomic_read(&rdev->irq.pflip[0]))
7312 radeon_crtc_handle_flip(rdev, 0);
7313 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7314 DRM_DEBUG("IH: D1 vblank\n");
7315 }
7316 break;
7317 case 1: /* D1 vline */
7318 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7319 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7320 DRM_DEBUG("IH: D1 vline\n");
7321 }
7322 break;
7323 default:
7324 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7325 break;
7326 }
7327 break;
7328 case 2: /* D2 vblank/vline */
7329 switch (src_data) {
7330 case 0: /* D2 vblank */
7331 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7332 if (rdev->irq.crtc_vblank_int[1]) {
7333 drm_handle_vblank(rdev->ddev, 1);
7334 rdev->pm.vblank_sync = true;
7335 wake_up(&rdev->irq.vblank_queue);
7336 }
7337 if (atomic_read(&rdev->irq.pflip[1]))
7338 radeon_crtc_handle_flip(rdev, 1);
7339 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7340 DRM_DEBUG("IH: D2 vblank\n");
7341 }
7342 break;
7343 case 1: /* D2 vline */
7344 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7345 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7346 DRM_DEBUG("IH: D2 vline\n");
7347 }
7348 break;
7349 default:
7350 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7351 break;
7352 }
7353 break;
7354 case 3: /* D3 vblank/vline */
7355 switch (src_data) {
7356 case 0: /* D3 vblank */
7357 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7358 if (rdev->irq.crtc_vblank_int[2]) {
7359 drm_handle_vblank(rdev->ddev, 2);
7360 rdev->pm.vblank_sync = true;
7361 wake_up(&rdev->irq.vblank_queue);
7362 }
7363 if (atomic_read(&rdev->irq.pflip[2]))
7364 radeon_crtc_handle_flip(rdev, 2);
7365 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7366 DRM_DEBUG("IH: D3 vblank\n");
7367 }
7368 break;
7369 case 1: /* D3 vline */
7370 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7371 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7372 DRM_DEBUG("IH: D3 vline\n");
7373 }
7374 break;
7375 default:
7376 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7377 break;
7378 }
7379 break;
7380 case 4: /* D4 vblank/vline */
7381 switch (src_data) {
7382 case 0: /* D4 vblank */
7383 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7384 if (rdev->irq.crtc_vblank_int[3]) {
7385 drm_handle_vblank(rdev->ddev, 3);
7386 rdev->pm.vblank_sync = true;
7387 wake_up(&rdev->irq.vblank_queue);
7388 }
7389 if (atomic_read(&rdev->irq.pflip[3]))
7390 radeon_crtc_handle_flip(rdev, 3);
7391 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7392 DRM_DEBUG("IH: D4 vblank\n");
7393 }
7394 break;
7395 case 1: /* D4 vline */
7396 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7397 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7398 DRM_DEBUG("IH: D4 vline\n");
7399 }
7400 break;
7401 default:
7402 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7403 break;
7404 }
7405 break;
7406 case 5: /* D5 vblank/vline */
7407 switch (src_data) {
7408 case 0: /* D5 vblank */
7409 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7410 if (rdev->irq.crtc_vblank_int[4]) {
7411 drm_handle_vblank(rdev->ddev, 4);
7412 rdev->pm.vblank_sync = true;
7413 wake_up(&rdev->irq.vblank_queue);
7414 }
7415 if (atomic_read(&rdev->irq.pflip[4]))
7416 radeon_crtc_handle_flip(rdev, 4);
7417 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7418 DRM_DEBUG("IH: D5 vblank\n");
7419 }
7420 break;
7421 case 1: /* D5 vline */
7422 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7423 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7424 DRM_DEBUG("IH: D5 vline\n");
7425 }
7426 break;
7427 default:
7428 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7429 break;
7430 }
7431 break;
7432 case 6: /* D6 vblank/vline */
7433 switch (src_data) {
7434 case 0: /* D6 vblank */
7435 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7436 if (rdev->irq.crtc_vblank_int[5]) {
7437 drm_handle_vblank(rdev->ddev, 5);
7438 rdev->pm.vblank_sync = true;
7439 wake_up(&rdev->irq.vblank_queue);
7440 }
7441 if (atomic_read(&rdev->irq.pflip[5]))
7442 radeon_crtc_handle_flip(rdev, 5);
7443 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7444 DRM_DEBUG("IH: D6 vblank\n");
7445 }
7446 break;
7447 case 1: /* D6 vline */
7448 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7449 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7450 DRM_DEBUG("IH: D6 vline\n");
7451 }
7452 break;
7453 default:
7454 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7455 break;
7456 }
7457 break;
7458 case 42: /* HPD hotplug */
7459 switch (src_data) {
7460 case 0:
7461 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7462 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7463 queue_hotplug = true;
7464 DRM_DEBUG("IH: HPD1\n");
7465 }
7466 break;
7467 case 1:
7468 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7469 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7470 queue_hotplug = true;
7471 DRM_DEBUG("IH: HPD2\n");
7472 }
7473 break;
7474 case 2:
7475 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7476 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7477 queue_hotplug = true;
7478 DRM_DEBUG("IH: HPD3\n");
7479 }
7480 break;
7481 case 3:
7482 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7483 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7484 queue_hotplug = true;
7485 DRM_DEBUG("IH: HPD4\n");
7486 }
7487 break;
7488 case 4:
7489 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7490 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7491 queue_hotplug = true;
7492 DRM_DEBUG("IH: HPD5\n");
7493 }
7494 break;
7495 case 5:
7496 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7497 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7498 queue_hotplug = true;
7499 DRM_DEBUG("IH: HPD6\n");
7500 }
7501 break;
7502 default:
7503 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7504 break;
7505 }
7506 break;
7507 case 124: /* UVD */
7508 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7509 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7510 break;
7511 case 146:
7512 case 147:
7513 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7514 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7515 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7516 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7517 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7518 addr);
7519 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7520 status);
7521 cik_vm_decode_fault(rdev, status, addr, mc_client);
7522 /* reset addr and status */
7523 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7524 break;
7525 case 167: /* VCE */
7526 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7527 switch (src_data) {
7528 case 0:
7529 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7530 break;
7531 case 1:
7532 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7533 break;
7534 default:
7535 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7536 break;
7537 }
7538 break;
7539 case 176: /* GFX RB CP_INT */
7540 case 177: /* GFX IB CP_INT */
7541 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7542 break;
7543 case 181: /* CP EOP event */
7544 DRM_DEBUG("IH: CP EOP\n");
7545 /* XXX check the bitfield order! */
7546 me_id = (ring_id & 0x60) >> 5;
7547 pipe_id = (ring_id & 0x18) >> 3;
7548 queue_id = (ring_id & 0x7) >> 0;
7549 switch (me_id) {
7550 case 0:
7551 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7552 break;
7553 case 1:
7554 case 2:
7555 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7556 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7557 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7558 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7559 break;
7560 }
7561 break;
7562 case 184: /* CP Privileged reg access */
7563 DRM_ERROR("Illegal register access in command stream\n");
7564 /* XXX check the bitfield order! */
7565 me_id = (ring_id & 0x60) >> 5;
7566 pipe_id = (ring_id & 0x18) >> 3;
7567 queue_id = (ring_id & 0x7) >> 0;
7568 switch (me_id) {
7569 case 0:
7570 /* This results in a full GPU reset, but all we need to do is soft
7571 * reset the CP for gfx
7572 */
7573 queue_reset = true;
7574 break;
7575 case 1:
7576 /* XXX compute */
7577 queue_reset = true;
7578 break;
7579 case 2:
7580 /* XXX compute */
7581 queue_reset = true;
7582 break;
7583 }
7584 break;
7585 case 185: /* CP Privileged inst */
7586 DRM_ERROR("Illegal instruction in command stream\n");
7587 /* XXX check the bitfield order! */
7588 me_id = (ring_id & 0x60) >> 5;
7589 pipe_id = (ring_id & 0x18) >> 3;
7590 queue_id = (ring_id & 0x7) >> 0;
7591 switch (me_id) {
7592 case 0:
7593 /* This results in a full GPU reset, but all we need to do is soft
7594 * reset the CP for gfx
7595 */
7596 queue_reset = true;
7597 break;
7598 case 1:
7599 /* XXX compute */
7600 queue_reset = true;
7601 break;
7602 case 2:
7603 /* XXX compute */
7604 queue_reset = true;
7605 break;
7606 }
7607 break;
7608 case 224: /* SDMA trap event */
7609 /* XXX check the bitfield order! */
7610 me_id = (ring_id & 0x3) >> 0;
7611 queue_id = (ring_id & 0xc) >> 2;
7612 DRM_DEBUG("IH: SDMA trap\n");
7613 switch (me_id) {
7614 case 0:
7615 switch (queue_id) {
7616 case 0:
7617 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7618 break;
7619 case 1:
7620 /* XXX compute */
7621 break;
7622 case 2:
7623 /* XXX compute */
7624 break;
7625 }
7626 break;
7627 case 1:
7628 switch (queue_id) {
7629 case 0:
7630 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7631 break;
7632 case 1:
7633 /* XXX compute */
7634 break;
7635 case 2:
7636 /* XXX compute */
7637 break;
7638 }
7639 break;
7640 }
7641 break;
7642 case 230: /* thermal low to high */
7643 DRM_DEBUG("IH: thermal low to high\n");
7644 rdev->pm.dpm.thermal.high_to_low = false;
7645 queue_thermal = true;
7646 break;
7647 case 231: /* thermal high to low */
7648 DRM_DEBUG("IH: thermal high to low\n");
7649 rdev->pm.dpm.thermal.high_to_low = true;
7650 queue_thermal = true;
7651 break;
7652 case 233: /* GUI IDLE */
7653 DRM_DEBUG("IH: GUI idle\n");
7654 break;
7655 case 241: /* SDMA Privileged inst */
7656 case 247: /* SDMA Privileged inst */
7657 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7658 /* XXX check the bitfield order! */
7659 me_id = (ring_id & 0x3) >> 0;
7660 queue_id = (ring_id & 0xc) >> 2;
7661 switch (me_id) {
7662 case 0:
7663 switch (queue_id) {
7664 case 0:
7665 queue_reset = true;
7666 break;
7667 case 1:
7668 /* XXX compute */
7669 queue_reset = true;
7670 break;
7671 case 2:
7672 /* XXX compute */
7673 queue_reset = true;
7674 break;
7675 }
7676 break;
7677 case 1:
7678 switch (queue_id) {
7679 case 0:
7680 queue_reset = true;
7681 break;
7682 case 1:
7683 /* XXX compute */
7684 queue_reset = true;
7685 break;
7686 case 2:
7687 /* XXX compute */
7688 queue_reset = true;
7689 break;
7690 }
7691 break;
7692 }
7693 break;
7694 default:
7695 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7696 break;
7697 }
7698
7699 /* wptr/rptr are in bytes! */
7700 rptr += 16;
7701 rptr &= rdev->ih.ptr_mask;
7702 }
7703 if (queue_hotplug)
7704 schedule_work(&rdev->hotplug_work);
7705 if (queue_reset)
7706 schedule_work(&rdev->reset_work);
7707 if (queue_thermal)
7708 schedule_work(&rdev->pm.dpm.thermal.work);
7709 rdev->ih.rptr = rptr;
7710 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7711 atomic_set(&rdev->ih.lock, 0);
7712
7713 /* make sure wptr hasn't changed while processing */
7714 wptr = cik_get_ih_wptr(rdev);
7715 if (wptr != rptr)
7716 goto restart_ih;
7717
7718 return IRQ_HANDLED;
7719 }
7720
7721 /*
7722 * startup/shutdown callbacks
7723 */
7724 /**
7725 * cik_startup - program the asic to a functional state
7726 *
7727 * @rdev: radeon_device pointer
7728 *
7729 * Programs the asic to a functional state (CIK).
7730 * Called by cik_init() and cik_resume().
7731 * Returns 0 for success, error for failure.
7732 */
7733 static int cik_startup(struct radeon_device *rdev)
7734 {
7735 struct radeon_ring *ring;
7736 int r;
7737
7738 /* enable pcie gen2/3 link */
7739 cik_pcie_gen3_enable(rdev);
7740 /* enable aspm */
7741 cik_program_aspm(rdev);
7742
7743 /* scratch needs to be initialized before MC */
7744 r = r600_vram_scratch_init(rdev);
7745 if (r)
7746 return r;
7747
7748 cik_mc_program(rdev);
7749
7750 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7751 r = ci_mc_load_microcode(rdev);
7752 if (r) {
7753 DRM_ERROR("Failed to load MC firmware!\n");
7754 return r;
7755 }
7756 }
7757
7758 r = cik_pcie_gart_enable(rdev);
7759 if (r)
7760 return r;
7761 cik_gpu_init(rdev);
7762
7763 /* allocate rlc buffers */
7764 if (rdev->flags & RADEON_IS_IGP) {
7765 if (rdev->family == CHIP_KAVERI) {
7766 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7767 rdev->rlc.reg_list_size =
7768 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7769 } else {
7770 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7771 rdev->rlc.reg_list_size =
7772 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7773 }
7774 }
7775 rdev->rlc.cs_data = ci_cs_data;
7776 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7777 r = sumo_rlc_init(rdev);
7778 if (r) {
7779 DRM_ERROR("Failed to init rlc BOs!\n");
7780 return r;
7781 }
7782
7783 /* allocate wb buffer */
7784 r = radeon_wb_init(rdev);
7785 if (r)
7786 return r;
7787
7788 /* allocate mec buffers */
7789 r = cik_mec_init(rdev);
7790 if (r) {
7791 DRM_ERROR("Failed to init MEC BOs!\n");
7792 return r;
7793 }
7794
7795 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7796 if (r) {
7797 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7798 return r;
7799 }
7800
7801 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7802 if (r) {
7803 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7804 return r;
7805 }
7806
7807 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7808 if (r) {
7809 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7810 return r;
7811 }
7812
7813 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7814 if (r) {
7815 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7816 return r;
7817 }
7818
7819 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7820 if (r) {
7821 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7822 return r;
7823 }
7824
7825 r = radeon_uvd_resume(rdev);
7826 if (!r) {
7827 r = uvd_v4_2_resume(rdev);
7828 if (!r) {
7829 r = radeon_fence_driver_start_ring(rdev,
7830 R600_RING_TYPE_UVD_INDEX);
7831 if (r)
7832 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7833 }
7834 }
7835 if (r)
7836 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7837
7838 r = radeon_vce_resume(rdev);
7839 if (!r) {
7840 r = vce_v2_0_resume(rdev);
7841 if (!r)
7842 r = radeon_fence_driver_start_ring(rdev,
7843 TN_RING_TYPE_VCE1_INDEX);
7844 if (!r)
7845 r = radeon_fence_driver_start_ring(rdev,
7846 TN_RING_TYPE_VCE2_INDEX);
7847 }
7848 if (r) {
7849 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7850 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7851 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7852 }
7853
7854 /* Enable IRQ */
7855 if (!rdev->irq.installed) {
7856 r = radeon_irq_kms_init(rdev);
7857 if (r)
7858 return r;
7859 }
7860
7861 r = cik_irq_init(rdev);
7862 if (r) {
7863 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7864 radeon_irq_kms_fini(rdev);
7865 return r;
7866 }
7867 cik_irq_set(rdev);
7868
7869 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7870 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7871 PACKET3(PACKET3_NOP, 0x3FFF));
7872 if (r)
7873 return r;
7874
7875 /* set up the compute queues */
7876 /* type-2 packets are deprecated on MEC, use type-3 instead */
7877 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7878 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7879 PACKET3(PACKET3_NOP, 0x3FFF));
7880 if (r)
7881 return r;
7882 ring->me = 1; /* first MEC */
7883 ring->pipe = 0; /* first pipe */
7884 ring->queue = 0; /* first queue */
7885 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7886
7887 /* type-2 packets are deprecated on MEC, use type-3 instead */
7888 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7889 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7890 PACKET3(PACKET3_NOP, 0x3FFF));
7891 if (r)
7892 return r;
7893 /* dGPU only have 1 MEC */
7894 ring->me = 1; /* first MEC */
7895 ring->pipe = 0; /* first pipe */
7896 ring->queue = 1; /* second queue */
7897 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7898
7899 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7900 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7901 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7902 if (r)
7903 return r;
7904
7905 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7906 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7907 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7908 if (r)
7909 return r;
7910
7911 r = cik_cp_resume(rdev);
7912 if (r)
7913 return r;
7914
7915 r = cik_sdma_resume(rdev);
7916 if (r)
7917 return r;
7918
7919 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7920 if (ring->ring_size) {
7921 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7922 RADEON_CP_PACKET2);
7923 if (!r)
7924 r = uvd_v1_0_init(rdev);
7925 if (r)
7926 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7927 }
7928
7929 r = -ENOENT;
7930
7931 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7932 if (ring->ring_size)
7933 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7934 VCE_CMD_NO_OP);
7935
7936 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7937 if (ring->ring_size)
7938 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7939 VCE_CMD_NO_OP);
7940
7941 if (!r)
7942 r = vce_v1_0_init(rdev);
7943 else if (r != -ENOENT)
7944 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7945
7946 r = radeon_ib_pool_init(rdev);
7947 if (r) {
7948 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7949 return r;
7950 }
7951
7952 r = radeon_vm_manager_init(rdev);
7953 if (r) {
7954 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7955 return r;
7956 }
7957
7958 r = dce6_audio_init(rdev);
7959 if (r)
7960 return r;
7961
7962 return 0;
7963 }
7964
7965 /**
7966 * cik_resume - resume the asic to a functional state
7967 *
7968 * @rdev: radeon_device pointer
7969 *
7970 * Programs the asic to a functional state (CIK).
7971 * Called at resume.
7972 * Returns 0 for success, error for failure.
7973 */
7974 int cik_resume(struct radeon_device *rdev)
7975 {
7976 int r;
7977
7978 /* post card */
7979 atom_asic_init(rdev->mode_info.atom_context);
7980
7981 /* init golden registers */
7982 cik_init_golden_registers(rdev);
7983
7984 if (rdev->pm.pm_method == PM_METHOD_DPM)
7985 radeon_pm_resume(rdev);
7986
7987 rdev->accel_working = true;
7988 r = cik_startup(rdev);
7989 if (r) {
7990 DRM_ERROR("cik startup failed on resume\n");
7991 rdev->accel_working = false;
7992 return r;
7993 }
7994
7995 return r;
7996
7997 }
7998
7999 /**
8000 * cik_suspend - suspend the asic
8001 *
8002 * @rdev: radeon_device pointer
8003 *
8004 * Bring the chip into a state suitable for suspend (CIK).
8005 * Called at suspend.
8006 * Returns 0 for success.
8007 */
8008 int cik_suspend(struct radeon_device *rdev)
8009 {
8010 radeon_pm_suspend(rdev);
8011 dce6_audio_fini(rdev);
8012 radeon_vm_manager_fini(rdev);
8013 cik_cp_enable(rdev, false);
8014 cik_sdma_enable(rdev, false);
8015 uvd_v1_0_fini(rdev);
8016 radeon_uvd_suspend(rdev);
8017 radeon_vce_suspend(rdev);
8018 cik_fini_pg(rdev);
8019 cik_fini_cg(rdev);
8020 cik_irq_suspend(rdev);
8021 radeon_wb_disable(rdev);
8022 cik_pcie_gart_disable(rdev);
8023 return 0;
8024 }
8025
8026 /* Plan is to move initialization in that function and use
8027 * helper function so that radeon_device_init pretty much
8028 * do nothing more than calling asic specific function. This
8029 * should also allow to remove a bunch of callback function
8030 * like vram_info.
8031 */
8032 /**
8033 * cik_init - asic specific driver and hw init
8034 *
8035 * @rdev: radeon_device pointer
8036 *
8037 * Setup asic specific driver variables and program the hw
8038 * to a functional state (CIK).
8039 * Called at driver startup.
8040 * Returns 0 for success, errors for failure.
8041 */
8042 int cik_init(struct radeon_device *rdev)
8043 {
8044 struct radeon_ring *ring;
8045 int r;
8046
8047 /* Read BIOS */
8048 if (!radeon_get_bios(rdev)) {
8049 if (ASIC_IS_AVIVO(rdev))
8050 return -EINVAL;
8051 }
8052 /* Must be an ATOMBIOS */
8053 if (!rdev->is_atom_bios) {
8054 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8055 return -EINVAL;
8056 }
8057 r = radeon_atombios_init(rdev);
8058 if (r)
8059 return r;
8060
8061 /* Post card if necessary */
8062 if (!radeon_card_posted(rdev)) {
8063 if (!rdev->bios) {
8064 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8065 return -EINVAL;
8066 }
8067 DRM_INFO("GPU not posted. posting now...\n");
8068 atom_asic_init(rdev->mode_info.atom_context);
8069 }
8070 /* init golden registers */
8071 cik_init_golden_registers(rdev);
8072 /* Initialize scratch registers */
8073 cik_scratch_init(rdev);
8074 /* Initialize surface registers */
8075 radeon_surface_init(rdev);
8076 /* Initialize clocks */
8077 radeon_get_clock_info(rdev->ddev);
8078
8079 /* Fence driver */
8080 r = radeon_fence_driver_init(rdev);
8081 if (r)
8082 return r;
8083
8084 /* initialize memory controller */
8085 r = cik_mc_init(rdev);
8086 if (r)
8087 return r;
8088 /* Memory manager */
8089 r = radeon_bo_init(rdev);
8090 if (r)
8091 return r;
8092
8093 if (rdev->flags & RADEON_IS_IGP) {
8094 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8095 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8096 r = cik_init_microcode(rdev);
8097 if (r) {
8098 DRM_ERROR("Failed to load firmware!\n");
8099 return r;
8100 }
8101 }
8102 } else {
8103 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8104 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8105 !rdev->mc_fw) {
8106 r = cik_init_microcode(rdev);
8107 if (r) {
8108 DRM_ERROR("Failed to load firmware!\n");
8109 return r;
8110 }
8111 }
8112 }
8113
8114 /* Initialize power management */
8115 radeon_pm_init(rdev);
8116
8117 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8118 ring->ring_obj = NULL;
8119 r600_ring_init(rdev, ring, 1024 * 1024);
8120
8121 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8122 ring->ring_obj = NULL;
8123 r600_ring_init(rdev, ring, 1024 * 1024);
8124 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8125 if (r)
8126 return r;
8127
8128 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8129 ring->ring_obj = NULL;
8130 r600_ring_init(rdev, ring, 1024 * 1024);
8131 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8132 if (r)
8133 return r;
8134
8135 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8136 ring->ring_obj = NULL;
8137 r600_ring_init(rdev, ring, 256 * 1024);
8138
8139 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8140 ring->ring_obj = NULL;
8141 r600_ring_init(rdev, ring, 256 * 1024);
8142
8143 r = radeon_uvd_init(rdev);
8144 if (!r) {
8145 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8146 ring->ring_obj = NULL;
8147 r600_ring_init(rdev, ring, 4096);
8148 }
8149
8150 r = radeon_vce_init(rdev);
8151 if (!r) {
8152 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8153 ring->ring_obj = NULL;
8154 r600_ring_init(rdev, ring, 4096);
8155
8156 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8157 ring->ring_obj = NULL;
8158 r600_ring_init(rdev, ring, 4096);
8159 }
8160
8161 rdev->ih.ring_obj = NULL;
8162 r600_ih_ring_init(rdev, 64 * 1024);
8163
8164 r = r600_pcie_gart_init(rdev);
8165 if (r)
8166 return r;
8167
8168 rdev->accel_working = true;
8169 r = cik_startup(rdev);
8170 if (r) {
8171 dev_err(rdev->dev, "disabling GPU acceleration\n");
8172 cik_cp_fini(rdev);
8173 cik_sdma_fini(rdev);
8174 cik_irq_fini(rdev);
8175 sumo_rlc_fini(rdev);
8176 cik_mec_fini(rdev);
8177 radeon_wb_fini(rdev);
8178 radeon_ib_pool_fini(rdev);
8179 radeon_vm_manager_fini(rdev);
8180 radeon_irq_kms_fini(rdev);
8181 cik_pcie_gart_fini(rdev);
8182 rdev->accel_working = false;
8183 }
8184
8185 /* Don't start up if the MC ucode is missing.
8186 * The default clocks and voltages before the MC ucode
8187 * is loaded are not suffient for advanced operations.
8188 */
8189 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8190 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8191 return -EINVAL;
8192 }
8193
8194 return 0;
8195 }
8196
8197 /**
8198 * cik_fini - asic specific driver and hw fini
8199 *
8200 * @rdev: radeon_device pointer
8201 *
8202 * Tear down the asic specific driver variables and program the hw
8203 * to an idle state (CIK).
8204 * Called at driver unload.
8205 */
8206 void cik_fini(struct radeon_device *rdev)
8207 {
8208 radeon_pm_fini(rdev);
8209 cik_cp_fini(rdev);
8210 cik_sdma_fini(rdev);
8211 cik_fini_pg(rdev);
8212 cik_fini_cg(rdev);
8213 cik_irq_fini(rdev);
8214 sumo_rlc_fini(rdev);
8215 cik_mec_fini(rdev);
8216 radeon_wb_fini(rdev);
8217 radeon_vm_manager_fini(rdev);
8218 radeon_ib_pool_fini(rdev);
8219 radeon_irq_kms_fini(rdev);
8220 uvd_v1_0_fini(rdev);
8221 radeon_uvd_fini(rdev);
8222 radeon_vce_fini(rdev);
8223 cik_pcie_gart_fini(rdev);
8224 r600_vram_scratch_fini(rdev);
8225 radeon_gem_fini(rdev);
8226 radeon_fence_driver_fini(rdev);
8227 radeon_bo_fini(rdev);
8228 radeon_atombios_fini(rdev);
8229 kfree(rdev->bios);
8230 rdev->bios = NULL;
8231 }
8232
8233 void dce8_program_fmt(struct drm_encoder *encoder)
8234 {
8235 struct drm_device *dev = encoder->dev;
8236 struct radeon_device *rdev = dev->dev_private;
8237 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8238 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8239 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8240 int bpc = 0;
8241 u32 tmp = 0;
8242 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8243
8244 if (connector) {
8245 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8246 bpc = radeon_get_monitor_bpc(connector);
8247 dither = radeon_connector->dither;
8248 }
8249
8250 /* LVDS/eDP FMT is set up by atom */
8251 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8252 return;
8253
8254 /* not needed for analog */
8255 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8256 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8257 return;
8258
8259 if (bpc == 0)
8260 return;
8261
8262 switch (bpc) {
8263 case 6:
8264 if (dither == RADEON_FMT_DITHER_ENABLE)
8265 /* XXX sort out optimal dither settings */
8266 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8267 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8268 else
8269 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8270 break;
8271 case 8:
8272 if (dither == RADEON_FMT_DITHER_ENABLE)
8273 /* XXX sort out optimal dither settings */
8274 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8275 FMT_RGB_RANDOM_ENABLE |
8276 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8277 else
8278 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8279 break;
8280 case 10:
8281 if (dither == RADEON_FMT_DITHER_ENABLE)
8282 /* XXX sort out optimal dither settings */
8283 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8284 FMT_RGB_RANDOM_ENABLE |
8285 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8286 else
8287 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8288 break;
8289 default:
8290 /* not needed */
8291 break;
8292 }
8293
8294 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8295 }
8296
8297 /* display watermark setup */
8298 /**
8299 * dce8_line_buffer_adjust - Set up the line buffer
8300 *
8301 * @rdev: radeon_device pointer
8302 * @radeon_crtc: the selected display controller
8303 * @mode: the current display mode on the selected display
8304 * controller
8305 *
8306 * Setup up the line buffer allocation for
8307 * the selected display controller (CIK).
8308 * Returns the line buffer size in pixels.
8309 */
8310 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8311 struct radeon_crtc *radeon_crtc,
8312 struct drm_display_mode *mode)
8313 {
8314 u32 tmp, buffer_alloc, i;
8315 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8316 /*
8317 * Line Buffer Setup
8318 * There are 6 line buffers, one for each display controllers.
8319 * There are 3 partitions per LB. Select the number of partitions
8320 * to enable based on the display width. For display widths larger
8321 * than 4096, you need use to use 2 display controllers and combine
8322 * them using the stereo blender.
8323 */
8324 if (radeon_crtc->base.enabled && mode) {
8325 if (mode->crtc_hdisplay < 1920) {
8326 tmp = 1;
8327 buffer_alloc = 2;
8328 } else if (mode->crtc_hdisplay < 2560) {
8329 tmp = 2;
8330 buffer_alloc = 2;
8331 } else if (mode->crtc_hdisplay < 4096) {
8332 tmp = 0;
8333 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8334 } else {
8335 DRM_DEBUG_KMS("Mode too big for LB!\n");
8336 tmp = 0;
8337 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8338 }
8339 } else {
8340 tmp = 1;
8341 buffer_alloc = 0;
8342 }
8343
8344 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8345 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8346
8347 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8348 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8349 for (i = 0; i < rdev->usec_timeout; i++) {
8350 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8351 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8352 break;
8353 udelay(1);
8354 }
8355
8356 if (radeon_crtc->base.enabled && mode) {
8357 switch (tmp) {
8358 case 0:
8359 default:
8360 return 4096 * 2;
8361 case 1:
8362 return 1920 * 2;
8363 case 2:
8364 return 2560 * 2;
8365 }
8366 }
8367
8368 /* controller not enabled, so no lb used */
8369 return 0;
8370 }
8371
8372 /**
8373 * cik_get_number_of_dram_channels - get the number of dram channels
8374 *
8375 * @rdev: radeon_device pointer
8376 *
8377 * Look up the number of video ram channels (CIK).
8378 * Used for display watermark bandwidth calculations
8379 * Returns the number of dram channels
8380 */
8381 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8382 {
8383 u32 tmp = RREG32(MC_SHARED_CHMAP);
8384
8385 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8386 case 0:
8387 default:
8388 return 1;
8389 case 1:
8390 return 2;
8391 case 2:
8392 return 4;
8393 case 3:
8394 return 8;
8395 case 4:
8396 return 3;
8397 case 5:
8398 return 6;
8399 case 6:
8400 return 10;
8401 case 7:
8402 return 12;
8403 case 8:
8404 return 16;
8405 }
8406 }
8407
8408 struct dce8_wm_params {
8409 u32 dram_channels; /* number of dram channels */
8410 u32 yclk; /* bandwidth per dram data pin in kHz */
8411 u32 sclk; /* engine clock in kHz */
8412 u32 disp_clk; /* display clock in kHz */
8413 u32 src_width; /* viewport width */
8414 u32 active_time; /* active display time in ns */
8415 u32 blank_time; /* blank time in ns */
8416 bool interlaced; /* mode is interlaced */
8417 fixed20_12 vsc; /* vertical scale ratio */
8418 u32 num_heads; /* number of active crtcs */
8419 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8420 u32 lb_size; /* line buffer allocated to pipe */
8421 u32 vtaps; /* vertical scaler taps */
8422 };
8423
8424 /**
8425 * dce8_dram_bandwidth - get the dram bandwidth
8426 *
8427 * @wm: watermark calculation data
8428 *
8429 * Calculate the raw dram bandwidth (CIK).
8430 * Used for display watermark bandwidth calculations
8431 * Returns the dram bandwidth in MBytes/s
8432 */
8433 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8434 {
8435 /* Calculate raw DRAM Bandwidth */
8436 fixed20_12 dram_efficiency; /* 0.7 */
8437 fixed20_12 yclk, dram_channels, bandwidth;
8438 fixed20_12 a;
8439
8440 a.full = dfixed_const(1000);
8441 yclk.full = dfixed_const(wm->yclk);
8442 yclk.full = dfixed_div(yclk, a);
8443 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8444 a.full = dfixed_const(10);
8445 dram_efficiency.full = dfixed_const(7);
8446 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8447 bandwidth.full = dfixed_mul(dram_channels, yclk);
8448 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8449
8450 return dfixed_trunc(bandwidth);
8451 }
8452
8453 /**
8454 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8455 *
8456 * @wm: watermark calculation data
8457 *
8458 * Calculate the dram bandwidth used for display (CIK).
8459 * Used for display watermark bandwidth calculations
8460 * Returns the dram bandwidth for display in MBytes/s
8461 */
8462 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8463 {
8464 /* Calculate DRAM Bandwidth and the part allocated to display. */
8465 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8466 fixed20_12 yclk, dram_channels, bandwidth;
8467 fixed20_12 a;
8468
8469 a.full = dfixed_const(1000);
8470 yclk.full = dfixed_const(wm->yclk);
8471 yclk.full = dfixed_div(yclk, a);
8472 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8473 a.full = dfixed_const(10);
8474 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8475 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8476 bandwidth.full = dfixed_mul(dram_channels, yclk);
8477 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8478
8479 return dfixed_trunc(bandwidth);
8480 }
8481
8482 /**
8483 * dce8_data_return_bandwidth - get the data return bandwidth
8484 *
8485 * @wm: watermark calculation data
8486 *
8487 * Calculate the data return bandwidth used for display (CIK).
8488 * Used for display watermark bandwidth calculations
8489 * Returns the data return bandwidth in MBytes/s
8490 */
8491 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8492 {
8493 /* Calculate the display Data return Bandwidth */
8494 fixed20_12 return_efficiency; /* 0.8 */
8495 fixed20_12 sclk, bandwidth;
8496 fixed20_12 a;
8497
8498 a.full = dfixed_const(1000);
8499 sclk.full = dfixed_const(wm->sclk);
8500 sclk.full = dfixed_div(sclk, a);
8501 a.full = dfixed_const(10);
8502 return_efficiency.full = dfixed_const(8);
8503 return_efficiency.full = dfixed_div(return_efficiency, a);
8504 a.full = dfixed_const(32);
8505 bandwidth.full = dfixed_mul(a, sclk);
8506 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8507
8508 return dfixed_trunc(bandwidth);
8509 }
8510
8511 /**
8512 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8513 *
8514 * @wm: watermark calculation data
8515 *
8516 * Calculate the dmif bandwidth used for display (CIK).
8517 * Used for display watermark bandwidth calculations
8518 * Returns the dmif bandwidth in MBytes/s
8519 */
8520 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8521 {
8522 /* Calculate the DMIF Request Bandwidth */
8523 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8524 fixed20_12 disp_clk, bandwidth;
8525 fixed20_12 a, b;
8526
8527 a.full = dfixed_const(1000);
8528 disp_clk.full = dfixed_const(wm->disp_clk);
8529 disp_clk.full = dfixed_div(disp_clk, a);
8530 a.full = dfixed_const(32);
8531 b.full = dfixed_mul(a, disp_clk);
8532
8533 a.full = dfixed_const(10);
8534 disp_clk_request_efficiency.full = dfixed_const(8);
8535 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8536
8537 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8538
8539 return dfixed_trunc(bandwidth);
8540 }
8541
8542 /**
8543 * dce8_available_bandwidth - get the min available bandwidth
8544 *
8545 * @wm: watermark calculation data
8546 *
8547 * Calculate the min available bandwidth used for display (CIK).
8548 * Used for display watermark bandwidth calculations
8549 * Returns the min available bandwidth in MBytes/s
8550 */
8551 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8552 {
8553 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8554 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8555 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8556 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8557
8558 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8559 }
8560
8561 /**
8562 * dce8_average_bandwidth - get the average available bandwidth
8563 *
8564 * @wm: watermark calculation data
8565 *
8566 * Calculate the average available bandwidth used for display (CIK).
8567 * Used for display watermark bandwidth calculations
8568 * Returns the average available bandwidth in MBytes/s
8569 */
8570 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8571 {
8572 /* Calculate the display mode Average Bandwidth
8573 * DisplayMode should contain the source and destination dimensions,
8574 * timing, etc.
8575 */
8576 fixed20_12 bpp;
8577 fixed20_12 line_time;
8578 fixed20_12 src_width;
8579 fixed20_12 bandwidth;
8580 fixed20_12 a;
8581
8582 a.full = dfixed_const(1000);
8583 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8584 line_time.full = dfixed_div(line_time, a);
8585 bpp.full = dfixed_const(wm->bytes_per_pixel);
8586 src_width.full = dfixed_const(wm->src_width);
8587 bandwidth.full = dfixed_mul(src_width, bpp);
8588 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8589 bandwidth.full = dfixed_div(bandwidth, line_time);
8590
8591 return dfixed_trunc(bandwidth);
8592 }
8593
8594 /**
8595 * dce8_latency_watermark - get the latency watermark
8596 *
8597 * @wm: watermark calculation data
8598 *
8599 * Calculate the latency watermark (CIK).
8600 * Used for display watermark bandwidth calculations
8601 * Returns the latency watermark in ns
8602 */
8603 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8604 {
8605 /* First calculate the latency in ns */
8606 u32 mc_latency = 2000; /* 2000 ns. */
8607 u32 available_bandwidth = dce8_available_bandwidth(wm);
8608 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8609 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8610 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8611 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8612 (wm->num_heads * cursor_line_pair_return_time);
8613 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8614 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8615 u32 tmp, dmif_size = 12288;
8616 fixed20_12 a, b, c;
8617
8618 if (wm->num_heads == 0)
8619 return 0;
8620
8621 a.full = dfixed_const(2);
8622 b.full = dfixed_const(1);
8623 if ((wm->vsc.full > a.full) ||
8624 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8625 (wm->vtaps >= 5) ||
8626 ((wm->vsc.full >= a.full) && wm->interlaced))
8627 max_src_lines_per_dst_line = 4;
8628 else
8629 max_src_lines_per_dst_line = 2;
8630
8631 a.full = dfixed_const(available_bandwidth);
8632 b.full = dfixed_const(wm->num_heads);
8633 a.full = dfixed_div(a, b);
8634
8635 b.full = dfixed_const(mc_latency + 512);
8636 c.full = dfixed_const(wm->disp_clk);
8637 b.full = dfixed_div(b, c);
8638
8639 c.full = dfixed_const(dmif_size);
8640 b.full = dfixed_div(c, b);
8641
8642 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8643
8644 b.full = dfixed_const(1000);
8645 c.full = dfixed_const(wm->disp_clk);
8646 b.full = dfixed_div(c, b);
8647 c.full = dfixed_const(wm->bytes_per_pixel);
8648 b.full = dfixed_mul(b, c);
8649
8650 lb_fill_bw = min(tmp, dfixed_trunc(b));
8651
8652 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8653 b.full = dfixed_const(1000);
8654 c.full = dfixed_const(lb_fill_bw);
8655 b.full = dfixed_div(c, b);
8656 a.full = dfixed_div(a, b);
8657 line_fill_time = dfixed_trunc(a);
8658
8659 if (line_fill_time < wm->active_time)
8660 return latency;
8661 else
8662 return latency + (line_fill_time - wm->active_time);
8663
8664 }
8665
8666 /**
8667 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8668 * average and available dram bandwidth
8669 *
8670 * @wm: watermark calculation data
8671 *
8672 * Check if the display average bandwidth fits in the display
8673 * dram bandwidth (CIK).
8674 * Used for display watermark bandwidth calculations
8675 * Returns true if the display fits, false if not.
8676 */
8677 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8678 {
8679 if (dce8_average_bandwidth(wm) <=
8680 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8681 return true;
8682 else
8683 return false;
8684 }
8685
8686 /**
8687 * dce8_average_bandwidth_vs_available_bandwidth - check
8688 * average and available bandwidth
8689 *
8690 * @wm: watermark calculation data
8691 *
8692 * Check if the display average bandwidth fits in the display
8693 * available bandwidth (CIK).
8694 * Used for display watermark bandwidth calculations
8695 * Returns true if the display fits, false if not.
8696 */
8697 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8698 {
8699 if (dce8_average_bandwidth(wm) <=
8700 (dce8_available_bandwidth(wm) / wm->num_heads))
8701 return true;
8702 else
8703 return false;
8704 }
8705
8706 /**
8707 * dce8_check_latency_hiding - check latency hiding
8708 *
8709 * @wm: watermark calculation data
8710 *
8711 * Check latency hiding (CIK).
8712 * Used for display watermark bandwidth calculations
8713 * Returns true if the display fits, false if not.
8714 */
8715 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8716 {
8717 u32 lb_partitions = wm->lb_size / wm->src_width;
8718 u32 line_time = wm->active_time + wm->blank_time;
8719 u32 latency_tolerant_lines;
8720 u32 latency_hiding;
8721 fixed20_12 a;
8722
8723 a.full = dfixed_const(1);
8724 if (wm->vsc.full > a.full)
8725 latency_tolerant_lines = 1;
8726 else {
8727 if (lb_partitions <= (wm->vtaps + 1))
8728 latency_tolerant_lines = 1;
8729 else
8730 latency_tolerant_lines = 2;
8731 }
8732
8733 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8734
8735 if (dce8_latency_watermark(wm) <= latency_hiding)
8736 return true;
8737 else
8738 return false;
8739 }
8740
8741 /**
8742 * dce8_program_watermarks - program display watermarks
8743 *
8744 * @rdev: radeon_device pointer
8745 * @radeon_crtc: the selected display controller
8746 * @lb_size: line buffer size
8747 * @num_heads: number of display controllers in use
8748 *
8749 * Calculate and program the display watermarks for the
8750 * selected display controller (CIK).
8751 */
8752 static void dce8_program_watermarks(struct radeon_device *rdev,
8753 struct radeon_crtc *radeon_crtc,
8754 u32 lb_size, u32 num_heads)
8755 {
8756 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8757 struct dce8_wm_params wm_low, wm_high;
8758 u32 pixel_period;
8759 u32 line_time = 0;
8760 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8761 u32 tmp, wm_mask;
8762
8763 if (radeon_crtc->base.enabled && num_heads && mode) {
8764 pixel_period = 1000000 / (u32)mode->clock;
8765 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8766
8767 /* watermark for high clocks */
8768 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8769 rdev->pm.dpm_enabled) {
8770 wm_high.yclk =
8771 radeon_dpm_get_mclk(rdev, false) * 10;
8772 wm_high.sclk =
8773 radeon_dpm_get_sclk(rdev, false) * 10;
8774 } else {
8775 wm_high.yclk = rdev->pm.current_mclk * 10;
8776 wm_high.sclk = rdev->pm.current_sclk * 10;
8777 }
8778
8779 wm_high.disp_clk = mode->clock;
8780 wm_high.src_width = mode->crtc_hdisplay;
8781 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8782 wm_high.blank_time = line_time - wm_high.active_time;
8783 wm_high.interlaced = false;
8784 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8785 wm_high.interlaced = true;
8786 wm_high.vsc = radeon_crtc->vsc;
8787 wm_high.vtaps = 1;
8788 if (radeon_crtc->rmx_type != RMX_OFF)
8789 wm_high.vtaps = 2;
8790 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8791 wm_high.lb_size = lb_size;
8792 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8793 wm_high.num_heads = num_heads;
8794
8795 /* set for high clocks */
8796 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8797
8798 /* possibly force display priority to high */
8799 /* should really do this at mode validation time... */
8800 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8801 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8802 !dce8_check_latency_hiding(&wm_high) ||
8803 (rdev->disp_priority == 2)) {
8804 DRM_DEBUG_KMS("force priority to high\n");
8805 }
8806
8807 /* watermark for low clocks */
8808 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8809 rdev->pm.dpm_enabled) {
8810 wm_low.yclk =
8811 radeon_dpm_get_mclk(rdev, true) * 10;
8812 wm_low.sclk =
8813 radeon_dpm_get_sclk(rdev, true) * 10;
8814 } else {
8815 wm_low.yclk = rdev->pm.current_mclk * 10;
8816 wm_low.sclk = rdev->pm.current_sclk * 10;
8817 }
8818
8819 wm_low.disp_clk = mode->clock;
8820 wm_low.src_width = mode->crtc_hdisplay;
8821 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8822 wm_low.blank_time = line_time - wm_low.active_time;
8823 wm_low.interlaced = false;
8824 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8825 wm_low.interlaced = true;
8826 wm_low.vsc = radeon_crtc->vsc;
8827 wm_low.vtaps = 1;
8828 if (radeon_crtc->rmx_type != RMX_OFF)
8829 wm_low.vtaps = 2;
8830 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8831 wm_low.lb_size = lb_size;
8832 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8833 wm_low.num_heads = num_heads;
8834
8835 /* set for low clocks */
8836 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8837
8838 /* possibly force display priority to high */
8839 /* should really do this at mode validation time... */
8840 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8841 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8842 !dce8_check_latency_hiding(&wm_low) ||
8843 (rdev->disp_priority == 2)) {
8844 DRM_DEBUG_KMS("force priority to high\n");
8845 }
8846 }
8847
8848 /* select wm A */
8849 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8850 tmp = wm_mask;
8851 tmp &= ~LATENCY_WATERMARK_MASK(3);
8852 tmp |= LATENCY_WATERMARK_MASK(1);
8853 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8854 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8855 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8856 LATENCY_HIGH_WATERMARK(line_time)));
8857 /* select wm B */
8858 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8859 tmp &= ~LATENCY_WATERMARK_MASK(3);
8860 tmp |= LATENCY_WATERMARK_MASK(2);
8861 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8862 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8863 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8864 LATENCY_HIGH_WATERMARK(line_time)));
8865 /* restore original selection */
8866 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8867
8868 /* save values for DPM */
8869 radeon_crtc->line_time = line_time;
8870 radeon_crtc->wm_high = latency_watermark_a;
8871 radeon_crtc->wm_low = latency_watermark_b;
8872 }
8873
8874 /**
8875 * dce8_bandwidth_update - program display watermarks
8876 *
8877 * @rdev: radeon_device pointer
8878 *
8879 * Calculate and program the display watermarks and line
8880 * buffer allocation (CIK).
8881 */
8882 void dce8_bandwidth_update(struct radeon_device *rdev)
8883 {
8884 struct drm_display_mode *mode = NULL;
8885 u32 num_heads = 0, lb_size;
8886 int i;
8887
8888 radeon_update_display_priority(rdev);
8889
8890 for (i = 0; i < rdev->num_crtc; i++) {
8891 if (rdev->mode_info.crtcs[i]->base.enabled)
8892 num_heads++;
8893 }
8894 for (i = 0; i < rdev->num_crtc; i++) {
8895 mode = &rdev->mode_info.crtcs[i]->base.mode;
8896 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8897 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8898 }
8899 }
8900
8901 /**
8902 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8903 *
8904 * @rdev: radeon_device pointer
8905 *
8906 * Fetches a GPU clock counter snapshot (SI).
8907 * Returns the 64 bit clock counter snapshot.
8908 */
8909 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8910 {
8911 uint64_t clock;
8912
8913 mutex_lock(&rdev->gpu_clock_mutex);
8914 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8915 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8916 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8917 mutex_unlock(&rdev->gpu_clock_mutex);
8918 return clock;
8919 }
8920
8921 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8922 u32 cntl_reg, u32 status_reg)
8923 {
8924 int r, i;
8925 struct atom_clock_dividers dividers;
8926 uint32_t tmp;
8927
8928 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8929 clock, false, &dividers);
8930 if (r)
8931 return r;
8932
8933 tmp = RREG32_SMC(cntl_reg);
8934 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8935 tmp |= dividers.post_divider;
8936 WREG32_SMC(cntl_reg, tmp);
8937
8938 for (i = 0; i < 100; i++) {
8939 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8940 break;
8941 mdelay(10);
8942 }
8943 if (i == 100)
8944 return -ETIMEDOUT;
8945
8946 return 0;
8947 }
8948
8949 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8950 {
8951 int r = 0;
8952
8953 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8954 if (r)
8955 return r;
8956
8957 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8958 return r;
8959 }
8960
8961 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
8962 {
8963 int r, i;
8964 struct atom_clock_dividers dividers;
8965 u32 tmp;
8966
8967 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8968 ecclk, false, &dividers);
8969 if (r)
8970 return r;
8971
8972 for (i = 0; i < 100; i++) {
8973 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8974 break;
8975 mdelay(10);
8976 }
8977 if (i == 100)
8978 return -ETIMEDOUT;
8979
8980 tmp = RREG32_SMC(CG_ECLK_CNTL);
8981 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
8982 tmp |= dividers.post_divider;
8983 WREG32_SMC(CG_ECLK_CNTL, tmp);
8984
8985 for (i = 0; i < 100; i++) {
8986 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8987 break;
8988 mdelay(10);
8989 }
8990 if (i == 100)
8991 return -ETIMEDOUT;
8992
8993 return 0;
8994 }
8995
8996 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8997 {
8998 struct pci_dev *root = rdev->pdev->bus->self;
8999 int bridge_pos, gpu_pos;
9000 u32 speed_cntl, mask, current_data_rate;
9001 int ret, i;
9002 u16 tmp16;
9003
9004 if (radeon_pcie_gen2 == 0)
9005 return;
9006
9007 if (rdev->flags & RADEON_IS_IGP)
9008 return;
9009
9010 if (!(rdev->flags & RADEON_IS_PCIE))
9011 return;
9012
9013 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9014 if (ret != 0)
9015 return;
9016
9017 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9018 return;
9019
9020 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9021 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9022 LC_CURRENT_DATA_RATE_SHIFT;
9023 if (mask & DRM_PCIE_SPEED_80) {
9024 if (current_data_rate == 2) {
9025 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9026 return;
9027 }
9028 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9029 } else if (mask & DRM_PCIE_SPEED_50) {
9030 if (current_data_rate == 1) {
9031 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9032 return;
9033 }
9034 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9035 }
9036
9037 bridge_pos = pci_pcie_cap(root);
9038 if (!bridge_pos)
9039 return;
9040
9041 gpu_pos = pci_pcie_cap(rdev->pdev);
9042 if (!gpu_pos)
9043 return;
9044
9045 if (mask & DRM_PCIE_SPEED_80) {
9046 /* re-try equalization if gen3 is not already enabled */
9047 if (current_data_rate != 2) {
9048 u16 bridge_cfg, gpu_cfg;
9049 u16 bridge_cfg2, gpu_cfg2;
9050 u32 max_lw, current_lw, tmp;
9051
9052 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9053 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9054
9055 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9056 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9057
9058 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9059 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9060
9061 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9062 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9063 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9064
9065 if (current_lw < max_lw) {
9066 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9067 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9068 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9069 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9070 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9071 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9072 }
9073 }
9074
9075 for (i = 0; i < 10; i++) {
9076 /* check status */
9077 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9078 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9079 break;
9080
9081 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9082 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9083
9084 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9085 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9086
9087 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9088 tmp |= LC_SET_QUIESCE;
9089 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9090
9091 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9092 tmp |= LC_REDO_EQ;
9093 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9094
9095 mdelay(100);
9096
9097 /* linkctl */
9098 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9099 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9100 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9101 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9102
9103 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9104 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9105 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9106 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9107
9108 /* linkctl2 */
9109 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9110 tmp16 &= ~((1 << 4) | (7 << 9));
9111 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9112 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9113
9114 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9115 tmp16 &= ~((1 << 4) | (7 << 9));
9116 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9117 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9118
9119 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9120 tmp &= ~LC_SET_QUIESCE;
9121 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9122 }
9123 }
9124 }
9125
9126 /* set the link speed */
9127 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9128 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9129 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9130
9131 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9132 tmp16 &= ~0xf;
9133 if (mask & DRM_PCIE_SPEED_80)
9134 tmp16 |= 3; /* gen3 */
9135 else if (mask & DRM_PCIE_SPEED_50)
9136 tmp16 |= 2; /* gen2 */
9137 else
9138 tmp16 |= 1; /* gen1 */
9139 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9140
9141 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9142 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9143 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9144
9145 for (i = 0; i < rdev->usec_timeout; i++) {
9146 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9147 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9148 break;
9149 udelay(1);
9150 }
9151 }
9152
9153 static void cik_program_aspm(struct radeon_device *rdev)
9154 {
9155 u32 data, orig;
9156 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9157 bool disable_clkreq = false;
9158
9159 if (radeon_aspm == 0)
9160 return;
9161
9162 /* XXX double check IGPs */
9163 if (rdev->flags & RADEON_IS_IGP)
9164 return;
9165
9166 if (!(rdev->flags & RADEON_IS_PCIE))
9167 return;
9168
9169 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9170 data &= ~LC_XMIT_N_FTS_MASK;
9171 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9172 if (orig != data)
9173 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9174
9175 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9176 data |= LC_GO_TO_RECOVERY;
9177 if (orig != data)
9178 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9179
9180 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9181 data |= P_IGNORE_EDB_ERR;
9182 if (orig != data)
9183 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9184
9185 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9186 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9187 data |= LC_PMI_TO_L1_DIS;
9188 if (!disable_l0s)
9189 data |= LC_L0S_INACTIVITY(7);
9190
9191 if (!disable_l1) {
9192 data |= LC_L1_INACTIVITY(7);
9193 data &= ~LC_PMI_TO_L1_DIS;
9194 if (orig != data)
9195 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9196
9197 if (!disable_plloff_in_l1) {
9198 bool clk_req_support;
9199
9200 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9201 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9202 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9203 if (orig != data)
9204 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9205
9206 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9207 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9208 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9209 if (orig != data)
9210 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9211
9212 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9213 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9214 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9215 if (orig != data)
9216 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9217
9218 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9219 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9220 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9221 if (orig != data)
9222 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9223
9224 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9225 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9226 data |= LC_DYN_LANES_PWR_STATE(3);
9227 if (orig != data)
9228 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9229
9230 if (!disable_clkreq) {
9231 struct pci_dev *root = rdev->pdev->bus->self;
9232 u32 lnkcap;
9233
9234 clk_req_support = false;
9235 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9236 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9237 clk_req_support = true;
9238 } else {
9239 clk_req_support = false;
9240 }
9241
9242 if (clk_req_support) {
9243 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9244 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9245 if (orig != data)
9246 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9247
9248 orig = data = RREG32_SMC(THM_CLK_CNTL);
9249 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9250 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9251 if (orig != data)
9252 WREG32_SMC(THM_CLK_CNTL, data);
9253
9254 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9255 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9256 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9257 if (orig != data)
9258 WREG32_SMC(MISC_CLK_CTRL, data);
9259
9260 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9261 data &= ~BCLK_AS_XCLK;
9262 if (orig != data)
9263 WREG32_SMC(CG_CLKPIN_CNTL, data);
9264
9265 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9266 data &= ~FORCE_BIF_REFCLK_EN;
9267 if (orig != data)
9268 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9269
9270 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9271 data &= ~MPLL_CLKOUT_SEL_MASK;
9272 data |= MPLL_CLKOUT_SEL(4);
9273 if (orig != data)
9274 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9275 }
9276 }
9277 } else {
9278 if (orig != data)
9279 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9280 }
9281
9282 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9283 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9284 if (orig != data)
9285 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9286
9287 if (!disable_l0s) {
9288 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9289 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9290 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9291 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9292 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9293 data &= ~LC_L0S_INACTIVITY_MASK;
9294 if (orig != data)
9295 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9296 }
9297 }
9298 }
9299 }
Linux kernel - Deliverables
This page took 0.3223 seconds and 5 git commands to generate.