2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
52 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
58 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59 MODULE_FIRMWARE("radeon/KABINI_me.bin");
60 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
63 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
65 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
66 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
67 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
68 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
69 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
70 extern void sumo_rlc_fini(struct radeon_device
*rdev
);
71 extern int sumo_rlc_init(struct radeon_device
*rdev
);
72 extern void si_vram_gtt_location(struct radeon_device
*rdev
, struct radeon_mc
*mc
);
73 extern void si_rlc_reset(struct radeon_device
*rdev
);
74 extern void si_init_uvd_internal_cg(struct radeon_device
*rdev
);
75 extern int cik_sdma_resume(struct radeon_device
*rdev
);
76 extern void cik_sdma_enable(struct radeon_device
*rdev
, bool enable
);
77 extern void cik_sdma_fini(struct radeon_device
*rdev
);
78 extern void vce_v2_0_enable_mgcg(struct radeon_device
*rdev
, bool enable
);
79 static void cik_rlc_stop(struct radeon_device
*rdev
);
80 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
);
81 static void cik_program_aspm(struct radeon_device
*rdev
);
82 static void cik_init_pg(struct radeon_device
*rdev
);
83 static void cik_init_cg(struct radeon_device
*rdev
);
84 static void cik_fini_pg(struct radeon_device
*rdev
);
85 static void cik_fini_cg(struct radeon_device
*rdev
);
86 static void cik_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
89 /* get temperature in millidegrees */
90 int ci_get_temp(struct radeon_device
*rdev
)
95 temp
= (RREG32_SMC(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
101 actual_temp
= temp
& 0x1ff;
103 actual_temp
= actual_temp
* 1000;
108 /* get temperature in millidegrees */
109 int kv_get_temp(struct radeon_device
*rdev
)
114 temp
= RREG32_SMC(0xC0300E0C);
117 actual_temp
= (temp
/ 8) - 49;
121 actual_temp
= actual_temp
* 1000;
127 * Indirect registers accessor
129 u32
cik_pciep_rreg(struct radeon_device
*rdev
, u32 reg
)
134 spin_lock_irqsave(&rdev
->pciep_idx_lock
, flags
);
135 WREG32(PCIE_INDEX
, reg
);
136 (void)RREG32(PCIE_INDEX
);
137 r
= RREG32(PCIE_DATA
);
138 spin_unlock_irqrestore(&rdev
->pciep_idx_lock
, flags
);
142 void cik_pciep_wreg(struct radeon_device
*rdev
, u32 reg
, u32 v
)
146 spin_lock_irqsave(&rdev
->pciep_idx_lock
, flags
);
147 WREG32(PCIE_INDEX
, reg
);
148 (void)RREG32(PCIE_INDEX
);
149 WREG32(PCIE_DATA
, v
);
150 (void)RREG32(PCIE_DATA
);
151 spin_unlock_irqrestore(&rdev
->pciep_idx_lock
, flags
);
154 static const u32 spectre_rlc_save_restore_register_list
[] =
156 (0x0e00 << 16) | (0xc12c >> 2),
158 (0x0e00 << 16) | (0xc140 >> 2),
160 (0x0e00 << 16) | (0xc150 >> 2),
162 (0x0e00 << 16) | (0xc15c >> 2),
164 (0x0e00 << 16) | (0xc168 >> 2),
166 (0x0e00 << 16) | (0xc170 >> 2),
168 (0x0e00 << 16) | (0xc178 >> 2),
170 (0x0e00 << 16) | (0xc204 >> 2),
172 (0x0e00 << 16) | (0xc2b4 >> 2),
174 (0x0e00 << 16) | (0xc2b8 >> 2),
176 (0x0e00 << 16) | (0xc2bc >> 2),
178 (0x0e00 << 16) | (0xc2c0 >> 2),
180 (0x0e00 << 16) | (0x8228 >> 2),
182 (0x0e00 << 16) | (0x829c >> 2),
184 (0x0e00 << 16) | (0x869c >> 2),
186 (0x0600 << 16) | (0x98f4 >> 2),
188 (0x0e00 << 16) | (0x98f8 >> 2),
190 (0x0e00 << 16) | (0x9900 >> 2),
192 (0x0e00 << 16) | (0xc260 >> 2),
194 (0x0e00 << 16) | (0x90e8 >> 2),
196 (0x0e00 << 16) | (0x3c000 >> 2),
198 (0x0e00 << 16) | (0x3c00c >> 2),
200 (0x0e00 << 16) | (0x8c1c >> 2),
202 (0x0e00 << 16) | (0x9700 >> 2),
204 (0x0e00 << 16) | (0xcd20 >> 2),
206 (0x4e00 << 16) | (0xcd20 >> 2),
208 (0x5e00 << 16) | (0xcd20 >> 2),
210 (0x6e00 << 16) | (0xcd20 >> 2),
212 (0x7e00 << 16) | (0xcd20 >> 2),
214 (0x8e00 << 16) | (0xcd20 >> 2),
216 (0x9e00 << 16) | (0xcd20 >> 2),
218 (0xae00 << 16) | (0xcd20 >> 2),
220 (0xbe00 << 16) | (0xcd20 >> 2),
222 (0x0e00 << 16) | (0x89bc >> 2),
224 (0x0e00 << 16) | (0x8900 >> 2),
227 (0x0e00 << 16) | (0xc130 >> 2),
229 (0x0e00 << 16) | (0xc134 >> 2),
231 (0x0e00 << 16) | (0xc1fc >> 2),
233 (0x0e00 << 16) | (0xc208 >> 2),
235 (0x0e00 << 16) | (0xc264 >> 2),
237 (0x0e00 << 16) | (0xc268 >> 2),
239 (0x0e00 << 16) | (0xc26c >> 2),
241 (0x0e00 << 16) | (0xc270 >> 2),
243 (0x0e00 << 16) | (0xc274 >> 2),
245 (0x0e00 << 16) | (0xc278 >> 2),
247 (0x0e00 << 16) | (0xc27c >> 2),
249 (0x0e00 << 16) | (0xc280 >> 2),
251 (0x0e00 << 16) | (0xc284 >> 2),
253 (0x0e00 << 16) | (0xc288 >> 2),
255 (0x0e00 << 16) | (0xc28c >> 2),
257 (0x0e00 << 16) | (0xc290 >> 2),
259 (0x0e00 << 16) | (0xc294 >> 2),
261 (0x0e00 << 16) | (0xc298 >> 2),
263 (0x0e00 << 16) | (0xc29c >> 2),
265 (0x0e00 << 16) | (0xc2a0 >> 2),
267 (0x0e00 << 16) | (0xc2a4 >> 2),
269 (0x0e00 << 16) | (0xc2a8 >> 2),
271 (0x0e00 << 16) | (0xc2ac >> 2),
273 (0x0e00 << 16) | (0xc2b0 >> 2),
275 (0x0e00 << 16) | (0x301d0 >> 2),
277 (0x0e00 << 16) | (0x30238 >> 2),
279 (0x0e00 << 16) | (0x30250 >> 2),
281 (0x0e00 << 16) | (0x30254 >> 2),
283 (0x0e00 << 16) | (0x30258 >> 2),
285 (0x0e00 << 16) | (0x3025c >> 2),
287 (0x4e00 << 16) | (0xc900 >> 2),
289 (0x5e00 << 16) | (0xc900 >> 2),
291 (0x6e00 << 16) | (0xc900 >> 2),
293 (0x7e00 << 16) | (0xc900 >> 2),
295 (0x8e00 << 16) | (0xc900 >> 2),
297 (0x9e00 << 16) | (0xc900 >> 2),
299 (0xae00 << 16) | (0xc900 >> 2),
301 (0xbe00 << 16) | (0xc900 >> 2),
303 (0x4e00 << 16) | (0xc904 >> 2),
305 (0x5e00 << 16) | (0xc904 >> 2),
307 (0x6e00 << 16) | (0xc904 >> 2),
309 (0x7e00 << 16) | (0xc904 >> 2),
311 (0x8e00 << 16) | (0xc904 >> 2),
313 (0x9e00 << 16) | (0xc904 >> 2),
315 (0xae00 << 16) | (0xc904 >> 2),
317 (0xbe00 << 16) | (0xc904 >> 2),
319 (0x4e00 << 16) | (0xc908 >> 2),
321 (0x5e00 << 16) | (0xc908 >> 2),
323 (0x6e00 << 16) | (0xc908 >> 2),
325 (0x7e00 << 16) | (0xc908 >> 2),
327 (0x8e00 << 16) | (0xc908 >> 2),
329 (0x9e00 << 16) | (0xc908 >> 2),
331 (0xae00 << 16) | (0xc908 >> 2),
333 (0xbe00 << 16) | (0xc908 >> 2),
335 (0x4e00 << 16) | (0xc90c >> 2),
337 (0x5e00 << 16) | (0xc90c >> 2),
339 (0x6e00 << 16) | (0xc90c >> 2),
341 (0x7e00 << 16) | (0xc90c >> 2),
343 (0x8e00 << 16) | (0xc90c >> 2),
345 (0x9e00 << 16) | (0xc90c >> 2),
347 (0xae00 << 16) | (0xc90c >> 2),
349 (0xbe00 << 16) | (0xc90c >> 2),
351 (0x4e00 << 16) | (0xc910 >> 2),
353 (0x5e00 << 16) | (0xc910 >> 2),
355 (0x6e00 << 16) | (0xc910 >> 2),
357 (0x7e00 << 16) | (0xc910 >> 2),
359 (0x8e00 << 16) | (0xc910 >> 2),
361 (0x9e00 << 16) | (0xc910 >> 2),
363 (0xae00 << 16) | (0xc910 >> 2),
365 (0xbe00 << 16) | (0xc910 >> 2),
367 (0x0e00 << 16) | (0xc99c >> 2),
369 (0x0e00 << 16) | (0x9834 >> 2),
371 (0x0000 << 16) | (0x30f00 >> 2),
373 (0x0001 << 16) | (0x30f00 >> 2),
375 (0x0000 << 16) | (0x30f04 >> 2),
377 (0x0001 << 16) | (0x30f04 >> 2),
379 (0x0000 << 16) | (0x30f08 >> 2),
381 (0x0001 << 16) | (0x30f08 >> 2),
383 (0x0000 << 16) | (0x30f0c >> 2),
385 (0x0001 << 16) | (0x30f0c >> 2),
387 (0x0600 << 16) | (0x9b7c >> 2),
389 (0x0e00 << 16) | (0x8a14 >> 2),
391 (0x0e00 << 16) | (0x8a18 >> 2),
393 (0x0600 << 16) | (0x30a00 >> 2),
395 (0x0e00 << 16) | (0x8bf0 >> 2),
397 (0x0e00 << 16) | (0x8bcc >> 2),
399 (0x0e00 << 16) | (0x8b24 >> 2),
401 (0x0e00 << 16) | (0x30a04 >> 2),
403 (0x0600 << 16) | (0x30a10 >> 2),
405 (0x0600 << 16) | (0x30a14 >> 2),
407 (0x0600 << 16) | (0x30a18 >> 2),
409 (0x0600 << 16) | (0x30a2c >> 2),
411 (0x0e00 << 16) | (0xc700 >> 2),
413 (0x0e00 << 16) | (0xc704 >> 2),
415 (0x0e00 << 16) | (0xc708 >> 2),
417 (0x0e00 << 16) | (0xc768 >> 2),
419 (0x0400 << 16) | (0xc770 >> 2),
421 (0x0400 << 16) | (0xc774 >> 2),
423 (0x0400 << 16) | (0xc778 >> 2),
425 (0x0400 << 16) | (0xc77c >> 2),
427 (0x0400 << 16) | (0xc780 >> 2),
429 (0x0400 << 16) | (0xc784 >> 2),
431 (0x0400 << 16) | (0xc788 >> 2),
433 (0x0400 << 16) | (0xc78c >> 2),
435 (0x0400 << 16) | (0xc798 >> 2),
437 (0x0400 << 16) | (0xc79c >> 2),
439 (0x0400 << 16) | (0xc7a0 >> 2),
441 (0x0400 << 16) | (0xc7a4 >> 2),
443 (0x0400 << 16) | (0xc7a8 >> 2),
445 (0x0400 << 16) | (0xc7ac >> 2),
447 (0x0400 << 16) | (0xc7b0 >> 2),
449 (0x0400 << 16) | (0xc7b4 >> 2),
451 (0x0e00 << 16) | (0x9100 >> 2),
453 (0x0e00 << 16) | (0x3c010 >> 2),
455 (0x0e00 << 16) | (0x92a8 >> 2),
457 (0x0e00 << 16) | (0x92ac >> 2),
459 (0x0e00 << 16) | (0x92b4 >> 2),
461 (0x0e00 << 16) | (0x92b8 >> 2),
463 (0x0e00 << 16) | (0x92bc >> 2),
465 (0x0e00 << 16) | (0x92c0 >> 2),
467 (0x0e00 << 16) | (0x92c4 >> 2),
469 (0x0e00 << 16) | (0x92c8 >> 2),
471 (0x0e00 << 16) | (0x92cc >> 2),
473 (0x0e00 << 16) | (0x92d0 >> 2),
475 (0x0e00 << 16) | (0x8c00 >> 2),
477 (0x0e00 << 16) | (0x8c04 >> 2),
479 (0x0e00 << 16) | (0x8c20 >> 2),
481 (0x0e00 << 16) | (0x8c38 >> 2),
483 (0x0e00 << 16) | (0x8c3c >> 2),
485 (0x0e00 << 16) | (0xae00 >> 2),
487 (0x0e00 << 16) | (0x9604 >> 2),
489 (0x0e00 << 16) | (0xac08 >> 2),
491 (0x0e00 << 16) | (0xac0c >> 2),
493 (0x0e00 << 16) | (0xac10 >> 2),
495 (0x0e00 << 16) | (0xac14 >> 2),
497 (0x0e00 << 16) | (0xac58 >> 2),
499 (0x0e00 << 16) | (0xac68 >> 2),
501 (0x0e00 << 16) | (0xac6c >> 2),
503 (0x0e00 << 16) | (0xac70 >> 2),
505 (0x0e00 << 16) | (0xac74 >> 2),
507 (0x0e00 << 16) | (0xac78 >> 2),
509 (0x0e00 << 16) | (0xac7c >> 2),
511 (0x0e00 << 16) | (0xac80 >> 2),
513 (0x0e00 << 16) | (0xac84 >> 2),
515 (0x0e00 << 16) | (0xac88 >> 2),
517 (0x0e00 << 16) | (0xac8c >> 2),
519 (0x0e00 << 16) | (0x970c >> 2),
521 (0x0e00 << 16) | (0x9714 >> 2),
523 (0x0e00 << 16) | (0x9718 >> 2),
525 (0x0e00 << 16) | (0x971c >> 2),
527 (0x0e00 << 16) | (0x31068 >> 2),
529 (0x4e00 << 16) | (0x31068 >> 2),
531 (0x5e00 << 16) | (0x31068 >> 2),
533 (0x6e00 << 16) | (0x31068 >> 2),
535 (0x7e00 << 16) | (0x31068 >> 2),
537 (0x8e00 << 16) | (0x31068 >> 2),
539 (0x9e00 << 16) | (0x31068 >> 2),
541 (0xae00 << 16) | (0x31068 >> 2),
543 (0xbe00 << 16) | (0x31068 >> 2),
545 (0x0e00 << 16) | (0xcd10 >> 2),
547 (0x0e00 << 16) | (0xcd14 >> 2),
549 (0x0e00 << 16) | (0x88b0 >> 2),
551 (0x0e00 << 16) | (0x88b4 >> 2),
553 (0x0e00 << 16) | (0x88b8 >> 2),
555 (0x0e00 << 16) | (0x88bc >> 2),
557 (0x0400 << 16) | (0x89c0 >> 2),
559 (0x0e00 << 16) | (0x88c4 >> 2),
561 (0x0e00 << 16) | (0x88c8 >> 2),
563 (0x0e00 << 16) | (0x88d0 >> 2),
565 (0x0e00 << 16) | (0x88d4 >> 2),
567 (0x0e00 << 16) | (0x88d8 >> 2),
569 (0x0e00 << 16) | (0x8980 >> 2),
571 (0x0e00 << 16) | (0x30938 >> 2),
573 (0x0e00 << 16) | (0x3093c >> 2),
575 (0x0e00 << 16) | (0x30940 >> 2),
577 (0x0e00 << 16) | (0x89a0 >> 2),
579 (0x0e00 << 16) | (0x30900 >> 2),
581 (0x0e00 << 16) | (0x30904 >> 2),
583 (0x0e00 << 16) | (0x89b4 >> 2),
585 (0x0e00 << 16) | (0x3c210 >> 2),
587 (0x0e00 << 16) | (0x3c214 >> 2),
589 (0x0e00 << 16) | (0x3c218 >> 2),
591 (0x0e00 << 16) | (0x8904 >> 2),
594 (0x0e00 << 16) | (0x8c28 >> 2),
595 (0x0e00 << 16) | (0x8c2c >> 2),
596 (0x0e00 << 16) | (0x8c30 >> 2),
597 (0x0e00 << 16) | (0x8c34 >> 2),
598 (0x0e00 << 16) | (0x9600 >> 2),
601 static const u32 kalindi_rlc_save_restore_register_list
[] =
603 (0x0e00 << 16) | (0xc12c >> 2),
605 (0x0e00 << 16) | (0xc140 >> 2),
607 (0x0e00 << 16) | (0xc150 >> 2),
609 (0x0e00 << 16) | (0xc15c >> 2),
611 (0x0e00 << 16) | (0xc168 >> 2),
613 (0x0e00 << 16) | (0xc170 >> 2),
615 (0x0e00 << 16) | (0xc204 >> 2),
617 (0x0e00 << 16) | (0xc2b4 >> 2),
619 (0x0e00 << 16) | (0xc2b8 >> 2),
621 (0x0e00 << 16) | (0xc2bc >> 2),
623 (0x0e00 << 16) | (0xc2c0 >> 2),
625 (0x0e00 << 16) | (0x8228 >> 2),
627 (0x0e00 << 16) | (0x829c >> 2),
629 (0x0e00 << 16) | (0x869c >> 2),
631 (0x0600 << 16) | (0x98f4 >> 2),
633 (0x0e00 << 16) | (0x98f8 >> 2),
635 (0x0e00 << 16) | (0x9900 >> 2),
637 (0x0e00 << 16) | (0xc260 >> 2),
639 (0x0e00 << 16) | (0x90e8 >> 2),
641 (0x0e00 << 16) | (0x3c000 >> 2),
643 (0x0e00 << 16) | (0x3c00c >> 2),
645 (0x0e00 << 16) | (0x8c1c >> 2),
647 (0x0e00 << 16) | (0x9700 >> 2),
649 (0x0e00 << 16) | (0xcd20 >> 2),
651 (0x4e00 << 16) | (0xcd20 >> 2),
653 (0x5e00 << 16) | (0xcd20 >> 2),
655 (0x6e00 << 16) | (0xcd20 >> 2),
657 (0x7e00 << 16) | (0xcd20 >> 2),
659 (0x0e00 << 16) | (0x89bc >> 2),
661 (0x0e00 << 16) | (0x8900 >> 2),
664 (0x0e00 << 16) | (0xc130 >> 2),
666 (0x0e00 << 16) | (0xc134 >> 2),
668 (0x0e00 << 16) | (0xc1fc >> 2),
670 (0x0e00 << 16) | (0xc208 >> 2),
672 (0x0e00 << 16) | (0xc264 >> 2),
674 (0x0e00 << 16) | (0xc268 >> 2),
676 (0x0e00 << 16) | (0xc26c >> 2),
678 (0x0e00 << 16) | (0xc270 >> 2),
680 (0x0e00 << 16) | (0xc274 >> 2),
682 (0x0e00 << 16) | (0xc28c >> 2),
684 (0x0e00 << 16) | (0xc290 >> 2),
686 (0x0e00 << 16) | (0xc294 >> 2),
688 (0x0e00 << 16) | (0xc298 >> 2),
690 (0x0e00 << 16) | (0xc2a0 >> 2),
692 (0x0e00 << 16) | (0xc2a4 >> 2),
694 (0x0e00 << 16) | (0xc2a8 >> 2),
696 (0x0e00 << 16) | (0xc2ac >> 2),
698 (0x0e00 << 16) | (0x301d0 >> 2),
700 (0x0e00 << 16) | (0x30238 >> 2),
702 (0x0e00 << 16) | (0x30250 >> 2),
704 (0x0e00 << 16) | (0x30254 >> 2),
706 (0x0e00 << 16) | (0x30258 >> 2),
708 (0x0e00 << 16) | (0x3025c >> 2),
710 (0x4e00 << 16) | (0xc900 >> 2),
712 (0x5e00 << 16) | (0xc900 >> 2),
714 (0x6e00 << 16) | (0xc900 >> 2),
716 (0x7e00 << 16) | (0xc900 >> 2),
718 (0x4e00 << 16) | (0xc904 >> 2),
720 (0x5e00 << 16) | (0xc904 >> 2),
722 (0x6e00 << 16) | (0xc904 >> 2),
724 (0x7e00 << 16) | (0xc904 >> 2),
726 (0x4e00 << 16) | (0xc908 >> 2),
728 (0x5e00 << 16) | (0xc908 >> 2),
730 (0x6e00 << 16) | (0xc908 >> 2),
732 (0x7e00 << 16) | (0xc908 >> 2),
734 (0x4e00 << 16) | (0xc90c >> 2),
736 (0x5e00 << 16) | (0xc90c >> 2),
738 (0x6e00 << 16) | (0xc90c >> 2),
740 (0x7e00 << 16) | (0xc90c >> 2),
742 (0x4e00 << 16) | (0xc910 >> 2),
744 (0x5e00 << 16) | (0xc910 >> 2),
746 (0x6e00 << 16) | (0xc910 >> 2),
748 (0x7e00 << 16) | (0xc910 >> 2),
750 (0x0e00 << 16) | (0xc99c >> 2),
752 (0x0e00 << 16) | (0x9834 >> 2),
754 (0x0000 << 16) | (0x30f00 >> 2),
756 (0x0000 << 16) | (0x30f04 >> 2),
758 (0x0000 << 16) | (0x30f08 >> 2),
760 (0x0000 << 16) | (0x30f0c >> 2),
762 (0x0600 << 16) | (0x9b7c >> 2),
764 (0x0e00 << 16) | (0x8a14 >> 2),
766 (0x0e00 << 16) | (0x8a18 >> 2),
768 (0x0600 << 16) | (0x30a00 >> 2),
770 (0x0e00 << 16) | (0x8bf0 >> 2),
772 (0x0e00 << 16) | (0x8bcc >> 2),
774 (0x0e00 << 16) | (0x8b24 >> 2),
776 (0x0e00 << 16) | (0x30a04 >> 2),
778 (0x0600 << 16) | (0x30a10 >> 2),
780 (0x0600 << 16) | (0x30a14 >> 2),
782 (0x0600 << 16) | (0x30a18 >> 2),
784 (0x0600 << 16) | (0x30a2c >> 2),
786 (0x0e00 << 16) | (0xc700 >> 2),
788 (0x0e00 << 16) | (0xc704 >> 2),
790 (0x0e00 << 16) | (0xc708 >> 2),
792 (0x0e00 << 16) | (0xc768 >> 2),
794 (0x0400 << 16) | (0xc770 >> 2),
796 (0x0400 << 16) | (0xc774 >> 2),
798 (0x0400 << 16) | (0xc798 >> 2),
800 (0x0400 << 16) | (0xc79c >> 2),
802 (0x0e00 << 16) | (0x9100 >> 2),
804 (0x0e00 << 16) | (0x3c010 >> 2),
806 (0x0e00 << 16) | (0x8c00 >> 2),
808 (0x0e00 << 16) | (0x8c04 >> 2),
810 (0x0e00 << 16) | (0x8c20 >> 2),
812 (0x0e00 << 16) | (0x8c38 >> 2),
814 (0x0e00 << 16) | (0x8c3c >> 2),
816 (0x0e00 << 16) | (0xae00 >> 2),
818 (0x0e00 << 16) | (0x9604 >> 2),
820 (0x0e00 << 16) | (0xac08 >> 2),
822 (0x0e00 << 16) | (0xac0c >> 2),
824 (0x0e00 << 16) | (0xac10 >> 2),
826 (0x0e00 << 16) | (0xac14 >> 2),
828 (0x0e00 << 16) | (0xac58 >> 2),
830 (0x0e00 << 16) | (0xac68 >> 2),
832 (0x0e00 << 16) | (0xac6c >> 2),
834 (0x0e00 << 16) | (0xac70 >> 2),
836 (0x0e00 << 16) | (0xac74 >> 2),
838 (0x0e00 << 16) | (0xac78 >> 2),
840 (0x0e00 << 16) | (0xac7c >> 2),
842 (0x0e00 << 16) | (0xac80 >> 2),
844 (0x0e00 << 16) | (0xac84 >> 2),
846 (0x0e00 << 16) | (0xac88 >> 2),
848 (0x0e00 << 16) | (0xac8c >> 2),
850 (0x0e00 << 16) | (0x970c >> 2),
852 (0x0e00 << 16) | (0x9714 >> 2),
854 (0x0e00 << 16) | (0x9718 >> 2),
856 (0x0e00 << 16) | (0x971c >> 2),
858 (0x0e00 << 16) | (0x31068 >> 2),
860 (0x4e00 << 16) | (0x31068 >> 2),
862 (0x5e00 << 16) | (0x31068 >> 2),
864 (0x6e00 << 16) | (0x31068 >> 2),
866 (0x7e00 << 16) | (0x31068 >> 2),
868 (0x0e00 << 16) | (0xcd10 >> 2),
870 (0x0e00 << 16) | (0xcd14 >> 2),
872 (0x0e00 << 16) | (0x88b0 >> 2),
874 (0x0e00 << 16) | (0x88b4 >> 2),
876 (0x0e00 << 16) | (0x88b8 >> 2),
878 (0x0e00 << 16) | (0x88bc >> 2),
880 (0x0400 << 16) | (0x89c0 >> 2),
882 (0x0e00 << 16) | (0x88c4 >> 2),
884 (0x0e00 << 16) | (0x88c8 >> 2),
886 (0x0e00 << 16) | (0x88d0 >> 2),
888 (0x0e00 << 16) | (0x88d4 >> 2),
890 (0x0e00 << 16) | (0x88d8 >> 2),
892 (0x0e00 << 16) | (0x8980 >> 2),
894 (0x0e00 << 16) | (0x30938 >> 2),
896 (0x0e00 << 16) | (0x3093c >> 2),
898 (0x0e00 << 16) | (0x30940 >> 2),
900 (0x0e00 << 16) | (0x89a0 >> 2),
902 (0x0e00 << 16) | (0x30900 >> 2),
904 (0x0e00 << 16) | (0x30904 >> 2),
906 (0x0e00 << 16) | (0x89b4 >> 2),
908 (0x0e00 << 16) | (0x3e1fc >> 2),
910 (0x0e00 << 16) | (0x3c210 >> 2),
912 (0x0e00 << 16) | (0x3c214 >> 2),
914 (0x0e00 << 16) | (0x3c218 >> 2),
916 (0x0e00 << 16) | (0x8904 >> 2),
919 (0x0e00 << 16) | (0x8c28 >> 2),
920 (0x0e00 << 16) | (0x8c2c >> 2),
921 (0x0e00 << 16) | (0x8c30 >> 2),
922 (0x0e00 << 16) | (0x8c34 >> 2),
923 (0x0e00 << 16) | (0x9600 >> 2),
926 static const u32 bonaire_golden_spm_registers
[] =
928 0x30800, 0xe0ffffff, 0xe0000000
931 static const u32 bonaire_golden_common_registers
[] =
933 0xc770, 0xffffffff, 0x00000800,
934 0xc774, 0xffffffff, 0x00000800,
935 0xc798, 0xffffffff, 0x00007fbf,
936 0xc79c, 0xffffffff, 0x00007faf
939 static const u32 bonaire_golden_registers
[] =
941 0x3354, 0x00000333, 0x00000333,
942 0x3350, 0x000c0fc0, 0x00040200,
943 0x9a10, 0x00010000, 0x00058208,
944 0x3c000, 0xffff1fff, 0x00140000,
945 0x3c200, 0xfdfc0fff, 0x00000100,
946 0x3c234, 0x40000000, 0x40000200,
947 0x9830, 0xffffffff, 0x00000000,
948 0x9834, 0xf00fffff, 0x00000400,
949 0x9838, 0x0002021c, 0x00020200,
950 0xc78, 0x00000080, 0x00000000,
951 0x5bb0, 0x000000f0, 0x00000070,
952 0x5bc0, 0xf0311fff, 0x80300000,
953 0x98f8, 0x73773777, 0x12010001,
954 0x350c, 0x00810000, 0x408af000,
955 0x7030, 0x31000111, 0x00000011,
956 0x2f48, 0x73773777, 0x12010001,
957 0x220c, 0x00007fb6, 0x0021a1b1,
958 0x2210, 0x00007fb6, 0x002021b1,
959 0x2180, 0x00007fb6, 0x00002191,
960 0x2218, 0x00007fb6, 0x002121b1,
961 0x221c, 0x00007fb6, 0x002021b1,
962 0x21dc, 0x00007fb6, 0x00002191,
963 0x21e0, 0x00007fb6, 0x00002191,
964 0x3628, 0x0000003f, 0x0000000a,
965 0x362c, 0x0000003f, 0x0000000a,
966 0x2ae4, 0x00073ffe, 0x000022a2,
967 0x240c, 0x000007ff, 0x00000000,
968 0x8a14, 0xf000003f, 0x00000007,
969 0x8bf0, 0x00002001, 0x00000001,
970 0x8b24, 0xffffffff, 0x00ffffff,
971 0x30a04, 0x0000ff0f, 0x00000000,
972 0x28a4c, 0x07ffffff, 0x06000000,
973 0x4d8, 0x00000fff, 0x00000100,
974 0x3e78, 0x00000001, 0x00000002,
975 0x9100, 0x03000000, 0x0362c688,
976 0x8c00, 0x000000ff, 0x00000001,
977 0xe40, 0x00001fff, 0x00001fff,
978 0x9060, 0x0000007f, 0x00000020,
979 0x9508, 0x00010000, 0x00010000,
980 0xac14, 0x000003ff, 0x000000f3,
981 0xac0c, 0xffffffff, 0x00001032
984 static const u32 bonaire_mgcg_cgcg_init
[] =
986 0xc420, 0xffffffff, 0xfffffffc,
987 0x30800, 0xffffffff, 0xe0000000,
988 0x3c2a0, 0xffffffff, 0x00000100,
989 0x3c208, 0xffffffff, 0x00000100,
990 0x3c2c0, 0xffffffff, 0xc0000100,
991 0x3c2c8, 0xffffffff, 0xc0000100,
992 0x3c2c4, 0xffffffff, 0xc0000100,
993 0x55e4, 0xffffffff, 0x00600100,
994 0x3c280, 0xffffffff, 0x00000100,
995 0x3c214, 0xffffffff, 0x06000100,
996 0x3c220, 0xffffffff, 0x00000100,
997 0x3c218, 0xffffffff, 0x06000100,
998 0x3c204, 0xffffffff, 0x00000100,
999 0x3c2e0, 0xffffffff, 0x00000100,
1000 0x3c224, 0xffffffff, 0x00000100,
1001 0x3c200, 0xffffffff, 0x00000100,
1002 0x3c230, 0xffffffff, 0x00000100,
1003 0x3c234, 0xffffffff, 0x00000100,
1004 0x3c250, 0xffffffff, 0x00000100,
1005 0x3c254, 0xffffffff, 0x00000100,
1006 0x3c258, 0xffffffff, 0x00000100,
1007 0x3c25c, 0xffffffff, 0x00000100,
1008 0x3c260, 0xffffffff, 0x00000100,
1009 0x3c27c, 0xffffffff, 0x00000100,
1010 0x3c278, 0xffffffff, 0x00000100,
1011 0x3c210, 0xffffffff, 0x06000100,
1012 0x3c290, 0xffffffff, 0x00000100,
1013 0x3c274, 0xffffffff, 0x00000100,
1014 0x3c2b4, 0xffffffff, 0x00000100,
1015 0x3c2b0, 0xffffffff, 0x00000100,
1016 0x3c270, 0xffffffff, 0x00000100,
1017 0x30800, 0xffffffff, 0xe0000000,
1018 0x3c020, 0xffffffff, 0x00010000,
1019 0x3c024, 0xffffffff, 0x00030002,
1020 0x3c028, 0xffffffff, 0x00040007,
1021 0x3c02c, 0xffffffff, 0x00060005,
1022 0x3c030, 0xffffffff, 0x00090008,
1023 0x3c034, 0xffffffff, 0x00010000,
1024 0x3c038, 0xffffffff, 0x00030002,
1025 0x3c03c, 0xffffffff, 0x00040007,
1026 0x3c040, 0xffffffff, 0x00060005,
1027 0x3c044, 0xffffffff, 0x00090008,
1028 0x3c048, 0xffffffff, 0x00010000,
1029 0x3c04c, 0xffffffff, 0x00030002,
1030 0x3c050, 0xffffffff, 0x00040007,
1031 0x3c054, 0xffffffff, 0x00060005,
1032 0x3c058, 0xffffffff, 0x00090008,
1033 0x3c05c, 0xffffffff, 0x00010000,
1034 0x3c060, 0xffffffff, 0x00030002,
1035 0x3c064, 0xffffffff, 0x00040007,
1036 0x3c068, 0xffffffff, 0x00060005,
1037 0x3c06c, 0xffffffff, 0x00090008,
1038 0x3c070, 0xffffffff, 0x00010000,
1039 0x3c074, 0xffffffff, 0x00030002,
1040 0x3c078, 0xffffffff, 0x00040007,
1041 0x3c07c, 0xffffffff, 0x00060005,
1042 0x3c080, 0xffffffff, 0x00090008,
1043 0x3c084, 0xffffffff, 0x00010000,
1044 0x3c088, 0xffffffff, 0x00030002,
1045 0x3c08c, 0xffffffff, 0x00040007,
1046 0x3c090, 0xffffffff, 0x00060005,
1047 0x3c094, 0xffffffff, 0x00090008,
1048 0x3c098, 0xffffffff, 0x00010000,
1049 0x3c09c, 0xffffffff, 0x00030002,
1050 0x3c0a0, 0xffffffff, 0x00040007,
1051 0x3c0a4, 0xffffffff, 0x00060005,
1052 0x3c0a8, 0xffffffff, 0x00090008,
1053 0x3c000, 0xffffffff, 0x96e00200,
1054 0x8708, 0xffffffff, 0x00900100,
1055 0xc424, 0xffffffff, 0x0020003f,
1056 0x38, 0xffffffff, 0x0140001c,
1057 0x3c, 0x000f0000, 0x000f0000,
1058 0x220, 0xffffffff, 0xC060000C,
1059 0x224, 0xc0000fff, 0x00000100,
1060 0xf90, 0xffffffff, 0x00000100,
1061 0xf98, 0x00000101, 0x00000000,
1062 0x20a8, 0xffffffff, 0x00000104,
1063 0x55e4, 0xff000fff, 0x00000100,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd00c, 0xff000ff0, 0x00000100,
1067 0xd80c, 0xff000ff0, 0x00000100
1070 static const u32 spectre_golden_spm_registers
[] =
1072 0x30800, 0xe0ffffff, 0xe0000000
1075 static const u32 spectre_golden_common_registers
[] =
1077 0xc770, 0xffffffff, 0x00000800,
1078 0xc774, 0xffffffff, 0x00000800,
1079 0xc798, 0xffffffff, 0x00007fbf,
1080 0xc79c, 0xffffffff, 0x00007faf
1083 static const u32 spectre_golden_registers
[] =
1085 0x3c000, 0xffff1fff, 0x96940200,
1086 0x3c00c, 0xffff0001, 0xff000000,
1087 0x3c200, 0xfffc0fff, 0x00000100,
1088 0x6ed8, 0x00010101, 0x00010000,
1089 0x9834, 0xf00fffff, 0x00000400,
1090 0x9838, 0xfffffffc, 0x00020200,
1091 0x5bb0, 0x000000f0, 0x00000070,
1092 0x5bc0, 0xf0311fff, 0x80300000,
1093 0x98f8, 0x73773777, 0x12010001,
1094 0x9b7c, 0x00ff0000, 0x00fc0000,
1095 0x2f48, 0x73773777, 0x12010001,
1096 0x8a14, 0xf000003f, 0x00000007,
1097 0x8b24, 0xffffffff, 0x00ffffff,
1098 0x28350, 0x3f3f3fff, 0x00000082,
1099 0x28354, 0x0000003f, 0x00000000,
1100 0x3e78, 0x00000001, 0x00000002,
1101 0x913c, 0xffff03df, 0x00000004,
1102 0xc768, 0x00000008, 0x00000008,
1103 0x8c00, 0x000008ff, 0x00000800,
1104 0x9508, 0x00010000, 0x00010000,
1105 0xac0c, 0xffffffff, 0x54763210,
1106 0x214f8, 0x01ff01ff, 0x00000002,
1107 0x21498, 0x007ff800, 0x00200000,
1108 0x2015c, 0xffffffff, 0x00000f40,
1109 0x30934, 0xffffffff, 0x00000001
1112 static const u32 spectre_mgcg_cgcg_init
[] =
1114 0xc420, 0xffffffff, 0xfffffffc,
1115 0x30800, 0xffffffff, 0xe0000000,
1116 0x3c2a0, 0xffffffff, 0x00000100,
1117 0x3c208, 0xffffffff, 0x00000100,
1118 0x3c2c0, 0xffffffff, 0x00000100,
1119 0x3c2c8, 0xffffffff, 0x00000100,
1120 0x3c2c4, 0xffffffff, 0x00000100,
1121 0x55e4, 0xffffffff, 0x00600100,
1122 0x3c280, 0xffffffff, 0x00000100,
1123 0x3c214, 0xffffffff, 0x06000100,
1124 0x3c220, 0xffffffff, 0x00000100,
1125 0x3c218, 0xffffffff, 0x06000100,
1126 0x3c204, 0xffffffff, 0x00000100,
1127 0x3c2e0, 0xffffffff, 0x00000100,
1128 0x3c224, 0xffffffff, 0x00000100,
1129 0x3c200, 0xffffffff, 0x00000100,
1130 0x3c230, 0xffffffff, 0x00000100,
1131 0x3c234, 0xffffffff, 0x00000100,
1132 0x3c250, 0xffffffff, 0x00000100,
1133 0x3c254, 0xffffffff, 0x00000100,
1134 0x3c258, 0xffffffff, 0x00000100,
1135 0x3c25c, 0xffffffff, 0x00000100,
1136 0x3c260, 0xffffffff, 0x00000100,
1137 0x3c27c, 0xffffffff, 0x00000100,
1138 0x3c278, 0xffffffff, 0x00000100,
1139 0x3c210, 0xffffffff, 0x06000100,
1140 0x3c290, 0xffffffff, 0x00000100,
1141 0x3c274, 0xffffffff, 0x00000100,
1142 0x3c2b4, 0xffffffff, 0x00000100,
1143 0x3c2b0, 0xffffffff, 0x00000100,
1144 0x3c270, 0xffffffff, 0x00000100,
1145 0x30800, 0xffffffff, 0xe0000000,
1146 0x3c020, 0xffffffff, 0x00010000,
1147 0x3c024, 0xffffffff, 0x00030002,
1148 0x3c028, 0xffffffff, 0x00040007,
1149 0x3c02c, 0xffffffff, 0x00060005,
1150 0x3c030, 0xffffffff, 0x00090008,
1151 0x3c034, 0xffffffff, 0x00010000,
1152 0x3c038, 0xffffffff, 0x00030002,
1153 0x3c03c, 0xffffffff, 0x00040007,
1154 0x3c040, 0xffffffff, 0x00060005,
1155 0x3c044, 0xffffffff, 0x00090008,
1156 0x3c048, 0xffffffff, 0x00010000,
1157 0x3c04c, 0xffffffff, 0x00030002,
1158 0x3c050, 0xffffffff, 0x00040007,
1159 0x3c054, 0xffffffff, 0x00060005,
1160 0x3c058, 0xffffffff, 0x00090008,
1161 0x3c05c, 0xffffffff, 0x00010000,
1162 0x3c060, 0xffffffff, 0x00030002,
1163 0x3c064, 0xffffffff, 0x00040007,
1164 0x3c068, 0xffffffff, 0x00060005,
1165 0x3c06c, 0xffffffff, 0x00090008,
1166 0x3c070, 0xffffffff, 0x00010000,
1167 0x3c074, 0xffffffff, 0x00030002,
1168 0x3c078, 0xffffffff, 0x00040007,
1169 0x3c07c, 0xffffffff, 0x00060005,
1170 0x3c080, 0xffffffff, 0x00090008,
1171 0x3c084, 0xffffffff, 0x00010000,
1172 0x3c088, 0xffffffff, 0x00030002,
1173 0x3c08c, 0xffffffff, 0x00040007,
1174 0x3c090, 0xffffffff, 0x00060005,
1175 0x3c094, 0xffffffff, 0x00090008,
1176 0x3c098, 0xffffffff, 0x00010000,
1177 0x3c09c, 0xffffffff, 0x00030002,
1178 0x3c0a0, 0xffffffff, 0x00040007,
1179 0x3c0a4, 0xffffffff, 0x00060005,
1180 0x3c0a8, 0xffffffff, 0x00090008,
1181 0x3c0ac, 0xffffffff, 0x00010000,
1182 0x3c0b0, 0xffffffff, 0x00030002,
1183 0x3c0b4, 0xffffffff, 0x00040007,
1184 0x3c0b8, 0xffffffff, 0x00060005,
1185 0x3c0bc, 0xffffffff, 0x00090008,
1186 0x3c000, 0xffffffff, 0x96e00200,
1187 0x8708, 0xffffffff, 0x00900100,
1188 0xc424, 0xffffffff, 0x0020003f,
1189 0x38, 0xffffffff, 0x0140001c,
1190 0x3c, 0x000f0000, 0x000f0000,
1191 0x220, 0xffffffff, 0xC060000C,
1192 0x224, 0xc0000fff, 0x00000100,
1193 0xf90, 0xffffffff, 0x00000100,
1194 0xf98, 0x00000101, 0x00000000,
1195 0x20a8, 0xffffffff, 0x00000104,
1196 0x55e4, 0xff000fff, 0x00000100,
1197 0x30cc, 0xc0000fff, 0x00000104,
1198 0xc1e4, 0x00000001, 0x00000001,
1199 0xd00c, 0xff000ff0, 0x00000100,
1200 0xd80c, 0xff000ff0, 0x00000100
1203 static const u32 kalindi_golden_spm_registers
[] =
1205 0x30800, 0xe0ffffff, 0xe0000000
1208 static const u32 kalindi_golden_common_registers
[] =
1210 0xc770, 0xffffffff, 0x00000800,
1211 0xc774, 0xffffffff, 0x00000800,
1212 0xc798, 0xffffffff, 0x00007fbf,
1213 0xc79c, 0xffffffff, 0x00007faf
1216 static const u32 kalindi_golden_registers
[] =
1218 0x3c000, 0xffffdfff, 0x6e944040,
1219 0x55e4, 0xff607fff, 0xfc000100,
1220 0x3c220, 0xff000fff, 0x00000100,
1221 0x3c224, 0xff000fff, 0x00000100,
1222 0x3c200, 0xfffc0fff, 0x00000100,
1223 0x6ed8, 0x00010101, 0x00010000,
1224 0x9830, 0xffffffff, 0x00000000,
1225 0x9834, 0xf00fffff, 0x00000400,
1226 0x5bb0, 0x000000f0, 0x00000070,
1227 0x5bc0, 0xf0311fff, 0x80300000,
1228 0x98f8, 0x73773777, 0x12010001,
1229 0x98fc, 0xffffffff, 0x00000010,
1230 0x9b7c, 0x00ff0000, 0x00fc0000,
1231 0x8030, 0x00001f0f, 0x0000100a,
1232 0x2f48, 0x73773777, 0x12010001,
1233 0x2408, 0x000fffff, 0x000c007f,
1234 0x8a14, 0xf000003f, 0x00000007,
1235 0x8b24, 0x3fff3fff, 0x00ffcfff,
1236 0x30a04, 0x0000ff0f, 0x00000000,
1237 0x28a4c, 0x07ffffff, 0x06000000,
1238 0x4d8, 0x00000fff, 0x00000100,
1239 0x3e78, 0x00000001, 0x00000002,
1240 0xc768, 0x00000008, 0x00000008,
1241 0x8c00, 0x000000ff, 0x00000003,
1242 0x214f8, 0x01ff01ff, 0x00000002,
1243 0x21498, 0x007ff800, 0x00200000,
1244 0x2015c, 0xffffffff, 0x00000f40,
1245 0x88c4, 0x001f3ae3, 0x00000082,
1246 0x88d4, 0x0000001f, 0x00000010,
1247 0x30934, 0xffffffff, 0x00000000
1250 static const u32 kalindi_mgcg_cgcg_init
[] =
1252 0xc420, 0xffffffff, 0xfffffffc,
1253 0x30800, 0xffffffff, 0xe0000000,
1254 0x3c2a0, 0xffffffff, 0x00000100,
1255 0x3c208, 0xffffffff, 0x00000100,
1256 0x3c2c0, 0xffffffff, 0x00000100,
1257 0x3c2c8, 0xffffffff, 0x00000100,
1258 0x3c2c4, 0xffffffff, 0x00000100,
1259 0x55e4, 0xffffffff, 0x00600100,
1260 0x3c280, 0xffffffff, 0x00000100,
1261 0x3c214, 0xffffffff, 0x06000100,
1262 0x3c220, 0xffffffff, 0x00000100,
1263 0x3c218, 0xffffffff, 0x06000100,
1264 0x3c204, 0xffffffff, 0x00000100,
1265 0x3c2e0, 0xffffffff, 0x00000100,
1266 0x3c224, 0xffffffff, 0x00000100,
1267 0x3c200, 0xffffffff, 0x00000100,
1268 0x3c230, 0xffffffff, 0x00000100,
1269 0x3c234, 0xffffffff, 0x00000100,
1270 0x3c250, 0xffffffff, 0x00000100,
1271 0x3c254, 0xffffffff, 0x00000100,
1272 0x3c258, 0xffffffff, 0x00000100,
1273 0x3c25c, 0xffffffff, 0x00000100,
1274 0x3c260, 0xffffffff, 0x00000100,
1275 0x3c27c, 0xffffffff, 0x00000100,
1276 0x3c278, 0xffffffff, 0x00000100,
1277 0x3c210, 0xffffffff, 0x06000100,
1278 0x3c290, 0xffffffff, 0x00000100,
1279 0x3c274, 0xffffffff, 0x00000100,
1280 0x3c2b4, 0xffffffff, 0x00000100,
1281 0x3c2b0, 0xffffffff, 0x00000100,
1282 0x3c270, 0xffffffff, 0x00000100,
1283 0x30800, 0xffffffff, 0xe0000000,
1284 0x3c020, 0xffffffff, 0x00010000,
1285 0x3c024, 0xffffffff, 0x00030002,
1286 0x3c028, 0xffffffff, 0x00040007,
1287 0x3c02c, 0xffffffff, 0x00060005,
1288 0x3c030, 0xffffffff, 0x00090008,
1289 0x3c034, 0xffffffff, 0x00010000,
1290 0x3c038, 0xffffffff, 0x00030002,
1291 0x3c03c, 0xffffffff, 0x00040007,
1292 0x3c040, 0xffffffff, 0x00060005,
1293 0x3c044, 0xffffffff, 0x00090008,
1294 0x3c000, 0xffffffff, 0x96e00200,
1295 0x8708, 0xffffffff, 0x00900100,
1296 0xc424, 0xffffffff, 0x0020003f,
1297 0x38, 0xffffffff, 0x0140001c,
1298 0x3c, 0x000f0000, 0x000f0000,
1299 0x220, 0xffffffff, 0xC060000C,
1300 0x224, 0xc0000fff, 0x00000100,
1301 0x20a8, 0xffffffff, 0x00000104,
1302 0x55e4, 0xff000fff, 0x00000100,
1303 0x30cc, 0xc0000fff, 0x00000104,
1304 0xc1e4, 0x00000001, 0x00000001,
1305 0xd00c, 0xff000ff0, 0x00000100,
1306 0xd80c, 0xff000ff0, 0x00000100
1309 static const u32 hawaii_golden_spm_registers
[] =
1311 0x30800, 0xe0ffffff, 0xe0000000
1314 static const u32 hawaii_golden_common_registers
[] =
1316 0x30800, 0xffffffff, 0xe0000000,
1317 0x28350, 0xffffffff, 0x3a00161a,
1318 0x28354, 0xffffffff, 0x0000002e,
1319 0x9a10, 0xffffffff, 0x00018208,
1320 0x98f8, 0xffffffff, 0x12011003
1323 static const u32 hawaii_golden_registers
[] =
1325 0x3354, 0x00000333, 0x00000333,
1326 0x9a10, 0x00010000, 0x00058208,
1327 0x9830, 0xffffffff, 0x00000000,
1328 0x9834, 0xf00fffff, 0x00000400,
1329 0x9838, 0x0002021c, 0x00020200,
1330 0xc78, 0x00000080, 0x00000000,
1331 0x5bb0, 0x000000f0, 0x00000070,
1332 0x5bc0, 0xf0311fff, 0x80300000,
1333 0x350c, 0x00810000, 0x408af000,
1334 0x7030, 0x31000111, 0x00000011,
1335 0x2f48, 0x73773777, 0x12010001,
1336 0x2120, 0x0000007f, 0x0000001b,
1337 0x21dc, 0x00007fb6, 0x00002191,
1338 0x3628, 0x0000003f, 0x0000000a,
1339 0x362c, 0x0000003f, 0x0000000a,
1340 0x2ae4, 0x00073ffe, 0x000022a2,
1341 0x240c, 0x000007ff, 0x00000000,
1342 0x8bf0, 0x00002001, 0x00000001,
1343 0x8b24, 0xffffffff, 0x00ffffff,
1344 0x30a04, 0x0000ff0f, 0x00000000,
1345 0x28a4c, 0x07ffffff, 0x06000000,
1346 0x3e78, 0x00000001, 0x00000002,
1347 0xc768, 0x00000008, 0x00000008,
1348 0xc770, 0x00000f00, 0x00000800,
1349 0xc774, 0x00000f00, 0x00000800,
1350 0xc798, 0x00ffffff, 0x00ff7fbf,
1351 0xc79c, 0x00ffffff, 0x00ff7faf,
1352 0x8c00, 0x000000ff, 0x00000800,
1353 0xe40, 0x00001fff, 0x00001fff,
1354 0x9060, 0x0000007f, 0x00000020,
1355 0x9508, 0x00010000, 0x00010000,
1356 0xae00, 0x00100000, 0x000ff07c,
1357 0xac14, 0x000003ff, 0x0000000f,
1358 0xac10, 0xffffffff, 0x7564fdec,
1359 0xac0c, 0xffffffff, 0x3120b9a8,
1360 0xac08, 0x20000000, 0x0f9c0000
1363 static const u32 hawaii_mgcg_cgcg_init
[] =
1365 0xc420, 0xffffffff, 0xfffffffd,
1366 0x30800, 0xffffffff, 0xe0000000,
1367 0x3c2a0, 0xffffffff, 0x00000100,
1368 0x3c208, 0xffffffff, 0x00000100,
1369 0x3c2c0, 0xffffffff, 0x00000100,
1370 0x3c2c8, 0xffffffff, 0x00000100,
1371 0x3c2c4, 0xffffffff, 0x00000100,
1372 0x55e4, 0xffffffff, 0x00200100,
1373 0x3c280, 0xffffffff, 0x00000100,
1374 0x3c214, 0xffffffff, 0x06000100,
1375 0x3c220, 0xffffffff, 0x00000100,
1376 0x3c218, 0xffffffff, 0x06000100,
1377 0x3c204, 0xffffffff, 0x00000100,
1378 0x3c2e0, 0xffffffff, 0x00000100,
1379 0x3c224, 0xffffffff, 0x00000100,
1380 0x3c200, 0xffffffff, 0x00000100,
1381 0x3c230, 0xffffffff, 0x00000100,
1382 0x3c234, 0xffffffff, 0x00000100,
1383 0x3c250, 0xffffffff, 0x00000100,
1384 0x3c254, 0xffffffff, 0x00000100,
1385 0x3c258, 0xffffffff, 0x00000100,
1386 0x3c25c, 0xffffffff, 0x00000100,
1387 0x3c260, 0xffffffff, 0x00000100,
1388 0x3c27c, 0xffffffff, 0x00000100,
1389 0x3c278, 0xffffffff, 0x00000100,
1390 0x3c210, 0xffffffff, 0x06000100,
1391 0x3c290, 0xffffffff, 0x00000100,
1392 0x3c274, 0xffffffff, 0x00000100,
1393 0x3c2b4, 0xffffffff, 0x00000100,
1394 0x3c2b0, 0xffffffff, 0x00000100,
1395 0x3c270, 0xffffffff, 0x00000100,
1396 0x30800, 0xffffffff, 0xe0000000,
1397 0x3c020, 0xffffffff, 0x00010000,
1398 0x3c024, 0xffffffff, 0x00030002,
1399 0x3c028, 0xffffffff, 0x00040007,
1400 0x3c02c, 0xffffffff, 0x00060005,
1401 0x3c030, 0xffffffff, 0x00090008,
1402 0x3c034, 0xffffffff, 0x00010000,
1403 0x3c038, 0xffffffff, 0x00030002,
1404 0x3c03c, 0xffffffff, 0x00040007,
1405 0x3c040, 0xffffffff, 0x00060005,
1406 0x3c044, 0xffffffff, 0x00090008,
1407 0x3c048, 0xffffffff, 0x00010000,
1408 0x3c04c, 0xffffffff, 0x00030002,
1409 0x3c050, 0xffffffff, 0x00040007,
1410 0x3c054, 0xffffffff, 0x00060005,
1411 0x3c058, 0xffffffff, 0x00090008,
1412 0x3c05c, 0xffffffff, 0x00010000,
1413 0x3c060, 0xffffffff, 0x00030002,
1414 0x3c064, 0xffffffff, 0x00040007,
1415 0x3c068, 0xffffffff, 0x00060005,
1416 0x3c06c, 0xffffffff, 0x00090008,
1417 0x3c070, 0xffffffff, 0x00010000,
1418 0x3c074, 0xffffffff, 0x00030002,
1419 0x3c078, 0xffffffff, 0x00040007,
1420 0x3c07c, 0xffffffff, 0x00060005,
1421 0x3c080, 0xffffffff, 0x00090008,
1422 0x3c084, 0xffffffff, 0x00010000,
1423 0x3c088, 0xffffffff, 0x00030002,
1424 0x3c08c, 0xffffffff, 0x00040007,
1425 0x3c090, 0xffffffff, 0x00060005,
1426 0x3c094, 0xffffffff, 0x00090008,
1427 0x3c098, 0xffffffff, 0x00010000,
1428 0x3c09c, 0xffffffff, 0x00030002,
1429 0x3c0a0, 0xffffffff, 0x00040007,
1430 0x3c0a4, 0xffffffff, 0x00060005,
1431 0x3c0a8, 0xffffffff, 0x00090008,
1432 0x3c0ac, 0xffffffff, 0x00010000,
1433 0x3c0b0, 0xffffffff, 0x00030002,
1434 0x3c0b4, 0xffffffff, 0x00040007,
1435 0x3c0b8, 0xffffffff, 0x00060005,
1436 0x3c0bc, 0xffffffff, 0x00090008,
1437 0x3c0c0, 0xffffffff, 0x00010000,
1438 0x3c0c4, 0xffffffff, 0x00030002,
1439 0x3c0c8, 0xffffffff, 0x00040007,
1440 0x3c0cc, 0xffffffff, 0x00060005,
1441 0x3c0d0, 0xffffffff, 0x00090008,
1442 0x3c0d4, 0xffffffff, 0x00010000,
1443 0x3c0d8, 0xffffffff, 0x00030002,
1444 0x3c0dc, 0xffffffff, 0x00040007,
1445 0x3c0e0, 0xffffffff, 0x00060005,
1446 0x3c0e4, 0xffffffff, 0x00090008,
1447 0x3c0e8, 0xffffffff, 0x00010000,
1448 0x3c0ec, 0xffffffff, 0x00030002,
1449 0x3c0f0, 0xffffffff, 0x00040007,
1450 0x3c0f4, 0xffffffff, 0x00060005,
1451 0x3c0f8, 0xffffffff, 0x00090008,
1452 0xc318, 0xffffffff, 0x00020200,
1453 0x3350, 0xffffffff, 0x00000200,
1454 0x15c0, 0xffffffff, 0x00000400,
1455 0x55e8, 0xffffffff, 0x00000000,
1456 0x2f50, 0xffffffff, 0x00000902,
1457 0x3c000, 0xffffffff, 0x96940200,
1458 0x8708, 0xffffffff, 0x00900100,
1459 0xc424, 0xffffffff, 0x0020003f,
1460 0x38, 0xffffffff, 0x0140001c,
1461 0x3c, 0x000f0000, 0x000f0000,
1462 0x220, 0xffffffff, 0xc060000c,
1463 0x224, 0xc0000fff, 0x00000100,
1464 0xf90, 0xffffffff, 0x00000100,
1465 0xf98, 0x00000101, 0x00000000,
1466 0x20a8, 0xffffffff, 0x00000104,
1467 0x55e4, 0xff000fff, 0x00000100,
1468 0x30cc, 0xc0000fff, 0x00000104,
1469 0xc1e4, 0x00000001, 0x00000001,
1470 0xd00c, 0xff000ff0, 0x00000100,
1471 0xd80c, 0xff000ff0, 0x00000100
1474 static void cik_init_golden_registers(struct radeon_device
*rdev
)
1476 switch (rdev
->family
) {
1478 radeon_program_register_sequence(rdev
,
1479 bonaire_mgcg_cgcg_init
,
1480 (const u32
)ARRAY_SIZE(bonaire_mgcg_cgcg_init
));
1481 radeon_program_register_sequence(rdev
,
1482 bonaire_golden_registers
,
1483 (const u32
)ARRAY_SIZE(bonaire_golden_registers
));
1484 radeon_program_register_sequence(rdev
,
1485 bonaire_golden_common_registers
,
1486 (const u32
)ARRAY_SIZE(bonaire_golden_common_registers
));
1487 radeon_program_register_sequence(rdev
,
1488 bonaire_golden_spm_registers
,
1489 (const u32
)ARRAY_SIZE(bonaire_golden_spm_registers
));
1492 radeon_program_register_sequence(rdev
,
1493 kalindi_mgcg_cgcg_init
,
1494 (const u32
)ARRAY_SIZE(kalindi_mgcg_cgcg_init
));
1495 radeon_program_register_sequence(rdev
,
1496 kalindi_golden_registers
,
1497 (const u32
)ARRAY_SIZE(kalindi_golden_registers
));
1498 radeon_program_register_sequence(rdev
,
1499 kalindi_golden_common_registers
,
1500 (const u32
)ARRAY_SIZE(kalindi_golden_common_registers
));
1501 radeon_program_register_sequence(rdev
,
1502 kalindi_golden_spm_registers
,
1503 (const u32
)ARRAY_SIZE(kalindi_golden_spm_registers
));
1506 radeon_program_register_sequence(rdev
,
1507 spectre_mgcg_cgcg_init
,
1508 (const u32
)ARRAY_SIZE(spectre_mgcg_cgcg_init
));
1509 radeon_program_register_sequence(rdev
,
1510 spectre_golden_registers
,
1511 (const u32
)ARRAY_SIZE(spectre_golden_registers
));
1512 radeon_program_register_sequence(rdev
,
1513 spectre_golden_common_registers
,
1514 (const u32
)ARRAY_SIZE(spectre_golden_common_registers
));
1515 radeon_program_register_sequence(rdev
,
1516 spectre_golden_spm_registers
,
1517 (const u32
)ARRAY_SIZE(spectre_golden_spm_registers
));
1520 radeon_program_register_sequence(rdev
,
1521 hawaii_mgcg_cgcg_init
,
1522 (const u32
)ARRAY_SIZE(hawaii_mgcg_cgcg_init
));
1523 radeon_program_register_sequence(rdev
,
1524 hawaii_golden_registers
,
1525 (const u32
)ARRAY_SIZE(hawaii_golden_registers
));
1526 radeon_program_register_sequence(rdev
,
1527 hawaii_golden_common_registers
,
1528 (const u32
)ARRAY_SIZE(hawaii_golden_common_registers
));
1529 radeon_program_register_sequence(rdev
,
1530 hawaii_golden_spm_registers
,
1531 (const u32
)ARRAY_SIZE(hawaii_golden_spm_registers
));
1539 * cik_get_xclk - get the xclk
1541 * @rdev: radeon_device pointer
1543 * Returns the reference clock used by the gfx engine
1546 u32
cik_get_xclk(struct radeon_device
*rdev
)
1548 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1550 if (rdev
->flags
& RADEON_IS_IGP
) {
1551 if (RREG32_SMC(GENERAL_PWRMGT
) & GPU_COUNTER_CLK
)
1552 return reference_clock
/ 2;
1554 if (RREG32_SMC(CG_CLKPIN_CNTL
) & XTALIN_DIVIDE
)
1555 return reference_clock
/ 4;
1557 return reference_clock
;
1561 * cik_mm_rdoorbell - read a doorbell dword
1563 * @rdev: radeon_device pointer
1564 * @index: doorbell index
1566 * Returns the value in the doorbell aperture at the
1567 * requested doorbell index (CIK).
1569 u32
cik_mm_rdoorbell(struct radeon_device
*rdev
, u32 index
)
1571 if (index
< rdev
->doorbell
.num_doorbells
) {
1572 return readl(rdev
->doorbell
.ptr
+ index
);
1574 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index
);
1580 * cik_mm_wdoorbell - write a doorbell dword
1582 * @rdev: radeon_device pointer
1583 * @index: doorbell index
1584 * @v: value to write
1586 * Writes @v to the doorbell aperture at the
1587 * requested doorbell index (CIK).
1589 void cik_mm_wdoorbell(struct radeon_device
*rdev
, u32 index
, u32 v
)
1591 if (index
< rdev
->doorbell
.num_doorbells
) {
1592 writel(v
, rdev
->doorbell
.ptr
+ index
);
1594 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index
);
1598 #define BONAIRE_IO_MC_REGS_SIZE 36
1600 static const u32 bonaire_io_mc_regs
[BONAIRE_IO_MC_REGS_SIZE
][2] =
1602 {0x00000070, 0x04400000},
1603 {0x00000071, 0x80c01803},
1604 {0x00000072, 0x00004004},
1605 {0x00000073, 0x00000100},
1606 {0x00000074, 0x00ff0000},
1607 {0x00000075, 0x34000000},
1608 {0x00000076, 0x08000014},
1609 {0x00000077, 0x00cc08ec},
1610 {0x00000078, 0x00000400},
1611 {0x00000079, 0x00000000},
1612 {0x0000007a, 0x04090000},
1613 {0x0000007c, 0x00000000},
1614 {0x0000007e, 0x4408a8e8},
1615 {0x0000007f, 0x00000304},
1616 {0x00000080, 0x00000000},
1617 {0x00000082, 0x00000001},
1618 {0x00000083, 0x00000002},
1619 {0x00000084, 0xf3e4f400},
1620 {0x00000085, 0x052024e3},
1621 {0x00000087, 0x00000000},
1622 {0x00000088, 0x01000000},
1623 {0x0000008a, 0x1c0a0000},
1624 {0x0000008b, 0xff010000},
1625 {0x0000008d, 0xffffefff},
1626 {0x0000008e, 0xfff3efff},
1627 {0x0000008f, 0xfff3efbf},
1628 {0x00000092, 0xf7ffffff},
1629 {0x00000093, 0xffffff7f},
1630 {0x00000095, 0x00101101},
1631 {0x00000096, 0x00000fff},
1632 {0x00000097, 0x00116fff},
1633 {0x00000098, 0x60010000},
1634 {0x00000099, 0x10010000},
1635 {0x0000009a, 0x00006000},
1636 {0x0000009b, 0x00001000},
1637 {0x0000009f, 0x00b48000}
1640 #define HAWAII_IO_MC_REGS_SIZE 22
1642 static const u32 hawaii_io_mc_regs
[HAWAII_IO_MC_REGS_SIZE
][2] =
1644 {0x0000007d, 0x40000000},
1645 {0x0000007e, 0x40180304},
1646 {0x0000007f, 0x0000ff00},
1647 {0x00000081, 0x00000000},
1648 {0x00000083, 0x00000800},
1649 {0x00000086, 0x00000000},
1650 {0x00000087, 0x00000100},
1651 {0x00000088, 0x00020100},
1652 {0x00000089, 0x00000000},
1653 {0x0000008b, 0x00040000},
1654 {0x0000008c, 0x00000100},
1655 {0x0000008e, 0xff010000},
1656 {0x00000090, 0xffffefff},
1657 {0x00000091, 0xfff3efff},
1658 {0x00000092, 0xfff3efbf},
1659 {0x00000093, 0xf7ffffff},
1660 {0x00000094, 0xffffff7f},
1661 {0x00000095, 0x00000fff},
1662 {0x00000096, 0x00116fff},
1663 {0x00000097, 0x60010000},
1664 {0x00000098, 0x10010000},
1665 {0x0000009f, 0x00c79000}
1670 * cik_srbm_select - select specific register instances
1672 * @rdev: radeon_device pointer
1673 * @me: selected ME (micro engine)
1678 * Switches the currently active registers instances. Some
1679 * registers are instanced per VMID, others are instanced per
1680 * me/pipe/queue combination.
1682 static void cik_srbm_select(struct radeon_device
*rdev
,
1683 u32 me
, u32 pipe
, u32 queue
, u32 vmid
)
1685 u32 srbm_gfx_cntl
= (PIPEID(pipe
& 0x3) |
1688 QUEUEID(queue
& 0x7));
1689 WREG32(SRBM_GFX_CNTL
, srbm_gfx_cntl
);
1694 * ci_mc_load_microcode - load MC ucode into the hw
1696 * @rdev: radeon_device pointer
1698 * Load the GDDR MC ucode into the hw (CIK).
1699 * Returns 0 on success, error on failure.
1701 int ci_mc_load_microcode(struct radeon_device
*rdev
)
1703 const __be32
*fw_data
;
1704 u32 running
, blackout
= 0;
1706 int i
, ucode_size
, regs_size
;
1711 switch (rdev
->family
) {
1713 io_mc_regs
= (u32
*)&bonaire_io_mc_regs
;
1714 ucode_size
= CIK_MC_UCODE_SIZE
;
1715 regs_size
= BONAIRE_IO_MC_REGS_SIZE
;
1718 io_mc_regs
= (u32
*)&hawaii_io_mc_regs
;
1719 ucode_size
= HAWAII_MC_UCODE_SIZE
;
1720 regs_size
= HAWAII_IO_MC_REGS_SIZE
;
1726 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1730 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1731 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1734 /* reset the engine and set to writable */
1735 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1736 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1738 /* load mc io regs */
1739 for (i
= 0; i
< regs_size
; i
++) {
1740 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1741 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1743 /* load the MC ucode */
1744 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1745 for (i
= 0; i
< ucode_size
; i
++)
1746 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1748 /* put the engine back into the active state */
1749 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1750 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1751 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1753 /* wait for training to complete */
1754 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1755 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1759 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1760 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1766 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1773 * cik_init_microcode - load ucode images from disk
1775 * @rdev: radeon_device pointer
1777 * Use the firmware interface to load the ucode images into
1778 * the driver (not loaded into hw).
1779 * Returns 0 on success, error on failure.
1781 static int cik_init_microcode(struct radeon_device
*rdev
)
1783 const char *chip_name
;
1784 size_t pfp_req_size
, me_req_size
, ce_req_size
,
1785 mec_req_size
, rlc_req_size
, mc_req_size
= 0,
1786 sdma_req_size
, smc_req_size
= 0;
1792 switch (rdev
->family
) {
1794 chip_name
= "BONAIRE";
1795 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1796 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1797 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1798 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1799 rlc_req_size
= BONAIRE_RLC_UCODE_SIZE
* 4;
1800 mc_req_size
= CIK_MC_UCODE_SIZE
* 4;
1801 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1802 smc_req_size
= ALIGN(BONAIRE_SMC_UCODE_SIZE
, 4);
1805 chip_name
= "HAWAII";
1806 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1807 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1808 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1809 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1810 rlc_req_size
= BONAIRE_RLC_UCODE_SIZE
* 4;
1811 mc_req_size
= HAWAII_MC_UCODE_SIZE
* 4;
1812 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1813 smc_req_size
= ALIGN(HAWAII_SMC_UCODE_SIZE
, 4);
1816 chip_name
= "KAVERI";
1817 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1818 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1819 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1820 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1821 rlc_req_size
= KV_RLC_UCODE_SIZE
* 4;
1822 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1825 chip_name
= "KABINI";
1826 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1827 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1828 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1829 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1830 rlc_req_size
= KB_RLC_UCODE_SIZE
* 4;
1831 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1836 DRM_INFO("Loading %s Microcode\n", chip_name
);
1838 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1839 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, rdev
->dev
);
1842 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1844 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1845 rdev
->pfp_fw
->size
, fw_name
);
1850 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1851 err
= request_firmware(&rdev
->me_fw
, fw_name
, rdev
->dev
);
1854 if (rdev
->me_fw
->size
!= me_req_size
) {
1856 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1857 rdev
->me_fw
->size
, fw_name
);
1861 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1862 err
= request_firmware(&rdev
->ce_fw
, fw_name
, rdev
->dev
);
1865 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1867 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1868 rdev
->ce_fw
->size
, fw_name
);
1872 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mec.bin", chip_name
);
1873 err
= request_firmware(&rdev
->mec_fw
, fw_name
, rdev
->dev
);
1876 if (rdev
->mec_fw
->size
!= mec_req_size
) {
1878 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1879 rdev
->mec_fw
->size
, fw_name
);
1883 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", chip_name
);
1884 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, rdev
->dev
);
1887 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1889 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1890 rdev
->rlc_fw
->size
, fw_name
);
1894 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_sdma.bin", chip_name
);
1895 err
= request_firmware(&rdev
->sdma_fw
, fw_name
, rdev
->dev
);
1898 if (rdev
->sdma_fw
->size
!= sdma_req_size
) {
1900 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1901 rdev
->sdma_fw
->size
, fw_name
);
1905 /* No SMC, MC ucode on APUs */
1906 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
1907 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1908 err
= request_firmware(&rdev
->mc_fw
, fw_name
, rdev
->dev
);
1911 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1913 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1914 rdev
->mc_fw
->size
, fw_name
);
1918 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_smc.bin", chip_name
);
1919 err
= request_firmware(&rdev
->smc_fw
, fw_name
, rdev
->dev
);
1922 "smc: error loading firmware \"%s\"\n",
1924 release_firmware(rdev
->smc_fw
);
1925 rdev
->smc_fw
= NULL
;
1927 } else if (rdev
->smc_fw
->size
!= smc_req_size
) {
1929 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1930 rdev
->smc_fw
->size
, fw_name
);
1939 "cik_cp: Failed to load firmware \"%s\"\n",
1941 release_firmware(rdev
->pfp_fw
);
1942 rdev
->pfp_fw
= NULL
;
1943 release_firmware(rdev
->me_fw
);
1945 release_firmware(rdev
->ce_fw
);
1947 release_firmware(rdev
->rlc_fw
);
1948 rdev
->rlc_fw
= NULL
;
1949 release_firmware(rdev
->mc_fw
);
1951 release_firmware(rdev
->smc_fw
);
1952 rdev
->smc_fw
= NULL
;
1961 * cik_tiling_mode_table_init - init the hw tiling table
1963 * @rdev: radeon_device pointer
1965 * Starting with SI, the tiling setup is done globally in a
1966 * set of 32 tiling modes. Rather than selecting each set of
1967 * parameters per surface as on older asics, we just select
1968 * which index in the tiling table we want to use, and the
1969 * surface uses those parameters (CIK).
1971 static void cik_tiling_mode_table_init(struct radeon_device
*rdev
)
1973 const u32 num_tile_mode_states
= 32;
1974 const u32 num_secondary_tile_mode_states
= 16;
1975 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
1976 u32 num_pipe_configs
;
1977 u32 num_rbs
= rdev
->config
.cik
.max_backends_per_se
*
1978 rdev
->config
.cik
.max_shader_engines
;
1980 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
1982 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
1986 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
1989 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
1993 num_pipe_configs
= rdev
->config
.cik
.max_tile_pipes
;
1994 if (num_pipe_configs
> 8)
1995 num_pipe_configs
= 16;
1997 if (num_pipe_configs
== 16) {
1998 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
1999 switch (reg_offset
) {
2001 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2002 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2003 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2004 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2007 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2009 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2010 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2013 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2015 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2016 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2019 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2021 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2022 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2025 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2026 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2027 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2028 TILE_SPLIT(split_equal_to_row_size
));
2031 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2032 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2033 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2036 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2037 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2038 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2039 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2042 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2044 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2045 TILE_SPLIT(split_equal_to_row_size
));
2048 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2049 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
));
2052 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2053 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2054 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2057 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2058 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2059 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2060 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2063 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2064 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2065 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2066 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2069 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2070 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2071 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2072 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2075 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2076 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2080 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2081 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2082 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2083 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2086 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2088 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2092 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2094 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2098 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2099 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2100 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2103 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2104 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2105 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2106 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2109 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2111 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2112 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2115 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2116 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2117 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2118 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2124 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2125 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2127 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2128 switch (reg_offset
) {
2130 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2131 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2132 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2133 NUM_BANKS(ADDR_SURF_16_BANK
));
2136 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2137 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2138 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2139 NUM_BANKS(ADDR_SURF_16_BANK
));
2142 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2143 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2144 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2145 NUM_BANKS(ADDR_SURF_16_BANK
));
2148 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2149 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2150 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2151 NUM_BANKS(ADDR_SURF_16_BANK
));
2154 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2155 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2156 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2157 NUM_BANKS(ADDR_SURF_8_BANK
));
2160 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2163 NUM_BANKS(ADDR_SURF_4_BANK
));
2166 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2167 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2168 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2169 NUM_BANKS(ADDR_SURF_2_BANK
));
2172 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2173 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2174 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2175 NUM_BANKS(ADDR_SURF_16_BANK
));
2178 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2179 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2180 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2181 NUM_BANKS(ADDR_SURF_16_BANK
));
2184 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2185 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2186 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2187 NUM_BANKS(ADDR_SURF_16_BANK
));
2190 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2191 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2192 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2193 NUM_BANKS(ADDR_SURF_8_BANK
));
2196 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2199 NUM_BANKS(ADDR_SURF_4_BANK
));
2202 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2203 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2204 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2205 NUM_BANKS(ADDR_SURF_2_BANK
));
2208 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2209 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2210 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2211 NUM_BANKS(ADDR_SURF_2_BANK
));
2217 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2219 } else if (num_pipe_configs
== 8) {
2220 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2221 switch (reg_offset
) {
2223 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2224 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2225 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2226 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2229 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2230 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2231 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2235 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2236 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2237 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2238 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2241 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2242 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2243 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2244 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2247 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2248 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2249 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2250 TILE_SPLIT(split_equal_to_row_size
));
2253 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2255 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2258 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2264 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2265 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2267 TILE_SPLIT(split_equal_to_row_size
));
2270 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2271 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
));
2274 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2276 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2279 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2285 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2287 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2291 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2292 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2294 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2297 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2298 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2299 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2302 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2304 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2305 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2308 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2314 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2316 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2317 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2320 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2321 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2322 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2325 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2326 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2327 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2328 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2331 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2332 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2337 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2339 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2340 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2346 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2347 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2349 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2350 switch (reg_offset
) {
2352 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2355 NUM_BANKS(ADDR_SURF_16_BANK
));
2358 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2361 NUM_BANKS(ADDR_SURF_16_BANK
));
2364 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2367 NUM_BANKS(ADDR_SURF_16_BANK
));
2370 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2371 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2372 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2373 NUM_BANKS(ADDR_SURF_16_BANK
));
2376 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2379 NUM_BANKS(ADDR_SURF_8_BANK
));
2382 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2383 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2384 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2385 NUM_BANKS(ADDR_SURF_4_BANK
));
2388 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2391 NUM_BANKS(ADDR_SURF_2_BANK
));
2394 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2395 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2396 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2397 NUM_BANKS(ADDR_SURF_16_BANK
));
2400 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2403 NUM_BANKS(ADDR_SURF_16_BANK
));
2406 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2409 NUM_BANKS(ADDR_SURF_16_BANK
));
2412 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2413 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2414 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2415 NUM_BANKS(ADDR_SURF_16_BANK
));
2418 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2421 NUM_BANKS(ADDR_SURF_8_BANK
));
2424 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2425 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2426 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2427 NUM_BANKS(ADDR_SURF_4_BANK
));
2430 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2431 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2432 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2433 NUM_BANKS(ADDR_SURF_2_BANK
));
2439 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
2440 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2442 } else if (num_pipe_configs
== 4) {
2444 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2445 switch (reg_offset
) {
2447 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2448 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2449 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2450 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2453 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2454 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2455 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2456 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2459 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2461 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2465 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2466 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2467 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2468 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2471 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2474 TILE_SPLIT(split_equal_to_row_size
));
2477 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2478 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2482 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2483 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2484 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2485 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2488 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2490 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2491 TILE_SPLIT(split_equal_to_row_size
));
2494 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2495 PIPE_CONFIG(ADDR_SURF_P4_16x16
));
2498 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2499 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2503 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2509 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2511 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2512 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2515 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2517 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2518 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2521 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2522 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2523 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2526 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2532 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2534 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2535 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2538 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2539 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2540 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2541 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2544 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2545 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2549 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2551 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2555 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2556 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2557 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2558 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2561 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2562 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2563 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2564 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2570 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2571 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2573 } else if (num_rbs
< 4) {
2574 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2575 switch (reg_offset
) {
2577 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2579 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2583 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2584 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2589 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2590 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2591 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2595 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2597 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2601 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2603 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2604 TILE_SPLIT(split_equal_to_row_size
));
2607 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2608 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2609 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2612 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2613 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2614 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2618 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2621 TILE_SPLIT(split_equal_to_row_size
));
2624 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16
));
2628 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2629 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2633 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2639 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2641 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2645 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2646 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2651 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2652 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2656 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2657 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2658 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2659 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2662 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2668 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2674 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2676 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2679 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2681 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2685 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2687 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2688 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2691 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2692 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2693 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2694 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2700 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2701 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2704 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2705 switch (reg_offset
) {
2707 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2710 NUM_BANKS(ADDR_SURF_16_BANK
));
2713 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2716 NUM_BANKS(ADDR_SURF_16_BANK
));
2719 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2722 NUM_BANKS(ADDR_SURF_16_BANK
));
2725 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2728 NUM_BANKS(ADDR_SURF_16_BANK
));
2731 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2734 NUM_BANKS(ADDR_SURF_16_BANK
));
2737 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2740 NUM_BANKS(ADDR_SURF_8_BANK
));
2743 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2746 NUM_BANKS(ADDR_SURF_4_BANK
));
2749 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2752 NUM_BANKS(ADDR_SURF_16_BANK
));
2755 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2758 NUM_BANKS(ADDR_SURF_16_BANK
));
2761 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2764 NUM_BANKS(ADDR_SURF_16_BANK
));
2767 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2770 NUM_BANKS(ADDR_SURF_16_BANK
));
2773 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2776 NUM_BANKS(ADDR_SURF_16_BANK
));
2779 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2782 NUM_BANKS(ADDR_SURF_8_BANK
));
2785 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2786 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2787 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2788 NUM_BANKS(ADDR_SURF_4_BANK
));
2794 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
2795 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2797 } else if (num_pipe_configs
== 2) {
2798 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2799 switch (reg_offset
) {
2801 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2803 PIPE_CONFIG(ADDR_SURF_P2
) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2807 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2808 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2809 PIPE_CONFIG(ADDR_SURF_P2
) |
2810 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2813 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2814 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2815 PIPE_CONFIG(ADDR_SURF_P2
) |
2816 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2819 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2821 PIPE_CONFIG(ADDR_SURF_P2
) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2825 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2827 PIPE_CONFIG(ADDR_SURF_P2
) |
2828 TILE_SPLIT(split_equal_to_row_size
));
2831 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2832 PIPE_CONFIG(ADDR_SURF_P2
) |
2833 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2836 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2837 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2838 PIPE_CONFIG(ADDR_SURF_P2
) |
2839 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2842 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2844 PIPE_CONFIG(ADDR_SURF_P2
) |
2845 TILE_SPLIT(split_equal_to_row_size
));
2848 gb_tile_moden
= ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2849 PIPE_CONFIG(ADDR_SURF_P2
);
2852 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2854 PIPE_CONFIG(ADDR_SURF_P2
));
2857 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2859 PIPE_CONFIG(ADDR_SURF_P2
) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2863 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2865 PIPE_CONFIG(ADDR_SURF_P2
) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2869 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2870 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2871 PIPE_CONFIG(ADDR_SURF_P2
) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2875 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2876 PIPE_CONFIG(ADDR_SURF_P2
) |
2877 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2880 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2881 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2882 PIPE_CONFIG(ADDR_SURF_P2
) |
2883 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2886 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2888 PIPE_CONFIG(ADDR_SURF_P2
) |
2889 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2892 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2893 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2894 PIPE_CONFIG(ADDR_SURF_P2
) |
2895 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2898 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2900 PIPE_CONFIG(ADDR_SURF_P2
));
2903 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2905 PIPE_CONFIG(ADDR_SURF_P2
) |
2906 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2909 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2910 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2911 PIPE_CONFIG(ADDR_SURF_P2
) |
2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2915 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2916 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2917 PIPE_CONFIG(ADDR_SURF_P2
) |
2918 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2924 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2925 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2927 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2928 switch (reg_offset
) {
2930 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2933 NUM_BANKS(ADDR_SURF_16_BANK
));
2936 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2939 NUM_BANKS(ADDR_SURF_16_BANK
));
2942 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2945 NUM_BANKS(ADDR_SURF_16_BANK
));
2948 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2951 NUM_BANKS(ADDR_SURF_16_BANK
));
2954 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2957 NUM_BANKS(ADDR_SURF_16_BANK
));
2960 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2963 NUM_BANKS(ADDR_SURF_16_BANK
));
2966 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2969 NUM_BANKS(ADDR_SURF_8_BANK
));
2972 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2975 NUM_BANKS(ADDR_SURF_16_BANK
));
2978 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2981 NUM_BANKS(ADDR_SURF_16_BANK
));
2984 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2987 NUM_BANKS(ADDR_SURF_16_BANK
));
2990 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2991 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2992 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2993 NUM_BANKS(ADDR_SURF_16_BANK
));
2996 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2999 NUM_BANKS(ADDR_SURF_16_BANK
));
3002 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
3003 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
3004 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
3005 NUM_BANKS(ADDR_SURF_16_BANK
));
3008 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
3011 NUM_BANKS(ADDR_SURF_8_BANK
));
3017 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
3018 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
3021 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs
);
3025 * cik_select_se_sh - select which SE, SH to address
3027 * @rdev: radeon_device pointer
3028 * @se_num: shader engine to address
3029 * @sh_num: sh block to address
3031 * Select which SE, SH combinations to address. Certain
3032 * registers are instanced per SE or SH. 0xffffffff means
3033 * broadcast to all SEs or SHs (CIK).
3035 static void cik_select_se_sh(struct radeon_device
*rdev
,
3036 u32 se_num
, u32 sh_num
)
3038 u32 data
= INSTANCE_BROADCAST_WRITES
;
3040 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
3041 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
3042 else if (se_num
== 0xffffffff)
3043 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
3044 else if (sh_num
== 0xffffffff)
3045 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
3047 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
3048 WREG32(GRBM_GFX_INDEX
, data
);
3052 * cik_create_bitmask - create a bitmask
3054 * @bit_width: length of the mask
3056 * create a variable length bit mask (CIK).
3057 * Returns the bitmask.
3059 static u32
cik_create_bitmask(u32 bit_width
)
3063 for (i
= 0; i
< bit_width
; i
++) {
3071 * cik_get_rb_disabled - computes the mask of disabled RBs
3073 * @rdev: radeon_device pointer
3074 * @max_rb_num: max RBs (render backends) for the asic
3075 * @se_num: number of SEs (shader engines) for the asic
3076 * @sh_per_se: number of SH blocks per SE for the asic
3078 * Calculates the bitmask of disabled RBs (CIK).
3079 * Returns the disabled RB bitmask.
3081 static u32
cik_get_rb_disabled(struct radeon_device
*rdev
,
3082 u32 max_rb_num_per_se
,
3087 data
= RREG32(CC_RB_BACKEND_DISABLE
);
3089 data
&= BACKEND_DISABLE_MASK
;
3092 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
3094 data
>>= BACKEND_DISABLE_SHIFT
;
3096 mask
= cik_create_bitmask(max_rb_num_per_se
/ sh_per_se
);
3102 * cik_setup_rb - setup the RBs on the asic
3104 * @rdev: radeon_device pointer
3105 * @se_num: number of SEs (shader engines) for the asic
3106 * @sh_per_se: number of SH blocks per SE for the asic
3107 * @max_rb_num: max RBs (render backends) for the asic
3109 * Configures per-SE/SH RB registers (CIK).
3111 static void cik_setup_rb(struct radeon_device
*rdev
,
3112 u32 se_num
, u32 sh_per_se
,
3113 u32 max_rb_num_per_se
)
3117 u32 disabled_rbs
= 0;
3118 u32 enabled_rbs
= 0;
3120 for (i
= 0; i
< se_num
; i
++) {
3121 for (j
= 0; j
< sh_per_se
; j
++) {
3122 cik_select_se_sh(rdev
, i
, j
);
3123 data
= cik_get_rb_disabled(rdev
, max_rb_num_per_se
, sh_per_se
);
3124 if (rdev
->family
== CHIP_HAWAII
)
3125 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * HAWAII_RB_BITMAP_WIDTH_PER_SH
);
3127 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * CIK_RB_BITMAP_WIDTH_PER_SH
);
3130 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
3133 for (i
= 0; i
< max_rb_num_per_se
* se_num
; i
++) {
3134 if (!(disabled_rbs
& mask
))
3135 enabled_rbs
|= mask
;
3139 rdev
->config
.cik
.backend_enable_mask
= enabled_rbs
;
3141 for (i
= 0; i
< se_num
; i
++) {
3142 cik_select_se_sh(rdev
, i
, 0xffffffff);
3144 for (j
= 0; j
< sh_per_se
; j
++) {
3145 switch (enabled_rbs
& 3) {
3148 data
|= PKR_MAP(RASTER_CONFIG_RB_MAP_3
);
3150 data
|= PKR_MAP(RASTER_CONFIG_RB_MAP_0
);
3153 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
3156 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
3160 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
3165 WREG32(PA_SC_RASTER_CONFIG
, data
);
3167 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
3171 * cik_gpu_init - setup the 3D engine
3173 * @rdev: radeon_device pointer
3175 * Configures the 3D engine and tiling configuration
3176 * registers so that the 3D engine is usable.
3178 static void cik_gpu_init(struct radeon_device
*rdev
)
3180 u32 gb_addr_config
= RREG32(GB_ADDR_CONFIG
);
3181 u32 mc_shared_chmap
, mc_arb_ramcfg
;
3182 u32 hdp_host_path_cntl
;
3186 switch (rdev
->family
) {
3188 rdev
->config
.cik
.max_shader_engines
= 2;
3189 rdev
->config
.cik
.max_tile_pipes
= 4;
3190 rdev
->config
.cik
.max_cu_per_sh
= 7;
3191 rdev
->config
.cik
.max_sh_per_se
= 1;
3192 rdev
->config
.cik
.max_backends_per_se
= 2;
3193 rdev
->config
.cik
.max_texture_channel_caches
= 4;
3194 rdev
->config
.cik
.max_gprs
= 256;
3195 rdev
->config
.cik
.max_gs_threads
= 32;
3196 rdev
->config
.cik
.max_hw_contexts
= 8;
3198 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3199 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3200 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3201 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3202 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3205 rdev
->config
.cik
.max_shader_engines
= 4;
3206 rdev
->config
.cik
.max_tile_pipes
= 16;
3207 rdev
->config
.cik
.max_cu_per_sh
= 11;
3208 rdev
->config
.cik
.max_sh_per_se
= 1;
3209 rdev
->config
.cik
.max_backends_per_se
= 4;
3210 rdev
->config
.cik
.max_texture_channel_caches
= 16;
3211 rdev
->config
.cik
.max_gprs
= 256;
3212 rdev
->config
.cik
.max_gs_threads
= 32;
3213 rdev
->config
.cik
.max_hw_contexts
= 8;
3215 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3216 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3217 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3218 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3219 gb_addr_config
= HAWAII_GB_ADDR_CONFIG_GOLDEN
;
3222 rdev
->config
.cik
.max_shader_engines
= 1;
3223 rdev
->config
.cik
.max_tile_pipes
= 4;
3224 if ((rdev
->pdev
->device
== 0x1304) ||
3225 (rdev
->pdev
->device
== 0x1305) ||
3226 (rdev
->pdev
->device
== 0x130C) ||
3227 (rdev
->pdev
->device
== 0x130F) ||
3228 (rdev
->pdev
->device
== 0x1310) ||
3229 (rdev
->pdev
->device
== 0x1311) ||
3230 (rdev
->pdev
->device
== 0x131C)) {
3231 rdev
->config
.cik
.max_cu_per_sh
= 8;
3232 rdev
->config
.cik
.max_backends_per_se
= 2;
3233 } else if ((rdev
->pdev
->device
== 0x1309) ||
3234 (rdev
->pdev
->device
== 0x130A) ||
3235 (rdev
->pdev
->device
== 0x130D) ||
3236 (rdev
->pdev
->device
== 0x1313) ||
3237 (rdev
->pdev
->device
== 0x131D)) {
3238 rdev
->config
.cik
.max_cu_per_sh
= 6;
3239 rdev
->config
.cik
.max_backends_per_se
= 2;
3240 } else if ((rdev
->pdev
->device
== 0x1306) ||
3241 (rdev
->pdev
->device
== 0x1307) ||
3242 (rdev
->pdev
->device
== 0x130B) ||
3243 (rdev
->pdev
->device
== 0x130E) ||
3244 (rdev
->pdev
->device
== 0x1315) ||
3245 (rdev
->pdev
->device
== 0x131B)) {
3246 rdev
->config
.cik
.max_cu_per_sh
= 4;
3247 rdev
->config
.cik
.max_backends_per_se
= 1;
3249 rdev
->config
.cik
.max_cu_per_sh
= 3;
3250 rdev
->config
.cik
.max_backends_per_se
= 1;
3252 rdev
->config
.cik
.max_sh_per_se
= 1;
3253 rdev
->config
.cik
.max_texture_channel_caches
= 4;
3254 rdev
->config
.cik
.max_gprs
= 256;
3255 rdev
->config
.cik
.max_gs_threads
= 16;
3256 rdev
->config
.cik
.max_hw_contexts
= 8;
3258 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3259 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3260 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3261 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3262 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3266 rdev
->config
.cik
.max_shader_engines
= 1;
3267 rdev
->config
.cik
.max_tile_pipes
= 2;
3268 rdev
->config
.cik
.max_cu_per_sh
= 2;
3269 rdev
->config
.cik
.max_sh_per_se
= 1;
3270 rdev
->config
.cik
.max_backends_per_se
= 1;
3271 rdev
->config
.cik
.max_texture_channel_caches
= 2;
3272 rdev
->config
.cik
.max_gprs
= 256;
3273 rdev
->config
.cik
.max_gs_threads
= 16;
3274 rdev
->config
.cik
.max_hw_contexts
= 8;
3276 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3277 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3278 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3279 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3280 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3284 /* Initialize HDP */
3285 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
3286 WREG32((0x2c14 + j
), 0x00000000);
3287 WREG32((0x2c18 + j
), 0x00000000);
3288 WREG32((0x2c1c + j
), 0x00000000);
3289 WREG32((0x2c20 + j
), 0x00000000);
3290 WREG32((0x2c24 + j
), 0x00000000);
3293 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
3295 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
3297 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
3298 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
3300 rdev
->config
.cik
.num_tile_pipes
= rdev
->config
.cik
.max_tile_pipes
;
3301 rdev
->config
.cik
.mem_max_burst_length_bytes
= 256;
3302 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
3303 rdev
->config
.cik
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
3304 if (rdev
->config
.cik
.mem_row_size_in_kb
> 4)
3305 rdev
->config
.cik
.mem_row_size_in_kb
= 4;
3306 /* XXX use MC settings? */
3307 rdev
->config
.cik
.shader_engine_tile_size
= 32;
3308 rdev
->config
.cik
.num_gpus
= 1;
3309 rdev
->config
.cik
.multi_gpu_tile_size
= 64;
3311 /* fix up row size */
3312 gb_addr_config
&= ~ROW_SIZE_MASK
;
3313 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
3316 gb_addr_config
|= ROW_SIZE(0);
3319 gb_addr_config
|= ROW_SIZE(1);
3322 gb_addr_config
|= ROW_SIZE(2);
3326 /* setup tiling info dword. gb_addr_config is not adequate since it does
3327 * not have bank info, so create a custom tiling dword.
3328 * bits 3:0 num_pipes
3329 * bits 7:4 num_banks
3330 * bits 11:8 group_size
3331 * bits 15:12 row_size
3333 rdev
->config
.cik
.tile_config
= 0;
3334 switch (rdev
->config
.cik
.num_tile_pipes
) {
3336 rdev
->config
.cik
.tile_config
|= (0 << 0);
3339 rdev
->config
.cik
.tile_config
|= (1 << 0);
3342 rdev
->config
.cik
.tile_config
|= (2 << 0);
3346 /* XXX what about 12? */
3347 rdev
->config
.cik
.tile_config
|= (3 << 0);
3350 rdev
->config
.cik
.tile_config
|=
3351 ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) << 4;
3352 rdev
->config
.cik
.tile_config
|=
3353 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
3354 rdev
->config
.cik
.tile_config
|=
3355 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
3357 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
3358 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
3359 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
3360 WREG32(SDMA0_TILING_CONFIG
+ SDMA0_REGISTER_OFFSET
, gb_addr_config
& 0x70);
3361 WREG32(SDMA0_TILING_CONFIG
+ SDMA1_REGISTER_OFFSET
, gb_addr_config
& 0x70);
3362 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
3363 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
3364 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
3366 cik_tiling_mode_table_init(rdev
);
3368 cik_setup_rb(rdev
, rdev
->config
.cik
.max_shader_engines
,
3369 rdev
->config
.cik
.max_sh_per_se
,
3370 rdev
->config
.cik
.max_backends_per_se
);
3372 /* set HW defaults for 3D engine */
3373 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
3375 WREG32(SX_DEBUG_1
, 0x20);
3377 WREG32(TA_CNTL_AUX
, 0x00010000);
3379 tmp
= RREG32(SPI_CONFIG_CNTL
);
3381 WREG32(SPI_CONFIG_CNTL
, tmp
);
3383 WREG32(SQ_CONFIG
, 1);
3385 WREG32(DB_DEBUG
, 0);
3387 tmp
= RREG32(DB_DEBUG2
) & ~0xf00fffff;
3389 WREG32(DB_DEBUG2
, tmp
);
3391 tmp
= RREG32(DB_DEBUG3
) & ~0x0002021c;
3393 WREG32(DB_DEBUG3
, tmp
);
3395 tmp
= RREG32(CB_HW_CONTROL
) & ~0x00010000;
3397 WREG32(CB_HW_CONTROL
, tmp
);
3399 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
3401 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_frontend
) |
3402 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_backend
) |
3403 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_hiz_tile_fifo_size
) |
3404 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_earlyz_tile_fifo_size
)));
3406 WREG32(VGT_NUM_INSTANCES
, 1);
3408 WREG32(CP_PERFMON_CNTL
, 0);
3410 WREG32(SQ_CONFIG
, 0);
3412 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
3413 FORCE_EOV_MAX_REZ_CNT(255)));
3415 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
3416 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
3418 WREG32(VGT_GS_VERTEX_REUSE
, 16);
3419 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
3421 tmp
= RREG32(HDP_MISC_CNTL
);
3422 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
3423 WREG32(HDP_MISC_CNTL
, tmp
);
3425 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
3426 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
3428 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
3429 WREG32(PA_SC_ENHANCE
, ENABLE_PA_SC_OUT_OF_ORDER
);
3435 * GPU scratch registers helpers function.
3438 * cik_scratch_init - setup driver info for CP scratch regs
3440 * @rdev: radeon_device pointer
3442 * Set up the number and offset of the CP scratch registers.
3443 * NOTE: use of CP scratch registers is a legacy inferface and
3444 * is not used by default on newer asics (r6xx+). On newer asics,
3445 * memory buffers are used for fences rather than scratch regs.
3447 static void cik_scratch_init(struct radeon_device
*rdev
)
3451 rdev
->scratch
.num_reg
= 7;
3452 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
3453 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
3454 rdev
->scratch
.free
[i
] = true;
3455 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
3460 * cik_ring_test - basic gfx ring test
3462 * @rdev: radeon_device pointer
3463 * @ring: radeon_ring structure holding ring information
3465 * Allocate a scratch register and write to it using the gfx ring (CIK).
3466 * Provides a basic gfx ring test to verify that the ring is working.
3467 * Used by cik_cp_gfx_resume();
3468 * Returns 0 on success, error on failure.
3470 int cik_ring_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3477 r
= radeon_scratch_get(rdev
, &scratch
);
3479 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r
);
3482 WREG32(scratch
, 0xCAFEDEAD);
3483 r
= radeon_ring_lock(rdev
, ring
, 3);
3485 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring
->idx
, r
);
3486 radeon_scratch_free(rdev
, scratch
);
3489 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
3490 radeon_ring_write(ring
, ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2));
3491 radeon_ring_write(ring
, 0xDEADBEEF);
3492 radeon_ring_unlock_commit(rdev
, ring
);
3494 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3495 tmp
= RREG32(scratch
);
3496 if (tmp
== 0xDEADBEEF)
3500 if (i
< rdev
->usec_timeout
) {
3501 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring
->idx
, i
);
3503 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3504 ring
->idx
, scratch
, tmp
);
3507 radeon_scratch_free(rdev
, scratch
);
3512 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3514 * @rdev: radeon_device pointer
3515 * @ridx: radeon ring index
3517 * Emits an hdp flush on the cp.
3519 static void cik_hdp_flush_cp_ring_emit(struct radeon_device
*rdev
,
3522 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
3525 switch (ring
->idx
) {
3526 case CAYMAN_RING_TYPE_CP1_INDEX
:
3527 case CAYMAN_RING_TYPE_CP2_INDEX
:
3531 ref_and_mask
= CP2
<< ring
->pipe
;
3534 ref_and_mask
= CP6
<< ring
->pipe
;
3540 case RADEON_RING_TYPE_GFX_INDEX
:
3545 radeon_ring_write(ring
, PACKET3(PACKET3_WAIT_REG_MEM
, 5));
3546 radeon_ring_write(ring
, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3547 WAIT_REG_MEM_FUNCTION(3) | /* == */
3548 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3549 radeon_ring_write(ring
, GPU_HDP_FLUSH_REQ
>> 2);
3550 radeon_ring_write(ring
, GPU_HDP_FLUSH_DONE
>> 2);
3551 radeon_ring_write(ring
, ref_and_mask
);
3552 radeon_ring_write(ring
, ref_and_mask
);
3553 radeon_ring_write(ring
, 0x20); /* poll interval */
3557 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3559 * @rdev: radeon_device pointer
3560 * @fence: radeon fence object
3562 * Emits a fence sequnce number on the gfx ring and flushes
3565 void cik_fence_gfx_ring_emit(struct radeon_device
*rdev
,
3566 struct radeon_fence
*fence
)
3568 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3569 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3571 /* EVENT_WRITE_EOP - flush caches, send int */
3572 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
3573 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
3575 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
3577 radeon_ring_write(ring
, addr
& 0xfffffffc);
3578 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3579 radeon_ring_write(ring
, fence
->seq
);
3580 radeon_ring_write(ring
, 0);
3582 cik_hdp_flush_cp_ring_emit(rdev
, fence
->ring
);
3586 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3588 * @rdev: radeon_device pointer
3589 * @fence: radeon fence object
3591 * Emits a fence sequnce number on the compute ring and flushes
3594 void cik_fence_compute_ring_emit(struct radeon_device
*rdev
,
3595 struct radeon_fence
*fence
)
3597 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3598 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3600 /* RELEASE_MEM - flush caches, send int */
3601 radeon_ring_write(ring
, PACKET3(PACKET3_RELEASE_MEM
, 5));
3602 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
3604 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
3606 radeon_ring_write(ring
, DATA_SEL(1) | INT_SEL(2));
3607 radeon_ring_write(ring
, addr
& 0xfffffffc);
3608 radeon_ring_write(ring
, upper_32_bits(addr
));
3609 radeon_ring_write(ring
, fence
->seq
);
3610 radeon_ring_write(ring
, 0);
3612 cik_hdp_flush_cp_ring_emit(rdev
, fence
->ring
);
3615 bool cik_semaphore_ring_emit(struct radeon_device
*rdev
,
3616 struct radeon_ring
*ring
,
3617 struct radeon_semaphore
*semaphore
,
3620 uint64_t addr
= semaphore
->gpu_addr
;
3621 unsigned sel
= emit_wait
? PACKET3_SEM_SEL_WAIT
: PACKET3_SEM_SEL_SIGNAL
;
3623 radeon_ring_write(ring
, PACKET3(PACKET3_MEM_SEMAPHORE
, 1));
3624 radeon_ring_write(ring
, addr
& 0xffffffff);
3625 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | sel
);
3631 * cik_copy_cpdma - copy pages using the CP DMA engine
3633 * @rdev: radeon_device pointer
3634 * @src_offset: src GPU address
3635 * @dst_offset: dst GPU address
3636 * @num_gpu_pages: number of GPU pages to xfer
3637 * @fence: radeon fence object
3639 * Copy GPU paging using the CP DMA engine (CIK+).
3640 * Used by the radeon ttm implementation to move pages if
3641 * registered as the asic copy callback.
3643 int cik_copy_cpdma(struct radeon_device
*rdev
,
3644 uint64_t src_offset
, uint64_t dst_offset
,
3645 unsigned num_gpu_pages
,
3646 struct radeon_fence
**fence
)
3648 struct radeon_semaphore
*sem
= NULL
;
3649 int ring_index
= rdev
->asic
->copy
.blit_ring_index
;
3650 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
3651 u32 size_in_bytes
, cur_size_in_bytes
, control
;
3655 r
= radeon_semaphore_create(rdev
, &sem
);
3657 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3661 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
3662 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0x1fffff);
3663 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 7 + 18);
3665 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3666 radeon_semaphore_free(rdev
, &sem
, NULL
);
3670 radeon_semaphore_sync_to(sem
, *fence
);
3671 radeon_semaphore_sync_rings(rdev
, sem
, ring
->idx
);
3673 for (i
= 0; i
< num_loops
; i
++) {
3674 cur_size_in_bytes
= size_in_bytes
;
3675 if (cur_size_in_bytes
> 0x1fffff)
3676 cur_size_in_bytes
= 0x1fffff;
3677 size_in_bytes
-= cur_size_in_bytes
;
3679 if (size_in_bytes
== 0)
3680 control
|= PACKET3_DMA_DATA_CP_SYNC
;
3681 radeon_ring_write(ring
, PACKET3(PACKET3_DMA_DATA
, 5));
3682 radeon_ring_write(ring
, control
);
3683 radeon_ring_write(ring
, lower_32_bits(src_offset
));
3684 radeon_ring_write(ring
, upper_32_bits(src_offset
));
3685 radeon_ring_write(ring
, lower_32_bits(dst_offset
));
3686 radeon_ring_write(ring
, upper_32_bits(dst_offset
));
3687 radeon_ring_write(ring
, cur_size_in_bytes
);
3688 src_offset
+= cur_size_in_bytes
;
3689 dst_offset
+= cur_size_in_bytes
;
3692 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
3694 radeon_ring_unlock_undo(rdev
, ring
);
3698 radeon_ring_unlock_commit(rdev
, ring
);
3699 radeon_semaphore_free(rdev
, &sem
, *fence
);
3708 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3710 * @rdev: radeon_device pointer
3711 * @ib: radeon indirect buffer object
3713 * Emits an DE (drawing engine) or CE (constant engine) IB
3714 * on the gfx ring. IBs are usually generated by userspace
3715 * acceleration drivers and submitted to the kernel for
3716 * sheduling on the ring. This function schedules the IB
3717 * on the gfx ring for execution by the GPU.
3719 void cik_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
3721 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3722 u32 header
, control
= INDIRECT_BUFFER_VALID
;
3724 if (ib
->is_const_ib
) {
3725 /* set switch buffer packet before const IB */
3726 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
3727 radeon_ring_write(ring
, 0);
3729 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
3732 if (ring
->rptr_save_reg
) {
3733 next_rptr
= ring
->wptr
+ 3 + 4;
3734 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
3735 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
3736 PACKET3_SET_UCONFIG_REG_START
) >> 2));
3737 radeon_ring_write(ring
, next_rptr
);
3738 } else if (rdev
->wb
.enabled
) {
3739 next_rptr
= ring
->wptr
+ 5 + 4;
3740 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3741 radeon_ring_write(ring
, WRITE_DATA_DST_SEL(1));
3742 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3743 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
3744 radeon_ring_write(ring
, next_rptr
);
3747 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
3750 control
|= ib
->length_dw
|
3751 (ib
->vm
? (ib
->vm
->id
<< 24) : 0);
3753 radeon_ring_write(ring
, header
);
3754 radeon_ring_write(ring
,
3758 (ib
->gpu_addr
& 0xFFFFFFFC));
3759 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
3760 radeon_ring_write(ring
, control
);
3764 * cik_ib_test - basic gfx ring IB test
3766 * @rdev: radeon_device pointer
3767 * @ring: radeon_ring structure holding ring information
3769 * Allocate an IB and execute it on the gfx ring (CIK).
3770 * Provides a basic gfx ring test to verify that IBs are working.
3771 * Returns 0 on success, error on failure.
3773 int cik_ib_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3775 struct radeon_ib ib
;
3781 r
= radeon_scratch_get(rdev
, &scratch
);
3783 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r
);
3786 WREG32(scratch
, 0xCAFEDEAD);
3787 r
= radeon_ib_get(rdev
, ring
->idx
, &ib
, NULL
, 256);
3789 DRM_ERROR("radeon: failed to get ib (%d).\n", r
);
3790 radeon_scratch_free(rdev
, scratch
);
3793 ib
.ptr
[0] = PACKET3(PACKET3_SET_UCONFIG_REG
, 1);
3794 ib
.ptr
[1] = ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2);
3795 ib
.ptr
[2] = 0xDEADBEEF;
3797 r
= radeon_ib_schedule(rdev
, &ib
, NULL
);
3799 radeon_scratch_free(rdev
, scratch
);
3800 radeon_ib_free(rdev
, &ib
);
3801 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r
);
3804 r
= radeon_fence_wait(ib
.fence
, false);
3806 DRM_ERROR("radeon: fence wait failed (%d).\n", r
);
3807 radeon_scratch_free(rdev
, scratch
);
3808 radeon_ib_free(rdev
, &ib
);
3811 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3812 tmp
= RREG32(scratch
);
3813 if (tmp
== 0xDEADBEEF)
3817 if (i
< rdev
->usec_timeout
) {
3818 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib
.fence
->ring
, i
);
3820 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3824 radeon_scratch_free(rdev
, scratch
);
3825 radeon_ib_free(rdev
, &ib
);
3831 * On CIK, gfx and compute now have independant command processors.
3834 * Gfx consists of a single ring and can process both gfx jobs and
3835 * compute jobs. The gfx CP consists of three microengines (ME):
3836 * PFP - Pre-Fetch Parser
3838 * CE - Constant Engine
3839 * The PFP and ME make up what is considered the Drawing Engine (DE).
3840 * The CE is an asynchronous engine used for updating buffer desciptors
3841 * used by the DE so that they can be loaded into cache in parallel
3842 * while the DE is processing state update packets.
3845 * The compute CP consists of two microengines (ME):
3846 * MEC1 - Compute MicroEngine 1
3847 * MEC2 - Compute MicroEngine 2
3848 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3849 * The queues are exposed to userspace and are programmed directly
3850 * by the compute runtime.
3853 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3855 * @rdev: radeon_device pointer
3856 * @enable: enable or disable the MEs
3858 * Halts or unhalts the gfx MEs.
3860 static void cik_cp_gfx_enable(struct radeon_device
*rdev
, bool enable
)
3863 WREG32(CP_ME_CNTL
, 0);
3865 if (rdev
->asic
->copy
.copy_ring_index
== RADEON_RING_TYPE_GFX_INDEX
)
3866 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
3867 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
3868 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3874 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3876 * @rdev: radeon_device pointer
3878 * Loads the gfx PFP, ME, and CE ucode.
3879 * Returns 0 for success, -EINVAL if the ucode is not available.
3881 static int cik_cp_gfx_load_microcode(struct radeon_device
*rdev
)
3883 const __be32
*fw_data
;
3886 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
)
3889 cik_cp_gfx_enable(rdev
, false);
3892 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
3893 WREG32(CP_PFP_UCODE_ADDR
, 0);
3894 for (i
= 0; i
< CIK_PFP_UCODE_SIZE
; i
++)
3895 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
3896 WREG32(CP_PFP_UCODE_ADDR
, 0);
3899 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
3900 WREG32(CP_CE_UCODE_ADDR
, 0);
3901 for (i
= 0; i
< CIK_CE_UCODE_SIZE
; i
++)
3902 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
3903 WREG32(CP_CE_UCODE_ADDR
, 0);
3906 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
3907 WREG32(CP_ME_RAM_WADDR
, 0);
3908 for (i
= 0; i
< CIK_ME_UCODE_SIZE
; i
++)
3909 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
3910 WREG32(CP_ME_RAM_WADDR
, 0);
3912 WREG32(CP_PFP_UCODE_ADDR
, 0);
3913 WREG32(CP_CE_UCODE_ADDR
, 0);
3914 WREG32(CP_ME_RAM_WADDR
, 0);
3915 WREG32(CP_ME_RAM_RADDR
, 0);
3920 * cik_cp_gfx_start - start the gfx ring
3922 * @rdev: radeon_device pointer
3924 * Enables the ring and loads the clear state context and other
3925 * packets required to init the ring.
3926 * Returns 0 for success, error for failure.
3928 static int cik_cp_gfx_start(struct radeon_device
*rdev
)
3930 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3934 WREG32(CP_MAX_CONTEXT
, rdev
->config
.cik
.max_hw_contexts
- 1);
3935 WREG32(CP_ENDIAN_SWAP
, 0);
3936 WREG32(CP_DEVICE_ID
, 1);
3938 cik_cp_gfx_enable(rdev
, true);
3940 r
= radeon_ring_lock(rdev
, ring
, cik_default_size
+ 17);
3942 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3946 /* init the CE partitions. CE only used for gfx on CIK */
3947 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3948 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3949 radeon_ring_write(ring
, 0xc000);
3950 radeon_ring_write(ring
, 0xc000);
3952 /* setup clear context state */
3953 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3954 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3956 radeon_ring_write(ring
, PACKET3(PACKET3_CONTEXT_CONTROL
, 1));
3957 radeon_ring_write(ring
, 0x80000000);
3958 radeon_ring_write(ring
, 0x80000000);
3960 for (i
= 0; i
< cik_default_size
; i
++)
3961 radeon_ring_write(ring
, cik_default_state
[i
]);
3963 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3964 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3966 /* set clear context state */
3967 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3968 radeon_ring_write(ring
, 0);
3970 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3971 radeon_ring_write(ring
, 0x00000316);
3972 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3973 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3975 radeon_ring_unlock_commit(rdev
, ring
);
3981 * cik_cp_gfx_fini - stop the gfx ring
3983 * @rdev: radeon_device pointer
3985 * Stop the gfx ring and tear down the driver ring
3988 static void cik_cp_gfx_fini(struct radeon_device
*rdev
)
3990 cik_cp_gfx_enable(rdev
, false);
3991 radeon_ring_fini(rdev
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3995 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3997 * @rdev: radeon_device pointer
3999 * Program the location and size of the gfx ring buffer
4000 * and test it to make sure it's working.
4001 * Returns 0 for success, error for failure.
4003 static int cik_cp_gfx_resume(struct radeon_device
*rdev
)
4005 struct radeon_ring
*ring
;
4011 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
4012 if (rdev
->family
!= CHIP_HAWAII
)
4013 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
4015 /* Set the write pointer delay */
4016 WREG32(CP_RB_WPTR_DELAY
, 0);
4018 /* set the RB to use vmid 0 */
4019 WREG32(CP_RB_VMID
, 0);
4021 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
4023 /* ring 0 - compute and gfx */
4024 /* Set ring buffer size */
4025 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
4026 rb_bufsz
= order_base_2(ring
->ring_size
/ 8);
4027 tmp
= (order_base_2(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
4029 tmp
|= BUF_SWAP_32BIT
;
4031 WREG32(CP_RB0_CNTL
, tmp
);
4033 /* Initialize the ring buffer's read and write pointers */
4034 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
4036 WREG32(CP_RB0_WPTR
, ring
->wptr
);
4038 /* set the wb address wether it's enabled or not */
4039 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
4040 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
4042 /* scratch register shadowing is no longer supported */
4043 WREG32(SCRATCH_UMSK
, 0);
4045 if (!rdev
->wb
.enabled
)
4046 tmp
|= RB_NO_UPDATE
;
4049 WREG32(CP_RB0_CNTL
, tmp
);
4051 rb_addr
= ring
->gpu_addr
>> 8;
4052 WREG32(CP_RB0_BASE
, rb_addr
);
4053 WREG32(CP_RB0_BASE_HI
, upper_32_bits(rb_addr
));
4055 /* start the ring */
4056 cik_cp_gfx_start(rdev
);
4057 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
4058 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
4060 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
4064 if (rdev
->asic
->copy
.copy_ring_index
== RADEON_RING_TYPE_GFX_INDEX
)
4065 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.real_vram_size
);
4070 u32
cik_gfx_get_rptr(struct radeon_device
*rdev
,
4071 struct radeon_ring
*ring
)
4075 if (rdev
->wb
.enabled
)
4076 rptr
= rdev
->wb
.wb
[ring
->rptr_offs
/4];
4078 rptr
= RREG32(CP_RB0_RPTR
);
4083 u32
cik_gfx_get_wptr(struct radeon_device
*rdev
,
4084 struct radeon_ring
*ring
)
4088 wptr
= RREG32(CP_RB0_WPTR
);
4093 void cik_gfx_set_wptr(struct radeon_device
*rdev
,
4094 struct radeon_ring
*ring
)
4096 WREG32(CP_RB0_WPTR
, ring
->wptr
);
4097 (void)RREG32(CP_RB0_WPTR
);
4100 u32
cik_compute_get_rptr(struct radeon_device
*rdev
,
4101 struct radeon_ring
*ring
)
4105 if (rdev
->wb
.enabled
) {
4106 rptr
= rdev
->wb
.wb
[ring
->rptr_offs
/4];
4108 mutex_lock(&rdev
->srbm_mutex
);
4109 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
4110 rptr
= RREG32(CP_HQD_PQ_RPTR
);
4111 cik_srbm_select(rdev
, 0, 0, 0, 0);
4112 mutex_unlock(&rdev
->srbm_mutex
);
4118 u32
cik_compute_get_wptr(struct radeon_device
*rdev
,
4119 struct radeon_ring
*ring
)
4123 if (rdev
->wb
.enabled
) {
4124 /* XXX check if swapping is necessary on BE */
4125 wptr
= rdev
->wb
.wb
[ring
->wptr_offs
/4];
4127 mutex_lock(&rdev
->srbm_mutex
);
4128 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
4129 wptr
= RREG32(CP_HQD_PQ_WPTR
);
4130 cik_srbm_select(rdev
, 0, 0, 0, 0);
4131 mutex_unlock(&rdev
->srbm_mutex
);
4137 void cik_compute_set_wptr(struct radeon_device
*rdev
,
4138 struct radeon_ring
*ring
)
4140 /* XXX check if swapping is necessary on BE */
4141 rdev
->wb
.wb
[ring
->wptr_offs
/4] = ring
->wptr
;
4142 WDOORBELL32(ring
->doorbell_index
, ring
->wptr
);
4146 * cik_cp_compute_enable - enable/disable the compute CP MEs
4148 * @rdev: radeon_device pointer
4149 * @enable: enable or disable the MEs
4151 * Halts or unhalts the compute MEs.
4153 static void cik_cp_compute_enable(struct radeon_device
*rdev
, bool enable
)
4156 WREG32(CP_MEC_CNTL
, 0);
4158 WREG32(CP_MEC_CNTL
, (MEC_ME1_HALT
| MEC_ME2_HALT
));
4159 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
4160 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
4166 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4168 * @rdev: radeon_device pointer
4170 * Loads the compute MEC1&2 ucode.
4171 * Returns 0 for success, -EINVAL if the ucode is not available.
4173 static int cik_cp_compute_load_microcode(struct radeon_device
*rdev
)
4175 const __be32
*fw_data
;
4181 cik_cp_compute_enable(rdev
, false);
4184 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
4185 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
4186 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
4187 WREG32(CP_MEC_ME1_UCODE_DATA
, be32_to_cpup(fw_data
++));
4188 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
4190 if (rdev
->family
== CHIP_KAVERI
) {
4192 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
4193 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
4194 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
4195 WREG32(CP_MEC_ME2_UCODE_DATA
, be32_to_cpup(fw_data
++));
4196 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
4203 * cik_cp_compute_start - start the compute queues
4205 * @rdev: radeon_device pointer
4207 * Enable the compute queues.
4208 * Returns 0 for success, error for failure.
4210 static int cik_cp_compute_start(struct radeon_device
*rdev
)
4212 cik_cp_compute_enable(rdev
, true);
4218 * cik_cp_compute_fini - stop the compute queues
4220 * @rdev: radeon_device pointer
4222 * Stop the compute queues and tear down the driver queue
4225 static void cik_cp_compute_fini(struct radeon_device
*rdev
)
4229 cik_cp_compute_enable(rdev
, false);
4231 for (i
= 0; i
< 2; i
++) {
4233 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
4235 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
4237 if (rdev
->ring
[idx
].mqd_obj
) {
4238 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
4239 if (unlikely(r
!= 0))
4240 dev_warn(rdev
->dev
, "(%d) reserve MQD bo failed\n", r
);
4242 radeon_bo_unpin(rdev
->ring
[idx
].mqd_obj
);
4243 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
4245 radeon_bo_unref(&rdev
->ring
[idx
].mqd_obj
);
4246 rdev
->ring
[idx
].mqd_obj
= NULL
;
4251 static void cik_mec_fini(struct radeon_device
*rdev
)
4255 if (rdev
->mec
.hpd_eop_obj
) {
4256 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
4257 if (unlikely(r
!= 0))
4258 dev_warn(rdev
->dev
, "(%d) reserve HPD EOP bo failed\n", r
);
4259 radeon_bo_unpin(rdev
->mec
.hpd_eop_obj
);
4260 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
4262 radeon_bo_unref(&rdev
->mec
.hpd_eop_obj
);
4263 rdev
->mec
.hpd_eop_obj
= NULL
;
4267 #define MEC_HPD_SIZE 2048
4269 static int cik_mec_init(struct radeon_device
*rdev
)
4275 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4276 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4278 if (rdev
->family
== CHIP_KAVERI
)
4279 rdev
->mec
.num_mec
= 2;
4281 rdev
->mec
.num_mec
= 1;
4282 rdev
->mec
.num_pipe
= 4;
4283 rdev
->mec
.num_queue
= rdev
->mec
.num_mec
* rdev
->mec
.num_pipe
* 8;
4285 if (rdev
->mec
.hpd_eop_obj
== NULL
) {
4286 r
= radeon_bo_create(rdev
,
4287 rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2,
4289 RADEON_GEM_DOMAIN_GTT
, NULL
,
4290 &rdev
->mec
.hpd_eop_obj
);
4292 dev_warn(rdev
->dev
, "(%d) create HDP EOP bo failed\n", r
);
4297 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
4298 if (unlikely(r
!= 0)) {
4302 r
= radeon_bo_pin(rdev
->mec
.hpd_eop_obj
, RADEON_GEM_DOMAIN_GTT
,
4303 &rdev
->mec
.hpd_eop_gpu_addr
);
4305 dev_warn(rdev
->dev
, "(%d) pin HDP EOP bo failed\n", r
);
4309 r
= radeon_bo_kmap(rdev
->mec
.hpd_eop_obj
, (void **)&hpd
);
4311 dev_warn(rdev
->dev
, "(%d) map HDP EOP bo failed\n", r
);
4316 /* clear memory. Not sure if this is required or not */
4317 memset(hpd
, 0, rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2);
4319 radeon_bo_kunmap(rdev
->mec
.hpd_eop_obj
);
4320 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
4325 struct hqd_registers
4327 u32 cp_mqd_base_addr
;
4328 u32 cp_mqd_base_addr_hi
;
4331 u32 cp_hqd_persistent_state
;
4332 u32 cp_hqd_pipe_priority
;
4333 u32 cp_hqd_queue_priority
;
4336 u32 cp_hqd_pq_base_hi
;
4338 u32 cp_hqd_pq_rptr_report_addr
;
4339 u32 cp_hqd_pq_rptr_report_addr_hi
;
4340 u32 cp_hqd_pq_wptr_poll_addr
;
4341 u32 cp_hqd_pq_wptr_poll_addr_hi
;
4342 u32 cp_hqd_pq_doorbell_control
;
4344 u32 cp_hqd_pq_control
;
4345 u32 cp_hqd_ib_base_addr
;
4346 u32 cp_hqd_ib_base_addr_hi
;
4348 u32 cp_hqd_ib_control
;
4349 u32 cp_hqd_iq_timer
;
4351 u32 cp_hqd_dequeue_request
;
4352 u32 cp_hqd_dma_offload
;
4353 u32 cp_hqd_sema_cmd
;
4354 u32 cp_hqd_msg_type
;
4355 u32 cp_hqd_atomic0_preop_lo
;
4356 u32 cp_hqd_atomic0_preop_hi
;
4357 u32 cp_hqd_atomic1_preop_lo
;
4358 u32 cp_hqd_atomic1_preop_hi
;
4359 u32 cp_hqd_hq_scheduler0
;
4360 u32 cp_hqd_hq_scheduler1
;
4367 u32 dispatch_initiator
;
4371 u32 pipeline_stat_enable
;
4372 u32 perf_counter_enable
;
4378 u32 resource_limits
;
4379 u32 static_thread_mgmt01
[2];
4381 u32 static_thread_mgmt23
[2];
4383 u32 thread_trace_enable
;
4386 u32 vgtcs_invoke_count
[2];
4387 struct hqd_registers queue_state
;
4389 u32 interrupt_queue
[64];
4393 * cik_cp_compute_resume - setup the compute queue registers
4395 * @rdev: radeon_device pointer
4397 * Program the compute queues and test them to make sure they
4399 * Returns 0 for success, error for failure.
4401 static int cik_cp_compute_resume(struct radeon_device
*rdev
)
4405 bool use_doorbell
= true;
4411 struct bonaire_mqd
*mqd
;
4413 r
= cik_cp_compute_start(rdev
);
4417 /* fix up chicken bits */
4418 tmp
= RREG32(CP_CPF_DEBUG
);
4420 WREG32(CP_CPF_DEBUG
, tmp
);
4422 /* init the pipes */
4423 mutex_lock(&rdev
->srbm_mutex
);
4424 for (i
= 0; i
< (rdev
->mec
.num_pipe
* rdev
->mec
.num_mec
); i
++) {
4425 int me
= (i
< 4) ? 1 : 2;
4426 int pipe
= (i
< 4) ? i
: (i
- 4);
4428 eop_gpu_addr
= rdev
->mec
.hpd_eop_gpu_addr
+ (i
* MEC_HPD_SIZE
* 2);
4430 cik_srbm_select(rdev
, me
, pipe
, 0, 0);
4432 /* write the EOP addr */
4433 WREG32(CP_HPD_EOP_BASE_ADDR
, eop_gpu_addr
>> 8);
4434 WREG32(CP_HPD_EOP_BASE_ADDR_HI
, upper_32_bits(eop_gpu_addr
) >> 8);
4436 /* set the VMID assigned */
4437 WREG32(CP_HPD_EOP_VMID
, 0);
4439 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4440 tmp
= RREG32(CP_HPD_EOP_CONTROL
);
4441 tmp
&= ~EOP_SIZE_MASK
;
4442 tmp
|= order_base_2(MEC_HPD_SIZE
/ 8);
4443 WREG32(CP_HPD_EOP_CONTROL
, tmp
);
4445 cik_srbm_select(rdev
, 0, 0, 0, 0);
4446 mutex_unlock(&rdev
->srbm_mutex
);
4448 /* init the queues. Just two for now. */
4449 for (i
= 0; i
< 2; i
++) {
4451 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
4453 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
4455 if (rdev
->ring
[idx
].mqd_obj
== NULL
) {
4456 r
= radeon_bo_create(rdev
,
4457 sizeof(struct bonaire_mqd
),
4459 RADEON_GEM_DOMAIN_GTT
, NULL
,
4460 &rdev
->ring
[idx
].mqd_obj
);
4462 dev_warn(rdev
->dev
, "(%d) create MQD bo failed\n", r
);
4467 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
4468 if (unlikely(r
!= 0)) {
4469 cik_cp_compute_fini(rdev
);
4472 r
= radeon_bo_pin(rdev
->ring
[idx
].mqd_obj
, RADEON_GEM_DOMAIN_GTT
,
4475 dev_warn(rdev
->dev
, "(%d) pin MQD bo failed\n", r
);
4476 cik_cp_compute_fini(rdev
);
4479 r
= radeon_bo_kmap(rdev
->ring
[idx
].mqd_obj
, (void **)&buf
);
4481 dev_warn(rdev
->dev
, "(%d) map MQD bo failed\n", r
);
4482 cik_cp_compute_fini(rdev
);
4486 /* init the mqd struct */
4487 memset(buf
, 0, sizeof(struct bonaire_mqd
));
4489 mqd
= (struct bonaire_mqd
*)buf
;
4490 mqd
->header
= 0xC0310800;
4491 mqd
->static_thread_mgmt01
[0] = 0xffffffff;
4492 mqd
->static_thread_mgmt01
[1] = 0xffffffff;
4493 mqd
->static_thread_mgmt23
[0] = 0xffffffff;
4494 mqd
->static_thread_mgmt23
[1] = 0xffffffff;
4496 mutex_lock(&rdev
->srbm_mutex
);
4497 cik_srbm_select(rdev
, rdev
->ring
[idx
].me
,
4498 rdev
->ring
[idx
].pipe
,
4499 rdev
->ring
[idx
].queue
, 0);
4501 /* disable wptr polling */
4502 tmp
= RREG32(CP_PQ_WPTR_POLL_CNTL
);
4503 tmp
&= ~WPTR_POLL_EN
;
4504 WREG32(CP_PQ_WPTR_POLL_CNTL
, tmp
);
4506 /* enable doorbell? */
4507 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
4508 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
4510 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
4512 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_EN
;
4513 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
4514 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
4516 /* disable the queue if it's active */
4517 mqd
->queue_state
.cp_hqd_dequeue_request
= 0;
4518 mqd
->queue_state
.cp_hqd_pq_rptr
= 0;
4519 mqd
->queue_state
.cp_hqd_pq_wptr
= 0;
4520 if (RREG32(CP_HQD_ACTIVE
) & 1) {
4521 WREG32(CP_HQD_DEQUEUE_REQUEST
, 1);
4522 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4523 if (!(RREG32(CP_HQD_ACTIVE
) & 1))
4527 WREG32(CP_HQD_DEQUEUE_REQUEST
, mqd
->queue_state
.cp_hqd_dequeue_request
);
4528 WREG32(CP_HQD_PQ_RPTR
, mqd
->queue_state
.cp_hqd_pq_rptr
);
4529 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
4532 /* set the pointer to the MQD */
4533 mqd
->queue_state
.cp_mqd_base_addr
= mqd_gpu_addr
& 0xfffffffc;
4534 mqd
->queue_state
.cp_mqd_base_addr_hi
= upper_32_bits(mqd_gpu_addr
);
4535 WREG32(CP_MQD_BASE_ADDR
, mqd
->queue_state
.cp_mqd_base_addr
);
4536 WREG32(CP_MQD_BASE_ADDR_HI
, mqd
->queue_state
.cp_mqd_base_addr_hi
);
4537 /* set MQD vmid to 0 */
4538 mqd
->queue_state
.cp_mqd_control
= RREG32(CP_MQD_CONTROL
);
4539 mqd
->queue_state
.cp_mqd_control
&= ~MQD_VMID_MASK
;
4540 WREG32(CP_MQD_CONTROL
, mqd
->queue_state
.cp_mqd_control
);
4542 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4543 hqd_gpu_addr
= rdev
->ring
[idx
].gpu_addr
>> 8;
4544 mqd
->queue_state
.cp_hqd_pq_base
= hqd_gpu_addr
;
4545 mqd
->queue_state
.cp_hqd_pq_base_hi
= upper_32_bits(hqd_gpu_addr
);
4546 WREG32(CP_HQD_PQ_BASE
, mqd
->queue_state
.cp_hqd_pq_base
);
4547 WREG32(CP_HQD_PQ_BASE_HI
, mqd
->queue_state
.cp_hqd_pq_base_hi
);
4549 /* set up the HQD, this is similar to CP_RB0_CNTL */
4550 mqd
->queue_state
.cp_hqd_pq_control
= RREG32(CP_HQD_PQ_CONTROL
);
4551 mqd
->queue_state
.cp_hqd_pq_control
&=
4552 ~(QUEUE_SIZE_MASK
| RPTR_BLOCK_SIZE_MASK
);
4554 mqd
->queue_state
.cp_hqd_pq_control
|=
4555 order_base_2(rdev
->ring
[idx
].ring_size
/ 8);
4556 mqd
->queue_state
.cp_hqd_pq_control
|=
4557 (order_base_2(RADEON_GPU_PAGE_SIZE
/8) << 8);
4559 mqd
->queue_state
.cp_hqd_pq_control
|= BUF_SWAP_32BIT
;
4561 mqd
->queue_state
.cp_hqd_pq_control
&=
4562 ~(UNORD_DISPATCH
| ROQ_PQ_IB_FLIP
| PQ_VOLATILE
);
4563 mqd
->queue_state
.cp_hqd_pq_control
|=
4564 PRIV_STATE
| KMD_QUEUE
; /* assuming kernel queue control */
4565 WREG32(CP_HQD_PQ_CONTROL
, mqd
->queue_state
.cp_hqd_pq_control
);
4567 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4569 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP1_WPTR_OFFSET
;
4571 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP2_WPTR_OFFSET
;
4572 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
= wb_gpu_addr
& 0xfffffffc;
4573 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
= upper_32_bits(wb_gpu_addr
) & 0xffff;
4574 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR
, mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
);
4575 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI
,
4576 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
);
4578 /* set the wb address wether it's enabled or not */
4580 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
;
4582 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
;
4583 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
= wb_gpu_addr
& 0xfffffffc;
4584 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
=
4585 upper_32_bits(wb_gpu_addr
) & 0xffff;
4586 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR
,
4587 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
);
4588 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI
,
4589 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
);
4591 /* enable the doorbell if requested */
4593 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
4594 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
4595 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_OFFSET_MASK
;
4596 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|=
4597 DOORBELL_OFFSET(rdev
->ring
[idx
].doorbell_index
);
4598 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
4599 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&=
4600 ~(DOORBELL_SOURCE
| DOORBELL_HIT
);
4603 mqd
->queue_state
.cp_hqd_pq_doorbell_control
= 0;
4605 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
4606 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
4608 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4609 rdev
->ring
[idx
].wptr
= 0;
4610 mqd
->queue_state
.cp_hqd_pq_wptr
= rdev
->ring
[idx
].wptr
;
4611 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
4612 mqd
->queue_state
.cp_hqd_pq_rptr
= RREG32(CP_HQD_PQ_RPTR
);
4614 /* set the vmid for the queue */
4615 mqd
->queue_state
.cp_hqd_vmid
= 0;
4616 WREG32(CP_HQD_VMID
, mqd
->queue_state
.cp_hqd_vmid
);
4618 /* activate the queue */
4619 mqd
->queue_state
.cp_hqd_active
= 1;
4620 WREG32(CP_HQD_ACTIVE
, mqd
->queue_state
.cp_hqd_active
);
4622 cik_srbm_select(rdev
, 0, 0, 0, 0);
4623 mutex_unlock(&rdev
->srbm_mutex
);
4625 radeon_bo_kunmap(rdev
->ring
[idx
].mqd_obj
);
4626 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
4628 rdev
->ring
[idx
].ready
= true;
4629 r
= radeon_ring_test(rdev
, idx
, &rdev
->ring
[idx
]);
4631 rdev
->ring
[idx
].ready
= false;
4637 static void cik_cp_enable(struct radeon_device
*rdev
, bool enable
)
4639 cik_cp_gfx_enable(rdev
, enable
);
4640 cik_cp_compute_enable(rdev
, enable
);
4643 static int cik_cp_load_microcode(struct radeon_device
*rdev
)
4647 r
= cik_cp_gfx_load_microcode(rdev
);
4650 r
= cik_cp_compute_load_microcode(rdev
);
4657 static void cik_cp_fini(struct radeon_device
*rdev
)
4659 cik_cp_gfx_fini(rdev
);
4660 cik_cp_compute_fini(rdev
);
4663 static int cik_cp_resume(struct radeon_device
*rdev
)
4667 cik_enable_gui_idle_interrupt(rdev
, false);
4669 r
= cik_cp_load_microcode(rdev
);
4673 r
= cik_cp_gfx_resume(rdev
);
4676 r
= cik_cp_compute_resume(rdev
);
4680 cik_enable_gui_idle_interrupt(rdev
, true);
4685 static void cik_print_gpu_status_regs(struct radeon_device
*rdev
)
4687 dev_info(rdev
->dev
, " GRBM_STATUS=0x%08X\n",
4688 RREG32(GRBM_STATUS
));
4689 dev_info(rdev
->dev
, " GRBM_STATUS2=0x%08X\n",
4690 RREG32(GRBM_STATUS2
));
4691 dev_info(rdev
->dev
, " GRBM_STATUS_SE0=0x%08X\n",
4692 RREG32(GRBM_STATUS_SE0
));
4693 dev_info(rdev
->dev
, " GRBM_STATUS_SE1=0x%08X\n",
4694 RREG32(GRBM_STATUS_SE1
));
4695 dev_info(rdev
->dev
, " GRBM_STATUS_SE2=0x%08X\n",
4696 RREG32(GRBM_STATUS_SE2
));
4697 dev_info(rdev
->dev
, " GRBM_STATUS_SE3=0x%08X\n",
4698 RREG32(GRBM_STATUS_SE3
));
4699 dev_info(rdev
->dev
, " SRBM_STATUS=0x%08X\n",
4700 RREG32(SRBM_STATUS
));
4701 dev_info(rdev
->dev
, " SRBM_STATUS2=0x%08X\n",
4702 RREG32(SRBM_STATUS2
));
4703 dev_info(rdev
->dev
, " SDMA0_STATUS_REG = 0x%08X\n",
4704 RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
));
4705 dev_info(rdev
->dev
, " SDMA1_STATUS_REG = 0x%08X\n",
4706 RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
));
4707 dev_info(rdev
->dev
, " CP_STAT = 0x%08x\n", RREG32(CP_STAT
));
4708 dev_info(rdev
->dev
, " CP_STALLED_STAT1 = 0x%08x\n",
4709 RREG32(CP_STALLED_STAT1
));
4710 dev_info(rdev
->dev
, " CP_STALLED_STAT2 = 0x%08x\n",
4711 RREG32(CP_STALLED_STAT2
));
4712 dev_info(rdev
->dev
, " CP_STALLED_STAT3 = 0x%08x\n",
4713 RREG32(CP_STALLED_STAT3
));
4714 dev_info(rdev
->dev
, " CP_CPF_BUSY_STAT = 0x%08x\n",
4715 RREG32(CP_CPF_BUSY_STAT
));
4716 dev_info(rdev
->dev
, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4717 RREG32(CP_CPF_STALLED_STAT1
));
4718 dev_info(rdev
->dev
, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS
));
4719 dev_info(rdev
->dev
, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT
));
4720 dev_info(rdev
->dev
, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4721 RREG32(CP_CPC_STALLED_STAT1
));
4722 dev_info(rdev
->dev
, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS
));
4726 * cik_gpu_check_soft_reset - check which blocks are busy
4728 * @rdev: radeon_device pointer
4730 * Check which blocks are busy and return the relevant reset
4731 * mask to be used by cik_gpu_soft_reset().
4732 * Returns a mask of the blocks to be reset.
4734 u32
cik_gpu_check_soft_reset(struct radeon_device
*rdev
)
4740 tmp
= RREG32(GRBM_STATUS
);
4741 if (tmp
& (PA_BUSY
| SC_BUSY
|
4742 BCI_BUSY
| SX_BUSY
|
4743 TA_BUSY
| VGT_BUSY
|
4745 GDS_BUSY
| SPI_BUSY
|
4746 IA_BUSY
| IA_BUSY_NO_DMA
))
4747 reset_mask
|= RADEON_RESET_GFX
;
4749 if (tmp
& (CP_BUSY
| CP_COHERENCY_BUSY
))
4750 reset_mask
|= RADEON_RESET_CP
;
4753 tmp
= RREG32(GRBM_STATUS2
);
4755 reset_mask
|= RADEON_RESET_RLC
;
4757 /* SDMA0_STATUS_REG */
4758 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
);
4759 if (!(tmp
& SDMA_IDLE
))
4760 reset_mask
|= RADEON_RESET_DMA
;
4762 /* SDMA1_STATUS_REG */
4763 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
);
4764 if (!(tmp
& SDMA_IDLE
))
4765 reset_mask
|= RADEON_RESET_DMA1
;
4768 tmp
= RREG32(SRBM_STATUS2
);
4769 if (tmp
& SDMA_BUSY
)
4770 reset_mask
|= RADEON_RESET_DMA
;
4772 if (tmp
& SDMA1_BUSY
)
4773 reset_mask
|= RADEON_RESET_DMA1
;
4776 tmp
= RREG32(SRBM_STATUS
);
4779 reset_mask
|= RADEON_RESET_IH
;
4782 reset_mask
|= RADEON_RESET_SEM
;
4784 if (tmp
& GRBM_RQ_PENDING
)
4785 reset_mask
|= RADEON_RESET_GRBM
;
4788 reset_mask
|= RADEON_RESET_VMC
;
4790 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
4791 MCC_BUSY
| MCD_BUSY
))
4792 reset_mask
|= RADEON_RESET_MC
;
4794 if (evergreen_is_display_hung(rdev
))
4795 reset_mask
|= RADEON_RESET_DISPLAY
;
4797 /* Skip MC reset as it's mostly likely not hung, just busy */
4798 if (reset_mask
& RADEON_RESET_MC
) {
4799 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
4800 reset_mask
&= ~RADEON_RESET_MC
;
4807 * cik_gpu_soft_reset - soft reset GPU
4809 * @rdev: radeon_device pointer
4810 * @reset_mask: mask of which blocks to reset
4812 * Soft reset the blocks specified in @reset_mask.
4814 static void cik_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
4816 struct evergreen_mc_save save
;
4817 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
4820 if (reset_mask
== 0)
4823 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
4825 cik_print_gpu_status_regs(rdev
);
4826 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4827 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
4828 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4829 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
4838 /* Disable GFX parsing/prefetching */
4839 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
4841 /* Disable MEC parsing/prefetching */
4842 WREG32(CP_MEC_CNTL
, MEC_ME1_HALT
| MEC_ME2_HALT
);
4844 if (reset_mask
& RADEON_RESET_DMA
) {
4846 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
);
4848 WREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
4850 if (reset_mask
& RADEON_RESET_DMA1
) {
4852 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
);
4854 WREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
4857 evergreen_mc_stop(rdev
, &save
);
4858 if (evergreen_mc_wait_for_idle(rdev
)) {
4859 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
4862 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
))
4863 grbm_soft_reset
= SOFT_RESET_CP
| SOFT_RESET_GFX
;
4865 if (reset_mask
& RADEON_RESET_CP
) {
4866 grbm_soft_reset
|= SOFT_RESET_CP
;
4868 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4871 if (reset_mask
& RADEON_RESET_DMA
)
4872 srbm_soft_reset
|= SOFT_RESET_SDMA
;
4874 if (reset_mask
& RADEON_RESET_DMA1
)
4875 srbm_soft_reset
|= SOFT_RESET_SDMA1
;
4877 if (reset_mask
& RADEON_RESET_DISPLAY
)
4878 srbm_soft_reset
|= SOFT_RESET_DC
;
4880 if (reset_mask
& RADEON_RESET_RLC
)
4881 grbm_soft_reset
|= SOFT_RESET_RLC
;
4883 if (reset_mask
& RADEON_RESET_SEM
)
4884 srbm_soft_reset
|= SOFT_RESET_SEM
;
4886 if (reset_mask
& RADEON_RESET_IH
)
4887 srbm_soft_reset
|= SOFT_RESET_IH
;
4889 if (reset_mask
& RADEON_RESET_GRBM
)
4890 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4892 if (reset_mask
& RADEON_RESET_VMC
)
4893 srbm_soft_reset
|= SOFT_RESET_VMC
;
4895 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
4896 if (reset_mask
& RADEON_RESET_MC
)
4897 srbm_soft_reset
|= SOFT_RESET_MC
;
4900 if (grbm_soft_reset
) {
4901 tmp
= RREG32(GRBM_SOFT_RESET
);
4902 tmp
|= grbm_soft_reset
;
4903 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
4904 WREG32(GRBM_SOFT_RESET
, tmp
);
4905 tmp
= RREG32(GRBM_SOFT_RESET
);
4909 tmp
&= ~grbm_soft_reset
;
4910 WREG32(GRBM_SOFT_RESET
, tmp
);
4911 tmp
= RREG32(GRBM_SOFT_RESET
);
4914 if (srbm_soft_reset
) {
4915 tmp
= RREG32(SRBM_SOFT_RESET
);
4916 tmp
|= srbm_soft_reset
;
4917 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
4918 WREG32(SRBM_SOFT_RESET
, tmp
);
4919 tmp
= RREG32(SRBM_SOFT_RESET
);
4923 tmp
&= ~srbm_soft_reset
;
4924 WREG32(SRBM_SOFT_RESET
, tmp
);
4925 tmp
= RREG32(SRBM_SOFT_RESET
);
4928 /* Wait a little for things to settle down */
4931 evergreen_mc_resume(rdev
, &save
);
4934 cik_print_gpu_status_regs(rdev
);
4937 struct kv_reset_save_regs
{
4938 u32 gmcon_reng_execute
;
4943 static void kv_save_regs_for_reset(struct radeon_device
*rdev
,
4944 struct kv_reset_save_regs
*save
)
4946 save
->gmcon_reng_execute
= RREG32(GMCON_RENG_EXECUTE
);
4947 save
->gmcon_misc
= RREG32(GMCON_MISC
);
4948 save
->gmcon_misc3
= RREG32(GMCON_MISC3
);
4950 WREG32(GMCON_RENG_EXECUTE
, save
->gmcon_reng_execute
& ~RENG_EXECUTE_ON_PWR_UP
);
4951 WREG32(GMCON_MISC
, save
->gmcon_misc
& ~(RENG_EXECUTE_ON_REG_UPDATE
|
4952 STCTRL_STUTTER_EN
));
4955 static void kv_restore_regs_for_reset(struct radeon_device
*rdev
,
4956 struct kv_reset_save_regs
*save
)
4960 WREG32(GMCON_PGFSM_WRITE
, 0);
4961 WREG32(GMCON_PGFSM_CONFIG
, 0x200010ff);
4963 for (i
= 0; i
< 5; i
++)
4964 WREG32(GMCON_PGFSM_WRITE
, 0);
4966 WREG32(GMCON_PGFSM_WRITE
, 0);
4967 WREG32(GMCON_PGFSM_CONFIG
, 0x300010ff);
4969 for (i
= 0; i
< 5; i
++)
4970 WREG32(GMCON_PGFSM_WRITE
, 0);
4972 WREG32(GMCON_PGFSM_WRITE
, 0x210000);
4973 WREG32(GMCON_PGFSM_CONFIG
, 0xa00010ff);
4975 for (i
= 0; i
< 5; i
++)
4976 WREG32(GMCON_PGFSM_WRITE
, 0);
4978 WREG32(GMCON_PGFSM_WRITE
, 0x21003);
4979 WREG32(GMCON_PGFSM_CONFIG
, 0xb00010ff);
4981 for (i
= 0; i
< 5; i
++)
4982 WREG32(GMCON_PGFSM_WRITE
, 0);
4984 WREG32(GMCON_PGFSM_WRITE
, 0x2b00);
4985 WREG32(GMCON_PGFSM_CONFIG
, 0xc00010ff);
4987 for (i
= 0; i
< 5; i
++)
4988 WREG32(GMCON_PGFSM_WRITE
, 0);
4990 WREG32(GMCON_PGFSM_WRITE
, 0);
4991 WREG32(GMCON_PGFSM_CONFIG
, 0xd00010ff);
4993 for (i
= 0; i
< 5; i
++)
4994 WREG32(GMCON_PGFSM_WRITE
, 0);
4996 WREG32(GMCON_PGFSM_WRITE
, 0x420000);
4997 WREG32(GMCON_PGFSM_CONFIG
, 0x100010ff);
4999 for (i
= 0; i
< 5; i
++)
5000 WREG32(GMCON_PGFSM_WRITE
, 0);
5002 WREG32(GMCON_PGFSM_WRITE
, 0x120202);
5003 WREG32(GMCON_PGFSM_CONFIG
, 0x500010ff);
5005 for (i
= 0; i
< 5; i
++)
5006 WREG32(GMCON_PGFSM_WRITE
, 0);
5008 WREG32(GMCON_PGFSM_WRITE
, 0x3e3e36);
5009 WREG32(GMCON_PGFSM_CONFIG
, 0x600010ff);
5011 for (i
= 0; i
< 5; i
++)
5012 WREG32(GMCON_PGFSM_WRITE
, 0);
5014 WREG32(GMCON_PGFSM_WRITE
, 0x373f3e);
5015 WREG32(GMCON_PGFSM_CONFIG
, 0x700010ff);
5017 for (i
= 0; i
< 5; i
++)
5018 WREG32(GMCON_PGFSM_WRITE
, 0);
5020 WREG32(GMCON_PGFSM_WRITE
, 0x3e1332);
5021 WREG32(GMCON_PGFSM_CONFIG
, 0xe00010ff);
5023 WREG32(GMCON_MISC3
, save
->gmcon_misc3
);
5024 WREG32(GMCON_MISC
, save
->gmcon_misc
);
5025 WREG32(GMCON_RENG_EXECUTE
, save
->gmcon_reng_execute
);
5028 static void cik_gpu_pci_config_reset(struct radeon_device
*rdev
)
5030 struct evergreen_mc_save save
;
5031 struct kv_reset_save_regs kv_save
= { 0 };
5034 dev_info(rdev
->dev
, "GPU pci config reset\n");
5042 /* Disable GFX parsing/prefetching */
5043 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
5045 /* Disable MEC parsing/prefetching */
5046 WREG32(CP_MEC_CNTL
, MEC_ME1_HALT
| MEC_ME2_HALT
);
5049 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
);
5051 WREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
5053 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
);
5055 WREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
5056 /* XXX other engines? */
5058 /* halt the rlc, disable cp internal ints */
5063 /* disable mem access */
5064 evergreen_mc_stop(rdev
, &save
);
5065 if (evergreen_mc_wait_for_idle(rdev
)) {
5066 dev_warn(rdev
->dev
, "Wait for MC idle timed out !\n");
5069 if (rdev
->flags
& RADEON_IS_IGP
)
5070 kv_save_regs_for_reset(rdev
, &kv_save
);
5073 pci_clear_master(rdev
->pdev
);
5075 radeon_pci_config_reset(rdev
);
5079 /* wait for asic to come out of reset */
5080 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5081 if (RREG32(CONFIG_MEMSIZE
) != 0xffffffff)
5086 /* does asic init need to be run first??? */
5087 if (rdev
->flags
& RADEON_IS_IGP
)
5088 kv_restore_regs_for_reset(rdev
, &kv_save
);
5092 * cik_asic_reset - soft reset GPU
5094 * @rdev: radeon_device pointer
5096 * Look up which blocks are hung and attempt
5098 * Returns 0 for success.
5100 int cik_asic_reset(struct radeon_device
*rdev
)
5104 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5107 r600_set_bios_scratch_engine_hung(rdev
, true);
5109 /* try soft reset */
5110 cik_gpu_soft_reset(rdev
, reset_mask
);
5112 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5114 /* try pci config reset */
5115 if (reset_mask
&& radeon_hard_reset
)
5116 cik_gpu_pci_config_reset(rdev
);
5118 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5121 r600_set_bios_scratch_engine_hung(rdev
, false);
5127 * cik_gfx_is_lockup - check if the 3D engine is locked up
5129 * @rdev: radeon_device pointer
5130 * @ring: radeon_ring structure holding ring information
5132 * Check if the 3D engine is locked up (CIK).
5133 * Returns true if the engine is locked, false if not.
5135 bool cik_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
5137 u32 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5139 if (!(reset_mask
& (RADEON_RESET_GFX
|
5140 RADEON_RESET_COMPUTE
|
5141 RADEON_RESET_CP
))) {
5142 radeon_ring_lockup_update(rdev
, ring
);
5145 return radeon_ring_test_lockup(rdev
, ring
);
5150 * cik_mc_program - program the GPU memory controller
5152 * @rdev: radeon_device pointer
5154 * Set the location of vram, gart, and AGP in the GPU's
5155 * physical address space (CIK).
5157 static void cik_mc_program(struct radeon_device
*rdev
)
5159 struct evergreen_mc_save save
;
5163 /* Initialize HDP */
5164 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
5165 WREG32((0x2c14 + j
), 0x00000000);
5166 WREG32((0x2c18 + j
), 0x00000000);
5167 WREG32((0x2c1c + j
), 0x00000000);
5168 WREG32((0x2c20 + j
), 0x00000000);
5169 WREG32((0x2c24 + j
), 0x00000000);
5171 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
5173 evergreen_mc_stop(rdev
, &save
);
5174 if (radeon_mc_wait_for_idle(rdev
)) {
5175 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
5177 /* Lockout access through VGA aperture*/
5178 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
5179 /* Update configuration */
5180 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
5181 rdev
->mc
.vram_start
>> 12);
5182 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
5183 rdev
->mc
.vram_end
>> 12);
5184 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
5185 rdev
->vram_scratch
.gpu_addr
>> 12);
5186 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
5187 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
5188 WREG32(MC_VM_FB_LOCATION
, tmp
);
5189 /* XXX double check these! */
5190 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
5191 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
5192 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
5193 WREG32(MC_VM_AGP_BASE
, 0);
5194 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
5195 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
5196 if (radeon_mc_wait_for_idle(rdev
)) {
5197 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
5199 evergreen_mc_resume(rdev
, &save
);
5200 /* we need to own VRAM, so turn off the VGA renderer here
5201 * to stop it overwriting our objects */
5202 rv515_vga_render_disable(rdev
);
5206 * cik_mc_init - initialize the memory controller driver params
5208 * @rdev: radeon_device pointer
5210 * Look up the amount of vram, vram width, and decide how to place
5211 * vram and gart within the GPU's physical address space (CIK).
5212 * Returns 0 for success.
5214 static int cik_mc_init(struct radeon_device
*rdev
)
5217 int chansize
, numchan
;
5219 /* Get VRAM informations */
5220 rdev
->mc
.vram_is_ddr
= true;
5221 tmp
= RREG32(MC_ARB_RAMCFG
);
5222 if (tmp
& CHANSIZE_MASK
) {
5227 tmp
= RREG32(MC_SHARED_CHMAP
);
5228 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
5258 rdev
->mc
.vram_width
= numchan
* chansize
;
5259 /* Could aper size report 0 ? */
5260 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
5261 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
5262 /* size in MB on si */
5263 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
5264 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
5265 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
5266 si_vram_gtt_location(rdev
, &rdev
->mc
);
5267 radeon_update_bandwidth_info(rdev
);
5274 * VMID 0 is the physical GPU addresses as used by the kernel.
5275 * VMIDs 1-15 are used for userspace clients and are handled
5276 * by the radeon vm/hsa code.
5279 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5281 * @rdev: radeon_device pointer
5283 * Flush the TLB for the VMID 0 page table (CIK).
5285 void cik_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
5287 /* flush hdp cache */
5288 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0);
5290 /* bits 0-15 are the VM contexts0-15 */
5291 WREG32(VM_INVALIDATE_REQUEST
, 0x1);
5295 * cik_pcie_gart_enable - gart enable
5297 * @rdev: radeon_device pointer
5299 * This sets up the TLBs, programs the page tables for VMID0,
5300 * sets up the hw for VMIDs 1-15 which are allocated on
5301 * demand, and sets up the global locations for the LDS, GDS,
5302 * and GPUVM for FSA64 clients (CIK).
5303 * Returns 0 for success, errors for failure.
5305 static int cik_pcie_gart_enable(struct radeon_device
*rdev
)
5309 if (rdev
->gart
.robj
== NULL
) {
5310 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
5313 r
= radeon_gart_table_vram_pin(rdev
);
5316 radeon_gart_restore(rdev
);
5317 /* Setup TLB control */
5318 WREG32(MC_VM_MX_L1_TLB_CNTL
,
5321 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5322 ENABLE_ADVANCED_DRIVER_MODEL
|
5323 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5324 /* Setup L2 cache */
5325 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
5326 ENABLE_L2_FRAGMENT_PROCESSING
|
5327 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5328 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5329 EFFECTIVE_L2_QUEUE_SIZE(7) |
5330 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5331 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
5332 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5333 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5334 /* setup context0 */
5335 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
5336 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
5337 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
5338 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
5339 (u32
)(rdev
->dummy_page
.addr
>> 12));
5340 WREG32(VM_CONTEXT0_CNTL2
, 0);
5341 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
5342 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
5348 /* empty context1-15 */
5349 /* FIXME start with 4G, once using 2 level pt switch to full
5352 /* set vm size, must be a multiple of 4 */
5353 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
5354 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
5355 for (i
= 1; i
< 16; i
++) {
5357 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
5358 rdev
->gart
.table_addr
>> 12);
5360 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
5361 rdev
->gart
.table_addr
>> 12);
5364 /* enable context1-15 */
5365 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
5366 (u32
)(rdev
->dummy_page
.addr
>> 12));
5367 WREG32(VM_CONTEXT1_CNTL2
, 4);
5368 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
5369 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5370 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5371 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5372 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5373 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5374 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
5375 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5376 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
5377 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5378 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
5379 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5380 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
5382 if (rdev
->family
== CHIP_KAVERI
) {
5383 u32 tmp
= RREG32(CHUB_CONTROL
);
5385 WREG32(CHUB_CONTROL
, tmp
);
5388 /* XXX SH_MEM regs */
5389 /* where to put LDS, scratch, GPUVM in FSA64 space */
5390 mutex_lock(&rdev
->srbm_mutex
);
5391 for (i
= 0; i
< 16; i
++) {
5392 cik_srbm_select(rdev
, 0, 0, 0, i
);
5393 /* CP and shaders */
5394 WREG32(SH_MEM_CONFIG
, 0);
5395 WREG32(SH_MEM_APE1_BASE
, 1);
5396 WREG32(SH_MEM_APE1_LIMIT
, 0);
5397 WREG32(SH_MEM_BASES
, 0);
5399 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA0_REGISTER_OFFSET
, 0);
5400 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA0_REGISTER_OFFSET
, 0);
5401 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA1_REGISTER_OFFSET
, 0);
5402 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA1_REGISTER_OFFSET
, 0);
5403 /* XXX SDMA RLC - todo */
5405 cik_srbm_select(rdev
, 0, 0, 0, 0);
5406 mutex_unlock(&rdev
->srbm_mutex
);
5408 cik_pcie_gart_tlb_flush(rdev
);
5409 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5410 (unsigned)(rdev
->mc
.gtt_size
>> 20),
5411 (unsigned long long)rdev
->gart
.table_addr
);
5412 rdev
->gart
.ready
= true;
5417 * cik_pcie_gart_disable - gart disable
5419 * @rdev: radeon_device pointer
5421 * This disables all VM page table (CIK).
5423 static void cik_pcie_gart_disable(struct radeon_device
*rdev
)
5425 /* Disable all tables */
5426 WREG32(VM_CONTEXT0_CNTL
, 0);
5427 WREG32(VM_CONTEXT1_CNTL
, 0);
5428 /* Setup TLB control */
5429 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5430 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5431 /* Setup L2 cache */
5433 ENABLE_L2_FRAGMENT_PROCESSING
|
5434 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5435 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5436 EFFECTIVE_L2_QUEUE_SIZE(7) |
5437 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5438 WREG32(VM_L2_CNTL2
, 0);
5439 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5440 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5441 radeon_gart_table_vram_unpin(rdev
);
5445 * cik_pcie_gart_fini - vm fini callback
5447 * @rdev: radeon_device pointer
5449 * Tears down the driver GART/VM setup (CIK).
5451 static void cik_pcie_gart_fini(struct radeon_device
*rdev
)
5453 cik_pcie_gart_disable(rdev
);
5454 radeon_gart_table_vram_free(rdev
);
5455 radeon_gart_fini(rdev
);
5460 * cik_ib_parse - vm ib_parse callback
5462 * @rdev: radeon_device pointer
5463 * @ib: indirect buffer pointer
5465 * CIK uses hw IB checking so this is a nop (CIK).
5467 int cik_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
5474 * VMID 0 is the physical GPU addresses as used by the kernel.
5475 * VMIDs 1-15 are used for userspace clients and are handled
5476 * by the radeon vm/hsa code.
5479 * cik_vm_init - cik vm init callback
5481 * @rdev: radeon_device pointer
5483 * Inits cik specific vm parameters (number of VMs, base of vram for
5484 * VMIDs 1-15) (CIK).
5485 * Returns 0 for success.
5487 int cik_vm_init(struct radeon_device
*rdev
)
5490 rdev
->vm_manager
.nvm
= 16;
5491 /* base offset of vram pages */
5492 if (rdev
->flags
& RADEON_IS_IGP
) {
5493 u64 tmp
= RREG32(MC_VM_FB_OFFSET
);
5495 rdev
->vm_manager
.vram_base_offset
= tmp
;
5497 rdev
->vm_manager
.vram_base_offset
= 0;
5503 * cik_vm_fini - cik vm fini callback
5505 * @rdev: radeon_device pointer
5507 * Tear down any asic specific VM setup (CIK).
5509 void cik_vm_fini(struct radeon_device
*rdev
)
5514 * cik_vm_decode_fault - print human readable fault info
5516 * @rdev: radeon_device pointer
5517 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5518 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5520 * Print human readable fault information (CIK).
5522 static void cik_vm_decode_fault(struct radeon_device
*rdev
,
5523 u32 status
, u32 addr
, u32 mc_client
)
5526 u32 vmid
= (status
& FAULT_VMID_MASK
) >> FAULT_VMID_SHIFT
;
5527 u32 protections
= (status
& PROTECTIONS_MASK
) >> PROTECTIONS_SHIFT
;
5528 char block
[5] = { mc_client
>> 24, (mc_client
>> 16) & 0xff,
5529 (mc_client
>> 8) & 0xff, mc_client
& 0xff, 0 };
5531 if (rdev
->family
== CHIP_HAWAII
)
5532 mc_id
= (status
& HAWAII_MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
5534 mc_id
= (status
& MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
5536 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5537 protections
, vmid
, addr
,
5538 (status
& MEMORY_CLIENT_RW_MASK
) ? "write" : "read",
5539 block
, mc_client
, mc_id
);
5543 * cik_vm_flush - cik vm flush using the CP
5545 * @rdev: radeon_device pointer
5547 * Update the page table base and flush the VM TLB
5548 * using the CP (CIK).
5550 void cik_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
5552 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
5557 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5558 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5559 WRITE_DATA_DST_SEL(0)));
5561 radeon_ring_write(ring
,
5562 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
5564 radeon_ring_write(ring
,
5565 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
5567 radeon_ring_write(ring
, 0);
5568 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
5570 /* update SH_MEM_* regs */
5571 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5572 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5573 WRITE_DATA_DST_SEL(0)));
5574 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5575 radeon_ring_write(ring
, 0);
5576 radeon_ring_write(ring
, VMID(vm
->id
));
5578 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 6));
5579 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5580 WRITE_DATA_DST_SEL(0)));
5581 radeon_ring_write(ring
, SH_MEM_BASES
>> 2);
5582 radeon_ring_write(ring
, 0);
5584 radeon_ring_write(ring
, 0); /* SH_MEM_BASES */
5585 radeon_ring_write(ring
, 0); /* SH_MEM_CONFIG */
5586 radeon_ring_write(ring
, 1); /* SH_MEM_APE1_BASE */
5587 radeon_ring_write(ring
, 0); /* SH_MEM_APE1_LIMIT */
5589 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5590 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5591 WRITE_DATA_DST_SEL(0)));
5592 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5593 radeon_ring_write(ring
, 0);
5594 radeon_ring_write(ring
, VMID(0));
5597 cik_hdp_flush_cp_ring_emit(rdev
, ridx
);
5599 /* bits 0-15 are the VM contexts0-15 */
5600 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5601 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5602 WRITE_DATA_DST_SEL(0)));
5603 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
5604 radeon_ring_write(ring
, 0);
5605 radeon_ring_write(ring
, 1 << vm
->id
);
5607 /* compute doesn't have PFP */
5608 if (ridx
== RADEON_RING_TYPE_GFX_INDEX
) {
5609 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5610 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
5611 radeon_ring_write(ring
, 0x0);
5617 * The RLC is a multi-purpose microengine that handles a
5618 * variety of functions, the most important of which is
5619 * the interrupt controller.
5621 static void cik_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
5624 u32 tmp
= RREG32(CP_INT_CNTL_RING0
);
5627 tmp
|= (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5629 tmp
&= ~(CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5630 WREG32(CP_INT_CNTL_RING0
, tmp
);
5633 static void cik_enable_lbpw(struct radeon_device
*rdev
, bool enable
)
5637 tmp
= RREG32(RLC_LB_CNTL
);
5639 tmp
|= LOAD_BALANCE_ENABLE
;
5641 tmp
&= ~LOAD_BALANCE_ENABLE
;
5642 WREG32(RLC_LB_CNTL
, tmp
);
5645 static void cik_wait_for_rlc_serdes(struct radeon_device
*rdev
)
5650 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
5651 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
5652 cik_select_se_sh(rdev
, i
, j
);
5653 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5654 if (RREG32(RLC_SERDES_CU_MASTER_BUSY
) == 0)
5660 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5662 mask
= SE_MASTER_BUSY_MASK
| GC_MASTER_BUSY
| TC0_MASTER_BUSY
| TC1_MASTER_BUSY
;
5663 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5664 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY
) & mask
) == 0)
5670 static void cik_update_rlc(struct radeon_device
*rdev
, u32 rlc
)
5674 tmp
= RREG32(RLC_CNTL
);
5676 WREG32(RLC_CNTL
, rlc
);
5679 static u32
cik_halt_rlc(struct radeon_device
*rdev
)
5683 orig
= data
= RREG32(RLC_CNTL
);
5685 if (data
& RLC_ENABLE
) {
5688 data
&= ~RLC_ENABLE
;
5689 WREG32(RLC_CNTL
, data
);
5691 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5692 if ((RREG32(RLC_GPM_STAT
) & RLC_GPM_BUSY
) == 0)
5697 cik_wait_for_rlc_serdes(rdev
);
5703 void cik_enter_rlc_safe_mode(struct radeon_device
*rdev
)
5707 tmp
= REQ
| MESSAGE(MSG_ENTER_RLC_SAFE_MODE
);
5708 WREG32(RLC_GPR_REG2
, tmp
);
5710 mask
= GFX_POWER_STATUS
| GFX_CLOCK_STATUS
;
5711 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5712 if ((RREG32(RLC_GPM_STAT
) & mask
) == mask
)
5717 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5718 if ((RREG32(RLC_GPR_REG2
) & REQ
) == 0)
5724 void cik_exit_rlc_safe_mode(struct radeon_device
*rdev
)
5728 tmp
= REQ
| MESSAGE(MSG_EXIT_RLC_SAFE_MODE
);
5729 WREG32(RLC_GPR_REG2
, tmp
);
5733 * cik_rlc_stop - stop the RLC ME
5735 * @rdev: radeon_device pointer
5737 * Halt the RLC ME (MicroEngine) (CIK).
5739 static void cik_rlc_stop(struct radeon_device
*rdev
)
5741 WREG32(RLC_CNTL
, 0);
5743 cik_enable_gui_idle_interrupt(rdev
, false);
5745 cik_wait_for_rlc_serdes(rdev
);
5749 * cik_rlc_start - start the RLC ME
5751 * @rdev: radeon_device pointer
5753 * Unhalt the RLC ME (MicroEngine) (CIK).
5755 static void cik_rlc_start(struct radeon_device
*rdev
)
5757 WREG32(RLC_CNTL
, RLC_ENABLE
);
5759 cik_enable_gui_idle_interrupt(rdev
, true);
5765 * cik_rlc_resume - setup the RLC hw
5767 * @rdev: radeon_device pointer
5769 * Initialize the RLC registers, load the ucode,
5770 * and start the RLC (CIK).
5771 * Returns 0 for success, -EINVAL if the ucode is not available.
5773 static int cik_rlc_resume(struct radeon_device
*rdev
)
5776 const __be32
*fw_data
;
5781 switch (rdev
->family
) {
5785 size
= BONAIRE_RLC_UCODE_SIZE
;
5788 size
= KV_RLC_UCODE_SIZE
;
5791 size
= KB_RLC_UCODE_SIZE
;
5798 tmp
= RREG32(RLC_CGCG_CGLS_CTRL
) & 0xfffffffc;
5799 WREG32(RLC_CGCG_CGLS_CTRL
, tmp
);
5807 WREG32(RLC_LB_CNTR_INIT
, 0);
5808 WREG32(RLC_LB_CNTR_MAX
, 0x00008000);
5810 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5811 WREG32(RLC_LB_INIT_CU_MASK
, 0xffffffff);
5812 WREG32(RLC_LB_PARAMS
, 0x00600408);
5813 WREG32(RLC_LB_CNTL
, 0x80000004);
5815 WREG32(RLC_MC_CNTL
, 0);
5816 WREG32(RLC_UCODE_CNTL
, 0);
5818 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
5819 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5820 for (i
= 0; i
< size
; i
++)
5821 WREG32(RLC_GPM_UCODE_DATA
, be32_to_cpup(fw_data
++));
5822 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5824 /* XXX - find out what chips support lbpw */
5825 cik_enable_lbpw(rdev
, false);
5827 if (rdev
->family
== CHIP_BONAIRE
)
5828 WREG32(RLC_DRIVER_DMA_STATUS
, 0);
5830 cik_rlc_start(rdev
);
5835 static void cik_enable_cgcg(struct radeon_device
*rdev
, bool enable
)
5837 u32 data
, orig
, tmp
, tmp2
;
5839 orig
= data
= RREG32(RLC_CGCG_CGLS_CTRL
);
5841 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGCG
)) {
5842 cik_enable_gui_idle_interrupt(rdev
, true);
5844 tmp
= cik_halt_rlc(rdev
);
5846 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5847 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5848 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5849 tmp2
= BPM_ADDR_MASK
| CGCG_OVERRIDE_0
| CGLS_ENABLE
;
5850 WREG32(RLC_SERDES_WR_CTRL
, tmp2
);
5852 cik_update_rlc(rdev
, tmp
);
5854 data
|= CGCG_EN
| CGLS_EN
;
5856 cik_enable_gui_idle_interrupt(rdev
, false);
5858 RREG32(CB_CGTT_SCLK_CTRL
);
5859 RREG32(CB_CGTT_SCLK_CTRL
);
5860 RREG32(CB_CGTT_SCLK_CTRL
);
5861 RREG32(CB_CGTT_SCLK_CTRL
);
5863 data
&= ~(CGCG_EN
| CGLS_EN
);
5867 WREG32(RLC_CGCG_CGLS_CTRL
, data
);
5871 static void cik_enable_mgcg(struct radeon_device
*rdev
, bool enable
)
5873 u32 data
, orig
, tmp
= 0;
5875 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGCG
)) {
5876 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGLS
) {
5877 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CP_LS
) {
5878 orig
= data
= RREG32(CP_MEM_SLP_CNTL
);
5879 data
|= CP_MEM_LS_EN
;
5881 WREG32(CP_MEM_SLP_CNTL
, data
);
5885 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5888 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5890 tmp
= cik_halt_rlc(rdev
);
5892 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5893 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5894 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5895 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_0
;
5896 WREG32(RLC_SERDES_WR_CTRL
, data
);
5898 cik_update_rlc(rdev
, tmp
);
5900 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGTS
) {
5901 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5902 data
&= ~SM_MODE_MASK
;
5903 data
|= SM_MODE(0x2);
5904 data
|= SM_MODE_ENABLE
;
5905 data
&= ~CGTS_OVERRIDE
;
5906 if ((rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGLS
) &&
5907 (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGTS_LS
))
5908 data
&= ~CGTS_LS_OVERRIDE
;
5909 data
&= ~ON_MONITOR_ADD_MASK
;
5910 data
|= ON_MONITOR_ADD_EN
;
5911 data
|= ON_MONITOR_ADD(0x96);
5913 WREG32(CGTS_SM_CTRL_REG
, data
);
5916 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5919 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5921 data
= RREG32(RLC_MEM_SLP_CNTL
);
5922 if (data
& RLC_MEM_LS_EN
) {
5923 data
&= ~RLC_MEM_LS_EN
;
5924 WREG32(RLC_MEM_SLP_CNTL
, data
);
5927 data
= RREG32(CP_MEM_SLP_CNTL
);
5928 if (data
& CP_MEM_LS_EN
) {
5929 data
&= ~CP_MEM_LS_EN
;
5930 WREG32(CP_MEM_SLP_CNTL
, data
);
5933 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5934 data
|= CGTS_OVERRIDE
| CGTS_LS_OVERRIDE
;
5936 WREG32(CGTS_SM_CTRL_REG
, data
);
5938 tmp
= cik_halt_rlc(rdev
);
5940 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5941 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5942 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5943 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_1
;
5944 WREG32(RLC_SERDES_WR_CTRL
, data
);
5946 cik_update_rlc(rdev
, tmp
);
5950 static const u32 mc_cg_registers
[] =
5963 static void cik_enable_mc_ls(struct radeon_device
*rdev
,
5969 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5970 orig
= data
= RREG32(mc_cg_registers
[i
]);
5971 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_MC_LS
))
5972 data
|= MC_LS_ENABLE
;
5974 data
&= ~MC_LS_ENABLE
;
5976 WREG32(mc_cg_registers
[i
], data
);
5980 static void cik_enable_mc_mgcg(struct radeon_device
*rdev
,
5986 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5987 orig
= data
= RREG32(mc_cg_registers
[i
]);
5988 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_MC_MGCG
))
5989 data
|= MC_CG_ENABLE
;
5991 data
&= ~MC_CG_ENABLE
;
5993 WREG32(mc_cg_registers
[i
], data
);
5997 static void cik_enable_sdma_mgcg(struct radeon_device
*rdev
,
6002 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_SDMA_MGCG
)) {
6003 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, 0x00000100);
6004 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, 0x00000100);
6006 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
);
6009 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, data
);
6011 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
);
6014 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, data
);
6018 static void cik_enable_sdma_mgls(struct radeon_device
*rdev
,
6023 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_SDMA_LS
)) {
6024 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
6027 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
6029 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
6032 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
6034 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
6037 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
6039 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
6042 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
6046 static void cik_enable_uvd_mgcg(struct radeon_device
*rdev
,
6051 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_UVD_MGCG
)) {
6052 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
6054 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
6056 orig
= data
= RREG32(UVD_CGC_CTRL
);
6059 WREG32(UVD_CGC_CTRL
, data
);
6061 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
6063 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
6065 orig
= data
= RREG32(UVD_CGC_CTRL
);
6068 WREG32(UVD_CGC_CTRL
, data
);
6072 static void cik_enable_bif_mgls(struct radeon_device
*rdev
,
6077 orig
= data
= RREG32_PCIE_PORT(PCIE_CNTL2
);
6079 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_BIF_LS
))
6080 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
|
6081 REPLAY_MEM_LS_EN
| SLV_MEM_AGGRESSIVE_LS_EN
;
6083 data
&= ~(SLV_MEM_LS_EN
| MST_MEM_LS_EN
|
6084 REPLAY_MEM_LS_EN
| SLV_MEM_AGGRESSIVE_LS_EN
);
6087 WREG32_PCIE_PORT(PCIE_CNTL2
, data
);
6090 static void cik_enable_hdp_mgcg(struct radeon_device
*rdev
,
6095 orig
= data
= RREG32(HDP_HOST_PATH_CNTL
);
6097 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_HDP_MGCG
))
6098 data
&= ~CLOCK_GATING_DIS
;
6100 data
|= CLOCK_GATING_DIS
;
6103 WREG32(HDP_HOST_PATH_CNTL
, data
);
6106 static void cik_enable_hdp_ls(struct radeon_device
*rdev
,
6111 orig
= data
= RREG32(HDP_MEM_POWER_LS
);
6113 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_HDP_LS
))
6114 data
|= HDP_LS_ENABLE
;
6116 data
&= ~HDP_LS_ENABLE
;
6119 WREG32(HDP_MEM_POWER_LS
, data
);
6122 void cik_update_cg(struct radeon_device
*rdev
,
6123 u32 block
, bool enable
)
6126 if (block
& RADEON_CG_BLOCK_GFX
) {
6127 cik_enable_gui_idle_interrupt(rdev
, false);
6128 /* order matters! */
6130 cik_enable_mgcg(rdev
, true);
6131 cik_enable_cgcg(rdev
, true);
6133 cik_enable_cgcg(rdev
, false);
6134 cik_enable_mgcg(rdev
, false);
6136 cik_enable_gui_idle_interrupt(rdev
, true);
6139 if (block
& RADEON_CG_BLOCK_MC
) {
6140 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
6141 cik_enable_mc_mgcg(rdev
, enable
);
6142 cik_enable_mc_ls(rdev
, enable
);
6146 if (block
& RADEON_CG_BLOCK_SDMA
) {
6147 cik_enable_sdma_mgcg(rdev
, enable
);
6148 cik_enable_sdma_mgls(rdev
, enable
);
6151 if (block
& RADEON_CG_BLOCK_BIF
) {
6152 cik_enable_bif_mgls(rdev
, enable
);
6155 if (block
& RADEON_CG_BLOCK_UVD
) {
6157 cik_enable_uvd_mgcg(rdev
, enable
);
6160 if (block
& RADEON_CG_BLOCK_HDP
) {
6161 cik_enable_hdp_mgcg(rdev
, enable
);
6162 cik_enable_hdp_ls(rdev
, enable
);
6165 if (block
& RADEON_CG_BLOCK_VCE
) {
6166 vce_v2_0_enable_mgcg(rdev
, enable
);
6170 static void cik_init_cg(struct radeon_device
*rdev
)
6173 cik_update_cg(rdev
, RADEON_CG_BLOCK_GFX
, true);
6176 si_init_uvd_internal_cg(rdev
);
6178 cik_update_cg(rdev
, (RADEON_CG_BLOCK_MC
|
6179 RADEON_CG_BLOCK_SDMA
|
6180 RADEON_CG_BLOCK_BIF
|
6181 RADEON_CG_BLOCK_UVD
|
6182 RADEON_CG_BLOCK_HDP
), true);
6185 static void cik_fini_cg(struct radeon_device
*rdev
)
6187 cik_update_cg(rdev
, (RADEON_CG_BLOCK_MC
|
6188 RADEON_CG_BLOCK_SDMA
|
6189 RADEON_CG_BLOCK_BIF
|
6190 RADEON_CG_BLOCK_UVD
|
6191 RADEON_CG_BLOCK_HDP
), false);
6193 cik_update_cg(rdev
, RADEON_CG_BLOCK_GFX
, false);
6196 static void cik_enable_sck_slowdown_on_pu(struct radeon_device
*rdev
,
6201 orig
= data
= RREG32(RLC_PG_CNTL
);
6202 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_RLC_SMU_HS
))
6203 data
|= SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6205 data
&= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6207 WREG32(RLC_PG_CNTL
, data
);
6210 static void cik_enable_sck_slowdown_on_pd(struct radeon_device
*rdev
,
6215 orig
= data
= RREG32(RLC_PG_CNTL
);
6216 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_RLC_SMU_HS
))
6217 data
|= SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6219 data
&= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6221 WREG32(RLC_PG_CNTL
, data
);
6224 static void cik_enable_cp_pg(struct radeon_device
*rdev
, bool enable
)
6228 orig
= data
= RREG32(RLC_PG_CNTL
);
6229 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_CP
))
6230 data
&= ~DISABLE_CP_PG
;
6232 data
|= DISABLE_CP_PG
;
6234 WREG32(RLC_PG_CNTL
, data
);
6237 static void cik_enable_gds_pg(struct radeon_device
*rdev
, bool enable
)
6241 orig
= data
= RREG32(RLC_PG_CNTL
);
6242 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GDS
))
6243 data
&= ~DISABLE_GDS_PG
;
6245 data
|= DISABLE_GDS_PG
;
6247 WREG32(RLC_PG_CNTL
, data
);
6250 #define CP_ME_TABLE_SIZE 96
6251 #define CP_ME_TABLE_OFFSET 2048
6252 #define CP_MEC_TABLE_OFFSET 4096
6254 void cik_init_cp_pg_table(struct radeon_device
*rdev
)
6256 const __be32
*fw_data
;
6257 volatile u32
*dst_ptr
;
6258 int me
, i
, max_me
= 4;
6262 if (rdev
->family
== CHIP_KAVERI
)
6265 if (rdev
->rlc
.cp_table_ptr
== NULL
)
6268 /* write the cp table buffer */
6269 dst_ptr
= rdev
->rlc
.cp_table_ptr
;
6270 for (me
= 0; me
< max_me
; me
++) {
6272 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
6273 table_offset
= CP_ME_TABLE_OFFSET
;
6274 } else if (me
== 1) {
6275 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
6276 table_offset
= CP_ME_TABLE_OFFSET
;
6277 } else if (me
== 2) {
6278 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
6279 table_offset
= CP_ME_TABLE_OFFSET
;
6281 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
6282 table_offset
= CP_MEC_TABLE_OFFSET
;
6285 for (i
= 0; i
< CP_ME_TABLE_SIZE
; i
++) {
6286 dst_ptr
[bo_offset
+ i
] = cpu_to_le32(be32_to_cpu(fw_data
[table_offset
+ i
]));
6288 bo_offset
+= CP_ME_TABLE_SIZE
;
6292 static void cik_enable_gfx_cgpg(struct radeon_device
*rdev
,
6297 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
)) {
6298 orig
= data
= RREG32(RLC_PG_CNTL
);
6299 data
|= GFX_PG_ENABLE
;
6301 WREG32(RLC_PG_CNTL
, data
);
6303 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6306 WREG32(RLC_AUTO_PG_CTRL
, data
);
6308 orig
= data
= RREG32(RLC_PG_CNTL
);
6309 data
&= ~GFX_PG_ENABLE
;
6311 WREG32(RLC_PG_CNTL
, data
);
6313 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6314 data
&= ~AUTO_PG_EN
;
6316 WREG32(RLC_AUTO_PG_CTRL
, data
);
6318 data
= RREG32(DB_RENDER_CONTROL
);
6322 static u32
cik_get_cu_active_bitmap(struct radeon_device
*rdev
, u32 se
, u32 sh
)
6324 u32 mask
= 0, tmp
, tmp1
;
6327 cik_select_se_sh(rdev
, se
, sh
);
6328 tmp
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
6329 tmp1
= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
6330 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
6337 for (i
= 0; i
< rdev
->config
.cik
.max_cu_per_sh
; i
++) {
6342 return (~tmp
) & mask
;
6345 static void cik_init_ao_cu_mask(struct radeon_device
*rdev
)
6347 u32 i
, j
, k
, active_cu_number
= 0;
6348 u32 mask
, counter
, cu_bitmap
;
6351 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
6352 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
6356 for (k
= 0; k
< rdev
->config
.cik
.max_cu_per_sh
; k
++) {
6357 if (cik_get_cu_active_bitmap(rdev
, i
, j
) & mask
) {
6365 active_cu_number
+= counter
;
6366 tmp
|= (cu_bitmap
<< (i
* 16 + j
* 8));
6370 WREG32(RLC_PG_AO_CU_MASK
, tmp
);
6372 tmp
= RREG32(RLC_MAX_PG_CU
);
6373 tmp
&= ~MAX_PU_CU_MASK
;
6374 tmp
|= MAX_PU_CU(active_cu_number
);
6375 WREG32(RLC_MAX_PG_CU
, tmp
);
6378 static void cik_enable_gfx_static_mgpg(struct radeon_device
*rdev
,
6383 orig
= data
= RREG32(RLC_PG_CNTL
);
6384 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_SMG
))
6385 data
|= STATIC_PER_CU_PG_ENABLE
;
6387 data
&= ~STATIC_PER_CU_PG_ENABLE
;
6389 WREG32(RLC_PG_CNTL
, data
);
6392 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device
*rdev
,
6397 orig
= data
= RREG32(RLC_PG_CNTL
);
6398 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_DMG
))
6399 data
|= DYN_PER_CU_PG_ENABLE
;
6401 data
&= ~DYN_PER_CU_PG_ENABLE
;
6403 WREG32(RLC_PG_CNTL
, data
);
6406 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6407 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6409 static void cik_init_gfx_cgpg(struct radeon_device
*rdev
)
6414 if (rdev
->rlc
.cs_data
) {
6415 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6416 WREG32(RLC_GPM_SCRATCH_DATA
, upper_32_bits(rdev
->rlc
.clear_state_gpu_addr
));
6417 WREG32(RLC_GPM_SCRATCH_DATA
, lower_32_bits(rdev
->rlc
.clear_state_gpu_addr
));
6418 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.clear_state_size
);
6420 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6421 for (i
= 0; i
< 3; i
++)
6422 WREG32(RLC_GPM_SCRATCH_DATA
, 0);
6424 if (rdev
->rlc
.reg_list
) {
6425 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_SAVE_AND_RESTORE_STARTING_OFFSET
);
6426 for (i
= 0; i
< rdev
->rlc
.reg_list_size
; i
++)
6427 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.reg_list
[i
]);
6430 orig
= data
= RREG32(RLC_PG_CNTL
);
6433 WREG32(RLC_PG_CNTL
, data
);
6435 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
6436 WREG32(RLC_CP_TABLE_RESTORE
, rdev
->rlc
.cp_table_gpu_addr
>> 8);
6438 data
= RREG32(CP_RB_WPTR_POLL_CNTL
);
6439 data
&= ~IDLE_POLL_COUNT_MASK
;
6440 data
|= IDLE_POLL_COUNT(0x60);
6441 WREG32(CP_RB_WPTR_POLL_CNTL
, data
);
6444 WREG32(RLC_PG_DELAY
, data
);
6446 data
= RREG32(RLC_PG_DELAY_2
);
6449 WREG32(RLC_PG_DELAY_2
, data
);
6451 data
= RREG32(RLC_AUTO_PG_CTRL
);
6452 data
&= ~GRBM_REG_SGIT_MASK
;
6453 data
|= GRBM_REG_SGIT(0x700);
6454 WREG32(RLC_AUTO_PG_CTRL
, data
);
6458 static void cik_update_gfx_pg(struct radeon_device
*rdev
, bool enable
)
6460 cik_enable_gfx_cgpg(rdev
, enable
);
6461 cik_enable_gfx_static_mgpg(rdev
, enable
);
6462 cik_enable_gfx_dynamic_mgpg(rdev
, enable
);
6465 u32
cik_get_csb_size(struct radeon_device
*rdev
)
6468 const struct cs_section_def
*sect
= NULL
;
6469 const struct cs_extent_def
*ext
= NULL
;
6471 if (rdev
->rlc
.cs_data
== NULL
)
6474 /* begin clear state */
6476 /* context control state */
6479 for (sect
= rdev
->rlc
.cs_data
; sect
->section
!= NULL
; ++sect
) {
6480 for (ext
= sect
->section
; ext
->extent
!= NULL
; ++ext
) {
6481 if (sect
->id
== SECT_CONTEXT
)
6482 count
+= 2 + ext
->reg_count
;
6487 /* pa_sc_raster_config/pa_sc_raster_config1 */
6489 /* end clear state */
6497 void cik_get_csb_buffer(struct radeon_device
*rdev
, volatile u32
*buffer
)
6500 const struct cs_section_def
*sect
= NULL
;
6501 const struct cs_extent_def
*ext
= NULL
;
6503 if (rdev
->rlc
.cs_data
== NULL
)
6508 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
6509 buffer
[count
++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
6511 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL
, 1));
6512 buffer
[count
++] = cpu_to_le32(0x80000000);
6513 buffer
[count
++] = cpu_to_le32(0x80000000);
6515 for (sect
= rdev
->rlc
.cs_data
; sect
->section
!= NULL
; ++sect
) {
6516 for (ext
= sect
->section
; ext
->extent
!= NULL
; ++ext
) {
6517 if (sect
->id
== SECT_CONTEXT
) {
6519 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG
, ext
->reg_count
));
6520 buffer
[count
++] = cpu_to_le32(ext
->reg_index
- 0xa000);
6521 for (i
= 0; i
< ext
->reg_count
; i
++)
6522 buffer
[count
++] = cpu_to_le32(ext
->extent
[i
]);
6529 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
6530 buffer
[count
++] = cpu_to_le32(PA_SC_RASTER_CONFIG
- PACKET3_SET_CONTEXT_REG_START
);
6531 switch (rdev
->family
) {
6533 buffer
[count
++] = cpu_to_le32(0x16000012);
6534 buffer
[count
++] = cpu_to_le32(0x00000000);
6537 buffer
[count
++] = cpu_to_le32(0x00000000); /* XXX */
6538 buffer
[count
++] = cpu_to_le32(0x00000000);
6541 buffer
[count
++] = cpu_to_le32(0x00000000); /* XXX */
6542 buffer
[count
++] = cpu_to_le32(0x00000000);
6545 buffer
[count
++] = cpu_to_le32(0x3a00161a);
6546 buffer
[count
++] = cpu_to_le32(0x0000002e);
6549 buffer
[count
++] = cpu_to_le32(0x00000000);
6550 buffer
[count
++] = cpu_to_le32(0x00000000);
6554 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
6555 buffer
[count
++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE
);
6557 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE
, 0));
6558 buffer
[count
++] = cpu_to_le32(0);
6561 static void cik_init_pg(struct radeon_device
*rdev
)
6563 if (rdev
->pg_flags
) {
6564 cik_enable_sck_slowdown_on_pu(rdev
, true);
6565 cik_enable_sck_slowdown_on_pd(rdev
, true);
6566 if (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
) {
6567 cik_init_gfx_cgpg(rdev
);
6568 cik_enable_cp_pg(rdev
, true);
6569 cik_enable_gds_pg(rdev
, true);
6571 cik_init_ao_cu_mask(rdev
);
6572 cik_update_gfx_pg(rdev
, true);
6576 static void cik_fini_pg(struct radeon_device
*rdev
)
6578 if (rdev
->pg_flags
) {
6579 cik_update_gfx_pg(rdev
, false);
6580 if (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
) {
6581 cik_enable_cp_pg(rdev
, false);
6582 cik_enable_gds_pg(rdev
, false);
6589 * Starting with r6xx, interrupts are handled via a ring buffer.
6590 * Ring buffers are areas of GPU accessible memory that the GPU
6591 * writes interrupt vectors into and the host reads vectors out of.
6592 * There is a rptr (read pointer) that determines where the
6593 * host is currently reading, and a wptr (write pointer)
6594 * which determines where the GPU has written. When the
6595 * pointers are equal, the ring is idle. When the GPU
6596 * writes vectors to the ring buffer, it increments the
6597 * wptr. When there is an interrupt, the host then starts
6598 * fetching commands and processing them until the pointers are
6599 * equal again at which point it updates the rptr.
6603 * cik_enable_interrupts - Enable the interrupt ring buffer
6605 * @rdev: radeon_device pointer
6607 * Enable the interrupt ring buffer (CIK).
6609 static void cik_enable_interrupts(struct radeon_device
*rdev
)
6611 u32 ih_cntl
= RREG32(IH_CNTL
);
6612 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6614 ih_cntl
|= ENABLE_INTR
;
6615 ih_rb_cntl
|= IH_RB_ENABLE
;
6616 WREG32(IH_CNTL
, ih_cntl
);
6617 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6618 rdev
->ih
.enabled
= true;
6622 * cik_disable_interrupts - Disable the interrupt ring buffer
6624 * @rdev: radeon_device pointer
6626 * Disable the interrupt ring buffer (CIK).
6628 static void cik_disable_interrupts(struct radeon_device
*rdev
)
6630 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6631 u32 ih_cntl
= RREG32(IH_CNTL
);
6633 ih_rb_cntl
&= ~IH_RB_ENABLE
;
6634 ih_cntl
&= ~ENABLE_INTR
;
6635 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6636 WREG32(IH_CNTL
, ih_cntl
);
6637 /* set rptr, wptr to 0 */
6638 WREG32(IH_RB_RPTR
, 0);
6639 WREG32(IH_RB_WPTR
, 0);
6640 rdev
->ih
.enabled
= false;
6645 * cik_disable_interrupt_state - Disable all interrupt sources
6647 * @rdev: radeon_device pointer
6649 * Clear all interrupt enable bits used by the driver (CIK).
6651 static void cik_disable_interrupt_state(struct radeon_device
*rdev
)
6656 tmp
= RREG32(CP_INT_CNTL_RING0
) &
6657 (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
6658 WREG32(CP_INT_CNTL_RING0
, tmp
);
6660 tmp
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6661 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
6662 tmp
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6663 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
6664 /* compute queues */
6665 WREG32(CP_ME1_PIPE0_INT_CNTL
, 0);
6666 WREG32(CP_ME1_PIPE1_INT_CNTL
, 0);
6667 WREG32(CP_ME1_PIPE2_INT_CNTL
, 0);
6668 WREG32(CP_ME1_PIPE3_INT_CNTL
, 0);
6669 WREG32(CP_ME2_PIPE0_INT_CNTL
, 0);
6670 WREG32(CP_ME2_PIPE1_INT_CNTL
, 0);
6671 WREG32(CP_ME2_PIPE2_INT_CNTL
, 0);
6672 WREG32(CP_ME2_PIPE3_INT_CNTL
, 0);
6674 WREG32(GRBM_INT_CNTL
, 0);
6675 /* vline/vblank, etc. */
6676 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
6677 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
6678 if (rdev
->num_crtc
>= 4) {
6679 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
6680 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
6682 if (rdev
->num_crtc
>= 6) {
6683 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
6684 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
6688 WREG32(DAC_AUTODETECT_INT_CONTROL
, 0);
6690 /* digital hotplug */
6691 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6692 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
6693 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6694 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
6695 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6696 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
6697 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6698 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
6699 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6700 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
6701 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6702 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
6707 * cik_irq_init - init and enable the interrupt ring
6709 * @rdev: radeon_device pointer
6711 * Allocate a ring buffer for the interrupt controller,
6712 * enable the RLC, disable interrupts, enable the IH
6713 * ring buffer and enable it (CIK).
6714 * Called at device load and reume.
6715 * Returns 0 for success, errors for failure.
6717 static int cik_irq_init(struct radeon_device
*rdev
)
6721 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
6724 ret
= r600_ih_ring_alloc(rdev
);
6729 cik_disable_interrupts(rdev
);
6732 ret
= cik_rlc_resume(rdev
);
6734 r600_ih_ring_fini(rdev
);
6738 /* setup interrupt control */
6739 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6740 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
6741 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
6742 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6743 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6745 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
6746 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6747 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
6748 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
6750 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
6751 rb_bufsz
= order_base_2(rdev
->ih
.ring_size
/ 4);
6753 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
6754 IH_WPTR_OVERFLOW_CLEAR
|
6757 if (rdev
->wb
.enabled
)
6758 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
6760 /* set the writeback address whether it's enabled or not */
6761 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
6762 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
6764 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6766 /* set rptr, wptr to 0 */
6767 WREG32(IH_RB_RPTR
, 0);
6768 WREG32(IH_RB_WPTR
, 0);
6770 /* Default settings for IH_CNTL (disabled at first) */
6771 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6772 /* RPTR_REARM only works if msi's are enabled */
6773 if (rdev
->msi_enabled
)
6774 ih_cntl
|= RPTR_REARM
;
6775 WREG32(IH_CNTL
, ih_cntl
);
6777 /* force the active interrupt state to all disabled */
6778 cik_disable_interrupt_state(rdev
);
6780 pci_set_master(rdev
->pdev
);
6783 cik_enable_interrupts(rdev
);
6789 * cik_irq_set - enable/disable interrupt sources
6791 * @rdev: radeon_device pointer
6793 * Enable interrupt sources on the GPU (vblanks, hpd,
6795 * Returns 0 for success, errors for failure.
6797 int cik_irq_set(struct radeon_device
*rdev
)
6800 u32 cp_m1p0
, cp_m1p1
, cp_m1p2
, cp_m1p3
;
6801 u32 cp_m2p0
, cp_m2p1
, cp_m2p2
, cp_m2p3
;
6802 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
6803 u32 hpd1
, hpd2
, hpd3
, hpd4
, hpd5
, hpd6
;
6804 u32 grbm_int_cntl
= 0;
6805 u32 dma_cntl
, dma_cntl1
;
6808 if (!rdev
->irq
.installed
) {
6809 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6812 /* don't enable anything if the ih is disabled */
6813 if (!rdev
->ih
.enabled
) {
6814 cik_disable_interrupts(rdev
);
6815 /* force the active interrupt state to all disabled */
6816 cik_disable_interrupt_state(rdev
);
6820 cp_int_cntl
= RREG32(CP_INT_CNTL_RING0
) &
6821 (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
6822 cp_int_cntl
|= PRIV_INSTR_INT_ENABLE
| PRIV_REG_INT_ENABLE
;
6824 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6825 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6826 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6827 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6828 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6829 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6831 dma_cntl
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6832 dma_cntl1
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6834 cp_m1p0
= RREG32(CP_ME1_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6835 cp_m1p1
= RREG32(CP_ME1_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6836 cp_m1p2
= RREG32(CP_ME1_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6837 cp_m1p3
= RREG32(CP_ME1_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6838 cp_m2p0
= RREG32(CP_ME2_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6839 cp_m2p1
= RREG32(CP_ME2_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6840 cp_m2p2
= RREG32(CP_ME2_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6841 cp_m2p3
= RREG32(CP_ME2_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6843 if (rdev
->flags
& RADEON_IS_IGP
)
6844 thermal_int
= RREG32_SMC(CG_THERMAL_INT_CTRL
) &
6845 ~(THERM_INTH_MASK
| THERM_INTL_MASK
);
6847 thermal_int
= RREG32_SMC(CG_THERMAL_INT
) &
6848 ~(THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
);
6850 /* enable CP interrupts on all rings */
6851 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
6852 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6853 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
6855 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
6856 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6857 DRM_DEBUG("si_irq_set: sw int cp1\n");
6858 if (ring
->me
== 1) {
6859 switch (ring
->pipe
) {
6861 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6864 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6867 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6870 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6873 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6876 } else if (ring
->me
== 2) {
6877 switch (ring
->pipe
) {
6879 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6882 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6885 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6888 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6891 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6895 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring
->me
);
6898 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
6899 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6900 DRM_DEBUG("si_irq_set: sw int cp2\n");
6901 if (ring
->me
== 1) {
6902 switch (ring
->pipe
) {
6904 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6907 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6910 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6913 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6916 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6919 } else if (ring
->me
== 2) {
6920 switch (ring
->pipe
) {
6922 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6925 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6928 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6931 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6934 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6938 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring
->me
);
6942 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
6943 DRM_DEBUG("cik_irq_set: sw int dma\n");
6944 dma_cntl
|= TRAP_ENABLE
;
6947 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
6948 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6949 dma_cntl1
|= TRAP_ENABLE
;
6952 if (rdev
->irq
.crtc_vblank_int
[0] ||
6953 atomic_read(&rdev
->irq
.pflip
[0])) {
6954 DRM_DEBUG("cik_irq_set: vblank 0\n");
6955 crtc1
|= VBLANK_INTERRUPT_MASK
;
6957 if (rdev
->irq
.crtc_vblank_int
[1] ||
6958 atomic_read(&rdev
->irq
.pflip
[1])) {
6959 DRM_DEBUG("cik_irq_set: vblank 1\n");
6960 crtc2
|= VBLANK_INTERRUPT_MASK
;
6962 if (rdev
->irq
.crtc_vblank_int
[2] ||
6963 atomic_read(&rdev
->irq
.pflip
[2])) {
6964 DRM_DEBUG("cik_irq_set: vblank 2\n");
6965 crtc3
|= VBLANK_INTERRUPT_MASK
;
6967 if (rdev
->irq
.crtc_vblank_int
[3] ||
6968 atomic_read(&rdev
->irq
.pflip
[3])) {
6969 DRM_DEBUG("cik_irq_set: vblank 3\n");
6970 crtc4
|= VBLANK_INTERRUPT_MASK
;
6972 if (rdev
->irq
.crtc_vblank_int
[4] ||
6973 atomic_read(&rdev
->irq
.pflip
[4])) {
6974 DRM_DEBUG("cik_irq_set: vblank 4\n");
6975 crtc5
|= VBLANK_INTERRUPT_MASK
;
6977 if (rdev
->irq
.crtc_vblank_int
[5] ||
6978 atomic_read(&rdev
->irq
.pflip
[5])) {
6979 DRM_DEBUG("cik_irq_set: vblank 5\n");
6980 crtc6
|= VBLANK_INTERRUPT_MASK
;
6982 if (rdev
->irq
.hpd
[0]) {
6983 DRM_DEBUG("cik_irq_set: hpd 1\n");
6984 hpd1
|= DC_HPDx_INT_EN
;
6986 if (rdev
->irq
.hpd
[1]) {
6987 DRM_DEBUG("cik_irq_set: hpd 2\n");
6988 hpd2
|= DC_HPDx_INT_EN
;
6990 if (rdev
->irq
.hpd
[2]) {
6991 DRM_DEBUG("cik_irq_set: hpd 3\n");
6992 hpd3
|= DC_HPDx_INT_EN
;
6994 if (rdev
->irq
.hpd
[3]) {
6995 DRM_DEBUG("cik_irq_set: hpd 4\n");
6996 hpd4
|= DC_HPDx_INT_EN
;
6998 if (rdev
->irq
.hpd
[4]) {
6999 DRM_DEBUG("cik_irq_set: hpd 5\n");
7000 hpd5
|= DC_HPDx_INT_EN
;
7002 if (rdev
->irq
.hpd
[5]) {
7003 DRM_DEBUG("cik_irq_set: hpd 6\n");
7004 hpd6
|= DC_HPDx_INT_EN
;
7007 if (rdev
->irq
.dpm_thermal
) {
7008 DRM_DEBUG("dpm thermal\n");
7009 if (rdev
->flags
& RADEON_IS_IGP
)
7010 thermal_int
|= THERM_INTH_MASK
| THERM_INTL_MASK
;
7012 thermal_int
|= THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
;
7015 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
7017 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, dma_cntl
);
7018 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, dma_cntl1
);
7020 WREG32(CP_ME1_PIPE0_INT_CNTL
, cp_m1p0
);
7021 WREG32(CP_ME1_PIPE1_INT_CNTL
, cp_m1p1
);
7022 WREG32(CP_ME1_PIPE2_INT_CNTL
, cp_m1p2
);
7023 WREG32(CP_ME1_PIPE3_INT_CNTL
, cp_m1p3
);
7024 WREG32(CP_ME2_PIPE0_INT_CNTL
, cp_m2p0
);
7025 WREG32(CP_ME2_PIPE1_INT_CNTL
, cp_m2p1
);
7026 WREG32(CP_ME2_PIPE2_INT_CNTL
, cp_m2p2
);
7027 WREG32(CP_ME2_PIPE3_INT_CNTL
, cp_m2p3
);
7029 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
7031 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
7032 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
7033 if (rdev
->num_crtc
>= 4) {
7034 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
7035 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
7037 if (rdev
->num_crtc
>= 6) {
7038 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
7039 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
7042 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
7043 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
7044 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
7045 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
7046 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
7047 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
7049 if (rdev
->flags
& RADEON_IS_IGP
)
7050 WREG32_SMC(CG_THERMAL_INT_CTRL
, thermal_int
);
7052 WREG32_SMC(CG_THERMAL_INT
, thermal_int
);
7058 * cik_irq_ack - ack interrupt sources
7060 * @rdev: radeon_device pointer
7062 * Ack interrupt sources on the GPU (vblanks, hpd,
7063 * etc.) (CIK). Certain interrupts sources are sw
7064 * generated and do not require an explicit ack.
7066 static inline void cik_irq_ack(struct radeon_device
*rdev
)
7070 rdev
->irq
.stat_regs
.cik
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
7071 rdev
->irq
.stat_regs
.cik
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
7072 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
7073 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
7074 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
7075 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
7076 rdev
->irq
.stat_regs
.cik
.disp_int_cont6
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE6
);
7078 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
7079 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
7080 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
)
7081 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
7082 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
7083 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
7084 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
7085 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
7087 if (rdev
->num_crtc
>= 4) {
7088 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
7089 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
7090 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
7091 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
7092 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
7093 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
7094 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
7095 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
7098 if (rdev
->num_crtc
>= 6) {
7099 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
7100 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
7101 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
7102 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
7103 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
7104 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
7105 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
7106 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
7109 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
7110 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
7111 tmp
|= DC_HPDx_INT_ACK
;
7112 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
7114 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
7115 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
7116 tmp
|= DC_HPDx_INT_ACK
;
7117 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
7119 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
7120 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
7121 tmp
|= DC_HPDx_INT_ACK
;
7122 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
7124 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
7125 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
7126 tmp
|= DC_HPDx_INT_ACK
;
7127 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
7129 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
7130 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
7131 tmp
|= DC_HPDx_INT_ACK
;
7132 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
7134 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
7135 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
7136 tmp
|= DC_HPDx_INT_ACK
;
7137 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
7142 * cik_irq_disable - disable interrupts
7144 * @rdev: radeon_device pointer
7146 * Disable interrupts on the hw (CIK).
7148 static void cik_irq_disable(struct radeon_device
*rdev
)
7150 cik_disable_interrupts(rdev
);
7151 /* Wait and acknowledge irq */
7154 cik_disable_interrupt_state(rdev
);
7158 * cik_irq_disable - disable interrupts for suspend
7160 * @rdev: radeon_device pointer
7162 * Disable interrupts and stop the RLC (CIK).
7165 static void cik_irq_suspend(struct radeon_device
*rdev
)
7167 cik_irq_disable(rdev
);
7172 * cik_irq_fini - tear down interrupt support
7174 * @rdev: radeon_device pointer
7176 * Disable interrupts on the hw and free the IH ring
7178 * Used for driver unload.
7180 static void cik_irq_fini(struct radeon_device
*rdev
)
7182 cik_irq_suspend(rdev
);
7183 r600_ih_ring_fini(rdev
);
7187 * cik_get_ih_wptr - get the IH ring buffer wptr
7189 * @rdev: radeon_device pointer
7191 * Get the IH ring buffer wptr from either the register
7192 * or the writeback memory buffer (CIK). Also check for
7193 * ring buffer overflow and deal with it.
7194 * Used by cik_irq_process().
7195 * Returns the value of the wptr.
7197 static inline u32
cik_get_ih_wptr(struct radeon_device
*rdev
)
7201 if (rdev
->wb
.enabled
)
7202 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
7204 wptr
= RREG32(IH_RB_WPTR
);
7206 if (wptr
& RB_OVERFLOW
) {
7207 /* When a ring buffer overflow happen start parsing interrupt
7208 * from the last not overwritten vector (wptr + 16). Hopefully
7209 * this should allow us to catchup.
7211 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7212 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
7213 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
7214 tmp
= RREG32(IH_RB_CNTL
);
7215 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
7216 WREG32(IH_RB_CNTL
, tmp
);
7218 return (wptr
& rdev
->ih
.ptr_mask
);
7222 * Each IV ring entry is 128 bits:
7223 * [7:0] - interrupt source id
7225 * [59:32] - interrupt source data
7226 * [63:60] - reserved
7229 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7230 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7231 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7232 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7233 * PIPE_ID - ME0 0=3D
7234 * - ME1&2 compute dispatcher (4 pipes each)
7236 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7237 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7238 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7241 * [127:96] - reserved
7244 * cik_irq_process - interrupt handler
7246 * @rdev: radeon_device pointer
7248 * Interrupt hander (CIK). Walk the IH ring,
7249 * ack interrupts and schedule work to handle
7251 * Returns irq process return code.
7253 int cik_irq_process(struct radeon_device
*rdev
)
7255 struct radeon_ring
*cp1_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7256 struct radeon_ring
*cp2_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7259 u32 src_id
, src_data
, ring_id
;
7260 u8 me_id
, pipe_id
, queue_id
;
7262 bool queue_hotplug
= false;
7263 bool queue_reset
= false;
7264 u32 addr
, status
, mc_client
;
7265 bool queue_thermal
= false;
7267 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
7270 wptr
= cik_get_ih_wptr(rdev
);
7273 /* is somebody else already processing irqs? */
7274 if (atomic_xchg(&rdev
->ih
.lock
, 1))
7277 rptr
= rdev
->ih
.rptr
;
7278 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
7280 /* Order reading of wptr vs. reading of IH ring data */
7283 /* display interrupts */
7286 while (rptr
!= wptr
) {
7287 /* wptr/rptr are in bytes! */
7288 ring_index
= rptr
/ 4;
7289 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
7290 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
7291 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
7294 case 1: /* D1 vblank/vline */
7296 case 0: /* D1 vblank */
7297 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
7298 if (rdev
->irq
.crtc_vblank_int
[0]) {
7299 drm_handle_vblank(rdev
->ddev
, 0);
7300 rdev
->pm
.vblank_sync
= true;
7301 wake_up(&rdev
->irq
.vblank_queue
);
7303 if (atomic_read(&rdev
->irq
.pflip
[0]))
7304 radeon_crtc_handle_flip(rdev
, 0);
7305 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
7306 DRM_DEBUG("IH: D1 vblank\n");
7309 case 1: /* D1 vline */
7310 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
7311 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
7312 DRM_DEBUG("IH: D1 vline\n");
7316 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7320 case 2: /* D2 vblank/vline */
7322 case 0: /* D2 vblank */
7323 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
7324 if (rdev
->irq
.crtc_vblank_int
[1]) {
7325 drm_handle_vblank(rdev
->ddev
, 1);
7326 rdev
->pm
.vblank_sync
= true;
7327 wake_up(&rdev
->irq
.vblank_queue
);
7329 if (atomic_read(&rdev
->irq
.pflip
[1]))
7330 radeon_crtc_handle_flip(rdev
, 1);
7331 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
7332 DRM_DEBUG("IH: D2 vblank\n");
7335 case 1: /* D2 vline */
7336 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
7337 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
7338 DRM_DEBUG("IH: D2 vline\n");
7342 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7346 case 3: /* D3 vblank/vline */
7348 case 0: /* D3 vblank */
7349 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
7350 if (rdev
->irq
.crtc_vblank_int
[2]) {
7351 drm_handle_vblank(rdev
->ddev
, 2);
7352 rdev
->pm
.vblank_sync
= true;
7353 wake_up(&rdev
->irq
.vblank_queue
);
7355 if (atomic_read(&rdev
->irq
.pflip
[2]))
7356 radeon_crtc_handle_flip(rdev
, 2);
7357 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
7358 DRM_DEBUG("IH: D3 vblank\n");
7361 case 1: /* D3 vline */
7362 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
7363 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
7364 DRM_DEBUG("IH: D3 vline\n");
7368 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7372 case 4: /* D4 vblank/vline */
7374 case 0: /* D4 vblank */
7375 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
7376 if (rdev
->irq
.crtc_vblank_int
[3]) {
7377 drm_handle_vblank(rdev
->ddev
, 3);
7378 rdev
->pm
.vblank_sync
= true;
7379 wake_up(&rdev
->irq
.vblank_queue
);
7381 if (atomic_read(&rdev
->irq
.pflip
[3]))
7382 radeon_crtc_handle_flip(rdev
, 3);
7383 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
7384 DRM_DEBUG("IH: D4 vblank\n");
7387 case 1: /* D4 vline */
7388 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
7389 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
7390 DRM_DEBUG("IH: D4 vline\n");
7394 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7398 case 5: /* D5 vblank/vline */
7400 case 0: /* D5 vblank */
7401 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
7402 if (rdev
->irq
.crtc_vblank_int
[4]) {
7403 drm_handle_vblank(rdev
->ddev
, 4);
7404 rdev
->pm
.vblank_sync
= true;
7405 wake_up(&rdev
->irq
.vblank_queue
);
7407 if (atomic_read(&rdev
->irq
.pflip
[4]))
7408 radeon_crtc_handle_flip(rdev
, 4);
7409 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
7410 DRM_DEBUG("IH: D5 vblank\n");
7413 case 1: /* D5 vline */
7414 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
7415 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
7416 DRM_DEBUG("IH: D5 vline\n");
7420 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7424 case 6: /* D6 vblank/vline */
7426 case 0: /* D6 vblank */
7427 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
7428 if (rdev
->irq
.crtc_vblank_int
[5]) {
7429 drm_handle_vblank(rdev
->ddev
, 5);
7430 rdev
->pm
.vblank_sync
= true;
7431 wake_up(&rdev
->irq
.vblank_queue
);
7433 if (atomic_read(&rdev
->irq
.pflip
[5]))
7434 radeon_crtc_handle_flip(rdev
, 5);
7435 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
7436 DRM_DEBUG("IH: D6 vblank\n");
7439 case 1: /* D6 vline */
7440 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
7441 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
7442 DRM_DEBUG("IH: D6 vline\n");
7446 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7450 case 42: /* HPD hotplug */
7453 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
7454 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~DC_HPD1_INTERRUPT
;
7455 queue_hotplug
= true;
7456 DRM_DEBUG("IH: HPD1\n");
7460 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
7461 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
7462 queue_hotplug
= true;
7463 DRM_DEBUG("IH: HPD2\n");
7467 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
7468 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
7469 queue_hotplug
= true;
7470 DRM_DEBUG("IH: HPD3\n");
7474 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
7475 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
7476 queue_hotplug
= true;
7477 DRM_DEBUG("IH: HPD4\n");
7481 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
7482 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
7483 queue_hotplug
= true;
7484 DRM_DEBUG("IH: HPD5\n");
7488 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
7489 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
7490 queue_hotplug
= true;
7491 DRM_DEBUG("IH: HPD6\n");
7495 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7500 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data
);
7501 radeon_fence_process(rdev
, R600_RING_TYPE_UVD_INDEX
);
7505 addr
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
);
7506 status
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
);
7507 mc_client
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT
);
7508 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
7509 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7511 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7513 cik_vm_decode_fault(rdev
, status
, addr
, mc_client
);
7514 /* reset addr and status */
7515 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
7518 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data
);
7521 radeon_fence_process(rdev
, TN_RING_TYPE_VCE1_INDEX
);
7524 radeon_fence_process(rdev
, TN_RING_TYPE_VCE2_INDEX
);
7527 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7531 case 176: /* GFX RB CP_INT */
7532 case 177: /* GFX IB CP_INT */
7533 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7535 case 181: /* CP EOP event */
7536 DRM_DEBUG("IH: CP EOP\n");
7537 /* XXX check the bitfield order! */
7538 me_id
= (ring_id
& 0x60) >> 5;
7539 pipe_id
= (ring_id
& 0x18) >> 3;
7540 queue_id
= (ring_id
& 0x7) >> 0;
7543 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7547 if ((cp1_ring
->me
== me_id
) & (cp1_ring
->pipe
== pipe_id
))
7548 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7549 if ((cp2_ring
->me
== me_id
) & (cp2_ring
->pipe
== pipe_id
))
7550 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7554 case 184: /* CP Privileged reg access */
7555 DRM_ERROR("Illegal register access in command stream\n");
7556 /* XXX check the bitfield order! */
7557 me_id
= (ring_id
& 0x60) >> 5;
7558 pipe_id
= (ring_id
& 0x18) >> 3;
7559 queue_id
= (ring_id
& 0x7) >> 0;
7562 /* This results in a full GPU reset, but all we need to do is soft
7563 * reset the CP for gfx
7577 case 185: /* CP Privileged inst */
7578 DRM_ERROR("Illegal instruction in command stream\n");
7579 /* XXX check the bitfield order! */
7580 me_id
= (ring_id
& 0x60) >> 5;
7581 pipe_id
= (ring_id
& 0x18) >> 3;
7582 queue_id
= (ring_id
& 0x7) >> 0;
7585 /* This results in a full GPU reset, but all we need to do is soft
7586 * reset the CP for gfx
7600 case 224: /* SDMA trap event */
7601 /* XXX check the bitfield order! */
7602 me_id
= (ring_id
& 0x3) >> 0;
7603 queue_id
= (ring_id
& 0xc) >> 2;
7604 DRM_DEBUG("IH: SDMA trap\n");
7609 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
7622 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7634 case 230: /* thermal low to high */
7635 DRM_DEBUG("IH: thermal low to high\n");
7636 rdev
->pm
.dpm
.thermal
.high_to_low
= false;
7637 queue_thermal
= true;
7639 case 231: /* thermal high to low */
7640 DRM_DEBUG("IH: thermal high to low\n");
7641 rdev
->pm
.dpm
.thermal
.high_to_low
= true;
7642 queue_thermal
= true;
7644 case 233: /* GUI IDLE */
7645 DRM_DEBUG("IH: GUI idle\n");
7647 case 241: /* SDMA Privileged inst */
7648 case 247: /* SDMA Privileged inst */
7649 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7650 /* XXX check the bitfield order! */
7651 me_id
= (ring_id
& 0x3) >> 0;
7652 queue_id
= (ring_id
& 0xc) >> 2;
7687 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7691 /* wptr/rptr are in bytes! */
7693 rptr
&= rdev
->ih
.ptr_mask
;
7696 schedule_work(&rdev
->hotplug_work
);
7698 schedule_work(&rdev
->reset_work
);
7700 schedule_work(&rdev
->pm
.dpm
.thermal
.work
);
7701 rdev
->ih
.rptr
= rptr
;
7702 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
7703 atomic_set(&rdev
->ih
.lock
, 0);
7705 /* make sure wptr hasn't changed while processing */
7706 wptr
= cik_get_ih_wptr(rdev
);
7714 * startup/shutdown callbacks
7717 * cik_startup - program the asic to a functional state
7719 * @rdev: radeon_device pointer
7721 * Programs the asic to a functional state (CIK).
7722 * Called by cik_init() and cik_resume().
7723 * Returns 0 for success, error for failure.
7725 static int cik_startup(struct radeon_device
*rdev
)
7727 struct radeon_ring
*ring
;
7730 /* enable pcie gen2/3 link */
7731 cik_pcie_gen3_enable(rdev
);
7733 cik_program_aspm(rdev
);
7735 /* scratch needs to be initialized before MC */
7736 r
= r600_vram_scratch_init(rdev
);
7740 cik_mc_program(rdev
);
7742 if (!(rdev
->flags
& RADEON_IS_IGP
) && !rdev
->pm
.dpm_enabled
) {
7743 r
= ci_mc_load_microcode(rdev
);
7745 DRM_ERROR("Failed to load MC firmware!\n");
7750 r
= cik_pcie_gart_enable(rdev
);
7755 /* allocate rlc buffers */
7756 if (rdev
->flags
& RADEON_IS_IGP
) {
7757 if (rdev
->family
== CHIP_KAVERI
) {
7758 rdev
->rlc
.reg_list
= spectre_rlc_save_restore_register_list
;
7759 rdev
->rlc
.reg_list_size
=
7760 (u32
)ARRAY_SIZE(spectre_rlc_save_restore_register_list
);
7762 rdev
->rlc
.reg_list
= kalindi_rlc_save_restore_register_list
;
7763 rdev
->rlc
.reg_list_size
=
7764 (u32
)ARRAY_SIZE(kalindi_rlc_save_restore_register_list
);
7767 rdev
->rlc
.cs_data
= ci_cs_data
;
7768 rdev
->rlc
.cp_table_size
= CP_ME_TABLE_SIZE
* 5 * 4;
7769 r
= sumo_rlc_init(rdev
);
7771 DRM_ERROR("Failed to init rlc BOs!\n");
7775 /* allocate wb buffer */
7776 r
= radeon_wb_init(rdev
);
7780 /* allocate mec buffers */
7781 r
= cik_mec_init(rdev
);
7783 DRM_ERROR("Failed to init MEC BOs!\n");
7787 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7789 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7793 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7795 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7799 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7801 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7805 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
7807 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7811 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7813 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7817 r
= radeon_uvd_resume(rdev
);
7819 r
= uvd_v4_2_resume(rdev
);
7821 r
= radeon_fence_driver_start_ring(rdev
,
7822 R600_RING_TYPE_UVD_INDEX
);
7824 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
7828 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
7830 r
= radeon_vce_resume(rdev
);
7832 r
= vce_v2_0_resume(rdev
);
7834 r
= radeon_fence_driver_start_ring(rdev
,
7835 TN_RING_TYPE_VCE1_INDEX
);
7837 r
= radeon_fence_driver_start_ring(rdev
,
7838 TN_RING_TYPE_VCE2_INDEX
);
7841 dev_err(rdev
->dev
, "VCE init error (%d).\n", r
);
7842 rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
].ring_size
= 0;
7843 rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
].ring_size
= 0;
7847 if (!rdev
->irq
.installed
) {
7848 r
= radeon_irq_kms_init(rdev
);
7853 r
= cik_irq_init(rdev
);
7855 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
7856 radeon_irq_kms_fini(rdev
);
7861 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
7862 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
7863 PACKET3(PACKET3_NOP
, 0x3FFF));
7867 /* set up the compute queues */
7868 /* type-2 packets are deprecated on MEC, use type-3 instead */
7869 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7870 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
7871 PACKET3(PACKET3_NOP
, 0x3FFF));
7874 ring
->me
= 1; /* first MEC */
7875 ring
->pipe
= 0; /* first pipe */
7876 ring
->queue
= 0; /* first queue */
7877 ring
->wptr_offs
= CIK_WB_CP1_WPTR_OFFSET
;
7879 /* type-2 packets are deprecated on MEC, use type-3 instead */
7880 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7881 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
7882 PACKET3(PACKET3_NOP
, 0x3FFF));
7885 /* dGPU only have 1 MEC */
7886 ring
->me
= 1; /* first MEC */
7887 ring
->pipe
= 0; /* first pipe */
7888 ring
->queue
= 1; /* second queue */
7889 ring
->wptr_offs
= CIK_WB_CP2_WPTR_OFFSET
;
7891 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
7892 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
7893 SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7897 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
7898 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
7899 SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7903 r
= cik_cp_resume(rdev
);
7907 r
= cik_sdma_resume(rdev
);
7911 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
7912 if (ring
->ring_size
) {
7913 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7916 r
= uvd_v1_0_init(rdev
);
7918 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
7923 ring
= &rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
];
7924 if (ring
->ring_size
)
7925 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7928 ring
= &rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
];
7929 if (ring
->ring_size
)
7930 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7934 r
= vce_v1_0_init(rdev
);
7935 else if (r
!= -ENOENT
)
7936 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r
);
7938 r
= radeon_ib_pool_init(rdev
);
7940 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
7944 r
= radeon_vm_manager_init(rdev
);
7946 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
7950 r
= dce6_audio_init(rdev
);
7958 * cik_resume - resume the asic to a functional state
7960 * @rdev: radeon_device pointer
7962 * Programs the asic to a functional state (CIK).
7964 * Returns 0 for success, error for failure.
7966 int cik_resume(struct radeon_device
*rdev
)
7971 atom_asic_init(rdev
->mode_info
.atom_context
);
7973 /* init golden registers */
7974 cik_init_golden_registers(rdev
);
7976 if (rdev
->pm
.pm_method
== PM_METHOD_DPM
)
7977 radeon_pm_resume(rdev
);
7979 rdev
->accel_working
= true;
7980 r
= cik_startup(rdev
);
7982 DRM_ERROR("cik startup failed on resume\n");
7983 rdev
->accel_working
= false;
7992 * cik_suspend - suspend the asic
7994 * @rdev: radeon_device pointer
7996 * Bring the chip into a state suitable for suspend (CIK).
7997 * Called at suspend.
7998 * Returns 0 for success.
8000 int cik_suspend(struct radeon_device
*rdev
)
8002 radeon_pm_suspend(rdev
);
8003 dce6_audio_fini(rdev
);
8004 radeon_vm_manager_fini(rdev
);
8005 cik_cp_enable(rdev
, false);
8006 cik_sdma_enable(rdev
, false);
8007 uvd_v1_0_fini(rdev
);
8008 radeon_uvd_suspend(rdev
);
8009 radeon_vce_suspend(rdev
);
8012 cik_irq_suspend(rdev
);
8013 radeon_wb_disable(rdev
);
8014 cik_pcie_gart_disable(rdev
);
8018 /* Plan is to move initialization in that function and use
8019 * helper function so that radeon_device_init pretty much
8020 * do nothing more than calling asic specific function. This
8021 * should also allow to remove a bunch of callback function
8025 * cik_init - asic specific driver and hw init
8027 * @rdev: radeon_device pointer
8029 * Setup asic specific driver variables and program the hw
8030 * to a functional state (CIK).
8031 * Called at driver startup.
8032 * Returns 0 for success, errors for failure.
8034 int cik_init(struct radeon_device
*rdev
)
8036 struct radeon_ring
*ring
;
8040 if (!radeon_get_bios(rdev
)) {
8041 if (ASIC_IS_AVIVO(rdev
))
8044 /* Must be an ATOMBIOS */
8045 if (!rdev
->is_atom_bios
) {
8046 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
8049 r
= radeon_atombios_init(rdev
);
8053 /* Post card if necessary */
8054 if (!radeon_card_posted(rdev
)) {
8056 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
8059 DRM_INFO("GPU not posted. posting now...\n");
8060 atom_asic_init(rdev
->mode_info
.atom_context
);
8062 /* init golden registers */
8063 cik_init_golden_registers(rdev
);
8064 /* Initialize scratch registers */
8065 cik_scratch_init(rdev
);
8066 /* Initialize surface registers */
8067 radeon_surface_init(rdev
);
8068 /* Initialize clocks */
8069 radeon_get_clock_info(rdev
->ddev
);
8072 r
= radeon_fence_driver_init(rdev
);
8076 /* initialize memory controller */
8077 r
= cik_mc_init(rdev
);
8080 /* Memory manager */
8081 r
= radeon_bo_init(rdev
);
8085 if (rdev
->flags
& RADEON_IS_IGP
) {
8086 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
8087 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
) {
8088 r
= cik_init_microcode(rdev
);
8090 DRM_ERROR("Failed to load firmware!\n");
8095 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
8096 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
||
8098 r
= cik_init_microcode(rdev
);
8100 DRM_ERROR("Failed to load firmware!\n");
8106 /* Initialize power management */
8107 radeon_pm_init(rdev
);
8109 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
8110 ring
->ring_obj
= NULL
;
8111 r600_ring_init(rdev
, ring
, 1024 * 1024);
8113 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
8114 ring
->ring_obj
= NULL
;
8115 r600_ring_init(rdev
, ring
, 1024 * 1024);
8116 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_index
);
8120 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
8121 ring
->ring_obj
= NULL
;
8122 r600_ring_init(rdev
, ring
, 1024 * 1024);
8123 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_index
);
8127 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
8128 ring
->ring_obj
= NULL
;
8129 r600_ring_init(rdev
, ring
, 256 * 1024);
8131 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
8132 ring
->ring_obj
= NULL
;
8133 r600_ring_init(rdev
, ring
, 256 * 1024);
8135 r
= radeon_uvd_init(rdev
);
8137 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
8138 ring
->ring_obj
= NULL
;
8139 r600_ring_init(rdev
, ring
, 4096);
8142 r
= radeon_vce_init(rdev
);
8144 ring
= &rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
];
8145 ring
->ring_obj
= NULL
;
8146 r600_ring_init(rdev
, ring
, 4096);
8148 ring
= &rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
];
8149 ring
->ring_obj
= NULL
;
8150 r600_ring_init(rdev
, ring
, 4096);
8153 rdev
->ih
.ring_obj
= NULL
;
8154 r600_ih_ring_init(rdev
, 64 * 1024);
8156 r
= r600_pcie_gart_init(rdev
);
8160 rdev
->accel_working
= true;
8161 r
= cik_startup(rdev
);
8163 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
8165 cik_sdma_fini(rdev
);
8167 sumo_rlc_fini(rdev
);
8169 radeon_wb_fini(rdev
);
8170 radeon_ib_pool_fini(rdev
);
8171 radeon_vm_manager_fini(rdev
);
8172 radeon_irq_kms_fini(rdev
);
8173 cik_pcie_gart_fini(rdev
);
8174 rdev
->accel_working
= false;
8177 /* Don't start up if the MC ucode is missing.
8178 * The default clocks and voltages before the MC ucode
8179 * is loaded are not suffient for advanced operations.
8181 if (!rdev
->mc_fw
&& !(rdev
->flags
& RADEON_IS_IGP
)) {
8182 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8190 * cik_fini - asic specific driver and hw fini
8192 * @rdev: radeon_device pointer
8194 * Tear down the asic specific driver variables and program the hw
8195 * to an idle state (CIK).
8196 * Called at driver unload.
8198 void cik_fini(struct radeon_device
*rdev
)
8200 radeon_pm_fini(rdev
);
8202 cik_sdma_fini(rdev
);
8206 sumo_rlc_fini(rdev
);
8208 radeon_wb_fini(rdev
);
8209 radeon_vm_manager_fini(rdev
);
8210 radeon_ib_pool_fini(rdev
);
8211 radeon_irq_kms_fini(rdev
);
8212 uvd_v1_0_fini(rdev
);
8213 radeon_uvd_fini(rdev
);
8214 radeon_vce_fini(rdev
);
8215 cik_pcie_gart_fini(rdev
);
8216 r600_vram_scratch_fini(rdev
);
8217 radeon_gem_fini(rdev
);
8218 radeon_fence_driver_fini(rdev
);
8219 radeon_bo_fini(rdev
);
8220 radeon_atombios_fini(rdev
);
8225 void dce8_program_fmt(struct drm_encoder
*encoder
)
8227 struct drm_device
*dev
= encoder
->dev
;
8228 struct radeon_device
*rdev
= dev
->dev_private
;
8229 struct radeon_encoder
*radeon_encoder
= to_radeon_encoder(encoder
);
8230 struct radeon_crtc
*radeon_crtc
= to_radeon_crtc(encoder
->crtc
);
8231 struct drm_connector
*connector
= radeon_get_connector_for_encoder(encoder
);
8234 enum radeon_connector_dither dither
= RADEON_FMT_DITHER_DISABLE
;
8237 struct radeon_connector
*radeon_connector
= to_radeon_connector(connector
);
8238 bpc
= radeon_get_monitor_bpc(connector
);
8239 dither
= radeon_connector
->dither
;
8242 /* LVDS/eDP FMT is set up by atom */
8243 if (radeon_encoder
->devices
& ATOM_DEVICE_LCD_SUPPORT
)
8246 /* not needed for analog */
8247 if ((radeon_encoder
->encoder_id
== ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1
) ||
8248 (radeon_encoder
->encoder_id
== ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2
))
8256 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8257 /* XXX sort out optimal dither settings */
8258 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8259 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(0));
8261 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(0));
8264 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8265 /* XXX sort out optimal dither settings */
8266 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8267 FMT_RGB_RANDOM_ENABLE
|
8268 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(1));
8270 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(1));
8273 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8274 /* XXX sort out optimal dither settings */
8275 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8276 FMT_RGB_RANDOM_ENABLE
|
8277 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(2));
8279 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(2));
8286 WREG32(FMT_BIT_DEPTH_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8289 /* display watermark setup */
8291 * dce8_line_buffer_adjust - Set up the line buffer
8293 * @rdev: radeon_device pointer
8294 * @radeon_crtc: the selected display controller
8295 * @mode: the current display mode on the selected display
8298 * Setup up the line buffer allocation for
8299 * the selected display controller (CIK).
8300 * Returns the line buffer size in pixels.
8302 static u32
dce8_line_buffer_adjust(struct radeon_device
*rdev
,
8303 struct radeon_crtc
*radeon_crtc
,
8304 struct drm_display_mode
*mode
)
8306 u32 tmp
, buffer_alloc
, i
;
8307 u32 pipe_offset
= radeon_crtc
->crtc_id
* 0x20;
8310 * There are 6 line buffers, one for each display controllers.
8311 * There are 3 partitions per LB. Select the number of partitions
8312 * to enable based on the display width. For display widths larger
8313 * than 4096, you need use to use 2 display controllers and combine
8314 * them using the stereo blender.
8316 if (radeon_crtc
->base
.enabled
&& mode
) {
8317 if (mode
->crtc_hdisplay
< 1920) {
8320 } else if (mode
->crtc_hdisplay
< 2560) {
8323 } else if (mode
->crtc_hdisplay
< 4096) {
8325 buffer_alloc
= (rdev
->flags
& RADEON_IS_IGP
) ? 2 : 4;
8327 DRM_DEBUG_KMS("Mode too big for LB!\n");
8329 buffer_alloc
= (rdev
->flags
& RADEON_IS_IGP
) ? 2 : 4;
8336 WREG32(LB_MEMORY_CTRL
+ radeon_crtc
->crtc_offset
,
8337 LB_MEMORY_CONFIG(tmp
) | LB_MEMORY_SIZE(0x6B0));
8339 WREG32(PIPE0_DMIF_BUFFER_CONTROL
+ pipe_offset
,
8340 DMIF_BUFFERS_ALLOCATED(buffer_alloc
));
8341 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
8342 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL
+ pipe_offset
) &
8343 DMIF_BUFFERS_ALLOCATED_COMPLETED
)
8348 if (radeon_crtc
->base
.enabled
&& mode
) {
8360 /* controller not enabled, so no lb used */
8365 * cik_get_number_of_dram_channels - get the number of dram channels
8367 * @rdev: radeon_device pointer
8369 * Look up the number of video ram channels (CIK).
8370 * Used for display watermark bandwidth calculations
8371 * Returns the number of dram channels
8373 static u32
cik_get_number_of_dram_channels(struct radeon_device
*rdev
)
8375 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
8377 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
8400 struct dce8_wm_params
{
8401 u32 dram_channels
; /* number of dram channels */
8402 u32 yclk
; /* bandwidth per dram data pin in kHz */
8403 u32 sclk
; /* engine clock in kHz */
8404 u32 disp_clk
; /* display clock in kHz */
8405 u32 src_width
; /* viewport width */
8406 u32 active_time
; /* active display time in ns */
8407 u32 blank_time
; /* blank time in ns */
8408 bool interlaced
; /* mode is interlaced */
8409 fixed20_12 vsc
; /* vertical scale ratio */
8410 u32 num_heads
; /* number of active crtcs */
8411 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
8412 u32 lb_size
; /* line buffer allocated to pipe */
8413 u32 vtaps
; /* vertical scaler taps */
8417 * dce8_dram_bandwidth - get the dram bandwidth
8419 * @wm: watermark calculation data
8421 * Calculate the raw dram bandwidth (CIK).
8422 * Used for display watermark bandwidth calculations
8423 * Returns the dram bandwidth in MBytes/s
8425 static u32
dce8_dram_bandwidth(struct dce8_wm_params
*wm
)
8427 /* Calculate raw DRAM Bandwidth */
8428 fixed20_12 dram_efficiency
; /* 0.7 */
8429 fixed20_12 yclk
, dram_channels
, bandwidth
;
8432 a
.full
= dfixed_const(1000);
8433 yclk
.full
= dfixed_const(wm
->yclk
);
8434 yclk
.full
= dfixed_div(yclk
, a
);
8435 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8436 a
.full
= dfixed_const(10);
8437 dram_efficiency
.full
= dfixed_const(7);
8438 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
8439 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8440 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
8442 return dfixed_trunc(bandwidth
);
8446 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8448 * @wm: watermark calculation data
8450 * Calculate the dram bandwidth used for display (CIK).
8451 * Used for display watermark bandwidth calculations
8452 * Returns the dram bandwidth for display in MBytes/s
8454 static u32
dce8_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8456 /* Calculate DRAM Bandwidth and the part allocated to display. */
8457 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
8458 fixed20_12 yclk
, dram_channels
, bandwidth
;
8461 a
.full
= dfixed_const(1000);
8462 yclk
.full
= dfixed_const(wm
->yclk
);
8463 yclk
.full
= dfixed_div(yclk
, a
);
8464 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8465 a
.full
= dfixed_const(10);
8466 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
8467 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
8468 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8469 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
8471 return dfixed_trunc(bandwidth
);
8475 * dce8_data_return_bandwidth - get the data return bandwidth
8477 * @wm: watermark calculation data
8479 * Calculate the data return bandwidth used for display (CIK).
8480 * Used for display watermark bandwidth calculations
8481 * Returns the data return bandwidth in MBytes/s
8483 static u32
dce8_data_return_bandwidth(struct dce8_wm_params
*wm
)
8485 /* Calculate the display Data return Bandwidth */
8486 fixed20_12 return_efficiency
; /* 0.8 */
8487 fixed20_12 sclk
, bandwidth
;
8490 a
.full
= dfixed_const(1000);
8491 sclk
.full
= dfixed_const(wm
->sclk
);
8492 sclk
.full
= dfixed_div(sclk
, a
);
8493 a
.full
= dfixed_const(10);
8494 return_efficiency
.full
= dfixed_const(8);
8495 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
8496 a
.full
= dfixed_const(32);
8497 bandwidth
.full
= dfixed_mul(a
, sclk
);
8498 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
8500 return dfixed_trunc(bandwidth
);
8504 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8506 * @wm: watermark calculation data
8508 * Calculate the dmif bandwidth used for display (CIK).
8509 * Used for display watermark bandwidth calculations
8510 * Returns the dmif bandwidth in MBytes/s
8512 static u32
dce8_dmif_request_bandwidth(struct dce8_wm_params
*wm
)
8514 /* Calculate the DMIF Request Bandwidth */
8515 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
8516 fixed20_12 disp_clk
, bandwidth
;
8519 a
.full
= dfixed_const(1000);
8520 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
8521 disp_clk
.full
= dfixed_div(disp_clk
, a
);
8522 a
.full
= dfixed_const(32);
8523 b
.full
= dfixed_mul(a
, disp_clk
);
8525 a
.full
= dfixed_const(10);
8526 disp_clk_request_efficiency
.full
= dfixed_const(8);
8527 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
8529 bandwidth
.full
= dfixed_mul(b
, disp_clk_request_efficiency
);
8531 return dfixed_trunc(bandwidth
);
8535 * dce8_available_bandwidth - get the min available bandwidth
8537 * @wm: watermark calculation data
8539 * Calculate the min available bandwidth used for display (CIK).
8540 * Used for display watermark bandwidth calculations
8541 * Returns the min available bandwidth in MBytes/s
8543 static u32
dce8_available_bandwidth(struct dce8_wm_params
*wm
)
8545 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8546 u32 dram_bandwidth
= dce8_dram_bandwidth(wm
);
8547 u32 data_return_bandwidth
= dce8_data_return_bandwidth(wm
);
8548 u32 dmif_req_bandwidth
= dce8_dmif_request_bandwidth(wm
);
8550 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
8554 * dce8_average_bandwidth - get the average available bandwidth
8556 * @wm: watermark calculation data
8558 * Calculate the average available bandwidth used for display (CIK).
8559 * Used for display watermark bandwidth calculations
8560 * Returns the average available bandwidth in MBytes/s
8562 static u32
dce8_average_bandwidth(struct dce8_wm_params
*wm
)
8564 /* Calculate the display mode Average Bandwidth
8565 * DisplayMode should contain the source and destination dimensions,
8569 fixed20_12 line_time
;
8570 fixed20_12 src_width
;
8571 fixed20_12 bandwidth
;
8574 a
.full
= dfixed_const(1000);
8575 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
8576 line_time
.full
= dfixed_div(line_time
, a
);
8577 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
8578 src_width
.full
= dfixed_const(wm
->src_width
);
8579 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
8580 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
8581 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
8583 return dfixed_trunc(bandwidth
);
8587 * dce8_latency_watermark - get the latency watermark
8589 * @wm: watermark calculation data
8591 * Calculate the latency watermark (CIK).
8592 * Used for display watermark bandwidth calculations
8593 * Returns the latency watermark in ns
8595 static u32
dce8_latency_watermark(struct dce8_wm_params
*wm
)
8597 /* First calculate the latency in ns */
8598 u32 mc_latency
= 2000; /* 2000 ns. */
8599 u32 available_bandwidth
= dce8_available_bandwidth(wm
);
8600 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
8601 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
8602 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
8603 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
8604 (wm
->num_heads
* cursor_line_pair_return_time
);
8605 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
8606 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
8607 u32 tmp
, dmif_size
= 12288;
8610 if (wm
->num_heads
== 0)
8613 a
.full
= dfixed_const(2);
8614 b
.full
= dfixed_const(1);
8615 if ((wm
->vsc
.full
> a
.full
) ||
8616 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
8618 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
8619 max_src_lines_per_dst_line
= 4;
8621 max_src_lines_per_dst_line
= 2;
8623 a
.full
= dfixed_const(available_bandwidth
);
8624 b
.full
= dfixed_const(wm
->num_heads
);
8625 a
.full
= dfixed_div(a
, b
);
8627 b
.full
= dfixed_const(mc_latency
+ 512);
8628 c
.full
= dfixed_const(wm
->disp_clk
);
8629 b
.full
= dfixed_div(b
, c
);
8631 c
.full
= dfixed_const(dmif_size
);
8632 b
.full
= dfixed_div(c
, b
);
8634 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
8636 b
.full
= dfixed_const(1000);
8637 c
.full
= dfixed_const(wm
->disp_clk
);
8638 b
.full
= dfixed_div(c
, b
);
8639 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
8640 b
.full
= dfixed_mul(b
, c
);
8642 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
8644 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
8645 b
.full
= dfixed_const(1000);
8646 c
.full
= dfixed_const(lb_fill_bw
);
8647 b
.full
= dfixed_div(c
, b
);
8648 a
.full
= dfixed_div(a
, b
);
8649 line_fill_time
= dfixed_trunc(a
);
8651 if (line_fill_time
< wm
->active_time
)
8654 return latency
+ (line_fill_time
- wm
->active_time
);
8659 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8660 * average and available dram bandwidth
8662 * @wm: watermark calculation data
8664 * Check if the display average bandwidth fits in the display
8665 * dram bandwidth (CIK).
8666 * Used for display watermark bandwidth calculations
8667 * Returns true if the display fits, false if not.
8669 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8671 if (dce8_average_bandwidth(wm
) <=
8672 (dce8_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
8679 * dce8_average_bandwidth_vs_available_bandwidth - check
8680 * average and available bandwidth
8682 * @wm: watermark calculation data
8684 * Check if the display average bandwidth fits in the display
8685 * available bandwidth (CIK).
8686 * Used for display watermark bandwidth calculations
8687 * Returns true if the display fits, false if not.
8689 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params
*wm
)
8691 if (dce8_average_bandwidth(wm
) <=
8692 (dce8_available_bandwidth(wm
) / wm
->num_heads
))
8699 * dce8_check_latency_hiding - check latency hiding
8701 * @wm: watermark calculation data
8703 * Check latency hiding (CIK).
8704 * Used for display watermark bandwidth calculations
8705 * Returns true if the display fits, false if not.
8707 static bool dce8_check_latency_hiding(struct dce8_wm_params
*wm
)
8709 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
8710 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
8711 u32 latency_tolerant_lines
;
8715 a
.full
= dfixed_const(1);
8716 if (wm
->vsc
.full
> a
.full
)
8717 latency_tolerant_lines
= 1;
8719 if (lb_partitions
<= (wm
->vtaps
+ 1))
8720 latency_tolerant_lines
= 1;
8722 latency_tolerant_lines
= 2;
8725 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
8727 if (dce8_latency_watermark(wm
) <= latency_hiding
)
8734 * dce8_program_watermarks - program display watermarks
8736 * @rdev: radeon_device pointer
8737 * @radeon_crtc: the selected display controller
8738 * @lb_size: line buffer size
8739 * @num_heads: number of display controllers in use
8741 * Calculate and program the display watermarks for the
8742 * selected display controller (CIK).
8744 static void dce8_program_watermarks(struct radeon_device
*rdev
,
8745 struct radeon_crtc
*radeon_crtc
,
8746 u32 lb_size
, u32 num_heads
)
8748 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
8749 struct dce8_wm_params wm_low
, wm_high
;
8752 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
8755 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
8756 pixel_period
= 1000000 / (u32
)mode
->clock
;
8757 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
8759 /* watermark for high clocks */
8760 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8761 rdev
->pm
.dpm_enabled
) {
8763 radeon_dpm_get_mclk(rdev
, false) * 10;
8765 radeon_dpm_get_sclk(rdev
, false) * 10;
8767 wm_high
.yclk
= rdev
->pm
.current_mclk
* 10;
8768 wm_high
.sclk
= rdev
->pm
.current_sclk
* 10;
8771 wm_high
.disp_clk
= mode
->clock
;
8772 wm_high
.src_width
= mode
->crtc_hdisplay
;
8773 wm_high
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8774 wm_high
.blank_time
= line_time
- wm_high
.active_time
;
8775 wm_high
.interlaced
= false;
8776 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8777 wm_high
.interlaced
= true;
8778 wm_high
.vsc
= radeon_crtc
->vsc
;
8780 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8782 wm_high
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8783 wm_high
.lb_size
= lb_size
;
8784 wm_high
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8785 wm_high
.num_heads
= num_heads
;
8787 /* set for high clocks */
8788 latency_watermark_a
= min(dce8_latency_watermark(&wm_high
), (u32
)65535);
8790 /* possibly force display priority to high */
8791 /* should really do this at mode validation time... */
8792 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high
) ||
8793 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high
) ||
8794 !dce8_check_latency_hiding(&wm_high
) ||
8795 (rdev
->disp_priority
== 2)) {
8796 DRM_DEBUG_KMS("force priority to high\n");
8799 /* watermark for low clocks */
8800 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8801 rdev
->pm
.dpm_enabled
) {
8803 radeon_dpm_get_mclk(rdev
, true) * 10;
8805 radeon_dpm_get_sclk(rdev
, true) * 10;
8807 wm_low
.yclk
= rdev
->pm
.current_mclk
* 10;
8808 wm_low
.sclk
= rdev
->pm
.current_sclk
* 10;
8811 wm_low
.disp_clk
= mode
->clock
;
8812 wm_low
.src_width
= mode
->crtc_hdisplay
;
8813 wm_low
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8814 wm_low
.blank_time
= line_time
- wm_low
.active_time
;
8815 wm_low
.interlaced
= false;
8816 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8817 wm_low
.interlaced
= true;
8818 wm_low
.vsc
= radeon_crtc
->vsc
;
8820 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8822 wm_low
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8823 wm_low
.lb_size
= lb_size
;
8824 wm_low
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8825 wm_low
.num_heads
= num_heads
;
8827 /* set for low clocks */
8828 latency_watermark_b
= min(dce8_latency_watermark(&wm_low
), (u32
)65535);
8830 /* possibly force display priority to high */
8831 /* should really do this at mode validation time... */
8832 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low
) ||
8833 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low
) ||
8834 !dce8_check_latency_hiding(&wm_low
) ||
8835 (rdev
->disp_priority
== 2)) {
8836 DRM_DEBUG_KMS("force priority to high\n");
8841 wm_mask
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8843 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8844 tmp
|= LATENCY_WATERMARK_MASK(1);
8845 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8846 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8847 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
8848 LATENCY_HIGH_WATERMARK(line_time
)));
8850 tmp
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8851 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8852 tmp
|= LATENCY_WATERMARK_MASK(2);
8853 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8854 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8855 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
8856 LATENCY_HIGH_WATERMARK(line_time
)));
8857 /* restore original selection */
8858 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, wm_mask
);
8860 /* save values for DPM */
8861 radeon_crtc
->line_time
= line_time
;
8862 radeon_crtc
->wm_high
= latency_watermark_a
;
8863 radeon_crtc
->wm_low
= latency_watermark_b
;
8867 * dce8_bandwidth_update - program display watermarks
8869 * @rdev: radeon_device pointer
8871 * Calculate and program the display watermarks and line
8872 * buffer allocation (CIK).
8874 void dce8_bandwidth_update(struct radeon_device
*rdev
)
8876 struct drm_display_mode
*mode
= NULL
;
8877 u32 num_heads
= 0, lb_size
;
8880 radeon_update_display_priority(rdev
);
8882 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8883 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
8886 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8887 mode
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
8888 lb_size
= dce8_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode
);
8889 dce8_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
8894 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8896 * @rdev: radeon_device pointer
8898 * Fetches a GPU clock counter snapshot (SI).
8899 * Returns the 64 bit clock counter snapshot.
8901 uint64_t cik_get_gpu_clock_counter(struct radeon_device
*rdev
)
8905 mutex_lock(&rdev
->gpu_clock_mutex
);
8906 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
8907 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
8908 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
8909 mutex_unlock(&rdev
->gpu_clock_mutex
);
8913 static int cik_set_uvd_clock(struct radeon_device
*rdev
, u32 clock
,
8914 u32 cntl_reg
, u32 status_reg
)
8917 struct atom_clock_dividers dividers
;
8920 r
= radeon_atom_get_clock_dividers(rdev
, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK
,
8921 clock
, false, ÷rs
);
8925 tmp
= RREG32_SMC(cntl_reg
);
8926 tmp
&= ~(DCLK_DIR_CNTL_EN
|DCLK_DIVIDER_MASK
);
8927 tmp
|= dividers
.post_divider
;
8928 WREG32_SMC(cntl_reg
, tmp
);
8930 for (i
= 0; i
< 100; i
++) {
8931 if (RREG32_SMC(status_reg
) & DCLK_STATUS
)
8941 int cik_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
8945 r
= cik_set_uvd_clock(rdev
, vclk
, CG_VCLK_CNTL
, CG_VCLK_STATUS
);
8949 r
= cik_set_uvd_clock(rdev
, dclk
, CG_DCLK_CNTL
, CG_DCLK_STATUS
);
8953 int cik_set_vce_clocks(struct radeon_device
*rdev
, u32 evclk
, u32 ecclk
)
8956 struct atom_clock_dividers dividers
;
8959 r
= radeon_atom_get_clock_dividers(rdev
, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK
,
8960 ecclk
, false, ÷rs
);
8964 for (i
= 0; i
< 100; i
++) {
8965 if (RREG32_SMC(CG_ECLK_STATUS
) & ECLK_STATUS
)
8972 tmp
= RREG32_SMC(CG_ECLK_CNTL
);
8973 tmp
&= ~(ECLK_DIR_CNTL_EN
|ECLK_DIVIDER_MASK
);
8974 tmp
|= dividers
.post_divider
;
8975 WREG32_SMC(CG_ECLK_CNTL
, tmp
);
8977 for (i
= 0; i
< 100; i
++) {
8978 if (RREG32_SMC(CG_ECLK_STATUS
) & ECLK_STATUS
)
8988 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
)
8990 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
8991 int bridge_pos
, gpu_pos
;
8992 u32 speed_cntl
, mask
, current_data_rate
;
8996 if (radeon_pcie_gen2
== 0)
8999 if (rdev
->flags
& RADEON_IS_IGP
)
9002 if (!(rdev
->flags
& RADEON_IS_PCIE
))
9005 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
9009 if (!(mask
& (DRM_PCIE_SPEED_50
| DRM_PCIE_SPEED_80
)))
9012 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
9013 current_data_rate
= (speed_cntl
& LC_CURRENT_DATA_RATE_MASK
) >>
9014 LC_CURRENT_DATA_RATE_SHIFT
;
9015 if (mask
& DRM_PCIE_SPEED_80
) {
9016 if (current_data_rate
== 2) {
9017 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9020 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9021 } else if (mask
& DRM_PCIE_SPEED_50
) {
9022 if (current_data_rate
== 1) {
9023 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9026 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9029 bridge_pos
= pci_pcie_cap(root
);
9033 gpu_pos
= pci_pcie_cap(rdev
->pdev
);
9037 if (mask
& DRM_PCIE_SPEED_80
) {
9038 /* re-try equalization if gen3 is not already enabled */
9039 if (current_data_rate
!= 2) {
9040 u16 bridge_cfg
, gpu_cfg
;
9041 u16 bridge_cfg2
, gpu_cfg2
;
9042 u32 max_lw
, current_lw
, tmp
;
9044 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
9045 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
9047 tmp16
= bridge_cfg
| PCI_EXP_LNKCTL_HAWD
;
9048 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9050 tmp16
= gpu_cfg
| PCI_EXP_LNKCTL_HAWD
;
9051 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9053 tmp
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
9054 max_lw
= (tmp
& LC_DETECTED_LINK_WIDTH_MASK
) >> LC_DETECTED_LINK_WIDTH_SHIFT
;
9055 current_lw
= (tmp
& LC_OPERATING_LINK_WIDTH_MASK
) >> LC_OPERATING_LINK_WIDTH_SHIFT
;
9057 if (current_lw
< max_lw
) {
9058 tmp
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
9059 if (tmp
& LC_RENEGOTIATION_SUPPORT
) {
9060 tmp
&= ~(LC_LINK_WIDTH_MASK
| LC_UPCONFIGURE_DIS
);
9061 tmp
|= (max_lw
<< LC_LINK_WIDTH_SHIFT
);
9062 tmp
|= LC_UPCONFIGURE_SUPPORT
| LC_RENEGOTIATE_EN
| LC_RECONFIG_NOW
;
9063 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, tmp
);
9067 for (i
= 0; i
< 10; i
++) {
9069 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_DEVSTA
, &tmp16
);
9070 if (tmp16
& PCI_EXP_DEVSTA_TRPND
)
9073 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
9074 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
9076 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &bridge_cfg2
);
9077 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &gpu_cfg2
);
9079 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9080 tmp
|= LC_SET_QUIESCE
;
9081 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9083 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9085 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9090 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
9091 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
9092 tmp16
|= (bridge_cfg
& PCI_EXP_LNKCTL_HAWD
);
9093 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9095 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
9096 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
9097 tmp16
|= (gpu_cfg
& PCI_EXP_LNKCTL_HAWD
);
9098 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9101 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9102 tmp16
&= ~((1 << 4) | (7 << 9));
9103 tmp16
|= (bridge_cfg2
& ((1 << 4) | (7 << 9)));
9104 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9106 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9107 tmp16
&= ~((1 << 4) | (7 << 9));
9108 tmp16
|= (gpu_cfg2
& ((1 << 4) | (7 << 9)));
9109 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9111 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9112 tmp
&= ~LC_SET_QUIESCE
;
9113 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9118 /* set the link speed */
9119 speed_cntl
|= LC_FORCE_EN_SW_SPEED_CHANGE
| LC_FORCE_DIS_HW_SPEED_CHANGE
;
9120 speed_cntl
&= ~LC_FORCE_DIS_SW_SPEED_CHANGE
;
9121 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
9123 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9125 if (mask
& DRM_PCIE_SPEED_80
)
9126 tmp16
|= 3; /* gen3 */
9127 else if (mask
& DRM_PCIE_SPEED_50
)
9128 tmp16
|= 2; /* gen2 */
9130 tmp16
|= 1; /* gen1 */
9131 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9133 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
9134 speed_cntl
|= LC_INITIATE_LINK_SPEED_CHANGE
;
9135 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
9137 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
9138 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
9139 if ((speed_cntl
& LC_INITIATE_LINK_SPEED_CHANGE
) == 0)
9145 static void cik_program_aspm(struct radeon_device
*rdev
)
9148 bool disable_l0s
= false, disable_l1
= false, disable_plloff_in_l1
= false;
9149 bool disable_clkreq
= false;
9151 if (radeon_aspm
== 0)
9154 /* XXX double check IGPs */
9155 if (rdev
->flags
& RADEON_IS_IGP
)
9158 if (!(rdev
->flags
& RADEON_IS_PCIE
))
9161 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
9162 data
&= ~LC_XMIT_N_FTS_MASK
;
9163 data
|= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN
;
9165 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
, data
);
9167 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL3
);
9168 data
|= LC_GO_TO_RECOVERY
;
9170 WREG32_PCIE_PORT(PCIE_LC_CNTL3
, data
);
9172 orig
= data
= RREG32_PCIE_PORT(PCIE_P_CNTL
);
9173 data
|= P_IGNORE_EDB_ERR
;
9175 WREG32_PCIE_PORT(PCIE_P_CNTL
, data
);
9177 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
9178 data
&= ~(LC_L0S_INACTIVITY_MASK
| LC_L1_INACTIVITY_MASK
);
9179 data
|= LC_PMI_TO_L1_DIS
;
9181 data
|= LC_L0S_INACTIVITY(7);
9184 data
|= LC_L1_INACTIVITY(7);
9185 data
&= ~LC_PMI_TO_L1_DIS
;
9187 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
9189 if (!disable_plloff_in_l1
) {
9190 bool clk_req_support
;
9192 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
);
9193 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
9194 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9196 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
, data
);
9198 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
);
9199 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
9200 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9202 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
, data
);
9204 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
);
9205 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
9206 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9208 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
, data
);
9210 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
);
9211 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
9212 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9214 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
, data
);
9216 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
9217 data
&= ~LC_DYN_LANES_PWR_STATE_MASK
;
9218 data
|= LC_DYN_LANES_PWR_STATE(3);
9220 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, data
);
9222 if (!disable_clkreq
) {
9223 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
9226 clk_req_support
= false;
9227 pcie_capability_read_dword(root
, PCI_EXP_LNKCAP
, &lnkcap
);
9228 if (lnkcap
& PCI_EXP_LNKCAP_CLKPM
)
9229 clk_req_support
= true;
9231 clk_req_support
= false;
9234 if (clk_req_support
) {
9235 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL2
);
9236 data
|= LC_ALLOW_PDWN_IN_L1
| LC_ALLOW_PDWN_IN_L23
;
9238 WREG32_PCIE_PORT(PCIE_LC_CNTL2
, data
);
9240 orig
= data
= RREG32_SMC(THM_CLK_CNTL
);
9241 data
&= ~(CMON_CLK_SEL_MASK
| TMON_CLK_SEL_MASK
);
9242 data
|= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9244 WREG32_SMC(THM_CLK_CNTL
, data
);
9246 orig
= data
= RREG32_SMC(MISC_CLK_CTRL
);
9247 data
&= ~(DEEP_SLEEP_CLK_SEL_MASK
| ZCLK_SEL_MASK
);
9248 data
|= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9250 WREG32_SMC(MISC_CLK_CTRL
, data
);
9252 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL
);
9253 data
&= ~BCLK_AS_XCLK
;
9255 WREG32_SMC(CG_CLKPIN_CNTL
, data
);
9257 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL_2
);
9258 data
&= ~FORCE_BIF_REFCLK_EN
;
9260 WREG32_SMC(CG_CLKPIN_CNTL_2
, data
);
9262 orig
= data
= RREG32_SMC(MPLL_BYPASSCLK_SEL
);
9263 data
&= ~MPLL_CLKOUT_SEL_MASK
;
9264 data
|= MPLL_CLKOUT_SEL(4);
9266 WREG32_SMC(MPLL_BYPASSCLK_SEL
, data
);
9271 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
9274 orig
= data
= RREG32_PCIE_PORT(PCIE_CNTL2
);
9275 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
| REPLAY_MEM_LS_EN
;
9277 WREG32_PCIE_PORT(PCIE_CNTL2
, data
);
9280 data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
9281 if((data
& LC_N_FTS_MASK
) == LC_N_FTS_MASK
) {
9282 data
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
9283 if ((data
& LC_REVERSE_XMIT
) && (data
& LC_REVERSE_RCVR
)) {
9284 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
9285 data
&= ~LC_L0S_INACTIVITY_MASK
;
9287 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);