2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
52 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
58 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59 MODULE_FIRMWARE("radeon/KABINI_me.bin");
60 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
63 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
65 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
66 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
67 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
68 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
69 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
70 extern void sumo_rlc_fini(struct radeon_device
*rdev
);
71 extern int sumo_rlc_init(struct radeon_device
*rdev
);
72 extern void si_vram_gtt_location(struct radeon_device
*rdev
, struct radeon_mc
*mc
);
73 extern void si_rlc_reset(struct radeon_device
*rdev
);
74 extern void si_init_uvd_internal_cg(struct radeon_device
*rdev
);
75 extern int cik_sdma_resume(struct radeon_device
*rdev
);
76 extern void cik_sdma_enable(struct radeon_device
*rdev
, bool enable
);
77 extern void cik_sdma_fini(struct radeon_device
*rdev
);
78 extern void vce_v2_0_enable_mgcg(struct radeon_device
*rdev
, bool enable
);
79 static void cik_rlc_stop(struct radeon_device
*rdev
);
80 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
);
81 static void cik_program_aspm(struct radeon_device
*rdev
);
82 static void cik_init_pg(struct radeon_device
*rdev
);
83 static void cik_init_cg(struct radeon_device
*rdev
);
84 static void cik_fini_pg(struct radeon_device
*rdev
);
85 static void cik_fini_cg(struct radeon_device
*rdev
);
86 static void cik_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
89 /* get temperature in millidegrees */
90 int ci_get_temp(struct radeon_device
*rdev
)
95 temp
= (RREG32_SMC(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
101 actual_temp
= temp
& 0x1ff;
103 actual_temp
= actual_temp
* 1000;
108 /* get temperature in millidegrees */
109 int kv_get_temp(struct radeon_device
*rdev
)
114 temp
= RREG32_SMC(0xC0300E0C);
117 actual_temp
= (temp
/ 8) - 49;
121 actual_temp
= actual_temp
* 1000;
127 * Indirect registers accessor
129 u32
cik_pciep_rreg(struct radeon_device
*rdev
, u32 reg
)
134 spin_lock_irqsave(&rdev
->pciep_idx_lock
, flags
);
135 WREG32(PCIE_INDEX
, reg
);
136 (void)RREG32(PCIE_INDEX
);
137 r
= RREG32(PCIE_DATA
);
138 spin_unlock_irqrestore(&rdev
->pciep_idx_lock
, flags
);
142 void cik_pciep_wreg(struct radeon_device
*rdev
, u32 reg
, u32 v
)
146 spin_lock_irqsave(&rdev
->pciep_idx_lock
, flags
);
147 WREG32(PCIE_INDEX
, reg
);
148 (void)RREG32(PCIE_INDEX
);
149 WREG32(PCIE_DATA
, v
);
150 (void)RREG32(PCIE_DATA
);
151 spin_unlock_irqrestore(&rdev
->pciep_idx_lock
, flags
);
154 static const u32 spectre_rlc_save_restore_register_list
[] =
156 (0x0e00 << 16) | (0xc12c >> 2),
158 (0x0e00 << 16) | (0xc140 >> 2),
160 (0x0e00 << 16) | (0xc150 >> 2),
162 (0x0e00 << 16) | (0xc15c >> 2),
164 (0x0e00 << 16) | (0xc168 >> 2),
166 (0x0e00 << 16) | (0xc170 >> 2),
168 (0x0e00 << 16) | (0xc178 >> 2),
170 (0x0e00 << 16) | (0xc204 >> 2),
172 (0x0e00 << 16) | (0xc2b4 >> 2),
174 (0x0e00 << 16) | (0xc2b8 >> 2),
176 (0x0e00 << 16) | (0xc2bc >> 2),
178 (0x0e00 << 16) | (0xc2c0 >> 2),
180 (0x0e00 << 16) | (0x8228 >> 2),
182 (0x0e00 << 16) | (0x829c >> 2),
184 (0x0e00 << 16) | (0x869c >> 2),
186 (0x0600 << 16) | (0x98f4 >> 2),
188 (0x0e00 << 16) | (0x98f8 >> 2),
190 (0x0e00 << 16) | (0x9900 >> 2),
192 (0x0e00 << 16) | (0xc260 >> 2),
194 (0x0e00 << 16) | (0x90e8 >> 2),
196 (0x0e00 << 16) | (0x3c000 >> 2),
198 (0x0e00 << 16) | (0x3c00c >> 2),
200 (0x0e00 << 16) | (0x8c1c >> 2),
202 (0x0e00 << 16) | (0x9700 >> 2),
204 (0x0e00 << 16) | (0xcd20 >> 2),
206 (0x4e00 << 16) | (0xcd20 >> 2),
208 (0x5e00 << 16) | (0xcd20 >> 2),
210 (0x6e00 << 16) | (0xcd20 >> 2),
212 (0x7e00 << 16) | (0xcd20 >> 2),
214 (0x8e00 << 16) | (0xcd20 >> 2),
216 (0x9e00 << 16) | (0xcd20 >> 2),
218 (0xae00 << 16) | (0xcd20 >> 2),
220 (0xbe00 << 16) | (0xcd20 >> 2),
222 (0x0e00 << 16) | (0x89bc >> 2),
224 (0x0e00 << 16) | (0x8900 >> 2),
227 (0x0e00 << 16) | (0xc130 >> 2),
229 (0x0e00 << 16) | (0xc134 >> 2),
231 (0x0e00 << 16) | (0xc1fc >> 2),
233 (0x0e00 << 16) | (0xc208 >> 2),
235 (0x0e00 << 16) | (0xc264 >> 2),
237 (0x0e00 << 16) | (0xc268 >> 2),
239 (0x0e00 << 16) | (0xc26c >> 2),
241 (0x0e00 << 16) | (0xc270 >> 2),
243 (0x0e00 << 16) | (0xc274 >> 2),
245 (0x0e00 << 16) | (0xc278 >> 2),
247 (0x0e00 << 16) | (0xc27c >> 2),
249 (0x0e00 << 16) | (0xc280 >> 2),
251 (0x0e00 << 16) | (0xc284 >> 2),
253 (0x0e00 << 16) | (0xc288 >> 2),
255 (0x0e00 << 16) | (0xc28c >> 2),
257 (0x0e00 << 16) | (0xc290 >> 2),
259 (0x0e00 << 16) | (0xc294 >> 2),
261 (0x0e00 << 16) | (0xc298 >> 2),
263 (0x0e00 << 16) | (0xc29c >> 2),
265 (0x0e00 << 16) | (0xc2a0 >> 2),
267 (0x0e00 << 16) | (0xc2a4 >> 2),
269 (0x0e00 << 16) | (0xc2a8 >> 2),
271 (0x0e00 << 16) | (0xc2ac >> 2),
273 (0x0e00 << 16) | (0xc2b0 >> 2),
275 (0x0e00 << 16) | (0x301d0 >> 2),
277 (0x0e00 << 16) | (0x30238 >> 2),
279 (0x0e00 << 16) | (0x30250 >> 2),
281 (0x0e00 << 16) | (0x30254 >> 2),
283 (0x0e00 << 16) | (0x30258 >> 2),
285 (0x0e00 << 16) | (0x3025c >> 2),
287 (0x4e00 << 16) | (0xc900 >> 2),
289 (0x5e00 << 16) | (0xc900 >> 2),
291 (0x6e00 << 16) | (0xc900 >> 2),
293 (0x7e00 << 16) | (0xc900 >> 2),
295 (0x8e00 << 16) | (0xc900 >> 2),
297 (0x9e00 << 16) | (0xc900 >> 2),
299 (0xae00 << 16) | (0xc900 >> 2),
301 (0xbe00 << 16) | (0xc900 >> 2),
303 (0x4e00 << 16) | (0xc904 >> 2),
305 (0x5e00 << 16) | (0xc904 >> 2),
307 (0x6e00 << 16) | (0xc904 >> 2),
309 (0x7e00 << 16) | (0xc904 >> 2),
311 (0x8e00 << 16) | (0xc904 >> 2),
313 (0x9e00 << 16) | (0xc904 >> 2),
315 (0xae00 << 16) | (0xc904 >> 2),
317 (0xbe00 << 16) | (0xc904 >> 2),
319 (0x4e00 << 16) | (0xc908 >> 2),
321 (0x5e00 << 16) | (0xc908 >> 2),
323 (0x6e00 << 16) | (0xc908 >> 2),
325 (0x7e00 << 16) | (0xc908 >> 2),
327 (0x8e00 << 16) | (0xc908 >> 2),
329 (0x9e00 << 16) | (0xc908 >> 2),
331 (0xae00 << 16) | (0xc908 >> 2),
333 (0xbe00 << 16) | (0xc908 >> 2),
335 (0x4e00 << 16) | (0xc90c >> 2),
337 (0x5e00 << 16) | (0xc90c >> 2),
339 (0x6e00 << 16) | (0xc90c >> 2),
341 (0x7e00 << 16) | (0xc90c >> 2),
343 (0x8e00 << 16) | (0xc90c >> 2),
345 (0x9e00 << 16) | (0xc90c >> 2),
347 (0xae00 << 16) | (0xc90c >> 2),
349 (0xbe00 << 16) | (0xc90c >> 2),
351 (0x4e00 << 16) | (0xc910 >> 2),
353 (0x5e00 << 16) | (0xc910 >> 2),
355 (0x6e00 << 16) | (0xc910 >> 2),
357 (0x7e00 << 16) | (0xc910 >> 2),
359 (0x8e00 << 16) | (0xc910 >> 2),
361 (0x9e00 << 16) | (0xc910 >> 2),
363 (0xae00 << 16) | (0xc910 >> 2),
365 (0xbe00 << 16) | (0xc910 >> 2),
367 (0x0e00 << 16) | (0xc99c >> 2),
369 (0x0e00 << 16) | (0x9834 >> 2),
371 (0x0000 << 16) | (0x30f00 >> 2),
373 (0x0001 << 16) | (0x30f00 >> 2),
375 (0x0000 << 16) | (0x30f04 >> 2),
377 (0x0001 << 16) | (0x30f04 >> 2),
379 (0x0000 << 16) | (0x30f08 >> 2),
381 (0x0001 << 16) | (0x30f08 >> 2),
383 (0x0000 << 16) | (0x30f0c >> 2),
385 (0x0001 << 16) | (0x30f0c >> 2),
387 (0x0600 << 16) | (0x9b7c >> 2),
389 (0x0e00 << 16) | (0x8a14 >> 2),
391 (0x0e00 << 16) | (0x8a18 >> 2),
393 (0x0600 << 16) | (0x30a00 >> 2),
395 (0x0e00 << 16) | (0x8bf0 >> 2),
397 (0x0e00 << 16) | (0x8bcc >> 2),
399 (0x0e00 << 16) | (0x8b24 >> 2),
401 (0x0e00 << 16) | (0x30a04 >> 2),
403 (0x0600 << 16) | (0x30a10 >> 2),
405 (0x0600 << 16) | (0x30a14 >> 2),
407 (0x0600 << 16) | (0x30a18 >> 2),
409 (0x0600 << 16) | (0x30a2c >> 2),
411 (0x0e00 << 16) | (0xc700 >> 2),
413 (0x0e00 << 16) | (0xc704 >> 2),
415 (0x0e00 << 16) | (0xc708 >> 2),
417 (0x0e00 << 16) | (0xc768 >> 2),
419 (0x0400 << 16) | (0xc770 >> 2),
421 (0x0400 << 16) | (0xc774 >> 2),
423 (0x0400 << 16) | (0xc778 >> 2),
425 (0x0400 << 16) | (0xc77c >> 2),
427 (0x0400 << 16) | (0xc780 >> 2),
429 (0x0400 << 16) | (0xc784 >> 2),
431 (0x0400 << 16) | (0xc788 >> 2),
433 (0x0400 << 16) | (0xc78c >> 2),
435 (0x0400 << 16) | (0xc798 >> 2),
437 (0x0400 << 16) | (0xc79c >> 2),
439 (0x0400 << 16) | (0xc7a0 >> 2),
441 (0x0400 << 16) | (0xc7a4 >> 2),
443 (0x0400 << 16) | (0xc7a8 >> 2),
445 (0x0400 << 16) | (0xc7ac >> 2),
447 (0x0400 << 16) | (0xc7b0 >> 2),
449 (0x0400 << 16) | (0xc7b4 >> 2),
451 (0x0e00 << 16) | (0x9100 >> 2),
453 (0x0e00 << 16) | (0x3c010 >> 2),
455 (0x0e00 << 16) | (0x92a8 >> 2),
457 (0x0e00 << 16) | (0x92ac >> 2),
459 (0x0e00 << 16) | (0x92b4 >> 2),
461 (0x0e00 << 16) | (0x92b8 >> 2),
463 (0x0e00 << 16) | (0x92bc >> 2),
465 (0x0e00 << 16) | (0x92c0 >> 2),
467 (0x0e00 << 16) | (0x92c4 >> 2),
469 (0x0e00 << 16) | (0x92c8 >> 2),
471 (0x0e00 << 16) | (0x92cc >> 2),
473 (0x0e00 << 16) | (0x92d0 >> 2),
475 (0x0e00 << 16) | (0x8c00 >> 2),
477 (0x0e00 << 16) | (0x8c04 >> 2),
479 (0x0e00 << 16) | (0x8c20 >> 2),
481 (0x0e00 << 16) | (0x8c38 >> 2),
483 (0x0e00 << 16) | (0x8c3c >> 2),
485 (0x0e00 << 16) | (0xae00 >> 2),
487 (0x0e00 << 16) | (0x9604 >> 2),
489 (0x0e00 << 16) | (0xac08 >> 2),
491 (0x0e00 << 16) | (0xac0c >> 2),
493 (0x0e00 << 16) | (0xac10 >> 2),
495 (0x0e00 << 16) | (0xac14 >> 2),
497 (0x0e00 << 16) | (0xac58 >> 2),
499 (0x0e00 << 16) | (0xac68 >> 2),
501 (0x0e00 << 16) | (0xac6c >> 2),
503 (0x0e00 << 16) | (0xac70 >> 2),
505 (0x0e00 << 16) | (0xac74 >> 2),
507 (0x0e00 << 16) | (0xac78 >> 2),
509 (0x0e00 << 16) | (0xac7c >> 2),
511 (0x0e00 << 16) | (0xac80 >> 2),
513 (0x0e00 << 16) | (0xac84 >> 2),
515 (0x0e00 << 16) | (0xac88 >> 2),
517 (0x0e00 << 16) | (0xac8c >> 2),
519 (0x0e00 << 16) | (0x970c >> 2),
521 (0x0e00 << 16) | (0x9714 >> 2),
523 (0x0e00 << 16) | (0x9718 >> 2),
525 (0x0e00 << 16) | (0x971c >> 2),
527 (0x0e00 << 16) | (0x31068 >> 2),
529 (0x4e00 << 16) | (0x31068 >> 2),
531 (0x5e00 << 16) | (0x31068 >> 2),
533 (0x6e00 << 16) | (0x31068 >> 2),
535 (0x7e00 << 16) | (0x31068 >> 2),
537 (0x8e00 << 16) | (0x31068 >> 2),
539 (0x9e00 << 16) | (0x31068 >> 2),
541 (0xae00 << 16) | (0x31068 >> 2),
543 (0xbe00 << 16) | (0x31068 >> 2),
545 (0x0e00 << 16) | (0xcd10 >> 2),
547 (0x0e00 << 16) | (0xcd14 >> 2),
549 (0x0e00 << 16) | (0x88b0 >> 2),
551 (0x0e00 << 16) | (0x88b4 >> 2),
553 (0x0e00 << 16) | (0x88b8 >> 2),
555 (0x0e00 << 16) | (0x88bc >> 2),
557 (0x0400 << 16) | (0x89c0 >> 2),
559 (0x0e00 << 16) | (0x88c4 >> 2),
561 (0x0e00 << 16) | (0x88c8 >> 2),
563 (0x0e00 << 16) | (0x88d0 >> 2),
565 (0x0e00 << 16) | (0x88d4 >> 2),
567 (0x0e00 << 16) | (0x88d8 >> 2),
569 (0x0e00 << 16) | (0x8980 >> 2),
571 (0x0e00 << 16) | (0x30938 >> 2),
573 (0x0e00 << 16) | (0x3093c >> 2),
575 (0x0e00 << 16) | (0x30940 >> 2),
577 (0x0e00 << 16) | (0x89a0 >> 2),
579 (0x0e00 << 16) | (0x30900 >> 2),
581 (0x0e00 << 16) | (0x30904 >> 2),
583 (0x0e00 << 16) | (0x89b4 >> 2),
585 (0x0e00 << 16) | (0x3c210 >> 2),
587 (0x0e00 << 16) | (0x3c214 >> 2),
589 (0x0e00 << 16) | (0x3c218 >> 2),
591 (0x0e00 << 16) | (0x8904 >> 2),
594 (0x0e00 << 16) | (0x8c28 >> 2),
595 (0x0e00 << 16) | (0x8c2c >> 2),
596 (0x0e00 << 16) | (0x8c30 >> 2),
597 (0x0e00 << 16) | (0x8c34 >> 2),
598 (0x0e00 << 16) | (0x9600 >> 2),
601 static const u32 kalindi_rlc_save_restore_register_list
[] =
603 (0x0e00 << 16) | (0xc12c >> 2),
605 (0x0e00 << 16) | (0xc140 >> 2),
607 (0x0e00 << 16) | (0xc150 >> 2),
609 (0x0e00 << 16) | (0xc15c >> 2),
611 (0x0e00 << 16) | (0xc168 >> 2),
613 (0x0e00 << 16) | (0xc170 >> 2),
615 (0x0e00 << 16) | (0xc204 >> 2),
617 (0x0e00 << 16) | (0xc2b4 >> 2),
619 (0x0e00 << 16) | (0xc2b8 >> 2),
621 (0x0e00 << 16) | (0xc2bc >> 2),
623 (0x0e00 << 16) | (0xc2c0 >> 2),
625 (0x0e00 << 16) | (0x8228 >> 2),
627 (0x0e00 << 16) | (0x829c >> 2),
629 (0x0e00 << 16) | (0x869c >> 2),
631 (0x0600 << 16) | (0x98f4 >> 2),
633 (0x0e00 << 16) | (0x98f8 >> 2),
635 (0x0e00 << 16) | (0x9900 >> 2),
637 (0x0e00 << 16) | (0xc260 >> 2),
639 (0x0e00 << 16) | (0x90e8 >> 2),
641 (0x0e00 << 16) | (0x3c000 >> 2),
643 (0x0e00 << 16) | (0x3c00c >> 2),
645 (0x0e00 << 16) | (0x8c1c >> 2),
647 (0x0e00 << 16) | (0x9700 >> 2),
649 (0x0e00 << 16) | (0xcd20 >> 2),
651 (0x4e00 << 16) | (0xcd20 >> 2),
653 (0x5e00 << 16) | (0xcd20 >> 2),
655 (0x6e00 << 16) | (0xcd20 >> 2),
657 (0x7e00 << 16) | (0xcd20 >> 2),
659 (0x0e00 << 16) | (0x89bc >> 2),
661 (0x0e00 << 16) | (0x8900 >> 2),
664 (0x0e00 << 16) | (0xc130 >> 2),
666 (0x0e00 << 16) | (0xc134 >> 2),
668 (0x0e00 << 16) | (0xc1fc >> 2),
670 (0x0e00 << 16) | (0xc208 >> 2),
672 (0x0e00 << 16) | (0xc264 >> 2),
674 (0x0e00 << 16) | (0xc268 >> 2),
676 (0x0e00 << 16) | (0xc26c >> 2),
678 (0x0e00 << 16) | (0xc270 >> 2),
680 (0x0e00 << 16) | (0xc274 >> 2),
682 (0x0e00 << 16) | (0xc28c >> 2),
684 (0x0e00 << 16) | (0xc290 >> 2),
686 (0x0e00 << 16) | (0xc294 >> 2),
688 (0x0e00 << 16) | (0xc298 >> 2),
690 (0x0e00 << 16) | (0xc2a0 >> 2),
692 (0x0e00 << 16) | (0xc2a4 >> 2),
694 (0x0e00 << 16) | (0xc2a8 >> 2),
696 (0x0e00 << 16) | (0xc2ac >> 2),
698 (0x0e00 << 16) | (0x301d0 >> 2),
700 (0x0e00 << 16) | (0x30238 >> 2),
702 (0x0e00 << 16) | (0x30250 >> 2),
704 (0x0e00 << 16) | (0x30254 >> 2),
706 (0x0e00 << 16) | (0x30258 >> 2),
708 (0x0e00 << 16) | (0x3025c >> 2),
710 (0x4e00 << 16) | (0xc900 >> 2),
712 (0x5e00 << 16) | (0xc900 >> 2),
714 (0x6e00 << 16) | (0xc900 >> 2),
716 (0x7e00 << 16) | (0xc900 >> 2),
718 (0x4e00 << 16) | (0xc904 >> 2),
720 (0x5e00 << 16) | (0xc904 >> 2),
722 (0x6e00 << 16) | (0xc904 >> 2),
724 (0x7e00 << 16) | (0xc904 >> 2),
726 (0x4e00 << 16) | (0xc908 >> 2),
728 (0x5e00 << 16) | (0xc908 >> 2),
730 (0x6e00 << 16) | (0xc908 >> 2),
732 (0x7e00 << 16) | (0xc908 >> 2),
734 (0x4e00 << 16) | (0xc90c >> 2),
736 (0x5e00 << 16) | (0xc90c >> 2),
738 (0x6e00 << 16) | (0xc90c >> 2),
740 (0x7e00 << 16) | (0xc90c >> 2),
742 (0x4e00 << 16) | (0xc910 >> 2),
744 (0x5e00 << 16) | (0xc910 >> 2),
746 (0x6e00 << 16) | (0xc910 >> 2),
748 (0x7e00 << 16) | (0xc910 >> 2),
750 (0x0e00 << 16) | (0xc99c >> 2),
752 (0x0e00 << 16) | (0x9834 >> 2),
754 (0x0000 << 16) | (0x30f00 >> 2),
756 (0x0000 << 16) | (0x30f04 >> 2),
758 (0x0000 << 16) | (0x30f08 >> 2),
760 (0x0000 << 16) | (0x30f0c >> 2),
762 (0x0600 << 16) | (0x9b7c >> 2),
764 (0x0e00 << 16) | (0x8a14 >> 2),
766 (0x0e00 << 16) | (0x8a18 >> 2),
768 (0x0600 << 16) | (0x30a00 >> 2),
770 (0x0e00 << 16) | (0x8bf0 >> 2),
772 (0x0e00 << 16) | (0x8bcc >> 2),
774 (0x0e00 << 16) | (0x8b24 >> 2),
776 (0x0e00 << 16) | (0x30a04 >> 2),
778 (0x0600 << 16) | (0x30a10 >> 2),
780 (0x0600 << 16) | (0x30a14 >> 2),
782 (0x0600 << 16) | (0x30a18 >> 2),
784 (0x0600 << 16) | (0x30a2c >> 2),
786 (0x0e00 << 16) | (0xc700 >> 2),
788 (0x0e00 << 16) | (0xc704 >> 2),
790 (0x0e00 << 16) | (0xc708 >> 2),
792 (0x0e00 << 16) | (0xc768 >> 2),
794 (0x0400 << 16) | (0xc770 >> 2),
796 (0x0400 << 16) | (0xc774 >> 2),
798 (0x0400 << 16) | (0xc798 >> 2),
800 (0x0400 << 16) | (0xc79c >> 2),
802 (0x0e00 << 16) | (0x9100 >> 2),
804 (0x0e00 << 16) | (0x3c010 >> 2),
806 (0x0e00 << 16) | (0x8c00 >> 2),
808 (0x0e00 << 16) | (0x8c04 >> 2),
810 (0x0e00 << 16) | (0x8c20 >> 2),
812 (0x0e00 << 16) | (0x8c38 >> 2),
814 (0x0e00 << 16) | (0x8c3c >> 2),
816 (0x0e00 << 16) | (0xae00 >> 2),
818 (0x0e00 << 16) | (0x9604 >> 2),
820 (0x0e00 << 16) | (0xac08 >> 2),
822 (0x0e00 << 16) | (0xac0c >> 2),
824 (0x0e00 << 16) | (0xac10 >> 2),
826 (0x0e00 << 16) | (0xac14 >> 2),
828 (0x0e00 << 16) | (0xac58 >> 2),
830 (0x0e00 << 16) | (0xac68 >> 2),
832 (0x0e00 << 16) | (0xac6c >> 2),
834 (0x0e00 << 16) | (0xac70 >> 2),
836 (0x0e00 << 16) | (0xac74 >> 2),
838 (0x0e00 << 16) | (0xac78 >> 2),
840 (0x0e00 << 16) | (0xac7c >> 2),
842 (0x0e00 << 16) | (0xac80 >> 2),
844 (0x0e00 << 16) | (0xac84 >> 2),
846 (0x0e00 << 16) | (0xac88 >> 2),
848 (0x0e00 << 16) | (0xac8c >> 2),
850 (0x0e00 << 16) | (0x970c >> 2),
852 (0x0e00 << 16) | (0x9714 >> 2),
854 (0x0e00 << 16) | (0x9718 >> 2),
856 (0x0e00 << 16) | (0x971c >> 2),
858 (0x0e00 << 16) | (0x31068 >> 2),
860 (0x4e00 << 16) | (0x31068 >> 2),
862 (0x5e00 << 16) | (0x31068 >> 2),
864 (0x6e00 << 16) | (0x31068 >> 2),
866 (0x7e00 << 16) | (0x31068 >> 2),
868 (0x0e00 << 16) | (0xcd10 >> 2),
870 (0x0e00 << 16) | (0xcd14 >> 2),
872 (0x0e00 << 16) | (0x88b0 >> 2),
874 (0x0e00 << 16) | (0x88b4 >> 2),
876 (0x0e00 << 16) | (0x88b8 >> 2),
878 (0x0e00 << 16) | (0x88bc >> 2),
880 (0x0400 << 16) | (0x89c0 >> 2),
882 (0x0e00 << 16) | (0x88c4 >> 2),
884 (0x0e00 << 16) | (0x88c8 >> 2),
886 (0x0e00 << 16) | (0x88d0 >> 2),
888 (0x0e00 << 16) | (0x88d4 >> 2),
890 (0x0e00 << 16) | (0x88d8 >> 2),
892 (0x0e00 << 16) | (0x8980 >> 2),
894 (0x0e00 << 16) | (0x30938 >> 2),
896 (0x0e00 << 16) | (0x3093c >> 2),
898 (0x0e00 << 16) | (0x30940 >> 2),
900 (0x0e00 << 16) | (0x89a0 >> 2),
902 (0x0e00 << 16) | (0x30900 >> 2),
904 (0x0e00 << 16) | (0x30904 >> 2),
906 (0x0e00 << 16) | (0x89b4 >> 2),
908 (0x0e00 << 16) | (0x3e1fc >> 2),
910 (0x0e00 << 16) | (0x3c210 >> 2),
912 (0x0e00 << 16) | (0x3c214 >> 2),
914 (0x0e00 << 16) | (0x3c218 >> 2),
916 (0x0e00 << 16) | (0x8904 >> 2),
919 (0x0e00 << 16) | (0x8c28 >> 2),
920 (0x0e00 << 16) | (0x8c2c >> 2),
921 (0x0e00 << 16) | (0x8c30 >> 2),
922 (0x0e00 << 16) | (0x8c34 >> 2),
923 (0x0e00 << 16) | (0x9600 >> 2),
926 static const u32 bonaire_golden_spm_registers
[] =
928 0x30800, 0xe0ffffff, 0xe0000000
931 static const u32 bonaire_golden_common_registers
[] =
933 0xc770, 0xffffffff, 0x00000800,
934 0xc774, 0xffffffff, 0x00000800,
935 0xc798, 0xffffffff, 0x00007fbf,
936 0xc79c, 0xffffffff, 0x00007faf
939 static const u32 bonaire_golden_registers
[] =
941 0x3354, 0x00000333, 0x00000333,
942 0x3350, 0x000c0fc0, 0x00040200,
943 0x9a10, 0x00010000, 0x00058208,
944 0x3c000, 0xffff1fff, 0x00140000,
945 0x3c200, 0xfdfc0fff, 0x00000100,
946 0x3c234, 0x40000000, 0x40000200,
947 0x9830, 0xffffffff, 0x00000000,
948 0x9834, 0xf00fffff, 0x00000400,
949 0x9838, 0x0002021c, 0x00020200,
950 0xc78, 0x00000080, 0x00000000,
951 0x5bb0, 0x000000f0, 0x00000070,
952 0x5bc0, 0xf0311fff, 0x80300000,
953 0x98f8, 0x73773777, 0x12010001,
954 0x350c, 0x00810000, 0x408af000,
955 0x7030, 0x31000111, 0x00000011,
956 0x2f48, 0x73773777, 0x12010001,
957 0x220c, 0x00007fb6, 0x0021a1b1,
958 0x2210, 0x00007fb6, 0x002021b1,
959 0x2180, 0x00007fb6, 0x00002191,
960 0x2218, 0x00007fb6, 0x002121b1,
961 0x221c, 0x00007fb6, 0x002021b1,
962 0x21dc, 0x00007fb6, 0x00002191,
963 0x21e0, 0x00007fb6, 0x00002191,
964 0x3628, 0x0000003f, 0x0000000a,
965 0x362c, 0x0000003f, 0x0000000a,
966 0x2ae4, 0x00073ffe, 0x000022a2,
967 0x240c, 0x000007ff, 0x00000000,
968 0x8a14, 0xf000003f, 0x00000007,
969 0x8bf0, 0x00002001, 0x00000001,
970 0x8b24, 0xffffffff, 0x00ffffff,
971 0x30a04, 0x0000ff0f, 0x00000000,
972 0x28a4c, 0x07ffffff, 0x06000000,
973 0x4d8, 0x00000fff, 0x00000100,
974 0x3e78, 0x00000001, 0x00000002,
975 0x9100, 0x03000000, 0x0362c688,
976 0x8c00, 0x000000ff, 0x00000001,
977 0xe40, 0x00001fff, 0x00001fff,
978 0x9060, 0x0000007f, 0x00000020,
979 0x9508, 0x00010000, 0x00010000,
980 0xac14, 0x000003ff, 0x000000f3,
981 0xac0c, 0xffffffff, 0x00001032
984 static const u32 bonaire_mgcg_cgcg_init
[] =
986 0xc420, 0xffffffff, 0xfffffffc,
987 0x30800, 0xffffffff, 0xe0000000,
988 0x3c2a0, 0xffffffff, 0x00000100,
989 0x3c208, 0xffffffff, 0x00000100,
990 0x3c2c0, 0xffffffff, 0xc0000100,
991 0x3c2c8, 0xffffffff, 0xc0000100,
992 0x3c2c4, 0xffffffff, 0xc0000100,
993 0x55e4, 0xffffffff, 0x00600100,
994 0x3c280, 0xffffffff, 0x00000100,
995 0x3c214, 0xffffffff, 0x06000100,
996 0x3c220, 0xffffffff, 0x00000100,
997 0x3c218, 0xffffffff, 0x06000100,
998 0x3c204, 0xffffffff, 0x00000100,
999 0x3c2e0, 0xffffffff, 0x00000100,
1000 0x3c224, 0xffffffff, 0x00000100,
1001 0x3c200, 0xffffffff, 0x00000100,
1002 0x3c230, 0xffffffff, 0x00000100,
1003 0x3c234, 0xffffffff, 0x00000100,
1004 0x3c250, 0xffffffff, 0x00000100,
1005 0x3c254, 0xffffffff, 0x00000100,
1006 0x3c258, 0xffffffff, 0x00000100,
1007 0x3c25c, 0xffffffff, 0x00000100,
1008 0x3c260, 0xffffffff, 0x00000100,
1009 0x3c27c, 0xffffffff, 0x00000100,
1010 0x3c278, 0xffffffff, 0x00000100,
1011 0x3c210, 0xffffffff, 0x06000100,
1012 0x3c290, 0xffffffff, 0x00000100,
1013 0x3c274, 0xffffffff, 0x00000100,
1014 0x3c2b4, 0xffffffff, 0x00000100,
1015 0x3c2b0, 0xffffffff, 0x00000100,
1016 0x3c270, 0xffffffff, 0x00000100,
1017 0x30800, 0xffffffff, 0xe0000000,
1018 0x3c020, 0xffffffff, 0x00010000,
1019 0x3c024, 0xffffffff, 0x00030002,
1020 0x3c028, 0xffffffff, 0x00040007,
1021 0x3c02c, 0xffffffff, 0x00060005,
1022 0x3c030, 0xffffffff, 0x00090008,
1023 0x3c034, 0xffffffff, 0x00010000,
1024 0x3c038, 0xffffffff, 0x00030002,
1025 0x3c03c, 0xffffffff, 0x00040007,
1026 0x3c040, 0xffffffff, 0x00060005,
1027 0x3c044, 0xffffffff, 0x00090008,
1028 0x3c048, 0xffffffff, 0x00010000,
1029 0x3c04c, 0xffffffff, 0x00030002,
1030 0x3c050, 0xffffffff, 0x00040007,
1031 0x3c054, 0xffffffff, 0x00060005,
1032 0x3c058, 0xffffffff, 0x00090008,
1033 0x3c05c, 0xffffffff, 0x00010000,
1034 0x3c060, 0xffffffff, 0x00030002,
1035 0x3c064, 0xffffffff, 0x00040007,
1036 0x3c068, 0xffffffff, 0x00060005,
1037 0x3c06c, 0xffffffff, 0x00090008,
1038 0x3c070, 0xffffffff, 0x00010000,
1039 0x3c074, 0xffffffff, 0x00030002,
1040 0x3c078, 0xffffffff, 0x00040007,
1041 0x3c07c, 0xffffffff, 0x00060005,
1042 0x3c080, 0xffffffff, 0x00090008,
1043 0x3c084, 0xffffffff, 0x00010000,
1044 0x3c088, 0xffffffff, 0x00030002,
1045 0x3c08c, 0xffffffff, 0x00040007,
1046 0x3c090, 0xffffffff, 0x00060005,
1047 0x3c094, 0xffffffff, 0x00090008,
1048 0x3c098, 0xffffffff, 0x00010000,
1049 0x3c09c, 0xffffffff, 0x00030002,
1050 0x3c0a0, 0xffffffff, 0x00040007,
1051 0x3c0a4, 0xffffffff, 0x00060005,
1052 0x3c0a8, 0xffffffff, 0x00090008,
1053 0x3c000, 0xffffffff, 0x96e00200,
1054 0x8708, 0xffffffff, 0x00900100,
1055 0xc424, 0xffffffff, 0x0020003f,
1056 0x38, 0xffffffff, 0x0140001c,
1057 0x3c, 0x000f0000, 0x000f0000,
1058 0x220, 0xffffffff, 0xC060000C,
1059 0x224, 0xc0000fff, 0x00000100,
1060 0xf90, 0xffffffff, 0x00000100,
1061 0xf98, 0x00000101, 0x00000000,
1062 0x20a8, 0xffffffff, 0x00000104,
1063 0x55e4, 0xff000fff, 0x00000100,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd00c, 0xff000ff0, 0x00000100,
1067 0xd80c, 0xff000ff0, 0x00000100
1070 static const u32 spectre_golden_spm_registers
[] =
1072 0x30800, 0xe0ffffff, 0xe0000000
1075 static const u32 spectre_golden_common_registers
[] =
1077 0xc770, 0xffffffff, 0x00000800,
1078 0xc774, 0xffffffff, 0x00000800,
1079 0xc798, 0xffffffff, 0x00007fbf,
1080 0xc79c, 0xffffffff, 0x00007faf
1083 static const u32 spectre_golden_registers
[] =
1085 0x3c000, 0xffff1fff, 0x96940200,
1086 0x3c00c, 0xffff0001, 0xff000000,
1087 0x3c200, 0xfffc0fff, 0x00000100,
1088 0x6ed8, 0x00010101, 0x00010000,
1089 0x9834, 0xf00fffff, 0x00000400,
1090 0x9838, 0xfffffffc, 0x00020200,
1091 0x5bb0, 0x000000f0, 0x00000070,
1092 0x5bc0, 0xf0311fff, 0x80300000,
1093 0x98f8, 0x73773777, 0x12010001,
1094 0x9b7c, 0x00ff0000, 0x00fc0000,
1095 0x2f48, 0x73773777, 0x12010001,
1096 0x8a14, 0xf000003f, 0x00000007,
1097 0x8b24, 0xffffffff, 0x00ffffff,
1098 0x28350, 0x3f3f3fff, 0x00000082,
1099 0x28355, 0x0000003f, 0x00000000,
1100 0x3e78, 0x00000001, 0x00000002,
1101 0x913c, 0xffff03df, 0x00000004,
1102 0xc768, 0x00000008, 0x00000008,
1103 0x8c00, 0x000008ff, 0x00000800,
1104 0x9508, 0x00010000, 0x00010000,
1105 0xac0c, 0xffffffff, 0x54763210,
1106 0x214f8, 0x01ff01ff, 0x00000002,
1107 0x21498, 0x007ff800, 0x00200000,
1108 0x2015c, 0xffffffff, 0x00000f40,
1109 0x30934, 0xffffffff, 0x00000001
1112 static const u32 spectre_mgcg_cgcg_init
[] =
1114 0xc420, 0xffffffff, 0xfffffffc,
1115 0x30800, 0xffffffff, 0xe0000000,
1116 0x3c2a0, 0xffffffff, 0x00000100,
1117 0x3c208, 0xffffffff, 0x00000100,
1118 0x3c2c0, 0xffffffff, 0x00000100,
1119 0x3c2c8, 0xffffffff, 0x00000100,
1120 0x3c2c4, 0xffffffff, 0x00000100,
1121 0x55e4, 0xffffffff, 0x00600100,
1122 0x3c280, 0xffffffff, 0x00000100,
1123 0x3c214, 0xffffffff, 0x06000100,
1124 0x3c220, 0xffffffff, 0x00000100,
1125 0x3c218, 0xffffffff, 0x06000100,
1126 0x3c204, 0xffffffff, 0x00000100,
1127 0x3c2e0, 0xffffffff, 0x00000100,
1128 0x3c224, 0xffffffff, 0x00000100,
1129 0x3c200, 0xffffffff, 0x00000100,
1130 0x3c230, 0xffffffff, 0x00000100,
1131 0x3c234, 0xffffffff, 0x00000100,
1132 0x3c250, 0xffffffff, 0x00000100,
1133 0x3c254, 0xffffffff, 0x00000100,
1134 0x3c258, 0xffffffff, 0x00000100,
1135 0x3c25c, 0xffffffff, 0x00000100,
1136 0x3c260, 0xffffffff, 0x00000100,
1137 0x3c27c, 0xffffffff, 0x00000100,
1138 0x3c278, 0xffffffff, 0x00000100,
1139 0x3c210, 0xffffffff, 0x06000100,
1140 0x3c290, 0xffffffff, 0x00000100,
1141 0x3c274, 0xffffffff, 0x00000100,
1142 0x3c2b4, 0xffffffff, 0x00000100,
1143 0x3c2b0, 0xffffffff, 0x00000100,
1144 0x3c270, 0xffffffff, 0x00000100,
1145 0x30800, 0xffffffff, 0xe0000000,
1146 0x3c020, 0xffffffff, 0x00010000,
1147 0x3c024, 0xffffffff, 0x00030002,
1148 0x3c028, 0xffffffff, 0x00040007,
1149 0x3c02c, 0xffffffff, 0x00060005,
1150 0x3c030, 0xffffffff, 0x00090008,
1151 0x3c034, 0xffffffff, 0x00010000,
1152 0x3c038, 0xffffffff, 0x00030002,
1153 0x3c03c, 0xffffffff, 0x00040007,
1154 0x3c040, 0xffffffff, 0x00060005,
1155 0x3c044, 0xffffffff, 0x00090008,
1156 0x3c048, 0xffffffff, 0x00010000,
1157 0x3c04c, 0xffffffff, 0x00030002,
1158 0x3c050, 0xffffffff, 0x00040007,
1159 0x3c054, 0xffffffff, 0x00060005,
1160 0x3c058, 0xffffffff, 0x00090008,
1161 0x3c05c, 0xffffffff, 0x00010000,
1162 0x3c060, 0xffffffff, 0x00030002,
1163 0x3c064, 0xffffffff, 0x00040007,
1164 0x3c068, 0xffffffff, 0x00060005,
1165 0x3c06c, 0xffffffff, 0x00090008,
1166 0x3c070, 0xffffffff, 0x00010000,
1167 0x3c074, 0xffffffff, 0x00030002,
1168 0x3c078, 0xffffffff, 0x00040007,
1169 0x3c07c, 0xffffffff, 0x00060005,
1170 0x3c080, 0xffffffff, 0x00090008,
1171 0x3c084, 0xffffffff, 0x00010000,
1172 0x3c088, 0xffffffff, 0x00030002,
1173 0x3c08c, 0xffffffff, 0x00040007,
1174 0x3c090, 0xffffffff, 0x00060005,
1175 0x3c094, 0xffffffff, 0x00090008,
1176 0x3c098, 0xffffffff, 0x00010000,
1177 0x3c09c, 0xffffffff, 0x00030002,
1178 0x3c0a0, 0xffffffff, 0x00040007,
1179 0x3c0a4, 0xffffffff, 0x00060005,
1180 0x3c0a8, 0xffffffff, 0x00090008,
1181 0x3c0ac, 0xffffffff, 0x00010000,
1182 0x3c0b0, 0xffffffff, 0x00030002,
1183 0x3c0b4, 0xffffffff, 0x00040007,
1184 0x3c0b8, 0xffffffff, 0x00060005,
1185 0x3c0bc, 0xffffffff, 0x00090008,
1186 0x3c000, 0xffffffff, 0x96e00200,
1187 0x8708, 0xffffffff, 0x00900100,
1188 0xc424, 0xffffffff, 0x0020003f,
1189 0x38, 0xffffffff, 0x0140001c,
1190 0x3c, 0x000f0000, 0x000f0000,
1191 0x220, 0xffffffff, 0xC060000C,
1192 0x224, 0xc0000fff, 0x00000100,
1193 0xf90, 0xffffffff, 0x00000100,
1194 0xf98, 0x00000101, 0x00000000,
1195 0x20a8, 0xffffffff, 0x00000104,
1196 0x55e4, 0xff000fff, 0x00000100,
1197 0x30cc, 0xc0000fff, 0x00000104,
1198 0xc1e4, 0x00000001, 0x00000001,
1199 0xd00c, 0xff000ff0, 0x00000100,
1200 0xd80c, 0xff000ff0, 0x00000100
1203 static const u32 kalindi_golden_spm_registers
[] =
1205 0x30800, 0xe0ffffff, 0xe0000000
1208 static const u32 kalindi_golden_common_registers
[] =
1210 0xc770, 0xffffffff, 0x00000800,
1211 0xc774, 0xffffffff, 0x00000800,
1212 0xc798, 0xffffffff, 0x00007fbf,
1213 0xc79c, 0xffffffff, 0x00007faf
1216 static const u32 kalindi_golden_registers
[] =
1218 0x3c000, 0xffffdfff, 0x6e944040,
1219 0x55e4, 0xff607fff, 0xfc000100,
1220 0x3c220, 0xff000fff, 0x00000100,
1221 0x3c224, 0xff000fff, 0x00000100,
1222 0x3c200, 0xfffc0fff, 0x00000100,
1223 0x6ed8, 0x00010101, 0x00010000,
1224 0x9830, 0xffffffff, 0x00000000,
1225 0x9834, 0xf00fffff, 0x00000400,
1226 0x5bb0, 0x000000f0, 0x00000070,
1227 0x5bc0, 0xf0311fff, 0x80300000,
1228 0x98f8, 0x73773777, 0x12010001,
1229 0x98fc, 0xffffffff, 0x00000010,
1230 0x9b7c, 0x00ff0000, 0x00fc0000,
1231 0x8030, 0x00001f0f, 0x0000100a,
1232 0x2f48, 0x73773777, 0x12010001,
1233 0x2408, 0x000fffff, 0x000c007f,
1234 0x8a14, 0xf000003f, 0x00000007,
1235 0x8b24, 0x3fff3fff, 0x00ffcfff,
1236 0x30a04, 0x0000ff0f, 0x00000000,
1237 0x28a4c, 0x07ffffff, 0x06000000,
1238 0x4d8, 0x00000fff, 0x00000100,
1239 0x3e78, 0x00000001, 0x00000002,
1240 0xc768, 0x00000008, 0x00000008,
1241 0x8c00, 0x000000ff, 0x00000003,
1242 0x214f8, 0x01ff01ff, 0x00000002,
1243 0x21498, 0x007ff800, 0x00200000,
1244 0x2015c, 0xffffffff, 0x00000f40,
1245 0x88c4, 0x001f3ae3, 0x00000082,
1246 0x88d4, 0x0000001f, 0x00000010,
1247 0x30934, 0xffffffff, 0x00000000
1250 static const u32 kalindi_mgcg_cgcg_init
[] =
1252 0xc420, 0xffffffff, 0xfffffffc,
1253 0x30800, 0xffffffff, 0xe0000000,
1254 0x3c2a0, 0xffffffff, 0x00000100,
1255 0x3c208, 0xffffffff, 0x00000100,
1256 0x3c2c0, 0xffffffff, 0x00000100,
1257 0x3c2c8, 0xffffffff, 0x00000100,
1258 0x3c2c4, 0xffffffff, 0x00000100,
1259 0x55e4, 0xffffffff, 0x00600100,
1260 0x3c280, 0xffffffff, 0x00000100,
1261 0x3c214, 0xffffffff, 0x06000100,
1262 0x3c220, 0xffffffff, 0x00000100,
1263 0x3c218, 0xffffffff, 0x06000100,
1264 0x3c204, 0xffffffff, 0x00000100,
1265 0x3c2e0, 0xffffffff, 0x00000100,
1266 0x3c224, 0xffffffff, 0x00000100,
1267 0x3c200, 0xffffffff, 0x00000100,
1268 0x3c230, 0xffffffff, 0x00000100,
1269 0x3c234, 0xffffffff, 0x00000100,
1270 0x3c250, 0xffffffff, 0x00000100,
1271 0x3c254, 0xffffffff, 0x00000100,
1272 0x3c258, 0xffffffff, 0x00000100,
1273 0x3c25c, 0xffffffff, 0x00000100,
1274 0x3c260, 0xffffffff, 0x00000100,
1275 0x3c27c, 0xffffffff, 0x00000100,
1276 0x3c278, 0xffffffff, 0x00000100,
1277 0x3c210, 0xffffffff, 0x06000100,
1278 0x3c290, 0xffffffff, 0x00000100,
1279 0x3c274, 0xffffffff, 0x00000100,
1280 0x3c2b4, 0xffffffff, 0x00000100,
1281 0x3c2b0, 0xffffffff, 0x00000100,
1282 0x3c270, 0xffffffff, 0x00000100,
1283 0x30800, 0xffffffff, 0xe0000000,
1284 0x3c020, 0xffffffff, 0x00010000,
1285 0x3c024, 0xffffffff, 0x00030002,
1286 0x3c028, 0xffffffff, 0x00040007,
1287 0x3c02c, 0xffffffff, 0x00060005,
1288 0x3c030, 0xffffffff, 0x00090008,
1289 0x3c034, 0xffffffff, 0x00010000,
1290 0x3c038, 0xffffffff, 0x00030002,
1291 0x3c03c, 0xffffffff, 0x00040007,
1292 0x3c040, 0xffffffff, 0x00060005,
1293 0x3c044, 0xffffffff, 0x00090008,
1294 0x3c000, 0xffffffff, 0x96e00200,
1295 0x8708, 0xffffffff, 0x00900100,
1296 0xc424, 0xffffffff, 0x0020003f,
1297 0x38, 0xffffffff, 0x0140001c,
1298 0x3c, 0x000f0000, 0x000f0000,
1299 0x220, 0xffffffff, 0xC060000C,
1300 0x224, 0xc0000fff, 0x00000100,
1301 0x20a8, 0xffffffff, 0x00000104,
1302 0x55e4, 0xff000fff, 0x00000100,
1303 0x30cc, 0xc0000fff, 0x00000104,
1304 0xc1e4, 0x00000001, 0x00000001,
1305 0xd00c, 0xff000ff0, 0x00000100,
1306 0xd80c, 0xff000ff0, 0x00000100
1309 static const u32 hawaii_golden_spm_registers
[] =
1311 0x30800, 0xe0ffffff, 0xe0000000
1314 static const u32 hawaii_golden_common_registers
[] =
1316 0x30800, 0xffffffff, 0xe0000000,
1317 0x28350, 0xffffffff, 0x3a00161a,
1318 0x28354, 0xffffffff, 0x0000002e,
1319 0x9a10, 0xffffffff, 0x00018208,
1320 0x98f8, 0xffffffff, 0x12011003
1323 static const u32 hawaii_golden_registers
[] =
1325 0x3354, 0x00000333, 0x00000333,
1326 0x9a10, 0x00010000, 0x00058208,
1327 0x9830, 0xffffffff, 0x00000000,
1328 0x9834, 0xf00fffff, 0x00000400,
1329 0x9838, 0x0002021c, 0x00020200,
1330 0xc78, 0x00000080, 0x00000000,
1331 0x5bb0, 0x000000f0, 0x00000070,
1332 0x5bc0, 0xf0311fff, 0x80300000,
1333 0x350c, 0x00810000, 0x408af000,
1334 0x7030, 0x31000111, 0x00000011,
1335 0x2f48, 0x73773777, 0x12010001,
1336 0x2120, 0x0000007f, 0x0000001b,
1337 0x21dc, 0x00007fb6, 0x00002191,
1338 0x3628, 0x0000003f, 0x0000000a,
1339 0x362c, 0x0000003f, 0x0000000a,
1340 0x2ae4, 0x00073ffe, 0x000022a2,
1341 0x240c, 0x000007ff, 0x00000000,
1342 0x8bf0, 0x00002001, 0x00000001,
1343 0x8b24, 0xffffffff, 0x00ffffff,
1344 0x30a04, 0x0000ff0f, 0x00000000,
1345 0x28a4c, 0x07ffffff, 0x06000000,
1346 0x3e78, 0x00000001, 0x00000002,
1347 0xc768, 0x00000008, 0x00000008,
1348 0xc770, 0x00000f00, 0x00000800,
1349 0xc774, 0x00000f00, 0x00000800,
1350 0xc798, 0x00ffffff, 0x00ff7fbf,
1351 0xc79c, 0x00ffffff, 0x00ff7faf,
1352 0x8c00, 0x000000ff, 0x00000800,
1353 0xe40, 0x00001fff, 0x00001fff,
1354 0x9060, 0x0000007f, 0x00000020,
1355 0x9508, 0x00010000, 0x00010000,
1356 0xae00, 0x00100000, 0x000ff07c,
1357 0xac14, 0x000003ff, 0x0000000f,
1358 0xac10, 0xffffffff, 0x7564fdec,
1359 0xac0c, 0xffffffff, 0x3120b9a8,
1360 0xac08, 0x20000000, 0x0f9c0000
1363 static const u32 hawaii_mgcg_cgcg_init
[] =
1365 0xc420, 0xffffffff, 0xfffffffd,
1366 0x30800, 0xffffffff, 0xe0000000,
1367 0x3c2a0, 0xffffffff, 0x00000100,
1368 0x3c208, 0xffffffff, 0x00000100,
1369 0x3c2c0, 0xffffffff, 0x00000100,
1370 0x3c2c8, 0xffffffff, 0x00000100,
1371 0x3c2c4, 0xffffffff, 0x00000100,
1372 0x55e4, 0xffffffff, 0x00200100,
1373 0x3c280, 0xffffffff, 0x00000100,
1374 0x3c214, 0xffffffff, 0x06000100,
1375 0x3c220, 0xffffffff, 0x00000100,
1376 0x3c218, 0xffffffff, 0x06000100,
1377 0x3c204, 0xffffffff, 0x00000100,
1378 0x3c2e0, 0xffffffff, 0x00000100,
1379 0x3c224, 0xffffffff, 0x00000100,
1380 0x3c200, 0xffffffff, 0x00000100,
1381 0x3c230, 0xffffffff, 0x00000100,
1382 0x3c234, 0xffffffff, 0x00000100,
1383 0x3c250, 0xffffffff, 0x00000100,
1384 0x3c254, 0xffffffff, 0x00000100,
1385 0x3c258, 0xffffffff, 0x00000100,
1386 0x3c25c, 0xffffffff, 0x00000100,
1387 0x3c260, 0xffffffff, 0x00000100,
1388 0x3c27c, 0xffffffff, 0x00000100,
1389 0x3c278, 0xffffffff, 0x00000100,
1390 0x3c210, 0xffffffff, 0x06000100,
1391 0x3c290, 0xffffffff, 0x00000100,
1392 0x3c274, 0xffffffff, 0x00000100,
1393 0x3c2b4, 0xffffffff, 0x00000100,
1394 0x3c2b0, 0xffffffff, 0x00000100,
1395 0x3c270, 0xffffffff, 0x00000100,
1396 0x30800, 0xffffffff, 0xe0000000,
1397 0x3c020, 0xffffffff, 0x00010000,
1398 0x3c024, 0xffffffff, 0x00030002,
1399 0x3c028, 0xffffffff, 0x00040007,
1400 0x3c02c, 0xffffffff, 0x00060005,
1401 0x3c030, 0xffffffff, 0x00090008,
1402 0x3c034, 0xffffffff, 0x00010000,
1403 0x3c038, 0xffffffff, 0x00030002,
1404 0x3c03c, 0xffffffff, 0x00040007,
1405 0x3c040, 0xffffffff, 0x00060005,
1406 0x3c044, 0xffffffff, 0x00090008,
1407 0x3c048, 0xffffffff, 0x00010000,
1408 0x3c04c, 0xffffffff, 0x00030002,
1409 0x3c050, 0xffffffff, 0x00040007,
1410 0x3c054, 0xffffffff, 0x00060005,
1411 0x3c058, 0xffffffff, 0x00090008,
1412 0x3c05c, 0xffffffff, 0x00010000,
1413 0x3c060, 0xffffffff, 0x00030002,
1414 0x3c064, 0xffffffff, 0x00040007,
1415 0x3c068, 0xffffffff, 0x00060005,
1416 0x3c06c, 0xffffffff, 0x00090008,
1417 0x3c070, 0xffffffff, 0x00010000,
1418 0x3c074, 0xffffffff, 0x00030002,
1419 0x3c078, 0xffffffff, 0x00040007,
1420 0x3c07c, 0xffffffff, 0x00060005,
1421 0x3c080, 0xffffffff, 0x00090008,
1422 0x3c084, 0xffffffff, 0x00010000,
1423 0x3c088, 0xffffffff, 0x00030002,
1424 0x3c08c, 0xffffffff, 0x00040007,
1425 0x3c090, 0xffffffff, 0x00060005,
1426 0x3c094, 0xffffffff, 0x00090008,
1427 0x3c098, 0xffffffff, 0x00010000,
1428 0x3c09c, 0xffffffff, 0x00030002,
1429 0x3c0a0, 0xffffffff, 0x00040007,
1430 0x3c0a4, 0xffffffff, 0x00060005,
1431 0x3c0a8, 0xffffffff, 0x00090008,
1432 0x3c0ac, 0xffffffff, 0x00010000,
1433 0x3c0b0, 0xffffffff, 0x00030002,
1434 0x3c0b4, 0xffffffff, 0x00040007,
1435 0x3c0b8, 0xffffffff, 0x00060005,
1436 0x3c0bc, 0xffffffff, 0x00090008,
1437 0x3c0c0, 0xffffffff, 0x00010000,
1438 0x3c0c4, 0xffffffff, 0x00030002,
1439 0x3c0c8, 0xffffffff, 0x00040007,
1440 0x3c0cc, 0xffffffff, 0x00060005,
1441 0x3c0d0, 0xffffffff, 0x00090008,
1442 0x3c0d4, 0xffffffff, 0x00010000,
1443 0x3c0d8, 0xffffffff, 0x00030002,
1444 0x3c0dc, 0xffffffff, 0x00040007,
1445 0x3c0e0, 0xffffffff, 0x00060005,
1446 0x3c0e4, 0xffffffff, 0x00090008,
1447 0x3c0e8, 0xffffffff, 0x00010000,
1448 0x3c0ec, 0xffffffff, 0x00030002,
1449 0x3c0f0, 0xffffffff, 0x00040007,
1450 0x3c0f4, 0xffffffff, 0x00060005,
1451 0x3c0f8, 0xffffffff, 0x00090008,
1452 0xc318, 0xffffffff, 0x00020200,
1453 0x3350, 0xffffffff, 0x00000200,
1454 0x15c0, 0xffffffff, 0x00000400,
1455 0x55e8, 0xffffffff, 0x00000000,
1456 0x2f50, 0xffffffff, 0x00000902,
1457 0x3c000, 0xffffffff, 0x96940200,
1458 0x8708, 0xffffffff, 0x00900100,
1459 0xc424, 0xffffffff, 0x0020003f,
1460 0x38, 0xffffffff, 0x0140001c,
1461 0x3c, 0x000f0000, 0x000f0000,
1462 0x220, 0xffffffff, 0xc060000c,
1463 0x224, 0xc0000fff, 0x00000100,
1464 0xf90, 0xffffffff, 0x00000100,
1465 0xf98, 0x00000101, 0x00000000,
1466 0x20a8, 0xffffffff, 0x00000104,
1467 0x55e4, 0xff000fff, 0x00000100,
1468 0x30cc, 0xc0000fff, 0x00000104,
1469 0xc1e4, 0x00000001, 0x00000001,
1470 0xd00c, 0xff000ff0, 0x00000100,
1471 0xd80c, 0xff000ff0, 0x00000100
1474 static void cik_init_golden_registers(struct radeon_device
*rdev
)
1476 switch (rdev
->family
) {
1478 radeon_program_register_sequence(rdev
,
1479 bonaire_mgcg_cgcg_init
,
1480 (const u32
)ARRAY_SIZE(bonaire_mgcg_cgcg_init
));
1481 radeon_program_register_sequence(rdev
,
1482 bonaire_golden_registers
,
1483 (const u32
)ARRAY_SIZE(bonaire_golden_registers
));
1484 radeon_program_register_sequence(rdev
,
1485 bonaire_golden_common_registers
,
1486 (const u32
)ARRAY_SIZE(bonaire_golden_common_registers
));
1487 radeon_program_register_sequence(rdev
,
1488 bonaire_golden_spm_registers
,
1489 (const u32
)ARRAY_SIZE(bonaire_golden_spm_registers
));
1492 radeon_program_register_sequence(rdev
,
1493 kalindi_mgcg_cgcg_init
,
1494 (const u32
)ARRAY_SIZE(kalindi_mgcg_cgcg_init
));
1495 radeon_program_register_sequence(rdev
,
1496 kalindi_golden_registers
,
1497 (const u32
)ARRAY_SIZE(kalindi_golden_registers
));
1498 radeon_program_register_sequence(rdev
,
1499 kalindi_golden_common_registers
,
1500 (const u32
)ARRAY_SIZE(kalindi_golden_common_registers
));
1501 radeon_program_register_sequence(rdev
,
1502 kalindi_golden_spm_registers
,
1503 (const u32
)ARRAY_SIZE(kalindi_golden_spm_registers
));
1506 radeon_program_register_sequence(rdev
,
1507 spectre_mgcg_cgcg_init
,
1508 (const u32
)ARRAY_SIZE(spectre_mgcg_cgcg_init
));
1509 radeon_program_register_sequence(rdev
,
1510 spectre_golden_registers
,
1511 (const u32
)ARRAY_SIZE(spectre_golden_registers
));
1512 radeon_program_register_sequence(rdev
,
1513 spectre_golden_common_registers
,
1514 (const u32
)ARRAY_SIZE(spectre_golden_common_registers
));
1515 radeon_program_register_sequence(rdev
,
1516 spectre_golden_spm_registers
,
1517 (const u32
)ARRAY_SIZE(spectre_golden_spm_registers
));
1520 radeon_program_register_sequence(rdev
,
1521 hawaii_mgcg_cgcg_init
,
1522 (const u32
)ARRAY_SIZE(hawaii_mgcg_cgcg_init
));
1523 radeon_program_register_sequence(rdev
,
1524 hawaii_golden_registers
,
1525 (const u32
)ARRAY_SIZE(hawaii_golden_registers
));
1526 radeon_program_register_sequence(rdev
,
1527 hawaii_golden_common_registers
,
1528 (const u32
)ARRAY_SIZE(hawaii_golden_common_registers
));
1529 radeon_program_register_sequence(rdev
,
1530 hawaii_golden_spm_registers
,
1531 (const u32
)ARRAY_SIZE(hawaii_golden_spm_registers
));
1539 * cik_get_xclk - get the xclk
1541 * @rdev: radeon_device pointer
1543 * Returns the reference clock used by the gfx engine
1546 u32
cik_get_xclk(struct radeon_device
*rdev
)
1548 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1550 if (rdev
->flags
& RADEON_IS_IGP
) {
1551 if (RREG32_SMC(GENERAL_PWRMGT
) & GPU_COUNTER_CLK
)
1552 return reference_clock
/ 2;
1554 if (RREG32_SMC(CG_CLKPIN_CNTL
) & XTALIN_DIVIDE
)
1555 return reference_clock
/ 4;
1557 return reference_clock
;
1561 * cik_mm_rdoorbell - read a doorbell dword
1563 * @rdev: radeon_device pointer
1564 * @index: doorbell index
1566 * Returns the value in the doorbell aperture at the
1567 * requested doorbell index (CIK).
1569 u32
cik_mm_rdoorbell(struct radeon_device
*rdev
, u32 index
)
1571 if (index
< rdev
->doorbell
.num_doorbells
) {
1572 return readl(rdev
->doorbell
.ptr
+ index
);
1574 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index
);
1580 * cik_mm_wdoorbell - write a doorbell dword
1582 * @rdev: radeon_device pointer
1583 * @index: doorbell index
1584 * @v: value to write
1586 * Writes @v to the doorbell aperture at the
1587 * requested doorbell index (CIK).
1589 void cik_mm_wdoorbell(struct radeon_device
*rdev
, u32 index
, u32 v
)
1591 if (index
< rdev
->doorbell
.num_doorbells
) {
1592 writel(v
, rdev
->doorbell
.ptr
+ index
);
1594 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index
);
1598 #define BONAIRE_IO_MC_REGS_SIZE 36
1600 static const u32 bonaire_io_mc_regs
[BONAIRE_IO_MC_REGS_SIZE
][2] =
1602 {0x00000070, 0x04400000},
1603 {0x00000071, 0x80c01803},
1604 {0x00000072, 0x00004004},
1605 {0x00000073, 0x00000100},
1606 {0x00000074, 0x00ff0000},
1607 {0x00000075, 0x34000000},
1608 {0x00000076, 0x08000014},
1609 {0x00000077, 0x00cc08ec},
1610 {0x00000078, 0x00000400},
1611 {0x00000079, 0x00000000},
1612 {0x0000007a, 0x04090000},
1613 {0x0000007c, 0x00000000},
1614 {0x0000007e, 0x4408a8e8},
1615 {0x0000007f, 0x00000304},
1616 {0x00000080, 0x00000000},
1617 {0x00000082, 0x00000001},
1618 {0x00000083, 0x00000002},
1619 {0x00000084, 0xf3e4f400},
1620 {0x00000085, 0x052024e3},
1621 {0x00000087, 0x00000000},
1622 {0x00000088, 0x01000000},
1623 {0x0000008a, 0x1c0a0000},
1624 {0x0000008b, 0xff010000},
1625 {0x0000008d, 0xffffefff},
1626 {0x0000008e, 0xfff3efff},
1627 {0x0000008f, 0xfff3efbf},
1628 {0x00000092, 0xf7ffffff},
1629 {0x00000093, 0xffffff7f},
1630 {0x00000095, 0x00101101},
1631 {0x00000096, 0x00000fff},
1632 {0x00000097, 0x00116fff},
1633 {0x00000098, 0x60010000},
1634 {0x00000099, 0x10010000},
1635 {0x0000009a, 0x00006000},
1636 {0x0000009b, 0x00001000},
1637 {0x0000009f, 0x00b48000}
1640 #define HAWAII_IO_MC_REGS_SIZE 22
1642 static const u32 hawaii_io_mc_regs
[HAWAII_IO_MC_REGS_SIZE
][2] =
1644 {0x0000007d, 0x40000000},
1645 {0x0000007e, 0x40180304},
1646 {0x0000007f, 0x0000ff00},
1647 {0x00000081, 0x00000000},
1648 {0x00000083, 0x00000800},
1649 {0x00000086, 0x00000000},
1650 {0x00000087, 0x00000100},
1651 {0x00000088, 0x00020100},
1652 {0x00000089, 0x00000000},
1653 {0x0000008b, 0x00040000},
1654 {0x0000008c, 0x00000100},
1655 {0x0000008e, 0xff010000},
1656 {0x00000090, 0xffffefff},
1657 {0x00000091, 0xfff3efff},
1658 {0x00000092, 0xfff3efbf},
1659 {0x00000093, 0xf7ffffff},
1660 {0x00000094, 0xffffff7f},
1661 {0x00000095, 0x00000fff},
1662 {0x00000096, 0x00116fff},
1663 {0x00000097, 0x60010000},
1664 {0x00000098, 0x10010000},
1665 {0x0000009f, 0x00c79000}
1670 * cik_srbm_select - select specific register instances
1672 * @rdev: radeon_device pointer
1673 * @me: selected ME (micro engine)
1678 * Switches the currently active registers instances. Some
1679 * registers are instanced per VMID, others are instanced per
1680 * me/pipe/queue combination.
1682 static void cik_srbm_select(struct radeon_device
*rdev
,
1683 u32 me
, u32 pipe
, u32 queue
, u32 vmid
)
1685 u32 srbm_gfx_cntl
= (PIPEID(pipe
& 0x3) |
1688 QUEUEID(queue
& 0x7));
1689 WREG32(SRBM_GFX_CNTL
, srbm_gfx_cntl
);
1694 * ci_mc_load_microcode - load MC ucode into the hw
1696 * @rdev: radeon_device pointer
1698 * Load the GDDR MC ucode into the hw (CIK).
1699 * Returns 0 on success, error on failure.
1701 int ci_mc_load_microcode(struct radeon_device
*rdev
)
1703 const __be32
*fw_data
;
1704 u32 running
, blackout
= 0;
1706 int i
, ucode_size
, regs_size
;
1711 switch (rdev
->family
) {
1713 io_mc_regs
= (u32
*)&bonaire_io_mc_regs
;
1714 ucode_size
= CIK_MC_UCODE_SIZE
;
1715 regs_size
= BONAIRE_IO_MC_REGS_SIZE
;
1718 io_mc_regs
= (u32
*)&hawaii_io_mc_regs
;
1719 ucode_size
= HAWAII_MC_UCODE_SIZE
;
1720 regs_size
= HAWAII_IO_MC_REGS_SIZE
;
1726 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1730 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1731 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1734 /* reset the engine and set to writable */
1735 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1736 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1738 /* load mc io regs */
1739 for (i
= 0; i
< regs_size
; i
++) {
1740 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1741 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1743 /* load the MC ucode */
1744 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1745 for (i
= 0; i
< ucode_size
; i
++)
1746 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1748 /* put the engine back into the active state */
1749 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1750 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1751 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1753 /* wait for training to complete */
1754 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1755 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1759 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1760 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1766 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1773 * cik_init_microcode - load ucode images from disk
1775 * @rdev: radeon_device pointer
1777 * Use the firmware interface to load the ucode images into
1778 * the driver (not loaded into hw).
1779 * Returns 0 on success, error on failure.
1781 static int cik_init_microcode(struct radeon_device
*rdev
)
1783 const char *chip_name
;
1784 size_t pfp_req_size
, me_req_size
, ce_req_size
,
1785 mec_req_size
, rlc_req_size
, mc_req_size
= 0,
1786 sdma_req_size
, smc_req_size
= 0;
1792 switch (rdev
->family
) {
1794 chip_name
= "BONAIRE";
1795 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1796 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1797 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1798 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1799 rlc_req_size
= BONAIRE_RLC_UCODE_SIZE
* 4;
1800 mc_req_size
= CIK_MC_UCODE_SIZE
* 4;
1801 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1802 smc_req_size
= ALIGN(BONAIRE_SMC_UCODE_SIZE
, 4);
1805 chip_name
= "HAWAII";
1806 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1807 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1808 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1809 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1810 rlc_req_size
= BONAIRE_RLC_UCODE_SIZE
* 4;
1811 mc_req_size
= HAWAII_MC_UCODE_SIZE
* 4;
1812 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1813 smc_req_size
= ALIGN(HAWAII_SMC_UCODE_SIZE
, 4);
1816 chip_name
= "KAVERI";
1817 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1818 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1819 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1820 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1821 rlc_req_size
= KV_RLC_UCODE_SIZE
* 4;
1822 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1825 chip_name
= "KABINI";
1826 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1827 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1828 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1829 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1830 rlc_req_size
= KB_RLC_UCODE_SIZE
* 4;
1831 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1836 DRM_INFO("Loading %s Microcode\n", chip_name
);
1838 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1839 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, rdev
->dev
);
1842 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1844 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1845 rdev
->pfp_fw
->size
, fw_name
);
1850 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1851 err
= request_firmware(&rdev
->me_fw
, fw_name
, rdev
->dev
);
1854 if (rdev
->me_fw
->size
!= me_req_size
) {
1856 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1857 rdev
->me_fw
->size
, fw_name
);
1861 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1862 err
= request_firmware(&rdev
->ce_fw
, fw_name
, rdev
->dev
);
1865 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1867 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1868 rdev
->ce_fw
->size
, fw_name
);
1872 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mec.bin", chip_name
);
1873 err
= request_firmware(&rdev
->mec_fw
, fw_name
, rdev
->dev
);
1876 if (rdev
->mec_fw
->size
!= mec_req_size
) {
1878 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1879 rdev
->mec_fw
->size
, fw_name
);
1883 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", chip_name
);
1884 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, rdev
->dev
);
1887 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1889 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1890 rdev
->rlc_fw
->size
, fw_name
);
1894 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_sdma.bin", chip_name
);
1895 err
= request_firmware(&rdev
->sdma_fw
, fw_name
, rdev
->dev
);
1898 if (rdev
->sdma_fw
->size
!= sdma_req_size
) {
1900 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1901 rdev
->sdma_fw
->size
, fw_name
);
1905 /* No SMC, MC ucode on APUs */
1906 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
1907 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1908 err
= request_firmware(&rdev
->mc_fw
, fw_name
, rdev
->dev
);
1911 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1913 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1914 rdev
->mc_fw
->size
, fw_name
);
1918 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_smc.bin", chip_name
);
1919 err
= request_firmware(&rdev
->smc_fw
, fw_name
, rdev
->dev
);
1922 "smc: error loading firmware \"%s\"\n",
1924 release_firmware(rdev
->smc_fw
);
1925 rdev
->smc_fw
= NULL
;
1927 } else if (rdev
->smc_fw
->size
!= smc_req_size
) {
1929 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1930 rdev
->smc_fw
->size
, fw_name
);
1939 "cik_cp: Failed to load firmware \"%s\"\n",
1941 release_firmware(rdev
->pfp_fw
);
1942 rdev
->pfp_fw
= NULL
;
1943 release_firmware(rdev
->me_fw
);
1945 release_firmware(rdev
->ce_fw
);
1947 release_firmware(rdev
->rlc_fw
);
1948 rdev
->rlc_fw
= NULL
;
1949 release_firmware(rdev
->mc_fw
);
1951 release_firmware(rdev
->smc_fw
);
1952 rdev
->smc_fw
= NULL
;
1961 * cik_tiling_mode_table_init - init the hw tiling table
1963 * @rdev: radeon_device pointer
1965 * Starting with SI, the tiling setup is done globally in a
1966 * set of 32 tiling modes. Rather than selecting each set of
1967 * parameters per surface as on older asics, we just select
1968 * which index in the tiling table we want to use, and the
1969 * surface uses those parameters (CIK).
1971 static void cik_tiling_mode_table_init(struct radeon_device
*rdev
)
1973 const u32 num_tile_mode_states
= 32;
1974 const u32 num_secondary_tile_mode_states
= 16;
1975 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
1976 u32 num_pipe_configs
;
1977 u32 num_rbs
= rdev
->config
.cik
.max_backends_per_se
*
1978 rdev
->config
.cik
.max_shader_engines
;
1980 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
1982 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
1986 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
1989 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
1993 num_pipe_configs
= rdev
->config
.cik
.max_tile_pipes
;
1994 if (num_pipe_configs
> 8)
1995 num_pipe_configs
= 16;
1997 if (num_pipe_configs
== 16) {
1998 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
1999 switch (reg_offset
) {
2001 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2002 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2003 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2004 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2007 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2009 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2010 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2013 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2015 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2016 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2019 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2021 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2022 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2025 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2026 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2027 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2028 TILE_SPLIT(split_equal_to_row_size
));
2031 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2032 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2035 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2036 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2037 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2038 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2041 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2042 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2043 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2044 TILE_SPLIT(split_equal_to_row_size
));
2047 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2048 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
));
2051 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2052 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2055 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2056 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2057 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2058 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2061 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2062 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2063 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2064 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2067 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2068 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2069 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2073 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2074 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2077 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2078 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2079 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2080 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2083 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2084 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2085 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2089 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2090 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2091 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2092 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2095 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2096 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2099 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2100 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2101 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2102 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2105 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2106 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2107 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16
) |
2108 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2111 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2112 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2113 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16
) |
2114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2120 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2121 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2123 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2124 switch (reg_offset
) {
2126 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2127 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2128 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2129 NUM_BANKS(ADDR_SURF_16_BANK
));
2132 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2133 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2134 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2135 NUM_BANKS(ADDR_SURF_16_BANK
));
2138 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2139 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2140 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2141 NUM_BANKS(ADDR_SURF_16_BANK
));
2144 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2147 NUM_BANKS(ADDR_SURF_16_BANK
));
2150 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2151 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2152 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2153 NUM_BANKS(ADDR_SURF_8_BANK
));
2156 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2159 NUM_BANKS(ADDR_SURF_4_BANK
));
2162 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2163 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2164 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2165 NUM_BANKS(ADDR_SURF_2_BANK
));
2168 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2169 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2170 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2171 NUM_BANKS(ADDR_SURF_16_BANK
));
2174 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2177 NUM_BANKS(ADDR_SURF_16_BANK
));
2180 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2183 NUM_BANKS(ADDR_SURF_16_BANK
));
2186 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2187 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2188 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2189 NUM_BANKS(ADDR_SURF_8_BANK
));
2192 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2193 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2194 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2195 NUM_BANKS(ADDR_SURF_4_BANK
));
2198 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2199 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2200 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2201 NUM_BANKS(ADDR_SURF_2_BANK
));
2204 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2207 NUM_BANKS(ADDR_SURF_2_BANK
));
2213 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2215 } else if (num_pipe_configs
== 8) {
2216 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2217 switch (reg_offset
) {
2219 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2220 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2221 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2222 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2225 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2226 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2227 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2228 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2231 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2232 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2233 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2237 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2238 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2239 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2243 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2244 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2245 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2246 TILE_SPLIT(split_equal_to_row_size
));
2249 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2250 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2253 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2254 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2255 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2256 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2259 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2260 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2261 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2262 TILE_SPLIT(split_equal_to_row_size
));
2265 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
));
2269 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2270 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2273 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2274 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2276 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2279 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2285 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2287 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2291 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2292 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2295 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2296 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2301 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2307 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2308 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2309 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2313 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2314 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2317 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2319 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2320 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2323 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2324 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2325 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2329 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
2332 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2338 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2339 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2341 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2342 switch (reg_offset
) {
2344 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2345 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2346 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2347 NUM_BANKS(ADDR_SURF_16_BANK
));
2350 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2351 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2352 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2353 NUM_BANKS(ADDR_SURF_16_BANK
));
2356 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2359 NUM_BANKS(ADDR_SURF_16_BANK
));
2362 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2365 NUM_BANKS(ADDR_SURF_16_BANK
));
2368 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2371 NUM_BANKS(ADDR_SURF_8_BANK
));
2374 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2377 NUM_BANKS(ADDR_SURF_4_BANK
));
2380 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2383 NUM_BANKS(ADDR_SURF_2_BANK
));
2386 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2389 NUM_BANKS(ADDR_SURF_16_BANK
));
2392 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2395 NUM_BANKS(ADDR_SURF_16_BANK
));
2398 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2401 NUM_BANKS(ADDR_SURF_16_BANK
));
2404 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2407 NUM_BANKS(ADDR_SURF_16_BANK
));
2410 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2413 NUM_BANKS(ADDR_SURF_8_BANK
));
2416 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2419 NUM_BANKS(ADDR_SURF_4_BANK
));
2422 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2425 NUM_BANKS(ADDR_SURF_2_BANK
));
2431 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
2432 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2434 } else if (num_pipe_configs
== 4) {
2436 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2437 switch (reg_offset
) {
2439 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2440 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2441 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2445 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2446 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2447 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2448 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2451 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2452 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2453 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2457 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2459 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2463 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2465 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2466 TILE_SPLIT(split_equal_to_row_size
));
2469 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2473 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2475 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2476 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2479 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2480 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2481 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2482 TILE_SPLIT(split_equal_to_row_size
));
2485 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2486 PIPE_CONFIG(ADDR_SURF_P4_16x16
));
2489 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2490 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2493 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2494 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2495 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2499 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2501 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2502 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2505 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2506 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2507 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2508 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2511 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2512 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2515 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2517 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2518 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2521 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2523 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2524 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2527 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2528 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2529 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2530 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2533 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2537 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2539 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2540 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2543 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2546 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2549 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2551 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2558 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2559 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2561 } else if (num_rbs
< 4) {
2562 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2563 switch (reg_offset
) {
2565 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2568 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2571 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2573 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2574 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2577 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2579 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2583 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2584 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2589 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2590 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2591 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2592 TILE_SPLIT(split_equal_to_row_size
));
2595 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2599 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2601 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2602 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2605 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2608 TILE_SPLIT(split_equal_to_row_size
));
2611 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16
));
2615 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2619 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2621 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2622 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2625 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2626 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2627 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2628 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2631 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2632 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2633 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2634 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2637 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2641 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2643 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2647 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2649 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2653 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2659 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2660 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2663 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2664 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2666 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2669 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2670 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2671 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2672 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2675 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2676 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2677 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2678 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2684 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2685 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2688 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2689 switch (reg_offset
) {
2691 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2692 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2693 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2694 NUM_BANKS(ADDR_SURF_16_BANK
));
2697 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2700 NUM_BANKS(ADDR_SURF_16_BANK
));
2703 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2704 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2705 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2706 NUM_BANKS(ADDR_SURF_16_BANK
));
2709 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2710 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2711 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2712 NUM_BANKS(ADDR_SURF_16_BANK
));
2715 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2718 NUM_BANKS(ADDR_SURF_16_BANK
));
2721 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2722 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2723 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2724 NUM_BANKS(ADDR_SURF_8_BANK
));
2727 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2730 NUM_BANKS(ADDR_SURF_4_BANK
));
2733 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2734 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2735 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2736 NUM_BANKS(ADDR_SURF_16_BANK
));
2739 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2740 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2741 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2742 NUM_BANKS(ADDR_SURF_16_BANK
));
2745 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2746 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2747 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2748 NUM_BANKS(ADDR_SURF_16_BANK
));
2751 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2752 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2753 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2754 NUM_BANKS(ADDR_SURF_16_BANK
));
2757 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2758 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2759 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2760 NUM_BANKS(ADDR_SURF_16_BANK
));
2763 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2764 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2765 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2766 NUM_BANKS(ADDR_SURF_8_BANK
));
2769 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2770 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2771 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2772 NUM_BANKS(ADDR_SURF_4_BANK
));
2778 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
2779 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2781 } else if (num_pipe_configs
== 2) {
2782 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2783 switch (reg_offset
) {
2785 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2786 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2787 PIPE_CONFIG(ADDR_SURF_P2
) |
2788 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2791 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2792 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2793 PIPE_CONFIG(ADDR_SURF_P2
) |
2794 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2797 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2798 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2799 PIPE_CONFIG(ADDR_SURF_P2
) |
2800 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2803 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2805 PIPE_CONFIG(ADDR_SURF_P2
) |
2806 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2809 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2811 PIPE_CONFIG(ADDR_SURF_P2
) |
2812 TILE_SPLIT(split_equal_to_row_size
));
2815 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2819 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2821 PIPE_CONFIG(ADDR_SURF_P2
) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2825 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2827 PIPE_CONFIG(ADDR_SURF_P2
) |
2828 TILE_SPLIT(split_equal_to_row_size
));
2831 gb_tile_moden
= ARRAY_MODE(ARRAY_LINEAR_ALIGNED
);
2834 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2838 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2840 PIPE_CONFIG(ADDR_SURF_P2
) |
2841 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2844 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2845 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2846 PIPE_CONFIG(ADDR_SURF_P2
) |
2847 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2850 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2851 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2852 PIPE_CONFIG(ADDR_SURF_P2
) |
2853 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2856 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2857 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2860 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2861 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2862 PIPE_CONFIG(ADDR_SURF_P2
) |
2863 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2866 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2867 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2868 PIPE_CONFIG(ADDR_SURF_P2
) |
2869 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2872 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2873 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2874 PIPE_CONFIG(ADDR_SURF_P2
) |
2875 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2878 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2879 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2882 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2883 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2884 PIPE_CONFIG(ADDR_SURF_P2
) |
2885 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2888 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2890 PIPE_CONFIG(ADDR_SURF_P2
) |
2891 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2894 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2896 PIPE_CONFIG(ADDR_SURF_P2
) |
2897 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2903 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2904 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2906 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2907 switch (reg_offset
) {
2909 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2910 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2911 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2912 NUM_BANKS(ADDR_SURF_16_BANK
));
2915 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2916 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2917 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2918 NUM_BANKS(ADDR_SURF_16_BANK
));
2921 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2922 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2923 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2924 NUM_BANKS(ADDR_SURF_16_BANK
));
2927 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2928 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2929 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2930 NUM_BANKS(ADDR_SURF_16_BANK
));
2933 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2934 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2935 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2936 NUM_BANKS(ADDR_SURF_16_BANK
));
2939 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2940 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2941 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2942 NUM_BANKS(ADDR_SURF_16_BANK
));
2945 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2946 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2947 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2948 NUM_BANKS(ADDR_SURF_8_BANK
));
2951 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2952 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2953 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2954 NUM_BANKS(ADDR_SURF_16_BANK
));
2957 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2958 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2959 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2960 NUM_BANKS(ADDR_SURF_16_BANK
));
2963 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2966 NUM_BANKS(ADDR_SURF_16_BANK
));
2969 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2970 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2971 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2972 NUM_BANKS(ADDR_SURF_16_BANK
));
2975 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2976 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2977 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2978 NUM_BANKS(ADDR_SURF_16_BANK
));
2981 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2982 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2983 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2984 NUM_BANKS(ADDR_SURF_16_BANK
));
2987 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2990 NUM_BANKS(ADDR_SURF_8_BANK
));
2996 rdev
->config
.cik
.macrotile_mode_array
[reg_offset
] = gb_tile_moden
;
2997 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
3000 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs
);
3004 * cik_select_se_sh - select which SE, SH to address
3006 * @rdev: radeon_device pointer
3007 * @se_num: shader engine to address
3008 * @sh_num: sh block to address
3010 * Select which SE, SH combinations to address. Certain
3011 * registers are instanced per SE or SH. 0xffffffff means
3012 * broadcast to all SEs or SHs (CIK).
3014 static void cik_select_se_sh(struct radeon_device
*rdev
,
3015 u32 se_num
, u32 sh_num
)
3017 u32 data
= INSTANCE_BROADCAST_WRITES
;
3019 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
3020 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
3021 else if (se_num
== 0xffffffff)
3022 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
3023 else if (sh_num
== 0xffffffff)
3024 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
3026 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
3027 WREG32(GRBM_GFX_INDEX
, data
);
3031 * cik_create_bitmask - create a bitmask
3033 * @bit_width: length of the mask
3035 * create a variable length bit mask (CIK).
3036 * Returns the bitmask.
3038 static u32
cik_create_bitmask(u32 bit_width
)
3042 for (i
= 0; i
< bit_width
; i
++) {
3050 * cik_get_rb_disabled - computes the mask of disabled RBs
3052 * @rdev: radeon_device pointer
3053 * @max_rb_num: max RBs (render backends) for the asic
3054 * @se_num: number of SEs (shader engines) for the asic
3055 * @sh_per_se: number of SH blocks per SE for the asic
3057 * Calculates the bitmask of disabled RBs (CIK).
3058 * Returns the disabled RB bitmask.
3060 static u32
cik_get_rb_disabled(struct radeon_device
*rdev
,
3061 u32 max_rb_num_per_se
,
3066 data
= RREG32(CC_RB_BACKEND_DISABLE
);
3068 data
&= BACKEND_DISABLE_MASK
;
3071 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
3073 data
>>= BACKEND_DISABLE_SHIFT
;
3075 mask
= cik_create_bitmask(max_rb_num_per_se
/ sh_per_se
);
3081 * cik_setup_rb - setup the RBs on the asic
3083 * @rdev: radeon_device pointer
3084 * @se_num: number of SEs (shader engines) for the asic
3085 * @sh_per_se: number of SH blocks per SE for the asic
3086 * @max_rb_num: max RBs (render backends) for the asic
3088 * Configures per-SE/SH RB registers (CIK).
3090 static void cik_setup_rb(struct radeon_device
*rdev
,
3091 u32 se_num
, u32 sh_per_se
,
3092 u32 max_rb_num_per_se
)
3096 u32 disabled_rbs
= 0;
3097 u32 enabled_rbs
= 0;
3099 for (i
= 0; i
< se_num
; i
++) {
3100 for (j
= 0; j
< sh_per_se
; j
++) {
3101 cik_select_se_sh(rdev
, i
, j
);
3102 data
= cik_get_rb_disabled(rdev
, max_rb_num_per_se
, sh_per_se
);
3103 if (rdev
->family
== CHIP_HAWAII
)
3104 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * HAWAII_RB_BITMAP_WIDTH_PER_SH
);
3106 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * CIK_RB_BITMAP_WIDTH_PER_SH
);
3109 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
3112 for (i
= 0; i
< max_rb_num_per_se
* se_num
; i
++) {
3113 if (!(disabled_rbs
& mask
))
3114 enabled_rbs
|= mask
;
3118 rdev
->config
.cik
.backend_enable_mask
= enabled_rbs
;
3120 for (i
= 0; i
< se_num
; i
++) {
3121 cik_select_se_sh(rdev
, i
, 0xffffffff);
3123 for (j
= 0; j
< sh_per_se
; j
++) {
3124 switch (enabled_rbs
& 3) {
3127 data
|= PKR_MAP(RASTER_CONFIG_RB_MAP_3
);
3129 data
|= PKR_MAP(RASTER_CONFIG_RB_MAP_0
);
3132 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
3135 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
3139 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
3144 WREG32(PA_SC_RASTER_CONFIG
, data
);
3146 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
3150 * cik_gpu_init - setup the 3D engine
3152 * @rdev: radeon_device pointer
3154 * Configures the 3D engine and tiling configuration
3155 * registers so that the 3D engine is usable.
3157 static void cik_gpu_init(struct radeon_device
*rdev
)
3159 u32 gb_addr_config
= RREG32(GB_ADDR_CONFIG
);
3160 u32 mc_shared_chmap
, mc_arb_ramcfg
;
3161 u32 hdp_host_path_cntl
;
3165 switch (rdev
->family
) {
3167 rdev
->config
.cik
.max_shader_engines
= 2;
3168 rdev
->config
.cik
.max_tile_pipes
= 4;
3169 rdev
->config
.cik
.max_cu_per_sh
= 7;
3170 rdev
->config
.cik
.max_sh_per_se
= 1;
3171 rdev
->config
.cik
.max_backends_per_se
= 2;
3172 rdev
->config
.cik
.max_texture_channel_caches
= 4;
3173 rdev
->config
.cik
.max_gprs
= 256;
3174 rdev
->config
.cik
.max_gs_threads
= 32;
3175 rdev
->config
.cik
.max_hw_contexts
= 8;
3177 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3178 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3179 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3180 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3181 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3184 rdev
->config
.cik
.max_shader_engines
= 4;
3185 rdev
->config
.cik
.max_tile_pipes
= 16;
3186 rdev
->config
.cik
.max_cu_per_sh
= 11;
3187 rdev
->config
.cik
.max_sh_per_se
= 1;
3188 rdev
->config
.cik
.max_backends_per_se
= 4;
3189 rdev
->config
.cik
.max_texture_channel_caches
= 16;
3190 rdev
->config
.cik
.max_gprs
= 256;
3191 rdev
->config
.cik
.max_gs_threads
= 32;
3192 rdev
->config
.cik
.max_hw_contexts
= 8;
3194 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3195 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3196 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3197 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3198 gb_addr_config
= HAWAII_GB_ADDR_CONFIG_GOLDEN
;
3201 rdev
->config
.cik
.max_shader_engines
= 1;
3202 rdev
->config
.cik
.max_tile_pipes
= 4;
3203 if ((rdev
->pdev
->device
== 0x1304) ||
3204 (rdev
->pdev
->device
== 0x1305) ||
3205 (rdev
->pdev
->device
== 0x130C) ||
3206 (rdev
->pdev
->device
== 0x130F) ||
3207 (rdev
->pdev
->device
== 0x1310) ||
3208 (rdev
->pdev
->device
== 0x1311) ||
3209 (rdev
->pdev
->device
== 0x131C)) {
3210 rdev
->config
.cik
.max_cu_per_sh
= 8;
3211 rdev
->config
.cik
.max_backends_per_se
= 2;
3212 } else if ((rdev
->pdev
->device
== 0x1309) ||
3213 (rdev
->pdev
->device
== 0x130A) ||
3214 (rdev
->pdev
->device
== 0x130D) ||
3215 (rdev
->pdev
->device
== 0x1313) ||
3216 (rdev
->pdev
->device
== 0x131D)) {
3217 rdev
->config
.cik
.max_cu_per_sh
= 6;
3218 rdev
->config
.cik
.max_backends_per_se
= 2;
3219 } else if ((rdev
->pdev
->device
== 0x1306) ||
3220 (rdev
->pdev
->device
== 0x1307) ||
3221 (rdev
->pdev
->device
== 0x130B) ||
3222 (rdev
->pdev
->device
== 0x130E) ||
3223 (rdev
->pdev
->device
== 0x1315) ||
3224 (rdev
->pdev
->device
== 0x131B)) {
3225 rdev
->config
.cik
.max_cu_per_sh
= 4;
3226 rdev
->config
.cik
.max_backends_per_se
= 1;
3228 rdev
->config
.cik
.max_cu_per_sh
= 3;
3229 rdev
->config
.cik
.max_backends_per_se
= 1;
3231 rdev
->config
.cik
.max_sh_per_se
= 1;
3232 rdev
->config
.cik
.max_texture_channel_caches
= 4;
3233 rdev
->config
.cik
.max_gprs
= 256;
3234 rdev
->config
.cik
.max_gs_threads
= 16;
3235 rdev
->config
.cik
.max_hw_contexts
= 8;
3237 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3238 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3239 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3240 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3241 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3245 rdev
->config
.cik
.max_shader_engines
= 1;
3246 rdev
->config
.cik
.max_tile_pipes
= 2;
3247 rdev
->config
.cik
.max_cu_per_sh
= 2;
3248 rdev
->config
.cik
.max_sh_per_se
= 1;
3249 rdev
->config
.cik
.max_backends_per_se
= 1;
3250 rdev
->config
.cik
.max_texture_channel_caches
= 2;
3251 rdev
->config
.cik
.max_gprs
= 256;
3252 rdev
->config
.cik
.max_gs_threads
= 16;
3253 rdev
->config
.cik
.max_hw_contexts
= 8;
3255 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
3256 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
3257 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
3258 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
3259 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
3263 /* Initialize HDP */
3264 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
3265 WREG32((0x2c14 + j
), 0x00000000);
3266 WREG32((0x2c18 + j
), 0x00000000);
3267 WREG32((0x2c1c + j
), 0x00000000);
3268 WREG32((0x2c20 + j
), 0x00000000);
3269 WREG32((0x2c24 + j
), 0x00000000);
3272 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
3274 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
3276 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
3277 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
3279 rdev
->config
.cik
.num_tile_pipes
= rdev
->config
.cik
.max_tile_pipes
;
3280 rdev
->config
.cik
.mem_max_burst_length_bytes
= 256;
3281 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
3282 rdev
->config
.cik
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
3283 if (rdev
->config
.cik
.mem_row_size_in_kb
> 4)
3284 rdev
->config
.cik
.mem_row_size_in_kb
= 4;
3285 /* XXX use MC settings? */
3286 rdev
->config
.cik
.shader_engine_tile_size
= 32;
3287 rdev
->config
.cik
.num_gpus
= 1;
3288 rdev
->config
.cik
.multi_gpu_tile_size
= 64;
3290 /* fix up row size */
3291 gb_addr_config
&= ~ROW_SIZE_MASK
;
3292 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
3295 gb_addr_config
|= ROW_SIZE(0);
3298 gb_addr_config
|= ROW_SIZE(1);
3301 gb_addr_config
|= ROW_SIZE(2);
3305 /* setup tiling info dword. gb_addr_config is not adequate since it does
3306 * not have bank info, so create a custom tiling dword.
3307 * bits 3:0 num_pipes
3308 * bits 7:4 num_banks
3309 * bits 11:8 group_size
3310 * bits 15:12 row_size
3312 rdev
->config
.cik
.tile_config
= 0;
3313 switch (rdev
->config
.cik
.num_tile_pipes
) {
3315 rdev
->config
.cik
.tile_config
|= (0 << 0);
3318 rdev
->config
.cik
.tile_config
|= (1 << 0);
3321 rdev
->config
.cik
.tile_config
|= (2 << 0);
3325 /* XXX what about 12? */
3326 rdev
->config
.cik
.tile_config
|= (3 << 0);
3329 rdev
->config
.cik
.tile_config
|=
3330 ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) << 4;
3331 rdev
->config
.cik
.tile_config
|=
3332 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
3333 rdev
->config
.cik
.tile_config
|=
3334 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
3336 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
3337 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
3338 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
3339 WREG32(SDMA0_TILING_CONFIG
+ SDMA0_REGISTER_OFFSET
, gb_addr_config
& 0x70);
3340 WREG32(SDMA0_TILING_CONFIG
+ SDMA1_REGISTER_OFFSET
, gb_addr_config
& 0x70);
3341 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
3342 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
3343 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
3345 cik_tiling_mode_table_init(rdev
);
3347 cik_setup_rb(rdev
, rdev
->config
.cik
.max_shader_engines
,
3348 rdev
->config
.cik
.max_sh_per_se
,
3349 rdev
->config
.cik
.max_backends_per_se
);
3351 /* set HW defaults for 3D engine */
3352 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
3354 WREG32(SX_DEBUG_1
, 0x20);
3356 WREG32(TA_CNTL_AUX
, 0x00010000);
3358 tmp
= RREG32(SPI_CONFIG_CNTL
);
3360 WREG32(SPI_CONFIG_CNTL
, tmp
);
3362 WREG32(SQ_CONFIG
, 1);
3364 WREG32(DB_DEBUG
, 0);
3366 tmp
= RREG32(DB_DEBUG2
) & ~0xf00fffff;
3368 WREG32(DB_DEBUG2
, tmp
);
3370 tmp
= RREG32(DB_DEBUG3
) & ~0x0002021c;
3372 WREG32(DB_DEBUG3
, tmp
);
3374 tmp
= RREG32(CB_HW_CONTROL
) & ~0x00010000;
3376 WREG32(CB_HW_CONTROL
, tmp
);
3378 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
3380 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_frontend
) |
3381 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_backend
) |
3382 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_hiz_tile_fifo_size
) |
3383 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_earlyz_tile_fifo_size
)));
3385 WREG32(VGT_NUM_INSTANCES
, 1);
3387 WREG32(CP_PERFMON_CNTL
, 0);
3389 WREG32(SQ_CONFIG
, 0);
3391 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
3392 FORCE_EOV_MAX_REZ_CNT(255)));
3394 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
3395 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
3397 WREG32(VGT_GS_VERTEX_REUSE
, 16);
3398 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
3400 tmp
= RREG32(HDP_MISC_CNTL
);
3401 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
3402 WREG32(HDP_MISC_CNTL
, tmp
);
3404 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
3405 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
3407 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
3408 WREG32(PA_SC_ENHANCE
, ENABLE_PA_SC_OUT_OF_ORDER
);
3414 * GPU scratch registers helpers function.
3417 * cik_scratch_init - setup driver info for CP scratch regs
3419 * @rdev: radeon_device pointer
3421 * Set up the number and offset of the CP scratch registers.
3422 * NOTE: use of CP scratch registers is a legacy inferface and
3423 * is not used by default on newer asics (r6xx+). On newer asics,
3424 * memory buffers are used for fences rather than scratch regs.
3426 static void cik_scratch_init(struct radeon_device
*rdev
)
3430 rdev
->scratch
.num_reg
= 7;
3431 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
3432 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
3433 rdev
->scratch
.free
[i
] = true;
3434 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
3439 * cik_ring_test - basic gfx ring test
3441 * @rdev: radeon_device pointer
3442 * @ring: radeon_ring structure holding ring information
3444 * Allocate a scratch register and write to it using the gfx ring (CIK).
3445 * Provides a basic gfx ring test to verify that the ring is working.
3446 * Used by cik_cp_gfx_resume();
3447 * Returns 0 on success, error on failure.
3449 int cik_ring_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3456 r
= radeon_scratch_get(rdev
, &scratch
);
3458 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r
);
3461 WREG32(scratch
, 0xCAFEDEAD);
3462 r
= radeon_ring_lock(rdev
, ring
, 3);
3464 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring
->idx
, r
);
3465 radeon_scratch_free(rdev
, scratch
);
3468 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
3469 radeon_ring_write(ring
, ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2));
3470 radeon_ring_write(ring
, 0xDEADBEEF);
3471 radeon_ring_unlock_commit(rdev
, ring
);
3473 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3474 tmp
= RREG32(scratch
);
3475 if (tmp
== 0xDEADBEEF)
3479 if (i
< rdev
->usec_timeout
) {
3480 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring
->idx
, i
);
3482 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3483 ring
->idx
, scratch
, tmp
);
3486 radeon_scratch_free(rdev
, scratch
);
3491 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3493 * @rdev: radeon_device pointer
3494 * @ridx: radeon ring index
3496 * Emits an hdp flush on the cp.
3498 static void cik_hdp_flush_cp_ring_emit(struct radeon_device
*rdev
,
3501 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
3504 switch (ring
->idx
) {
3505 case CAYMAN_RING_TYPE_CP1_INDEX
:
3506 case CAYMAN_RING_TYPE_CP2_INDEX
:
3510 ref_and_mask
= CP2
<< ring
->pipe
;
3513 ref_and_mask
= CP6
<< ring
->pipe
;
3519 case RADEON_RING_TYPE_GFX_INDEX
:
3524 radeon_ring_write(ring
, PACKET3(PACKET3_WAIT_REG_MEM
, 5));
3525 radeon_ring_write(ring
, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3526 WAIT_REG_MEM_FUNCTION(3) | /* == */
3527 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3528 radeon_ring_write(ring
, GPU_HDP_FLUSH_REQ
>> 2);
3529 radeon_ring_write(ring
, GPU_HDP_FLUSH_DONE
>> 2);
3530 radeon_ring_write(ring
, ref_and_mask
);
3531 radeon_ring_write(ring
, ref_and_mask
);
3532 radeon_ring_write(ring
, 0x20); /* poll interval */
3536 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3538 * @rdev: radeon_device pointer
3539 * @fence: radeon fence object
3541 * Emits a fence sequnce number on the gfx ring and flushes
3544 void cik_fence_gfx_ring_emit(struct radeon_device
*rdev
,
3545 struct radeon_fence
*fence
)
3547 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3548 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3550 /* EVENT_WRITE_EOP - flush caches, send int */
3551 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
3552 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
3554 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
3556 radeon_ring_write(ring
, addr
& 0xfffffffc);
3557 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3558 radeon_ring_write(ring
, fence
->seq
);
3559 radeon_ring_write(ring
, 0);
3561 cik_hdp_flush_cp_ring_emit(rdev
, fence
->ring
);
3565 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3567 * @rdev: radeon_device pointer
3568 * @fence: radeon fence object
3570 * Emits a fence sequnce number on the compute ring and flushes
3573 void cik_fence_compute_ring_emit(struct radeon_device
*rdev
,
3574 struct radeon_fence
*fence
)
3576 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3577 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3579 /* RELEASE_MEM - flush caches, send int */
3580 radeon_ring_write(ring
, PACKET3(PACKET3_RELEASE_MEM
, 5));
3581 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
3583 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
3585 radeon_ring_write(ring
, DATA_SEL(1) | INT_SEL(2));
3586 radeon_ring_write(ring
, addr
& 0xfffffffc);
3587 radeon_ring_write(ring
, upper_32_bits(addr
));
3588 radeon_ring_write(ring
, fence
->seq
);
3589 radeon_ring_write(ring
, 0);
3591 cik_hdp_flush_cp_ring_emit(rdev
, fence
->ring
);
3594 bool cik_semaphore_ring_emit(struct radeon_device
*rdev
,
3595 struct radeon_ring
*ring
,
3596 struct radeon_semaphore
*semaphore
,
3599 uint64_t addr
= semaphore
->gpu_addr
;
3600 unsigned sel
= emit_wait
? PACKET3_SEM_SEL_WAIT
: PACKET3_SEM_SEL_SIGNAL
;
3602 radeon_ring_write(ring
, PACKET3(PACKET3_MEM_SEMAPHORE
, 1));
3603 radeon_ring_write(ring
, addr
& 0xffffffff);
3604 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | sel
);
3610 * cik_copy_cpdma - copy pages using the CP DMA engine
3612 * @rdev: radeon_device pointer
3613 * @src_offset: src GPU address
3614 * @dst_offset: dst GPU address
3615 * @num_gpu_pages: number of GPU pages to xfer
3616 * @fence: radeon fence object
3618 * Copy GPU paging using the CP DMA engine (CIK+).
3619 * Used by the radeon ttm implementation to move pages if
3620 * registered as the asic copy callback.
3622 int cik_copy_cpdma(struct radeon_device
*rdev
,
3623 uint64_t src_offset
, uint64_t dst_offset
,
3624 unsigned num_gpu_pages
,
3625 struct radeon_fence
**fence
)
3627 struct radeon_semaphore
*sem
= NULL
;
3628 int ring_index
= rdev
->asic
->copy
.blit_ring_index
;
3629 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
3630 u32 size_in_bytes
, cur_size_in_bytes
, control
;
3634 r
= radeon_semaphore_create(rdev
, &sem
);
3636 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3640 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
3641 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0x1fffff);
3642 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 7 + 18);
3644 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3645 radeon_semaphore_free(rdev
, &sem
, NULL
);
3649 radeon_semaphore_sync_to(sem
, *fence
);
3650 radeon_semaphore_sync_rings(rdev
, sem
, ring
->idx
);
3652 for (i
= 0; i
< num_loops
; i
++) {
3653 cur_size_in_bytes
= size_in_bytes
;
3654 if (cur_size_in_bytes
> 0x1fffff)
3655 cur_size_in_bytes
= 0x1fffff;
3656 size_in_bytes
-= cur_size_in_bytes
;
3658 if (size_in_bytes
== 0)
3659 control
|= PACKET3_DMA_DATA_CP_SYNC
;
3660 radeon_ring_write(ring
, PACKET3(PACKET3_DMA_DATA
, 5));
3661 radeon_ring_write(ring
, control
);
3662 radeon_ring_write(ring
, lower_32_bits(src_offset
));
3663 radeon_ring_write(ring
, upper_32_bits(src_offset
));
3664 radeon_ring_write(ring
, lower_32_bits(dst_offset
));
3665 radeon_ring_write(ring
, upper_32_bits(dst_offset
));
3666 radeon_ring_write(ring
, cur_size_in_bytes
);
3667 src_offset
+= cur_size_in_bytes
;
3668 dst_offset
+= cur_size_in_bytes
;
3671 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
3673 radeon_ring_unlock_undo(rdev
, ring
);
3677 radeon_ring_unlock_commit(rdev
, ring
);
3678 radeon_semaphore_free(rdev
, &sem
, *fence
);
3687 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3689 * @rdev: radeon_device pointer
3690 * @ib: radeon indirect buffer object
3692 * Emits an DE (drawing engine) or CE (constant engine) IB
3693 * on the gfx ring. IBs are usually generated by userspace
3694 * acceleration drivers and submitted to the kernel for
3695 * sheduling on the ring. This function schedules the IB
3696 * on the gfx ring for execution by the GPU.
3698 void cik_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
3700 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3701 u32 header
, control
= INDIRECT_BUFFER_VALID
;
3703 if (ib
->is_const_ib
) {
3704 /* set switch buffer packet before const IB */
3705 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
3706 radeon_ring_write(ring
, 0);
3708 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
3711 if (ring
->rptr_save_reg
) {
3712 next_rptr
= ring
->wptr
+ 3 + 4;
3713 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
3714 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
3715 PACKET3_SET_UCONFIG_REG_START
) >> 2));
3716 radeon_ring_write(ring
, next_rptr
);
3717 } else if (rdev
->wb
.enabled
) {
3718 next_rptr
= ring
->wptr
+ 5 + 4;
3719 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3720 radeon_ring_write(ring
, WRITE_DATA_DST_SEL(1));
3721 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3722 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
3723 radeon_ring_write(ring
, next_rptr
);
3726 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
3729 control
|= ib
->length_dw
|
3730 (ib
->vm
? (ib
->vm
->id
<< 24) : 0);
3732 radeon_ring_write(ring
, header
);
3733 radeon_ring_write(ring
,
3737 (ib
->gpu_addr
& 0xFFFFFFFC));
3738 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
3739 radeon_ring_write(ring
, control
);
3743 * cik_ib_test - basic gfx ring IB test
3745 * @rdev: radeon_device pointer
3746 * @ring: radeon_ring structure holding ring information
3748 * Allocate an IB and execute it on the gfx ring (CIK).
3749 * Provides a basic gfx ring test to verify that IBs are working.
3750 * Returns 0 on success, error on failure.
3752 int cik_ib_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3754 struct radeon_ib ib
;
3760 r
= radeon_scratch_get(rdev
, &scratch
);
3762 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r
);
3765 WREG32(scratch
, 0xCAFEDEAD);
3766 r
= radeon_ib_get(rdev
, ring
->idx
, &ib
, NULL
, 256);
3768 DRM_ERROR("radeon: failed to get ib (%d).\n", r
);
3769 radeon_scratch_free(rdev
, scratch
);
3772 ib
.ptr
[0] = PACKET3(PACKET3_SET_UCONFIG_REG
, 1);
3773 ib
.ptr
[1] = ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2);
3774 ib
.ptr
[2] = 0xDEADBEEF;
3776 r
= radeon_ib_schedule(rdev
, &ib
, NULL
);
3778 radeon_scratch_free(rdev
, scratch
);
3779 radeon_ib_free(rdev
, &ib
);
3780 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r
);
3783 r
= radeon_fence_wait(ib
.fence
, false);
3785 DRM_ERROR("radeon: fence wait failed (%d).\n", r
);
3786 radeon_scratch_free(rdev
, scratch
);
3787 radeon_ib_free(rdev
, &ib
);
3790 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3791 tmp
= RREG32(scratch
);
3792 if (tmp
== 0xDEADBEEF)
3796 if (i
< rdev
->usec_timeout
) {
3797 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib
.fence
->ring
, i
);
3799 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3803 radeon_scratch_free(rdev
, scratch
);
3804 radeon_ib_free(rdev
, &ib
);
3810 * On CIK, gfx and compute now have independant command processors.
3813 * Gfx consists of a single ring and can process both gfx jobs and
3814 * compute jobs. The gfx CP consists of three microengines (ME):
3815 * PFP - Pre-Fetch Parser
3817 * CE - Constant Engine
3818 * The PFP and ME make up what is considered the Drawing Engine (DE).
3819 * The CE is an asynchronous engine used for updating buffer desciptors
3820 * used by the DE so that they can be loaded into cache in parallel
3821 * while the DE is processing state update packets.
3824 * The compute CP consists of two microengines (ME):
3825 * MEC1 - Compute MicroEngine 1
3826 * MEC2 - Compute MicroEngine 2
3827 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3828 * The queues are exposed to userspace and are programmed directly
3829 * by the compute runtime.
3832 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3834 * @rdev: radeon_device pointer
3835 * @enable: enable or disable the MEs
3837 * Halts or unhalts the gfx MEs.
3839 static void cik_cp_gfx_enable(struct radeon_device
*rdev
, bool enable
)
3842 WREG32(CP_ME_CNTL
, 0);
3844 if (rdev
->asic
->copy
.copy_ring_index
== RADEON_RING_TYPE_GFX_INDEX
)
3845 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
3846 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
3847 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3853 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3855 * @rdev: radeon_device pointer
3857 * Loads the gfx PFP, ME, and CE ucode.
3858 * Returns 0 for success, -EINVAL if the ucode is not available.
3860 static int cik_cp_gfx_load_microcode(struct radeon_device
*rdev
)
3862 const __be32
*fw_data
;
3865 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
)
3868 cik_cp_gfx_enable(rdev
, false);
3871 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
3872 WREG32(CP_PFP_UCODE_ADDR
, 0);
3873 for (i
= 0; i
< CIK_PFP_UCODE_SIZE
; i
++)
3874 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
3875 WREG32(CP_PFP_UCODE_ADDR
, 0);
3878 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
3879 WREG32(CP_CE_UCODE_ADDR
, 0);
3880 for (i
= 0; i
< CIK_CE_UCODE_SIZE
; i
++)
3881 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
3882 WREG32(CP_CE_UCODE_ADDR
, 0);
3885 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
3886 WREG32(CP_ME_RAM_WADDR
, 0);
3887 for (i
= 0; i
< CIK_ME_UCODE_SIZE
; i
++)
3888 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
3889 WREG32(CP_ME_RAM_WADDR
, 0);
3891 WREG32(CP_PFP_UCODE_ADDR
, 0);
3892 WREG32(CP_CE_UCODE_ADDR
, 0);
3893 WREG32(CP_ME_RAM_WADDR
, 0);
3894 WREG32(CP_ME_RAM_RADDR
, 0);
3899 * cik_cp_gfx_start - start the gfx ring
3901 * @rdev: radeon_device pointer
3903 * Enables the ring and loads the clear state context and other
3904 * packets required to init the ring.
3905 * Returns 0 for success, error for failure.
3907 static int cik_cp_gfx_start(struct radeon_device
*rdev
)
3909 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3913 WREG32(CP_MAX_CONTEXT
, rdev
->config
.cik
.max_hw_contexts
- 1);
3914 WREG32(CP_ENDIAN_SWAP
, 0);
3915 WREG32(CP_DEVICE_ID
, 1);
3917 cik_cp_gfx_enable(rdev
, true);
3919 r
= radeon_ring_lock(rdev
, ring
, cik_default_size
+ 17);
3921 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3925 /* init the CE partitions. CE only used for gfx on CIK */
3926 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3927 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3928 radeon_ring_write(ring
, 0xc000);
3929 radeon_ring_write(ring
, 0xc000);
3931 /* setup clear context state */
3932 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3933 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3935 radeon_ring_write(ring
, PACKET3(PACKET3_CONTEXT_CONTROL
, 1));
3936 radeon_ring_write(ring
, 0x80000000);
3937 radeon_ring_write(ring
, 0x80000000);
3939 for (i
= 0; i
< cik_default_size
; i
++)
3940 radeon_ring_write(ring
, cik_default_state
[i
]);
3942 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3943 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3945 /* set clear context state */
3946 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3947 radeon_ring_write(ring
, 0);
3949 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3950 radeon_ring_write(ring
, 0x00000316);
3951 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3952 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3954 radeon_ring_unlock_commit(rdev
, ring
);
3960 * cik_cp_gfx_fini - stop the gfx ring
3962 * @rdev: radeon_device pointer
3964 * Stop the gfx ring and tear down the driver ring
3967 static void cik_cp_gfx_fini(struct radeon_device
*rdev
)
3969 cik_cp_gfx_enable(rdev
, false);
3970 radeon_ring_fini(rdev
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3974 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3976 * @rdev: radeon_device pointer
3978 * Program the location and size of the gfx ring buffer
3979 * and test it to make sure it's working.
3980 * Returns 0 for success, error for failure.
3982 static int cik_cp_gfx_resume(struct radeon_device
*rdev
)
3984 struct radeon_ring
*ring
;
3990 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
3991 if (rdev
->family
!= CHIP_HAWAII
)
3992 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
3994 /* Set the write pointer delay */
3995 WREG32(CP_RB_WPTR_DELAY
, 0);
3997 /* set the RB to use vmid 0 */
3998 WREG32(CP_RB_VMID
, 0);
4000 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
4002 /* ring 0 - compute and gfx */
4003 /* Set ring buffer size */
4004 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
4005 rb_bufsz
= order_base_2(ring
->ring_size
/ 8);
4006 tmp
= (order_base_2(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
4008 tmp
|= BUF_SWAP_32BIT
;
4010 WREG32(CP_RB0_CNTL
, tmp
);
4012 /* Initialize the ring buffer's read and write pointers */
4013 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
4015 WREG32(CP_RB0_WPTR
, ring
->wptr
);
4017 /* set the wb address wether it's enabled or not */
4018 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
4019 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
4021 /* scratch register shadowing is no longer supported */
4022 WREG32(SCRATCH_UMSK
, 0);
4024 if (!rdev
->wb
.enabled
)
4025 tmp
|= RB_NO_UPDATE
;
4028 WREG32(CP_RB0_CNTL
, tmp
);
4030 rb_addr
= ring
->gpu_addr
>> 8;
4031 WREG32(CP_RB0_BASE
, rb_addr
);
4032 WREG32(CP_RB0_BASE_HI
, upper_32_bits(rb_addr
));
4034 /* start the ring */
4035 cik_cp_gfx_start(rdev
);
4036 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
4037 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
4039 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
4043 if (rdev
->asic
->copy
.copy_ring_index
== RADEON_RING_TYPE_GFX_INDEX
)
4044 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.real_vram_size
);
4049 u32
cik_gfx_get_rptr(struct radeon_device
*rdev
,
4050 struct radeon_ring
*ring
)
4054 if (rdev
->wb
.enabled
)
4055 rptr
= rdev
->wb
.wb
[ring
->rptr_offs
/4];
4057 rptr
= RREG32(CP_RB0_RPTR
);
4062 u32
cik_gfx_get_wptr(struct radeon_device
*rdev
,
4063 struct radeon_ring
*ring
)
4067 wptr
= RREG32(CP_RB0_WPTR
);
4072 void cik_gfx_set_wptr(struct radeon_device
*rdev
,
4073 struct radeon_ring
*ring
)
4075 WREG32(CP_RB0_WPTR
, ring
->wptr
);
4076 (void)RREG32(CP_RB0_WPTR
);
4079 u32
cik_compute_get_rptr(struct radeon_device
*rdev
,
4080 struct radeon_ring
*ring
)
4084 if (rdev
->wb
.enabled
) {
4085 rptr
= rdev
->wb
.wb
[ring
->rptr_offs
/4];
4087 mutex_lock(&rdev
->srbm_mutex
);
4088 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
4089 rptr
= RREG32(CP_HQD_PQ_RPTR
);
4090 cik_srbm_select(rdev
, 0, 0, 0, 0);
4091 mutex_unlock(&rdev
->srbm_mutex
);
4097 u32
cik_compute_get_wptr(struct radeon_device
*rdev
,
4098 struct radeon_ring
*ring
)
4102 if (rdev
->wb
.enabled
) {
4103 /* XXX check if swapping is necessary on BE */
4104 wptr
= rdev
->wb
.wb
[ring
->wptr_offs
/4];
4106 mutex_lock(&rdev
->srbm_mutex
);
4107 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
4108 wptr
= RREG32(CP_HQD_PQ_WPTR
);
4109 cik_srbm_select(rdev
, 0, 0, 0, 0);
4110 mutex_unlock(&rdev
->srbm_mutex
);
4116 void cik_compute_set_wptr(struct radeon_device
*rdev
,
4117 struct radeon_ring
*ring
)
4119 /* XXX check if swapping is necessary on BE */
4120 rdev
->wb
.wb
[ring
->wptr_offs
/4] = ring
->wptr
;
4121 WDOORBELL32(ring
->doorbell_index
, ring
->wptr
);
4125 * cik_cp_compute_enable - enable/disable the compute CP MEs
4127 * @rdev: radeon_device pointer
4128 * @enable: enable or disable the MEs
4130 * Halts or unhalts the compute MEs.
4132 static void cik_cp_compute_enable(struct radeon_device
*rdev
, bool enable
)
4135 WREG32(CP_MEC_CNTL
, 0);
4137 WREG32(CP_MEC_CNTL
, (MEC_ME1_HALT
| MEC_ME2_HALT
));
4138 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
4139 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
4145 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4147 * @rdev: radeon_device pointer
4149 * Loads the compute MEC1&2 ucode.
4150 * Returns 0 for success, -EINVAL if the ucode is not available.
4152 static int cik_cp_compute_load_microcode(struct radeon_device
*rdev
)
4154 const __be32
*fw_data
;
4160 cik_cp_compute_enable(rdev
, false);
4163 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
4164 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
4165 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
4166 WREG32(CP_MEC_ME1_UCODE_DATA
, be32_to_cpup(fw_data
++));
4167 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
4169 if (rdev
->family
== CHIP_KAVERI
) {
4171 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
4172 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
4173 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
4174 WREG32(CP_MEC_ME2_UCODE_DATA
, be32_to_cpup(fw_data
++));
4175 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
4182 * cik_cp_compute_start - start the compute queues
4184 * @rdev: radeon_device pointer
4186 * Enable the compute queues.
4187 * Returns 0 for success, error for failure.
4189 static int cik_cp_compute_start(struct radeon_device
*rdev
)
4191 cik_cp_compute_enable(rdev
, true);
4197 * cik_cp_compute_fini - stop the compute queues
4199 * @rdev: radeon_device pointer
4201 * Stop the compute queues and tear down the driver queue
4204 static void cik_cp_compute_fini(struct radeon_device
*rdev
)
4208 cik_cp_compute_enable(rdev
, false);
4210 for (i
= 0; i
< 2; i
++) {
4212 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
4214 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
4216 if (rdev
->ring
[idx
].mqd_obj
) {
4217 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
4218 if (unlikely(r
!= 0))
4219 dev_warn(rdev
->dev
, "(%d) reserve MQD bo failed\n", r
);
4221 radeon_bo_unpin(rdev
->ring
[idx
].mqd_obj
);
4222 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
4224 radeon_bo_unref(&rdev
->ring
[idx
].mqd_obj
);
4225 rdev
->ring
[idx
].mqd_obj
= NULL
;
4230 static void cik_mec_fini(struct radeon_device
*rdev
)
4234 if (rdev
->mec
.hpd_eop_obj
) {
4235 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
4236 if (unlikely(r
!= 0))
4237 dev_warn(rdev
->dev
, "(%d) reserve HPD EOP bo failed\n", r
);
4238 radeon_bo_unpin(rdev
->mec
.hpd_eop_obj
);
4239 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
4241 radeon_bo_unref(&rdev
->mec
.hpd_eop_obj
);
4242 rdev
->mec
.hpd_eop_obj
= NULL
;
4246 #define MEC_HPD_SIZE 2048
4248 static int cik_mec_init(struct radeon_device
*rdev
)
4254 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4255 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4257 if (rdev
->family
== CHIP_KAVERI
)
4258 rdev
->mec
.num_mec
= 2;
4260 rdev
->mec
.num_mec
= 1;
4261 rdev
->mec
.num_pipe
= 4;
4262 rdev
->mec
.num_queue
= rdev
->mec
.num_mec
* rdev
->mec
.num_pipe
* 8;
4264 if (rdev
->mec
.hpd_eop_obj
== NULL
) {
4265 r
= radeon_bo_create(rdev
,
4266 rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2,
4268 RADEON_GEM_DOMAIN_GTT
, NULL
,
4269 &rdev
->mec
.hpd_eop_obj
);
4271 dev_warn(rdev
->dev
, "(%d) create HDP EOP bo failed\n", r
);
4276 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
4277 if (unlikely(r
!= 0)) {
4281 r
= radeon_bo_pin(rdev
->mec
.hpd_eop_obj
, RADEON_GEM_DOMAIN_GTT
,
4282 &rdev
->mec
.hpd_eop_gpu_addr
);
4284 dev_warn(rdev
->dev
, "(%d) pin HDP EOP bo failed\n", r
);
4288 r
= radeon_bo_kmap(rdev
->mec
.hpd_eop_obj
, (void **)&hpd
);
4290 dev_warn(rdev
->dev
, "(%d) map HDP EOP bo failed\n", r
);
4295 /* clear memory. Not sure if this is required or not */
4296 memset(hpd
, 0, rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2);
4298 radeon_bo_kunmap(rdev
->mec
.hpd_eop_obj
);
4299 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
4304 struct hqd_registers
4306 u32 cp_mqd_base_addr
;
4307 u32 cp_mqd_base_addr_hi
;
4310 u32 cp_hqd_persistent_state
;
4311 u32 cp_hqd_pipe_priority
;
4312 u32 cp_hqd_queue_priority
;
4315 u32 cp_hqd_pq_base_hi
;
4317 u32 cp_hqd_pq_rptr_report_addr
;
4318 u32 cp_hqd_pq_rptr_report_addr_hi
;
4319 u32 cp_hqd_pq_wptr_poll_addr
;
4320 u32 cp_hqd_pq_wptr_poll_addr_hi
;
4321 u32 cp_hqd_pq_doorbell_control
;
4323 u32 cp_hqd_pq_control
;
4324 u32 cp_hqd_ib_base_addr
;
4325 u32 cp_hqd_ib_base_addr_hi
;
4327 u32 cp_hqd_ib_control
;
4328 u32 cp_hqd_iq_timer
;
4330 u32 cp_hqd_dequeue_request
;
4331 u32 cp_hqd_dma_offload
;
4332 u32 cp_hqd_sema_cmd
;
4333 u32 cp_hqd_msg_type
;
4334 u32 cp_hqd_atomic0_preop_lo
;
4335 u32 cp_hqd_atomic0_preop_hi
;
4336 u32 cp_hqd_atomic1_preop_lo
;
4337 u32 cp_hqd_atomic1_preop_hi
;
4338 u32 cp_hqd_hq_scheduler0
;
4339 u32 cp_hqd_hq_scheduler1
;
4346 u32 dispatch_initiator
;
4350 u32 pipeline_stat_enable
;
4351 u32 perf_counter_enable
;
4357 u32 resource_limits
;
4358 u32 static_thread_mgmt01
[2];
4360 u32 static_thread_mgmt23
[2];
4362 u32 thread_trace_enable
;
4365 u32 vgtcs_invoke_count
[2];
4366 struct hqd_registers queue_state
;
4368 u32 interrupt_queue
[64];
4372 * cik_cp_compute_resume - setup the compute queue registers
4374 * @rdev: radeon_device pointer
4376 * Program the compute queues and test them to make sure they
4378 * Returns 0 for success, error for failure.
4380 static int cik_cp_compute_resume(struct radeon_device
*rdev
)
4384 bool use_doorbell
= true;
4390 struct bonaire_mqd
*mqd
;
4392 r
= cik_cp_compute_start(rdev
);
4396 /* fix up chicken bits */
4397 tmp
= RREG32(CP_CPF_DEBUG
);
4399 WREG32(CP_CPF_DEBUG
, tmp
);
4401 /* init the pipes */
4402 mutex_lock(&rdev
->srbm_mutex
);
4403 for (i
= 0; i
< (rdev
->mec
.num_pipe
* rdev
->mec
.num_mec
); i
++) {
4404 int me
= (i
< 4) ? 1 : 2;
4405 int pipe
= (i
< 4) ? i
: (i
- 4);
4407 eop_gpu_addr
= rdev
->mec
.hpd_eop_gpu_addr
+ (i
* MEC_HPD_SIZE
* 2);
4409 cik_srbm_select(rdev
, me
, pipe
, 0, 0);
4411 /* write the EOP addr */
4412 WREG32(CP_HPD_EOP_BASE_ADDR
, eop_gpu_addr
>> 8);
4413 WREG32(CP_HPD_EOP_BASE_ADDR_HI
, upper_32_bits(eop_gpu_addr
) >> 8);
4415 /* set the VMID assigned */
4416 WREG32(CP_HPD_EOP_VMID
, 0);
4418 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4419 tmp
= RREG32(CP_HPD_EOP_CONTROL
);
4420 tmp
&= ~EOP_SIZE_MASK
;
4421 tmp
|= order_base_2(MEC_HPD_SIZE
/ 8);
4422 WREG32(CP_HPD_EOP_CONTROL
, tmp
);
4424 cik_srbm_select(rdev
, 0, 0, 0, 0);
4425 mutex_unlock(&rdev
->srbm_mutex
);
4427 /* init the queues. Just two for now. */
4428 for (i
= 0; i
< 2; i
++) {
4430 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
4432 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
4434 if (rdev
->ring
[idx
].mqd_obj
== NULL
) {
4435 r
= radeon_bo_create(rdev
,
4436 sizeof(struct bonaire_mqd
),
4438 RADEON_GEM_DOMAIN_GTT
, NULL
,
4439 &rdev
->ring
[idx
].mqd_obj
);
4441 dev_warn(rdev
->dev
, "(%d) create MQD bo failed\n", r
);
4446 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
4447 if (unlikely(r
!= 0)) {
4448 cik_cp_compute_fini(rdev
);
4451 r
= radeon_bo_pin(rdev
->ring
[idx
].mqd_obj
, RADEON_GEM_DOMAIN_GTT
,
4454 dev_warn(rdev
->dev
, "(%d) pin MQD bo failed\n", r
);
4455 cik_cp_compute_fini(rdev
);
4458 r
= radeon_bo_kmap(rdev
->ring
[idx
].mqd_obj
, (void **)&buf
);
4460 dev_warn(rdev
->dev
, "(%d) map MQD bo failed\n", r
);
4461 cik_cp_compute_fini(rdev
);
4465 /* init the mqd struct */
4466 memset(buf
, 0, sizeof(struct bonaire_mqd
));
4468 mqd
= (struct bonaire_mqd
*)buf
;
4469 mqd
->header
= 0xC0310800;
4470 mqd
->static_thread_mgmt01
[0] = 0xffffffff;
4471 mqd
->static_thread_mgmt01
[1] = 0xffffffff;
4472 mqd
->static_thread_mgmt23
[0] = 0xffffffff;
4473 mqd
->static_thread_mgmt23
[1] = 0xffffffff;
4475 mutex_lock(&rdev
->srbm_mutex
);
4476 cik_srbm_select(rdev
, rdev
->ring
[idx
].me
,
4477 rdev
->ring
[idx
].pipe
,
4478 rdev
->ring
[idx
].queue
, 0);
4480 /* disable wptr polling */
4481 tmp
= RREG32(CP_PQ_WPTR_POLL_CNTL
);
4482 tmp
&= ~WPTR_POLL_EN
;
4483 WREG32(CP_PQ_WPTR_POLL_CNTL
, tmp
);
4485 /* enable doorbell? */
4486 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
4487 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
4489 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
4491 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_EN
;
4492 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
4493 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
4495 /* disable the queue if it's active */
4496 mqd
->queue_state
.cp_hqd_dequeue_request
= 0;
4497 mqd
->queue_state
.cp_hqd_pq_rptr
= 0;
4498 mqd
->queue_state
.cp_hqd_pq_wptr
= 0;
4499 if (RREG32(CP_HQD_ACTIVE
) & 1) {
4500 WREG32(CP_HQD_DEQUEUE_REQUEST
, 1);
4501 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4502 if (!(RREG32(CP_HQD_ACTIVE
) & 1))
4506 WREG32(CP_HQD_DEQUEUE_REQUEST
, mqd
->queue_state
.cp_hqd_dequeue_request
);
4507 WREG32(CP_HQD_PQ_RPTR
, mqd
->queue_state
.cp_hqd_pq_rptr
);
4508 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
4511 /* set the pointer to the MQD */
4512 mqd
->queue_state
.cp_mqd_base_addr
= mqd_gpu_addr
& 0xfffffffc;
4513 mqd
->queue_state
.cp_mqd_base_addr_hi
= upper_32_bits(mqd_gpu_addr
);
4514 WREG32(CP_MQD_BASE_ADDR
, mqd
->queue_state
.cp_mqd_base_addr
);
4515 WREG32(CP_MQD_BASE_ADDR_HI
, mqd
->queue_state
.cp_mqd_base_addr_hi
);
4516 /* set MQD vmid to 0 */
4517 mqd
->queue_state
.cp_mqd_control
= RREG32(CP_MQD_CONTROL
);
4518 mqd
->queue_state
.cp_mqd_control
&= ~MQD_VMID_MASK
;
4519 WREG32(CP_MQD_CONTROL
, mqd
->queue_state
.cp_mqd_control
);
4521 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4522 hqd_gpu_addr
= rdev
->ring
[idx
].gpu_addr
>> 8;
4523 mqd
->queue_state
.cp_hqd_pq_base
= hqd_gpu_addr
;
4524 mqd
->queue_state
.cp_hqd_pq_base_hi
= upper_32_bits(hqd_gpu_addr
);
4525 WREG32(CP_HQD_PQ_BASE
, mqd
->queue_state
.cp_hqd_pq_base
);
4526 WREG32(CP_HQD_PQ_BASE_HI
, mqd
->queue_state
.cp_hqd_pq_base_hi
);
4528 /* set up the HQD, this is similar to CP_RB0_CNTL */
4529 mqd
->queue_state
.cp_hqd_pq_control
= RREG32(CP_HQD_PQ_CONTROL
);
4530 mqd
->queue_state
.cp_hqd_pq_control
&=
4531 ~(QUEUE_SIZE_MASK
| RPTR_BLOCK_SIZE_MASK
);
4533 mqd
->queue_state
.cp_hqd_pq_control
|=
4534 order_base_2(rdev
->ring
[idx
].ring_size
/ 8);
4535 mqd
->queue_state
.cp_hqd_pq_control
|=
4536 (order_base_2(RADEON_GPU_PAGE_SIZE
/8) << 8);
4538 mqd
->queue_state
.cp_hqd_pq_control
|= BUF_SWAP_32BIT
;
4540 mqd
->queue_state
.cp_hqd_pq_control
&=
4541 ~(UNORD_DISPATCH
| ROQ_PQ_IB_FLIP
| PQ_VOLATILE
);
4542 mqd
->queue_state
.cp_hqd_pq_control
|=
4543 PRIV_STATE
| KMD_QUEUE
; /* assuming kernel queue control */
4544 WREG32(CP_HQD_PQ_CONTROL
, mqd
->queue_state
.cp_hqd_pq_control
);
4546 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4548 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP1_WPTR_OFFSET
;
4550 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP2_WPTR_OFFSET
;
4551 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
= wb_gpu_addr
& 0xfffffffc;
4552 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
= upper_32_bits(wb_gpu_addr
) & 0xffff;
4553 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR
, mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
);
4554 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI
,
4555 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
);
4557 /* set the wb address wether it's enabled or not */
4559 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
;
4561 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
;
4562 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
= wb_gpu_addr
& 0xfffffffc;
4563 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
=
4564 upper_32_bits(wb_gpu_addr
) & 0xffff;
4565 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR
,
4566 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
);
4567 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI
,
4568 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
);
4570 /* enable the doorbell if requested */
4572 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
4573 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
4574 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_OFFSET_MASK
;
4575 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|=
4576 DOORBELL_OFFSET(rdev
->ring
[idx
].doorbell_index
);
4577 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
4578 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&=
4579 ~(DOORBELL_SOURCE
| DOORBELL_HIT
);
4582 mqd
->queue_state
.cp_hqd_pq_doorbell_control
= 0;
4584 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
4585 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
4587 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4588 rdev
->ring
[idx
].wptr
= 0;
4589 mqd
->queue_state
.cp_hqd_pq_wptr
= rdev
->ring
[idx
].wptr
;
4590 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
4591 mqd
->queue_state
.cp_hqd_pq_rptr
= RREG32(CP_HQD_PQ_RPTR
);
4593 /* set the vmid for the queue */
4594 mqd
->queue_state
.cp_hqd_vmid
= 0;
4595 WREG32(CP_HQD_VMID
, mqd
->queue_state
.cp_hqd_vmid
);
4597 /* activate the queue */
4598 mqd
->queue_state
.cp_hqd_active
= 1;
4599 WREG32(CP_HQD_ACTIVE
, mqd
->queue_state
.cp_hqd_active
);
4601 cik_srbm_select(rdev
, 0, 0, 0, 0);
4602 mutex_unlock(&rdev
->srbm_mutex
);
4604 radeon_bo_kunmap(rdev
->ring
[idx
].mqd_obj
);
4605 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
4607 rdev
->ring
[idx
].ready
= true;
4608 r
= radeon_ring_test(rdev
, idx
, &rdev
->ring
[idx
]);
4610 rdev
->ring
[idx
].ready
= false;
4616 static void cik_cp_enable(struct radeon_device
*rdev
, bool enable
)
4618 cik_cp_gfx_enable(rdev
, enable
);
4619 cik_cp_compute_enable(rdev
, enable
);
4622 static int cik_cp_load_microcode(struct radeon_device
*rdev
)
4626 r
= cik_cp_gfx_load_microcode(rdev
);
4629 r
= cik_cp_compute_load_microcode(rdev
);
4636 static void cik_cp_fini(struct radeon_device
*rdev
)
4638 cik_cp_gfx_fini(rdev
);
4639 cik_cp_compute_fini(rdev
);
4642 static int cik_cp_resume(struct radeon_device
*rdev
)
4646 cik_enable_gui_idle_interrupt(rdev
, false);
4648 r
= cik_cp_load_microcode(rdev
);
4652 r
= cik_cp_gfx_resume(rdev
);
4655 r
= cik_cp_compute_resume(rdev
);
4659 cik_enable_gui_idle_interrupt(rdev
, true);
4664 static void cik_print_gpu_status_regs(struct radeon_device
*rdev
)
4666 dev_info(rdev
->dev
, " GRBM_STATUS=0x%08X\n",
4667 RREG32(GRBM_STATUS
));
4668 dev_info(rdev
->dev
, " GRBM_STATUS2=0x%08X\n",
4669 RREG32(GRBM_STATUS2
));
4670 dev_info(rdev
->dev
, " GRBM_STATUS_SE0=0x%08X\n",
4671 RREG32(GRBM_STATUS_SE0
));
4672 dev_info(rdev
->dev
, " GRBM_STATUS_SE1=0x%08X\n",
4673 RREG32(GRBM_STATUS_SE1
));
4674 dev_info(rdev
->dev
, " GRBM_STATUS_SE2=0x%08X\n",
4675 RREG32(GRBM_STATUS_SE2
));
4676 dev_info(rdev
->dev
, " GRBM_STATUS_SE3=0x%08X\n",
4677 RREG32(GRBM_STATUS_SE3
));
4678 dev_info(rdev
->dev
, " SRBM_STATUS=0x%08X\n",
4679 RREG32(SRBM_STATUS
));
4680 dev_info(rdev
->dev
, " SRBM_STATUS2=0x%08X\n",
4681 RREG32(SRBM_STATUS2
));
4682 dev_info(rdev
->dev
, " SDMA0_STATUS_REG = 0x%08X\n",
4683 RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
));
4684 dev_info(rdev
->dev
, " SDMA1_STATUS_REG = 0x%08X\n",
4685 RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
));
4686 dev_info(rdev
->dev
, " CP_STAT = 0x%08x\n", RREG32(CP_STAT
));
4687 dev_info(rdev
->dev
, " CP_STALLED_STAT1 = 0x%08x\n",
4688 RREG32(CP_STALLED_STAT1
));
4689 dev_info(rdev
->dev
, " CP_STALLED_STAT2 = 0x%08x\n",
4690 RREG32(CP_STALLED_STAT2
));
4691 dev_info(rdev
->dev
, " CP_STALLED_STAT3 = 0x%08x\n",
4692 RREG32(CP_STALLED_STAT3
));
4693 dev_info(rdev
->dev
, " CP_CPF_BUSY_STAT = 0x%08x\n",
4694 RREG32(CP_CPF_BUSY_STAT
));
4695 dev_info(rdev
->dev
, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4696 RREG32(CP_CPF_STALLED_STAT1
));
4697 dev_info(rdev
->dev
, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS
));
4698 dev_info(rdev
->dev
, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT
));
4699 dev_info(rdev
->dev
, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4700 RREG32(CP_CPC_STALLED_STAT1
));
4701 dev_info(rdev
->dev
, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS
));
4705 * cik_gpu_check_soft_reset - check which blocks are busy
4707 * @rdev: radeon_device pointer
4709 * Check which blocks are busy and return the relevant reset
4710 * mask to be used by cik_gpu_soft_reset().
4711 * Returns a mask of the blocks to be reset.
4713 u32
cik_gpu_check_soft_reset(struct radeon_device
*rdev
)
4719 tmp
= RREG32(GRBM_STATUS
);
4720 if (tmp
& (PA_BUSY
| SC_BUSY
|
4721 BCI_BUSY
| SX_BUSY
|
4722 TA_BUSY
| VGT_BUSY
|
4724 GDS_BUSY
| SPI_BUSY
|
4725 IA_BUSY
| IA_BUSY_NO_DMA
))
4726 reset_mask
|= RADEON_RESET_GFX
;
4728 if (tmp
& (CP_BUSY
| CP_COHERENCY_BUSY
))
4729 reset_mask
|= RADEON_RESET_CP
;
4732 tmp
= RREG32(GRBM_STATUS2
);
4734 reset_mask
|= RADEON_RESET_RLC
;
4736 /* SDMA0_STATUS_REG */
4737 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
);
4738 if (!(tmp
& SDMA_IDLE
))
4739 reset_mask
|= RADEON_RESET_DMA
;
4741 /* SDMA1_STATUS_REG */
4742 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
);
4743 if (!(tmp
& SDMA_IDLE
))
4744 reset_mask
|= RADEON_RESET_DMA1
;
4747 tmp
= RREG32(SRBM_STATUS2
);
4748 if (tmp
& SDMA_BUSY
)
4749 reset_mask
|= RADEON_RESET_DMA
;
4751 if (tmp
& SDMA1_BUSY
)
4752 reset_mask
|= RADEON_RESET_DMA1
;
4755 tmp
= RREG32(SRBM_STATUS
);
4758 reset_mask
|= RADEON_RESET_IH
;
4761 reset_mask
|= RADEON_RESET_SEM
;
4763 if (tmp
& GRBM_RQ_PENDING
)
4764 reset_mask
|= RADEON_RESET_GRBM
;
4767 reset_mask
|= RADEON_RESET_VMC
;
4769 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
4770 MCC_BUSY
| MCD_BUSY
))
4771 reset_mask
|= RADEON_RESET_MC
;
4773 if (evergreen_is_display_hung(rdev
))
4774 reset_mask
|= RADEON_RESET_DISPLAY
;
4776 /* Skip MC reset as it's mostly likely not hung, just busy */
4777 if (reset_mask
& RADEON_RESET_MC
) {
4778 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
4779 reset_mask
&= ~RADEON_RESET_MC
;
4786 * cik_gpu_soft_reset - soft reset GPU
4788 * @rdev: radeon_device pointer
4789 * @reset_mask: mask of which blocks to reset
4791 * Soft reset the blocks specified in @reset_mask.
4793 static void cik_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
4795 struct evergreen_mc_save save
;
4796 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
4799 if (reset_mask
== 0)
4802 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
4804 cik_print_gpu_status_regs(rdev
);
4805 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4806 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
4807 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4808 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
4817 /* Disable GFX parsing/prefetching */
4818 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
4820 /* Disable MEC parsing/prefetching */
4821 WREG32(CP_MEC_CNTL
, MEC_ME1_HALT
| MEC_ME2_HALT
);
4823 if (reset_mask
& RADEON_RESET_DMA
) {
4825 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
);
4827 WREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
4829 if (reset_mask
& RADEON_RESET_DMA1
) {
4831 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
);
4833 WREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
4836 evergreen_mc_stop(rdev
, &save
);
4837 if (evergreen_mc_wait_for_idle(rdev
)) {
4838 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
4841 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
))
4842 grbm_soft_reset
= SOFT_RESET_CP
| SOFT_RESET_GFX
;
4844 if (reset_mask
& RADEON_RESET_CP
) {
4845 grbm_soft_reset
|= SOFT_RESET_CP
;
4847 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4850 if (reset_mask
& RADEON_RESET_DMA
)
4851 srbm_soft_reset
|= SOFT_RESET_SDMA
;
4853 if (reset_mask
& RADEON_RESET_DMA1
)
4854 srbm_soft_reset
|= SOFT_RESET_SDMA1
;
4856 if (reset_mask
& RADEON_RESET_DISPLAY
)
4857 srbm_soft_reset
|= SOFT_RESET_DC
;
4859 if (reset_mask
& RADEON_RESET_RLC
)
4860 grbm_soft_reset
|= SOFT_RESET_RLC
;
4862 if (reset_mask
& RADEON_RESET_SEM
)
4863 srbm_soft_reset
|= SOFT_RESET_SEM
;
4865 if (reset_mask
& RADEON_RESET_IH
)
4866 srbm_soft_reset
|= SOFT_RESET_IH
;
4868 if (reset_mask
& RADEON_RESET_GRBM
)
4869 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4871 if (reset_mask
& RADEON_RESET_VMC
)
4872 srbm_soft_reset
|= SOFT_RESET_VMC
;
4874 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
4875 if (reset_mask
& RADEON_RESET_MC
)
4876 srbm_soft_reset
|= SOFT_RESET_MC
;
4879 if (grbm_soft_reset
) {
4880 tmp
= RREG32(GRBM_SOFT_RESET
);
4881 tmp
|= grbm_soft_reset
;
4882 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
4883 WREG32(GRBM_SOFT_RESET
, tmp
);
4884 tmp
= RREG32(GRBM_SOFT_RESET
);
4888 tmp
&= ~grbm_soft_reset
;
4889 WREG32(GRBM_SOFT_RESET
, tmp
);
4890 tmp
= RREG32(GRBM_SOFT_RESET
);
4893 if (srbm_soft_reset
) {
4894 tmp
= RREG32(SRBM_SOFT_RESET
);
4895 tmp
|= srbm_soft_reset
;
4896 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
4897 WREG32(SRBM_SOFT_RESET
, tmp
);
4898 tmp
= RREG32(SRBM_SOFT_RESET
);
4902 tmp
&= ~srbm_soft_reset
;
4903 WREG32(SRBM_SOFT_RESET
, tmp
);
4904 tmp
= RREG32(SRBM_SOFT_RESET
);
4907 /* Wait a little for things to settle down */
4910 evergreen_mc_resume(rdev
, &save
);
4913 cik_print_gpu_status_regs(rdev
);
4916 struct kv_reset_save_regs
{
4917 u32 gmcon_reng_execute
;
4922 static void kv_save_regs_for_reset(struct radeon_device
*rdev
,
4923 struct kv_reset_save_regs
*save
)
4925 save
->gmcon_reng_execute
= RREG32(GMCON_RENG_EXECUTE
);
4926 save
->gmcon_misc
= RREG32(GMCON_MISC
);
4927 save
->gmcon_misc3
= RREG32(GMCON_MISC3
);
4929 WREG32(GMCON_RENG_EXECUTE
, save
->gmcon_reng_execute
& ~RENG_EXECUTE_ON_PWR_UP
);
4930 WREG32(GMCON_MISC
, save
->gmcon_misc
& ~(RENG_EXECUTE_ON_REG_UPDATE
|
4931 STCTRL_STUTTER_EN
));
4934 static void kv_restore_regs_for_reset(struct radeon_device
*rdev
,
4935 struct kv_reset_save_regs
*save
)
4939 WREG32(GMCON_PGFSM_WRITE
, 0);
4940 WREG32(GMCON_PGFSM_CONFIG
, 0x200010ff);
4942 for (i
= 0; i
< 5; i
++)
4943 WREG32(GMCON_PGFSM_WRITE
, 0);
4945 WREG32(GMCON_PGFSM_WRITE
, 0);
4946 WREG32(GMCON_PGFSM_CONFIG
, 0x300010ff);
4948 for (i
= 0; i
< 5; i
++)
4949 WREG32(GMCON_PGFSM_WRITE
, 0);
4951 WREG32(GMCON_PGFSM_WRITE
, 0x210000);
4952 WREG32(GMCON_PGFSM_CONFIG
, 0xa00010ff);
4954 for (i
= 0; i
< 5; i
++)
4955 WREG32(GMCON_PGFSM_WRITE
, 0);
4957 WREG32(GMCON_PGFSM_WRITE
, 0x21003);
4958 WREG32(GMCON_PGFSM_CONFIG
, 0xb00010ff);
4960 for (i
= 0; i
< 5; i
++)
4961 WREG32(GMCON_PGFSM_WRITE
, 0);
4963 WREG32(GMCON_PGFSM_WRITE
, 0x2b00);
4964 WREG32(GMCON_PGFSM_CONFIG
, 0xc00010ff);
4966 for (i
= 0; i
< 5; i
++)
4967 WREG32(GMCON_PGFSM_WRITE
, 0);
4969 WREG32(GMCON_PGFSM_WRITE
, 0);
4970 WREG32(GMCON_PGFSM_CONFIG
, 0xd00010ff);
4972 for (i
= 0; i
< 5; i
++)
4973 WREG32(GMCON_PGFSM_WRITE
, 0);
4975 WREG32(GMCON_PGFSM_WRITE
, 0x420000);
4976 WREG32(GMCON_PGFSM_CONFIG
, 0x100010ff);
4978 for (i
= 0; i
< 5; i
++)
4979 WREG32(GMCON_PGFSM_WRITE
, 0);
4981 WREG32(GMCON_PGFSM_WRITE
, 0x120202);
4982 WREG32(GMCON_PGFSM_CONFIG
, 0x500010ff);
4984 for (i
= 0; i
< 5; i
++)
4985 WREG32(GMCON_PGFSM_WRITE
, 0);
4987 WREG32(GMCON_PGFSM_WRITE
, 0x3e3e36);
4988 WREG32(GMCON_PGFSM_CONFIG
, 0x600010ff);
4990 for (i
= 0; i
< 5; i
++)
4991 WREG32(GMCON_PGFSM_WRITE
, 0);
4993 WREG32(GMCON_PGFSM_WRITE
, 0x373f3e);
4994 WREG32(GMCON_PGFSM_CONFIG
, 0x700010ff);
4996 for (i
= 0; i
< 5; i
++)
4997 WREG32(GMCON_PGFSM_WRITE
, 0);
4999 WREG32(GMCON_PGFSM_WRITE
, 0x3e1332);
5000 WREG32(GMCON_PGFSM_CONFIG
, 0xe00010ff);
5002 WREG32(GMCON_MISC3
, save
->gmcon_misc3
);
5003 WREG32(GMCON_MISC
, save
->gmcon_misc
);
5004 WREG32(GMCON_RENG_EXECUTE
, save
->gmcon_reng_execute
);
5007 static void cik_gpu_pci_config_reset(struct radeon_device
*rdev
)
5009 struct evergreen_mc_save save
;
5010 struct kv_reset_save_regs kv_save
= { 0 };
5013 dev_info(rdev
->dev
, "GPU pci config reset\n");
5021 /* Disable GFX parsing/prefetching */
5022 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
5024 /* Disable MEC parsing/prefetching */
5025 WREG32(CP_MEC_CNTL
, MEC_ME1_HALT
| MEC_ME2_HALT
);
5028 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
);
5030 WREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
5032 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
);
5034 WREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
5035 /* XXX other engines? */
5037 /* halt the rlc, disable cp internal ints */
5042 /* disable mem access */
5043 evergreen_mc_stop(rdev
, &save
);
5044 if (evergreen_mc_wait_for_idle(rdev
)) {
5045 dev_warn(rdev
->dev
, "Wait for MC idle timed out !\n");
5048 if (rdev
->flags
& RADEON_IS_IGP
)
5049 kv_save_regs_for_reset(rdev
, &kv_save
);
5052 pci_clear_master(rdev
->pdev
);
5054 radeon_pci_config_reset(rdev
);
5058 /* wait for asic to come out of reset */
5059 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5060 if (RREG32(CONFIG_MEMSIZE
) != 0xffffffff)
5065 /* does asic init need to be run first??? */
5066 if (rdev
->flags
& RADEON_IS_IGP
)
5067 kv_restore_regs_for_reset(rdev
, &kv_save
);
5071 * cik_asic_reset - soft reset GPU
5073 * @rdev: radeon_device pointer
5075 * Look up which blocks are hung and attempt
5077 * Returns 0 for success.
5079 int cik_asic_reset(struct radeon_device
*rdev
)
5083 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5086 r600_set_bios_scratch_engine_hung(rdev
, true);
5088 /* try soft reset */
5089 cik_gpu_soft_reset(rdev
, reset_mask
);
5091 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5093 /* try pci config reset */
5094 if (reset_mask
&& radeon_hard_reset
)
5095 cik_gpu_pci_config_reset(rdev
);
5097 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5100 r600_set_bios_scratch_engine_hung(rdev
, false);
5106 * cik_gfx_is_lockup - check if the 3D engine is locked up
5108 * @rdev: radeon_device pointer
5109 * @ring: radeon_ring structure holding ring information
5111 * Check if the 3D engine is locked up (CIK).
5112 * Returns true if the engine is locked, false if not.
5114 bool cik_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
5116 u32 reset_mask
= cik_gpu_check_soft_reset(rdev
);
5118 if (!(reset_mask
& (RADEON_RESET_GFX
|
5119 RADEON_RESET_COMPUTE
|
5120 RADEON_RESET_CP
))) {
5121 radeon_ring_lockup_update(rdev
, ring
);
5124 return radeon_ring_test_lockup(rdev
, ring
);
5129 * cik_mc_program - program the GPU memory controller
5131 * @rdev: radeon_device pointer
5133 * Set the location of vram, gart, and AGP in the GPU's
5134 * physical address space (CIK).
5136 static void cik_mc_program(struct radeon_device
*rdev
)
5138 struct evergreen_mc_save save
;
5142 /* Initialize HDP */
5143 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
5144 WREG32((0x2c14 + j
), 0x00000000);
5145 WREG32((0x2c18 + j
), 0x00000000);
5146 WREG32((0x2c1c + j
), 0x00000000);
5147 WREG32((0x2c20 + j
), 0x00000000);
5148 WREG32((0x2c24 + j
), 0x00000000);
5150 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
5152 evergreen_mc_stop(rdev
, &save
);
5153 if (radeon_mc_wait_for_idle(rdev
)) {
5154 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
5156 /* Lockout access through VGA aperture*/
5157 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
5158 /* Update configuration */
5159 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
5160 rdev
->mc
.vram_start
>> 12);
5161 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
5162 rdev
->mc
.vram_end
>> 12);
5163 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
5164 rdev
->vram_scratch
.gpu_addr
>> 12);
5165 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
5166 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
5167 WREG32(MC_VM_FB_LOCATION
, tmp
);
5168 /* XXX double check these! */
5169 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
5170 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
5171 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
5172 WREG32(MC_VM_AGP_BASE
, 0);
5173 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
5174 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
5175 if (radeon_mc_wait_for_idle(rdev
)) {
5176 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
5178 evergreen_mc_resume(rdev
, &save
);
5179 /* we need to own VRAM, so turn off the VGA renderer here
5180 * to stop it overwriting our objects */
5181 rv515_vga_render_disable(rdev
);
5185 * cik_mc_init - initialize the memory controller driver params
5187 * @rdev: radeon_device pointer
5189 * Look up the amount of vram, vram width, and decide how to place
5190 * vram and gart within the GPU's physical address space (CIK).
5191 * Returns 0 for success.
5193 static int cik_mc_init(struct radeon_device
*rdev
)
5196 int chansize
, numchan
;
5198 /* Get VRAM informations */
5199 rdev
->mc
.vram_is_ddr
= true;
5200 tmp
= RREG32(MC_ARB_RAMCFG
);
5201 if (tmp
& CHANSIZE_MASK
) {
5206 tmp
= RREG32(MC_SHARED_CHMAP
);
5207 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
5237 rdev
->mc
.vram_width
= numchan
* chansize
;
5238 /* Could aper size report 0 ? */
5239 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
5240 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
5241 /* size in MB on si */
5242 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
5243 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
5244 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
5245 si_vram_gtt_location(rdev
, &rdev
->mc
);
5246 radeon_update_bandwidth_info(rdev
);
5253 * VMID 0 is the physical GPU addresses as used by the kernel.
5254 * VMIDs 1-15 are used for userspace clients and are handled
5255 * by the radeon vm/hsa code.
5258 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5260 * @rdev: radeon_device pointer
5262 * Flush the TLB for the VMID 0 page table (CIK).
5264 void cik_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
5266 /* flush hdp cache */
5267 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0);
5269 /* bits 0-15 are the VM contexts0-15 */
5270 WREG32(VM_INVALIDATE_REQUEST
, 0x1);
5274 * cik_pcie_gart_enable - gart enable
5276 * @rdev: radeon_device pointer
5278 * This sets up the TLBs, programs the page tables for VMID0,
5279 * sets up the hw for VMIDs 1-15 which are allocated on
5280 * demand, and sets up the global locations for the LDS, GDS,
5281 * and GPUVM for FSA64 clients (CIK).
5282 * Returns 0 for success, errors for failure.
5284 static int cik_pcie_gart_enable(struct radeon_device
*rdev
)
5288 if (rdev
->gart
.robj
== NULL
) {
5289 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
5292 r
= radeon_gart_table_vram_pin(rdev
);
5295 radeon_gart_restore(rdev
);
5296 /* Setup TLB control */
5297 WREG32(MC_VM_MX_L1_TLB_CNTL
,
5300 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5301 ENABLE_ADVANCED_DRIVER_MODEL
|
5302 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5303 /* Setup L2 cache */
5304 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
5305 ENABLE_L2_FRAGMENT_PROCESSING
|
5306 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5307 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5308 EFFECTIVE_L2_QUEUE_SIZE(7) |
5309 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5310 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
5311 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5312 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5313 /* setup context0 */
5314 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
5315 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
5316 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
5317 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
5318 (u32
)(rdev
->dummy_page
.addr
>> 12));
5319 WREG32(VM_CONTEXT0_CNTL2
, 0);
5320 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
5321 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
5327 /* empty context1-15 */
5328 /* FIXME start with 4G, once using 2 level pt switch to full
5331 /* set vm size, must be a multiple of 4 */
5332 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
5333 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
5334 for (i
= 1; i
< 16; i
++) {
5336 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
5337 rdev
->gart
.table_addr
>> 12);
5339 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
5340 rdev
->gart
.table_addr
>> 12);
5343 /* enable context1-15 */
5344 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
5345 (u32
)(rdev
->dummy_page
.addr
>> 12));
5346 WREG32(VM_CONTEXT1_CNTL2
, 4);
5347 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
5348 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5349 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5350 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5351 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5352 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5353 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
5354 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5355 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
5356 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5357 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
5358 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5359 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
5361 if (rdev
->family
== CHIP_KAVERI
) {
5362 u32 tmp
= RREG32(CHUB_CONTROL
);
5364 WREG32(CHUB_CONTROL
, tmp
);
5367 /* XXX SH_MEM regs */
5368 /* where to put LDS, scratch, GPUVM in FSA64 space */
5369 mutex_lock(&rdev
->srbm_mutex
);
5370 for (i
= 0; i
< 16; i
++) {
5371 cik_srbm_select(rdev
, 0, 0, 0, i
);
5372 /* CP and shaders */
5373 WREG32(SH_MEM_CONFIG
, 0);
5374 WREG32(SH_MEM_APE1_BASE
, 1);
5375 WREG32(SH_MEM_APE1_LIMIT
, 0);
5376 WREG32(SH_MEM_BASES
, 0);
5378 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA0_REGISTER_OFFSET
, 0);
5379 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA0_REGISTER_OFFSET
, 0);
5380 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA1_REGISTER_OFFSET
, 0);
5381 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA1_REGISTER_OFFSET
, 0);
5382 /* XXX SDMA RLC - todo */
5384 cik_srbm_select(rdev
, 0, 0, 0, 0);
5385 mutex_unlock(&rdev
->srbm_mutex
);
5387 cik_pcie_gart_tlb_flush(rdev
);
5388 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5389 (unsigned)(rdev
->mc
.gtt_size
>> 20),
5390 (unsigned long long)rdev
->gart
.table_addr
);
5391 rdev
->gart
.ready
= true;
5396 * cik_pcie_gart_disable - gart disable
5398 * @rdev: radeon_device pointer
5400 * This disables all VM page table (CIK).
5402 static void cik_pcie_gart_disable(struct radeon_device
*rdev
)
5404 /* Disable all tables */
5405 WREG32(VM_CONTEXT0_CNTL
, 0);
5406 WREG32(VM_CONTEXT1_CNTL
, 0);
5407 /* Setup TLB control */
5408 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5409 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5410 /* Setup L2 cache */
5412 ENABLE_L2_FRAGMENT_PROCESSING
|
5413 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5414 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5415 EFFECTIVE_L2_QUEUE_SIZE(7) |
5416 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5417 WREG32(VM_L2_CNTL2
, 0);
5418 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5419 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5420 radeon_gart_table_vram_unpin(rdev
);
5424 * cik_pcie_gart_fini - vm fini callback
5426 * @rdev: radeon_device pointer
5428 * Tears down the driver GART/VM setup (CIK).
5430 static void cik_pcie_gart_fini(struct radeon_device
*rdev
)
5432 cik_pcie_gart_disable(rdev
);
5433 radeon_gart_table_vram_free(rdev
);
5434 radeon_gart_fini(rdev
);
5439 * cik_ib_parse - vm ib_parse callback
5441 * @rdev: radeon_device pointer
5442 * @ib: indirect buffer pointer
5444 * CIK uses hw IB checking so this is a nop (CIK).
5446 int cik_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
5453 * VMID 0 is the physical GPU addresses as used by the kernel.
5454 * VMIDs 1-15 are used for userspace clients and are handled
5455 * by the radeon vm/hsa code.
5458 * cik_vm_init - cik vm init callback
5460 * @rdev: radeon_device pointer
5462 * Inits cik specific vm parameters (number of VMs, base of vram for
5463 * VMIDs 1-15) (CIK).
5464 * Returns 0 for success.
5466 int cik_vm_init(struct radeon_device
*rdev
)
5469 rdev
->vm_manager
.nvm
= 16;
5470 /* base offset of vram pages */
5471 if (rdev
->flags
& RADEON_IS_IGP
) {
5472 u64 tmp
= RREG32(MC_VM_FB_OFFSET
);
5474 rdev
->vm_manager
.vram_base_offset
= tmp
;
5476 rdev
->vm_manager
.vram_base_offset
= 0;
5482 * cik_vm_fini - cik vm fini callback
5484 * @rdev: radeon_device pointer
5486 * Tear down any asic specific VM setup (CIK).
5488 void cik_vm_fini(struct radeon_device
*rdev
)
5493 * cik_vm_decode_fault - print human readable fault info
5495 * @rdev: radeon_device pointer
5496 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5497 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5499 * Print human readable fault information (CIK).
5501 static void cik_vm_decode_fault(struct radeon_device
*rdev
,
5502 u32 status
, u32 addr
, u32 mc_client
)
5505 u32 vmid
= (status
& FAULT_VMID_MASK
) >> FAULT_VMID_SHIFT
;
5506 u32 protections
= (status
& PROTECTIONS_MASK
) >> PROTECTIONS_SHIFT
;
5507 char block
[5] = { mc_client
>> 24, (mc_client
>> 16) & 0xff,
5508 (mc_client
>> 8) & 0xff, mc_client
& 0xff, 0 };
5510 if (rdev
->family
== CHIP_HAWAII
)
5511 mc_id
= (status
& HAWAII_MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
5513 mc_id
= (status
& MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
5515 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5516 protections
, vmid
, addr
,
5517 (status
& MEMORY_CLIENT_RW_MASK
) ? "write" : "read",
5518 block
, mc_client
, mc_id
);
5522 * cik_vm_flush - cik vm flush using the CP
5524 * @rdev: radeon_device pointer
5526 * Update the page table base and flush the VM TLB
5527 * using the CP (CIK).
5529 void cik_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
5531 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
5536 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5537 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5538 WRITE_DATA_DST_SEL(0)));
5540 radeon_ring_write(ring
,
5541 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
5543 radeon_ring_write(ring
,
5544 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
5546 radeon_ring_write(ring
, 0);
5547 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
5549 /* update SH_MEM_* regs */
5550 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5551 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5552 WRITE_DATA_DST_SEL(0)));
5553 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5554 radeon_ring_write(ring
, 0);
5555 radeon_ring_write(ring
, VMID(vm
->id
));
5557 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 6));
5558 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5559 WRITE_DATA_DST_SEL(0)));
5560 radeon_ring_write(ring
, SH_MEM_BASES
>> 2);
5561 radeon_ring_write(ring
, 0);
5563 radeon_ring_write(ring
, 0); /* SH_MEM_BASES */
5564 radeon_ring_write(ring
, 0); /* SH_MEM_CONFIG */
5565 radeon_ring_write(ring
, 1); /* SH_MEM_APE1_BASE */
5566 radeon_ring_write(ring
, 0); /* SH_MEM_APE1_LIMIT */
5568 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5569 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5570 WRITE_DATA_DST_SEL(0)));
5571 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5572 radeon_ring_write(ring
, 0);
5573 radeon_ring_write(ring
, VMID(0));
5576 cik_hdp_flush_cp_ring_emit(rdev
, ridx
);
5578 /* bits 0-15 are the VM contexts0-15 */
5579 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5580 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5581 WRITE_DATA_DST_SEL(0)));
5582 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
5583 radeon_ring_write(ring
, 0);
5584 radeon_ring_write(ring
, 1 << vm
->id
);
5586 /* compute doesn't have PFP */
5587 if (ridx
== RADEON_RING_TYPE_GFX_INDEX
) {
5588 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5589 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
5590 radeon_ring_write(ring
, 0x0);
5596 * The RLC is a multi-purpose microengine that handles a
5597 * variety of functions, the most important of which is
5598 * the interrupt controller.
5600 static void cik_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
5603 u32 tmp
= RREG32(CP_INT_CNTL_RING0
);
5606 tmp
|= (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5608 tmp
&= ~(CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5609 WREG32(CP_INT_CNTL_RING0
, tmp
);
5612 static void cik_enable_lbpw(struct radeon_device
*rdev
, bool enable
)
5616 tmp
= RREG32(RLC_LB_CNTL
);
5618 tmp
|= LOAD_BALANCE_ENABLE
;
5620 tmp
&= ~LOAD_BALANCE_ENABLE
;
5621 WREG32(RLC_LB_CNTL
, tmp
);
5624 static void cik_wait_for_rlc_serdes(struct radeon_device
*rdev
)
5629 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
5630 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
5631 cik_select_se_sh(rdev
, i
, j
);
5632 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5633 if (RREG32(RLC_SERDES_CU_MASTER_BUSY
) == 0)
5639 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5641 mask
= SE_MASTER_BUSY_MASK
| GC_MASTER_BUSY
| TC0_MASTER_BUSY
| TC1_MASTER_BUSY
;
5642 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5643 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY
) & mask
) == 0)
5649 static void cik_update_rlc(struct radeon_device
*rdev
, u32 rlc
)
5653 tmp
= RREG32(RLC_CNTL
);
5655 WREG32(RLC_CNTL
, rlc
);
5658 static u32
cik_halt_rlc(struct radeon_device
*rdev
)
5662 orig
= data
= RREG32(RLC_CNTL
);
5664 if (data
& RLC_ENABLE
) {
5667 data
&= ~RLC_ENABLE
;
5668 WREG32(RLC_CNTL
, data
);
5670 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5671 if ((RREG32(RLC_GPM_STAT
) & RLC_GPM_BUSY
) == 0)
5676 cik_wait_for_rlc_serdes(rdev
);
5682 void cik_enter_rlc_safe_mode(struct radeon_device
*rdev
)
5686 tmp
= REQ
| MESSAGE(MSG_ENTER_RLC_SAFE_MODE
);
5687 WREG32(RLC_GPR_REG2
, tmp
);
5689 mask
= GFX_POWER_STATUS
| GFX_CLOCK_STATUS
;
5690 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5691 if ((RREG32(RLC_GPM_STAT
) & mask
) == mask
)
5696 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5697 if ((RREG32(RLC_GPR_REG2
) & REQ
) == 0)
5703 void cik_exit_rlc_safe_mode(struct radeon_device
*rdev
)
5707 tmp
= REQ
| MESSAGE(MSG_EXIT_RLC_SAFE_MODE
);
5708 WREG32(RLC_GPR_REG2
, tmp
);
5712 * cik_rlc_stop - stop the RLC ME
5714 * @rdev: radeon_device pointer
5716 * Halt the RLC ME (MicroEngine) (CIK).
5718 static void cik_rlc_stop(struct radeon_device
*rdev
)
5720 WREG32(RLC_CNTL
, 0);
5722 cik_enable_gui_idle_interrupt(rdev
, false);
5724 cik_wait_for_rlc_serdes(rdev
);
5728 * cik_rlc_start - start the RLC ME
5730 * @rdev: radeon_device pointer
5732 * Unhalt the RLC ME (MicroEngine) (CIK).
5734 static void cik_rlc_start(struct radeon_device
*rdev
)
5736 WREG32(RLC_CNTL
, RLC_ENABLE
);
5738 cik_enable_gui_idle_interrupt(rdev
, true);
5744 * cik_rlc_resume - setup the RLC hw
5746 * @rdev: radeon_device pointer
5748 * Initialize the RLC registers, load the ucode,
5749 * and start the RLC (CIK).
5750 * Returns 0 for success, -EINVAL if the ucode is not available.
5752 static int cik_rlc_resume(struct radeon_device
*rdev
)
5755 const __be32
*fw_data
;
5760 switch (rdev
->family
) {
5764 size
= BONAIRE_RLC_UCODE_SIZE
;
5767 size
= KV_RLC_UCODE_SIZE
;
5770 size
= KB_RLC_UCODE_SIZE
;
5777 tmp
= RREG32(RLC_CGCG_CGLS_CTRL
) & 0xfffffffc;
5778 WREG32(RLC_CGCG_CGLS_CTRL
, tmp
);
5786 WREG32(RLC_LB_CNTR_INIT
, 0);
5787 WREG32(RLC_LB_CNTR_MAX
, 0x00008000);
5789 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5790 WREG32(RLC_LB_INIT_CU_MASK
, 0xffffffff);
5791 WREG32(RLC_LB_PARAMS
, 0x00600408);
5792 WREG32(RLC_LB_CNTL
, 0x80000004);
5794 WREG32(RLC_MC_CNTL
, 0);
5795 WREG32(RLC_UCODE_CNTL
, 0);
5797 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
5798 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5799 for (i
= 0; i
< size
; i
++)
5800 WREG32(RLC_GPM_UCODE_DATA
, be32_to_cpup(fw_data
++));
5801 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5803 /* XXX - find out what chips support lbpw */
5804 cik_enable_lbpw(rdev
, false);
5806 if (rdev
->family
== CHIP_BONAIRE
)
5807 WREG32(RLC_DRIVER_DMA_STATUS
, 0);
5809 cik_rlc_start(rdev
);
5814 static void cik_enable_cgcg(struct radeon_device
*rdev
, bool enable
)
5816 u32 data
, orig
, tmp
, tmp2
;
5818 orig
= data
= RREG32(RLC_CGCG_CGLS_CTRL
);
5820 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGCG
)) {
5821 cik_enable_gui_idle_interrupt(rdev
, true);
5823 tmp
= cik_halt_rlc(rdev
);
5825 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5826 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5827 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5828 tmp2
= BPM_ADDR_MASK
| CGCG_OVERRIDE_0
| CGLS_ENABLE
;
5829 WREG32(RLC_SERDES_WR_CTRL
, tmp2
);
5831 cik_update_rlc(rdev
, tmp
);
5833 data
|= CGCG_EN
| CGLS_EN
;
5835 cik_enable_gui_idle_interrupt(rdev
, false);
5837 RREG32(CB_CGTT_SCLK_CTRL
);
5838 RREG32(CB_CGTT_SCLK_CTRL
);
5839 RREG32(CB_CGTT_SCLK_CTRL
);
5840 RREG32(CB_CGTT_SCLK_CTRL
);
5842 data
&= ~(CGCG_EN
| CGLS_EN
);
5846 WREG32(RLC_CGCG_CGLS_CTRL
, data
);
5850 static void cik_enable_mgcg(struct radeon_device
*rdev
, bool enable
)
5852 u32 data
, orig
, tmp
= 0;
5854 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGCG
)) {
5855 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGLS
) {
5856 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CP_LS
) {
5857 orig
= data
= RREG32(CP_MEM_SLP_CNTL
);
5858 data
|= CP_MEM_LS_EN
;
5860 WREG32(CP_MEM_SLP_CNTL
, data
);
5864 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5867 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5869 tmp
= cik_halt_rlc(rdev
);
5871 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5872 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5873 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5874 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_0
;
5875 WREG32(RLC_SERDES_WR_CTRL
, data
);
5877 cik_update_rlc(rdev
, tmp
);
5879 if (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGTS
) {
5880 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5881 data
&= ~SM_MODE_MASK
;
5882 data
|= SM_MODE(0x2);
5883 data
|= SM_MODE_ENABLE
;
5884 data
&= ~CGTS_OVERRIDE
;
5885 if ((rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_MGLS
) &&
5886 (rdev
->cg_flags
& RADEON_CG_SUPPORT_GFX_CGTS_LS
))
5887 data
&= ~CGTS_LS_OVERRIDE
;
5888 data
&= ~ON_MONITOR_ADD_MASK
;
5889 data
|= ON_MONITOR_ADD_EN
;
5890 data
|= ON_MONITOR_ADD(0x96);
5892 WREG32(CGTS_SM_CTRL_REG
, data
);
5895 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5898 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5900 data
= RREG32(RLC_MEM_SLP_CNTL
);
5901 if (data
& RLC_MEM_LS_EN
) {
5902 data
&= ~RLC_MEM_LS_EN
;
5903 WREG32(RLC_MEM_SLP_CNTL
, data
);
5906 data
= RREG32(CP_MEM_SLP_CNTL
);
5907 if (data
& CP_MEM_LS_EN
) {
5908 data
&= ~CP_MEM_LS_EN
;
5909 WREG32(CP_MEM_SLP_CNTL
, data
);
5912 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5913 data
|= CGTS_OVERRIDE
| CGTS_LS_OVERRIDE
;
5915 WREG32(CGTS_SM_CTRL_REG
, data
);
5917 tmp
= cik_halt_rlc(rdev
);
5919 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5920 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5921 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5922 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_1
;
5923 WREG32(RLC_SERDES_WR_CTRL
, data
);
5925 cik_update_rlc(rdev
, tmp
);
5929 static const u32 mc_cg_registers
[] =
5942 static void cik_enable_mc_ls(struct radeon_device
*rdev
,
5948 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5949 orig
= data
= RREG32(mc_cg_registers
[i
]);
5950 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_MC_LS
))
5951 data
|= MC_LS_ENABLE
;
5953 data
&= ~MC_LS_ENABLE
;
5955 WREG32(mc_cg_registers
[i
], data
);
5959 static void cik_enable_mc_mgcg(struct radeon_device
*rdev
,
5965 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5966 orig
= data
= RREG32(mc_cg_registers
[i
]);
5967 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_MC_MGCG
))
5968 data
|= MC_CG_ENABLE
;
5970 data
&= ~MC_CG_ENABLE
;
5972 WREG32(mc_cg_registers
[i
], data
);
5976 static void cik_enable_sdma_mgcg(struct radeon_device
*rdev
,
5981 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_SDMA_MGCG
)) {
5982 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, 0x00000100);
5983 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, 0x00000100);
5985 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
);
5988 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, data
);
5990 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
);
5993 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, data
);
5997 static void cik_enable_sdma_mgls(struct radeon_device
*rdev
,
6002 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_SDMA_LS
)) {
6003 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
6006 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
6008 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
6011 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
6013 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
6016 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
6018 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
6021 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
6025 static void cik_enable_uvd_mgcg(struct radeon_device
*rdev
,
6030 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_UVD_MGCG
)) {
6031 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
6033 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
6035 orig
= data
= RREG32(UVD_CGC_CTRL
);
6038 WREG32(UVD_CGC_CTRL
, data
);
6040 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
6042 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
6044 orig
= data
= RREG32(UVD_CGC_CTRL
);
6047 WREG32(UVD_CGC_CTRL
, data
);
6051 static void cik_enable_bif_mgls(struct radeon_device
*rdev
,
6056 orig
= data
= RREG32_PCIE_PORT(PCIE_CNTL2
);
6058 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_BIF_LS
))
6059 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
|
6060 REPLAY_MEM_LS_EN
| SLV_MEM_AGGRESSIVE_LS_EN
;
6062 data
&= ~(SLV_MEM_LS_EN
| MST_MEM_LS_EN
|
6063 REPLAY_MEM_LS_EN
| SLV_MEM_AGGRESSIVE_LS_EN
);
6066 WREG32_PCIE_PORT(PCIE_CNTL2
, data
);
6069 static void cik_enable_hdp_mgcg(struct radeon_device
*rdev
,
6074 orig
= data
= RREG32(HDP_HOST_PATH_CNTL
);
6076 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_HDP_MGCG
))
6077 data
&= ~CLOCK_GATING_DIS
;
6079 data
|= CLOCK_GATING_DIS
;
6082 WREG32(HDP_HOST_PATH_CNTL
, data
);
6085 static void cik_enable_hdp_ls(struct radeon_device
*rdev
,
6090 orig
= data
= RREG32(HDP_MEM_POWER_LS
);
6092 if (enable
&& (rdev
->cg_flags
& RADEON_CG_SUPPORT_HDP_LS
))
6093 data
|= HDP_LS_ENABLE
;
6095 data
&= ~HDP_LS_ENABLE
;
6098 WREG32(HDP_MEM_POWER_LS
, data
);
6101 void cik_update_cg(struct radeon_device
*rdev
,
6102 u32 block
, bool enable
)
6105 if (block
& RADEON_CG_BLOCK_GFX
) {
6106 cik_enable_gui_idle_interrupt(rdev
, false);
6107 /* order matters! */
6109 cik_enable_mgcg(rdev
, true);
6110 cik_enable_cgcg(rdev
, true);
6112 cik_enable_cgcg(rdev
, false);
6113 cik_enable_mgcg(rdev
, false);
6115 cik_enable_gui_idle_interrupt(rdev
, true);
6118 if (block
& RADEON_CG_BLOCK_MC
) {
6119 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
6120 cik_enable_mc_mgcg(rdev
, enable
);
6121 cik_enable_mc_ls(rdev
, enable
);
6125 if (block
& RADEON_CG_BLOCK_SDMA
) {
6126 cik_enable_sdma_mgcg(rdev
, enable
);
6127 cik_enable_sdma_mgls(rdev
, enable
);
6130 if (block
& RADEON_CG_BLOCK_BIF
) {
6131 cik_enable_bif_mgls(rdev
, enable
);
6134 if (block
& RADEON_CG_BLOCK_UVD
) {
6136 cik_enable_uvd_mgcg(rdev
, enable
);
6139 if (block
& RADEON_CG_BLOCK_HDP
) {
6140 cik_enable_hdp_mgcg(rdev
, enable
);
6141 cik_enable_hdp_ls(rdev
, enable
);
6144 if (block
& RADEON_CG_BLOCK_VCE
) {
6145 vce_v2_0_enable_mgcg(rdev
, enable
);
6149 static void cik_init_cg(struct radeon_device
*rdev
)
6152 cik_update_cg(rdev
, RADEON_CG_BLOCK_GFX
, true);
6155 si_init_uvd_internal_cg(rdev
);
6157 cik_update_cg(rdev
, (RADEON_CG_BLOCK_MC
|
6158 RADEON_CG_BLOCK_SDMA
|
6159 RADEON_CG_BLOCK_BIF
|
6160 RADEON_CG_BLOCK_UVD
|
6161 RADEON_CG_BLOCK_HDP
), true);
6164 static void cik_fini_cg(struct radeon_device
*rdev
)
6166 cik_update_cg(rdev
, (RADEON_CG_BLOCK_MC
|
6167 RADEON_CG_BLOCK_SDMA
|
6168 RADEON_CG_BLOCK_BIF
|
6169 RADEON_CG_BLOCK_UVD
|
6170 RADEON_CG_BLOCK_HDP
), false);
6172 cik_update_cg(rdev
, RADEON_CG_BLOCK_GFX
, false);
6175 static void cik_enable_sck_slowdown_on_pu(struct radeon_device
*rdev
,
6180 orig
= data
= RREG32(RLC_PG_CNTL
);
6181 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_RLC_SMU_HS
))
6182 data
|= SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6184 data
&= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6186 WREG32(RLC_PG_CNTL
, data
);
6189 static void cik_enable_sck_slowdown_on_pd(struct radeon_device
*rdev
,
6194 orig
= data
= RREG32(RLC_PG_CNTL
);
6195 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_RLC_SMU_HS
))
6196 data
|= SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6198 data
&= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6200 WREG32(RLC_PG_CNTL
, data
);
6203 static void cik_enable_cp_pg(struct radeon_device
*rdev
, bool enable
)
6207 orig
= data
= RREG32(RLC_PG_CNTL
);
6208 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_CP
))
6209 data
&= ~DISABLE_CP_PG
;
6211 data
|= DISABLE_CP_PG
;
6213 WREG32(RLC_PG_CNTL
, data
);
6216 static void cik_enable_gds_pg(struct radeon_device
*rdev
, bool enable
)
6220 orig
= data
= RREG32(RLC_PG_CNTL
);
6221 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GDS
))
6222 data
&= ~DISABLE_GDS_PG
;
6224 data
|= DISABLE_GDS_PG
;
6226 WREG32(RLC_PG_CNTL
, data
);
6229 #define CP_ME_TABLE_SIZE 96
6230 #define CP_ME_TABLE_OFFSET 2048
6231 #define CP_MEC_TABLE_OFFSET 4096
6233 void cik_init_cp_pg_table(struct radeon_device
*rdev
)
6235 const __be32
*fw_data
;
6236 volatile u32
*dst_ptr
;
6237 int me
, i
, max_me
= 4;
6241 if (rdev
->family
== CHIP_KAVERI
)
6244 if (rdev
->rlc
.cp_table_ptr
== NULL
)
6247 /* write the cp table buffer */
6248 dst_ptr
= rdev
->rlc
.cp_table_ptr
;
6249 for (me
= 0; me
< max_me
; me
++) {
6251 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
6252 table_offset
= CP_ME_TABLE_OFFSET
;
6253 } else if (me
== 1) {
6254 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
6255 table_offset
= CP_ME_TABLE_OFFSET
;
6256 } else if (me
== 2) {
6257 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
6258 table_offset
= CP_ME_TABLE_OFFSET
;
6260 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
6261 table_offset
= CP_MEC_TABLE_OFFSET
;
6264 for (i
= 0; i
< CP_ME_TABLE_SIZE
; i
++) {
6265 dst_ptr
[bo_offset
+ i
] = cpu_to_le32(be32_to_cpu(fw_data
[table_offset
+ i
]));
6267 bo_offset
+= CP_ME_TABLE_SIZE
;
6271 static void cik_enable_gfx_cgpg(struct radeon_device
*rdev
,
6276 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
)) {
6277 orig
= data
= RREG32(RLC_PG_CNTL
);
6278 data
|= GFX_PG_ENABLE
;
6280 WREG32(RLC_PG_CNTL
, data
);
6282 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6285 WREG32(RLC_AUTO_PG_CTRL
, data
);
6287 orig
= data
= RREG32(RLC_PG_CNTL
);
6288 data
&= ~GFX_PG_ENABLE
;
6290 WREG32(RLC_PG_CNTL
, data
);
6292 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6293 data
&= ~AUTO_PG_EN
;
6295 WREG32(RLC_AUTO_PG_CTRL
, data
);
6297 data
= RREG32(DB_RENDER_CONTROL
);
6301 static u32
cik_get_cu_active_bitmap(struct radeon_device
*rdev
, u32 se
, u32 sh
)
6303 u32 mask
= 0, tmp
, tmp1
;
6306 cik_select_se_sh(rdev
, se
, sh
);
6307 tmp
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
6308 tmp1
= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
6309 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
6316 for (i
= 0; i
< rdev
->config
.cik
.max_cu_per_sh
; i
++) {
6321 return (~tmp
) & mask
;
6324 static void cik_init_ao_cu_mask(struct radeon_device
*rdev
)
6326 u32 i
, j
, k
, active_cu_number
= 0;
6327 u32 mask
, counter
, cu_bitmap
;
6330 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
6331 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
6335 for (k
= 0; k
< rdev
->config
.cik
.max_cu_per_sh
; k
++) {
6336 if (cik_get_cu_active_bitmap(rdev
, i
, j
) & mask
) {
6344 active_cu_number
+= counter
;
6345 tmp
|= (cu_bitmap
<< (i
* 16 + j
* 8));
6349 WREG32(RLC_PG_AO_CU_MASK
, tmp
);
6351 tmp
= RREG32(RLC_MAX_PG_CU
);
6352 tmp
&= ~MAX_PU_CU_MASK
;
6353 tmp
|= MAX_PU_CU(active_cu_number
);
6354 WREG32(RLC_MAX_PG_CU
, tmp
);
6357 static void cik_enable_gfx_static_mgpg(struct radeon_device
*rdev
,
6362 orig
= data
= RREG32(RLC_PG_CNTL
);
6363 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_SMG
))
6364 data
|= STATIC_PER_CU_PG_ENABLE
;
6366 data
&= ~STATIC_PER_CU_PG_ENABLE
;
6368 WREG32(RLC_PG_CNTL
, data
);
6371 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device
*rdev
,
6376 orig
= data
= RREG32(RLC_PG_CNTL
);
6377 if (enable
&& (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_DMG
))
6378 data
|= DYN_PER_CU_PG_ENABLE
;
6380 data
&= ~DYN_PER_CU_PG_ENABLE
;
6382 WREG32(RLC_PG_CNTL
, data
);
6385 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6386 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6388 static void cik_init_gfx_cgpg(struct radeon_device
*rdev
)
6393 if (rdev
->rlc
.cs_data
) {
6394 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6395 WREG32(RLC_GPM_SCRATCH_DATA
, upper_32_bits(rdev
->rlc
.clear_state_gpu_addr
));
6396 WREG32(RLC_GPM_SCRATCH_DATA
, lower_32_bits(rdev
->rlc
.clear_state_gpu_addr
));
6397 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.clear_state_size
);
6399 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6400 for (i
= 0; i
< 3; i
++)
6401 WREG32(RLC_GPM_SCRATCH_DATA
, 0);
6403 if (rdev
->rlc
.reg_list
) {
6404 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_SAVE_AND_RESTORE_STARTING_OFFSET
);
6405 for (i
= 0; i
< rdev
->rlc
.reg_list_size
; i
++)
6406 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.reg_list
[i
]);
6409 orig
= data
= RREG32(RLC_PG_CNTL
);
6412 WREG32(RLC_PG_CNTL
, data
);
6414 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
6415 WREG32(RLC_CP_TABLE_RESTORE
, rdev
->rlc
.cp_table_gpu_addr
>> 8);
6417 data
= RREG32(CP_RB_WPTR_POLL_CNTL
);
6418 data
&= ~IDLE_POLL_COUNT_MASK
;
6419 data
|= IDLE_POLL_COUNT(0x60);
6420 WREG32(CP_RB_WPTR_POLL_CNTL
, data
);
6423 WREG32(RLC_PG_DELAY
, data
);
6425 data
= RREG32(RLC_PG_DELAY_2
);
6428 WREG32(RLC_PG_DELAY_2
, data
);
6430 data
= RREG32(RLC_AUTO_PG_CTRL
);
6431 data
&= ~GRBM_REG_SGIT_MASK
;
6432 data
|= GRBM_REG_SGIT(0x700);
6433 WREG32(RLC_AUTO_PG_CTRL
, data
);
6437 static void cik_update_gfx_pg(struct radeon_device
*rdev
, bool enable
)
6439 cik_enable_gfx_cgpg(rdev
, enable
);
6440 cik_enable_gfx_static_mgpg(rdev
, enable
);
6441 cik_enable_gfx_dynamic_mgpg(rdev
, enable
);
6444 u32
cik_get_csb_size(struct radeon_device
*rdev
)
6447 const struct cs_section_def
*sect
= NULL
;
6448 const struct cs_extent_def
*ext
= NULL
;
6450 if (rdev
->rlc
.cs_data
== NULL
)
6453 /* begin clear state */
6455 /* context control state */
6458 for (sect
= rdev
->rlc
.cs_data
; sect
->section
!= NULL
; ++sect
) {
6459 for (ext
= sect
->section
; ext
->extent
!= NULL
; ++ext
) {
6460 if (sect
->id
== SECT_CONTEXT
)
6461 count
+= 2 + ext
->reg_count
;
6466 /* pa_sc_raster_config/pa_sc_raster_config1 */
6468 /* end clear state */
6476 void cik_get_csb_buffer(struct radeon_device
*rdev
, volatile u32
*buffer
)
6479 const struct cs_section_def
*sect
= NULL
;
6480 const struct cs_extent_def
*ext
= NULL
;
6482 if (rdev
->rlc
.cs_data
== NULL
)
6487 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
6488 buffer
[count
++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
6490 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL
, 1));
6491 buffer
[count
++] = cpu_to_le32(0x80000000);
6492 buffer
[count
++] = cpu_to_le32(0x80000000);
6494 for (sect
= rdev
->rlc
.cs_data
; sect
->section
!= NULL
; ++sect
) {
6495 for (ext
= sect
->section
; ext
->extent
!= NULL
; ++ext
) {
6496 if (sect
->id
== SECT_CONTEXT
) {
6498 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG
, ext
->reg_count
));
6499 buffer
[count
++] = cpu_to_le32(ext
->reg_index
- 0xa000);
6500 for (i
= 0; i
< ext
->reg_count
; i
++)
6501 buffer
[count
++] = cpu_to_le32(ext
->extent
[i
]);
6508 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
6509 buffer
[count
++] = cpu_to_le32(PA_SC_RASTER_CONFIG
- PACKET3_SET_CONTEXT_REG_START
);
6510 switch (rdev
->family
) {
6512 buffer
[count
++] = cpu_to_le32(0x16000012);
6513 buffer
[count
++] = cpu_to_le32(0x00000000);
6516 buffer
[count
++] = cpu_to_le32(0x00000000); /* XXX */
6517 buffer
[count
++] = cpu_to_le32(0x00000000);
6520 buffer
[count
++] = cpu_to_le32(0x00000000); /* XXX */
6521 buffer
[count
++] = cpu_to_le32(0x00000000);
6524 buffer
[count
++] = 0x3a00161a;
6525 buffer
[count
++] = 0x0000002e;
6528 buffer
[count
++] = cpu_to_le32(0x00000000);
6529 buffer
[count
++] = cpu_to_le32(0x00000000);
6533 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
6534 buffer
[count
++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE
);
6536 buffer
[count
++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE
, 0));
6537 buffer
[count
++] = cpu_to_le32(0);
6540 static void cik_init_pg(struct radeon_device
*rdev
)
6542 if (rdev
->pg_flags
) {
6543 cik_enable_sck_slowdown_on_pu(rdev
, true);
6544 cik_enable_sck_slowdown_on_pd(rdev
, true);
6545 if (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
) {
6546 cik_init_gfx_cgpg(rdev
);
6547 cik_enable_cp_pg(rdev
, true);
6548 cik_enable_gds_pg(rdev
, true);
6550 cik_init_ao_cu_mask(rdev
);
6551 cik_update_gfx_pg(rdev
, true);
6555 static void cik_fini_pg(struct radeon_device
*rdev
)
6557 if (rdev
->pg_flags
) {
6558 cik_update_gfx_pg(rdev
, false);
6559 if (rdev
->pg_flags
& RADEON_PG_SUPPORT_GFX_PG
) {
6560 cik_enable_cp_pg(rdev
, false);
6561 cik_enable_gds_pg(rdev
, false);
6568 * Starting with r6xx, interrupts are handled via a ring buffer.
6569 * Ring buffers are areas of GPU accessible memory that the GPU
6570 * writes interrupt vectors into and the host reads vectors out of.
6571 * There is a rptr (read pointer) that determines where the
6572 * host is currently reading, and a wptr (write pointer)
6573 * which determines where the GPU has written. When the
6574 * pointers are equal, the ring is idle. When the GPU
6575 * writes vectors to the ring buffer, it increments the
6576 * wptr. When there is an interrupt, the host then starts
6577 * fetching commands and processing them until the pointers are
6578 * equal again at which point it updates the rptr.
6582 * cik_enable_interrupts - Enable the interrupt ring buffer
6584 * @rdev: radeon_device pointer
6586 * Enable the interrupt ring buffer (CIK).
6588 static void cik_enable_interrupts(struct radeon_device
*rdev
)
6590 u32 ih_cntl
= RREG32(IH_CNTL
);
6591 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6593 ih_cntl
|= ENABLE_INTR
;
6594 ih_rb_cntl
|= IH_RB_ENABLE
;
6595 WREG32(IH_CNTL
, ih_cntl
);
6596 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6597 rdev
->ih
.enabled
= true;
6601 * cik_disable_interrupts - Disable the interrupt ring buffer
6603 * @rdev: radeon_device pointer
6605 * Disable the interrupt ring buffer (CIK).
6607 static void cik_disable_interrupts(struct radeon_device
*rdev
)
6609 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6610 u32 ih_cntl
= RREG32(IH_CNTL
);
6612 ih_rb_cntl
&= ~IH_RB_ENABLE
;
6613 ih_cntl
&= ~ENABLE_INTR
;
6614 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6615 WREG32(IH_CNTL
, ih_cntl
);
6616 /* set rptr, wptr to 0 */
6617 WREG32(IH_RB_RPTR
, 0);
6618 WREG32(IH_RB_WPTR
, 0);
6619 rdev
->ih
.enabled
= false;
6624 * cik_disable_interrupt_state - Disable all interrupt sources
6626 * @rdev: radeon_device pointer
6628 * Clear all interrupt enable bits used by the driver (CIK).
6630 static void cik_disable_interrupt_state(struct radeon_device
*rdev
)
6635 tmp
= RREG32(CP_INT_CNTL_RING0
) &
6636 (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
6637 WREG32(CP_INT_CNTL_RING0
, tmp
);
6639 tmp
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6640 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
6641 tmp
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6642 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
6643 /* compute queues */
6644 WREG32(CP_ME1_PIPE0_INT_CNTL
, 0);
6645 WREG32(CP_ME1_PIPE1_INT_CNTL
, 0);
6646 WREG32(CP_ME1_PIPE2_INT_CNTL
, 0);
6647 WREG32(CP_ME1_PIPE3_INT_CNTL
, 0);
6648 WREG32(CP_ME2_PIPE0_INT_CNTL
, 0);
6649 WREG32(CP_ME2_PIPE1_INT_CNTL
, 0);
6650 WREG32(CP_ME2_PIPE2_INT_CNTL
, 0);
6651 WREG32(CP_ME2_PIPE3_INT_CNTL
, 0);
6653 WREG32(GRBM_INT_CNTL
, 0);
6654 /* vline/vblank, etc. */
6655 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
6656 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
6657 if (rdev
->num_crtc
>= 4) {
6658 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
6659 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
6661 if (rdev
->num_crtc
>= 6) {
6662 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
6663 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
6667 WREG32(DAC_AUTODETECT_INT_CONTROL
, 0);
6669 /* digital hotplug */
6670 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6671 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
6672 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6673 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
6674 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6675 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
6676 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6677 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
6678 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6679 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
6680 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6681 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
6686 * cik_irq_init - init and enable the interrupt ring
6688 * @rdev: radeon_device pointer
6690 * Allocate a ring buffer for the interrupt controller,
6691 * enable the RLC, disable interrupts, enable the IH
6692 * ring buffer and enable it (CIK).
6693 * Called at device load and reume.
6694 * Returns 0 for success, errors for failure.
6696 static int cik_irq_init(struct radeon_device
*rdev
)
6700 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
6703 ret
= r600_ih_ring_alloc(rdev
);
6708 cik_disable_interrupts(rdev
);
6711 ret
= cik_rlc_resume(rdev
);
6713 r600_ih_ring_fini(rdev
);
6717 /* setup interrupt control */
6718 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6719 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
6720 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
6721 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6722 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6724 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
6725 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6726 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
6727 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
6729 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
6730 rb_bufsz
= order_base_2(rdev
->ih
.ring_size
/ 4);
6732 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
6733 IH_WPTR_OVERFLOW_CLEAR
|
6736 if (rdev
->wb
.enabled
)
6737 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
6739 /* set the writeback address whether it's enabled or not */
6740 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
6741 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
6743 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6745 /* set rptr, wptr to 0 */
6746 WREG32(IH_RB_RPTR
, 0);
6747 WREG32(IH_RB_WPTR
, 0);
6749 /* Default settings for IH_CNTL (disabled at first) */
6750 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6751 /* RPTR_REARM only works if msi's are enabled */
6752 if (rdev
->msi_enabled
)
6753 ih_cntl
|= RPTR_REARM
;
6754 WREG32(IH_CNTL
, ih_cntl
);
6756 /* force the active interrupt state to all disabled */
6757 cik_disable_interrupt_state(rdev
);
6759 pci_set_master(rdev
->pdev
);
6762 cik_enable_interrupts(rdev
);
6768 * cik_irq_set - enable/disable interrupt sources
6770 * @rdev: radeon_device pointer
6772 * Enable interrupt sources on the GPU (vblanks, hpd,
6774 * Returns 0 for success, errors for failure.
6776 int cik_irq_set(struct radeon_device
*rdev
)
6779 u32 cp_m1p0
, cp_m1p1
, cp_m1p2
, cp_m1p3
;
6780 u32 cp_m2p0
, cp_m2p1
, cp_m2p2
, cp_m2p3
;
6781 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
6782 u32 hpd1
, hpd2
, hpd3
, hpd4
, hpd5
, hpd6
;
6783 u32 grbm_int_cntl
= 0;
6784 u32 dma_cntl
, dma_cntl1
;
6787 if (!rdev
->irq
.installed
) {
6788 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6791 /* don't enable anything if the ih is disabled */
6792 if (!rdev
->ih
.enabled
) {
6793 cik_disable_interrupts(rdev
);
6794 /* force the active interrupt state to all disabled */
6795 cik_disable_interrupt_state(rdev
);
6799 cp_int_cntl
= RREG32(CP_INT_CNTL_RING0
) &
6800 (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
6801 cp_int_cntl
|= PRIV_INSTR_INT_ENABLE
| PRIV_REG_INT_ENABLE
;
6803 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6804 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6805 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6806 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6807 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6808 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6810 dma_cntl
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6811 dma_cntl1
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6813 cp_m1p0
= RREG32(CP_ME1_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6814 cp_m1p1
= RREG32(CP_ME1_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6815 cp_m1p2
= RREG32(CP_ME1_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6816 cp_m1p3
= RREG32(CP_ME1_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6817 cp_m2p0
= RREG32(CP_ME2_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6818 cp_m2p1
= RREG32(CP_ME2_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6819 cp_m2p2
= RREG32(CP_ME2_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6820 cp_m2p3
= RREG32(CP_ME2_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6822 if (rdev
->flags
& RADEON_IS_IGP
)
6823 thermal_int
= RREG32_SMC(CG_THERMAL_INT_CTRL
) &
6824 ~(THERM_INTH_MASK
| THERM_INTL_MASK
);
6826 thermal_int
= RREG32_SMC(CG_THERMAL_INT
) &
6827 ~(THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
);
6829 /* enable CP interrupts on all rings */
6830 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
6831 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6832 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
6834 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
6835 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6836 DRM_DEBUG("si_irq_set: sw int cp1\n");
6837 if (ring
->me
== 1) {
6838 switch (ring
->pipe
) {
6840 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6843 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6846 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6849 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6852 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6855 } else if (ring
->me
== 2) {
6856 switch (ring
->pipe
) {
6858 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6861 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6864 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6867 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6870 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6874 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring
->me
);
6877 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
6878 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6879 DRM_DEBUG("si_irq_set: sw int cp2\n");
6880 if (ring
->me
== 1) {
6881 switch (ring
->pipe
) {
6883 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6886 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6889 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6892 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6895 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6898 } else if (ring
->me
== 2) {
6899 switch (ring
->pipe
) {
6901 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6904 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6907 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6910 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6913 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6917 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring
->me
);
6921 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
6922 DRM_DEBUG("cik_irq_set: sw int dma\n");
6923 dma_cntl
|= TRAP_ENABLE
;
6926 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
6927 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6928 dma_cntl1
|= TRAP_ENABLE
;
6931 if (rdev
->irq
.crtc_vblank_int
[0] ||
6932 atomic_read(&rdev
->irq
.pflip
[0])) {
6933 DRM_DEBUG("cik_irq_set: vblank 0\n");
6934 crtc1
|= VBLANK_INTERRUPT_MASK
;
6936 if (rdev
->irq
.crtc_vblank_int
[1] ||
6937 atomic_read(&rdev
->irq
.pflip
[1])) {
6938 DRM_DEBUG("cik_irq_set: vblank 1\n");
6939 crtc2
|= VBLANK_INTERRUPT_MASK
;
6941 if (rdev
->irq
.crtc_vblank_int
[2] ||
6942 atomic_read(&rdev
->irq
.pflip
[2])) {
6943 DRM_DEBUG("cik_irq_set: vblank 2\n");
6944 crtc3
|= VBLANK_INTERRUPT_MASK
;
6946 if (rdev
->irq
.crtc_vblank_int
[3] ||
6947 atomic_read(&rdev
->irq
.pflip
[3])) {
6948 DRM_DEBUG("cik_irq_set: vblank 3\n");
6949 crtc4
|= VBLANK_INTERRUPT_MASK
;
6951 if (rdev
->irq
.crtc_vblank_int
[4] ||
6952 atomic_read(&rdev
->irq
.pflip
[4])) {
6953 DRM_DEBUG("cik_irq_set: vblank 4\n");
6954 crtc5
|= VBLANK_INTERRUPT_MASK
;
6956 if (rdev
->irq
.crtc_vblank_int
[5] ||
6957 atomic_read(&rdev
->irq
.pflip
[5])) {
6958 DRM_DEBUG("cik_irq_set: vblank 5\n");
6959 crtc6
|= VBLANK_INTERRUPT_MASK
;
6961 if (rdev
->irq
.hpd
[0]) {
6962 DRM_DEBUG("cik_irq_set: hpd 1\n");
6963 hpd1
|= DC_HPDx_INT_EN
;
6965 if (rdev
->irq
.hpd
[1]) {
6966 DRM_DEBUG("cik_irq_set: hpd 2\n");
6967 hpd2
|= DC_HPDx_INT_EN
;
6969 if (rdev
->irq
.hpd
[2]) {
6970 DRM_DEBUG("cik_irq_set: hpd 3\n");
6971 hpd3
|= DC_HPDx_INT_EN
;
6973 if (rdev
->irq
.hpd
[3]) {
6974 DRM_DEBUG("cik_irq_set: hpd 4\n");
6975 hpd4
|= DC_HPDx_INT_EN
;
6977 if (rdev
->irq
.hpd
[4]) {
6978 DRM_DEBUG("cik_irq_set: hpd 5\n");
6979 hpd5
|= DC_HPDx_INT_EN
;
6981 if (rdev
->irq
.hpd
[5]) {
6982 DRM_DEBUG("cik_irq_set: hpd 6\n");
6983 hpd6
|= DC_HPDx_INT_EN
;
6986 if (rdev
->irq
.dpm_thermal
) {
6987 DRM_DEBUG("dpm thermal\n");
6988 if (rdev
->flags
& RADEON_IS_IGP
)
6989 thermal_int
|= THERM_INTH_MASK
| THERM_INTL_MASK
;
6991 thermal_int
|= THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
;
6994 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
6996 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, dma_cntl
);
6997 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, dma_cntl1
);
6999 WREG32(CP_ME1_PIPE0_INT_CNTL
, cp_m1p0
);
7000 WREG32(CP_ME1_PIPE1_INT_CNTL
, cp_m1p1
);
7001 WREG32(CP_ME1_PIPE2_INT_CNTL
, cp_m1p2
);
7002 WREG32(CP_ME1_PIPE3_INT_CNTL
, cp_m1p3
);
7003 WREG32(CP_ME2_PIPE0_INT_CNTL
, cp_m2p0
);
7004 WREG32(CP_ME2_PIPE1_INT_CNTL
, cp_m2p1
);
7005 WREG32(CP_ME2_PIPE2_INT_CNTL
, cp_m2p2
);
7006 WREG32(CP_ME2_PIPE3_INT_CNTL
, cp_m2p3
);
7008 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
7010 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
7011 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
7012 if (rdev
->num_crtc
>= 4) {
7013 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
7014 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
7016 if (rdev
->num_crtc
>= 6) {
7017 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
7018 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
7021 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
7022 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
7023 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
7024 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
7025 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
7026 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
7028 if (rdev
->flags
& RADEON_IS_IGP
)
7029 WREG32_SMC(CG_THERMAL_INT_CTRL
, thermal_int
);
7031 WREG32_SMC(CG_THERMAL_INT
, thermal_int
);
7037 * cik_irq_ack - ack interrupt sources
7039 * @rdev: radeon_device pointer
7041 * Ack interrupt sources on the GPU (vblanks, hpd,
7042 * etc.) (CIK). Certain interrupts sources are sw
7043 * generated and do not require an explicit ack.
7045 static inline void cik_irq_ack(struct radeon_device
*rdev
)
7049 rdev
->irq
.stat_regs
.cik
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
7050 rdev
->irq
.stat_regs
.cik
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
7051 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
7052 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
7053 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
7054 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
7055 rdev
->irq
.stat_regs
.cik
.disp_int_cont6
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE6
);
7057 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
7058 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
7059 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
)
7060 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
7061 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
7062 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
7063 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
7064 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
7066 if (rdev
->num_crtc
>= 4) {
7067 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
7068 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
7069 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
7070 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
7071 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
7072 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
7073 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
7074 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
7077 if (rdev
->num_crtc
>= 6) {
7078 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
7079 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
7080 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
7081 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
7082 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
7083 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
7084 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
7085 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
7088 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
7089 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
7090 tmp
|= DC_HPDx_INT_ACK
;
7091 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
7093 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
7094 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
7095 tmp
|= DC_HPDx_INT_ACK
;
7096 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
7098 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
7099 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
7100 tmp
|= DC_HPDx_INT_ACK
;
7101 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
7103 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
7104 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
7105 tmp
|= DC_HPDx_INT_ACK
;
7106 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
7108 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
7109 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
7110 tmp
|= DC_HPDx_INT_ACK
;
7111 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
7113 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
7114 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
7115 tmp
|= DC_HPDx_INT_ACK
;
7116 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
7121 * cik_irq_disable - disable interrupts
7123 * @rdev: radeon_device pointer
7125 * Disable interrupts on the hw (CIK).
7127 static void cik_irq_disable(struct radeon_device
*rdev
)
7129 cik_disable_interrupts(rdev
);
7130 /* Wait and acknowledge irq */
7133 cik_disable_interrupt_state(rdev
);
7137 * cik_irq_disable - disable interrupts for suspend
7139 * @rdev: radeon_device pointer
7141 * Disable interrupts and stop the RLC (CIK).
7144 static void cik_irq_suspend(struct radeon_device
*rdev
)
7146 cik_irq_disable(rdev
);
7151 * cik_irq_fini - tear down interrupt support
7153 * @rdev: radeon_device pointer
7155 * Disable interrupts on the hw and free the IH ring
7157 * Used for driver unload.
7159 static void cik_irq_fini(struct radeon_device
*rdev
)
7161 cik_irq_suspend(rdev
);
7162 r600_ih_ring_fini(rdev
);
7166 * cik_get_ih_wptr - get the IH ring buffer wptr
7168 * @rdev: radeon_device pointer
7170 * Get the IH ring buffer wptr from either the register
7171 * or the writeback memory buffer (CIK). Also check for
7172 * ring buffer overflow and deal with it.
7173 * Used by cik_irq_process().
7174 * Returns the value of the wptr.
7176 static inline u32
cik_get_ih_wptr(struct radeon_device
*rdev
)
7180 if (rdev
->wb
.enabled
)
7181 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
7183 wptr
= RREG32(IH_RB_WPTR
);
7185 if (wptr
& RB_OVERFLOW
) {
7186 /* When a ring buffer overflow happen start parsing interrupt
7187 * from the last not overwritten vector (wptr + 16). Hopefully
7188 * this should allow us to catchup.
7190 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7191 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
7192 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
7193 tmp
= RREG32(IH_RB_CNTL
);
7194 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
7195 WREG32(IH_RB_CNTL
, tmp
);
7197 return (wptr
& rdev
->ih
.ptr_mask
);
7201 * Each IV ring entry is 128 bits:
7202 * [7:0] - interrupt source id
7204 * [59:32] - interrupt source data
7205 * [63:60] - reserved
7208 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7209 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7210 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7211 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7212 * PIPE_ID - ME0 0=3D
7213 * - ME1&2 compute dispatcher (4 pipes each)
7215 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7216 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7217 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7220 * [127:96] - reserved
7223 * cik_irq_process - interrupt handler
7225 * @rdev: radeon_device pointer
7227 * Interrupt hander (CIK). Walk the IH ring,
7228 * ack interrupts and schedule work to handle
7230 * Returns irq process return code.
7232 int cik_irq_process(struct radeon_device
*rdev
)
7234 struct radeon_ring
*cp1_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7235 struct radeon_ring
*cp2_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7238 u32 src_id
, src_data
, ring_id
;
7239 u8 me_id
, pipe_id
, queue_id
;
7241 bool queue_hotplug
= false;
7242 bool queue_reset
= false;
7243 u32 addr
, status
, mc_client
;
7244 bool queue_thermal
= false;
7246 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
7249 wptr
= cik_get_ih_wptr(rdev
);
7252 /* is somebody else already processing irqs? */
7253 if (atomic_xchg(&rdev
->ih
.lock
, 1))
7256 rptr
= rdev
->ih
.rptr
;
7257 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
7259 /* Order reading of wptr vs. reading of IH ring data */
7262 /* display interrupts */
7265 while (rptr
!= wptr
) {
7266 /* wptr/rptr are in bytes! */
7267 ring_index
= rptr
/ 4;
7268 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
7269 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
7270 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
7273 case 1: /* D1 vblank/vline */
7275 case 0: /* D1 vblank */
7276 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
7277 if (rdev
->irq
.crtc_vblank_int
[0]) {
7278 drm_handle_vblank(rdev
->ddev
, 0);
7279 rdev
->pm
.vblank_sync
= true;
7280 wake_up(&rdev
->irq
.vblank_queue
);
7282 if (atomic_read(&rdev
->irq
.pflip
[0]))
7283 radeon_crtc_handle_flip(rdev
, 0);
7284 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
7285 DRM_DEBUG("IH: D1 vblank\n");
7288 case 1: /* D1 vline */
7289 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
7290 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
7291 DRM_DEBUG("IH: D1 vline\n");
7295 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7299 case 2: /* D2 vblank/vline */
7301 case 0: /* D2 vblank */
7302 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
7303 if (rdev
->irq
.crtc_vblank_int
[1]) {
7304 drm_handle_vblank(rdev
->ddev
, 1);
7305 rdev
->pm
.vblank_sync
= true;
7306 wake_up(&rdev
->irq
.vblank_queue
);
7308 if (atomic_read(&rdev
->irq
.pflip
[1]))
7309 radeon_crtc_handle_flip(rdev
, 1);
7310 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
7311 DRM_DEBUG("IH: D2 vblank\n");
7314 case 1: /* D2 vline */
7315 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
7316 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
7317 DRM_DEBUG("IH: D2 vline\n");
7321 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7325 case 3: /* D3 vblank/vline */
7327 case 0: /* D3 vblank */
7328 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
7329 if (rdev
->irq
.crtc_vblank_int
[2]) {
7330 drm_handle_vblank(rdev
->ddev
, 2);
7331 rdev
->pm
.vblank_sync
= true;
7332 wake_up(&rdev
->irq
.vblank_queue
);
7334 if (atomic_read(&rdev
->irq
.pflip
[2]))
7335 radeon_crtc_handle_flip(rdev
, 2);
7336 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
7337 DRM_DEBUG("IH: D3 vblank\n");
7340 case 1: /* D3 vline */
7341 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
7342 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
7343 DRM_DEBUG("IH: D3 vline\n");
7347 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7351 case 4: /* D4 vblank/vline */
7353 case 0: /* D4 vblank */
7354 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
7355 if (rdev
->irq
.crtc_vblank_int
[3]) {
7356 drm_handle_vblank(rdev
->ddev
, 3);
7357 rdev
->pm
.vblank_sync
= true;
7358 wake_up(&rdev
->irq
.vblank_queue
);
7360 if (atomic_read(&rdev
->irq
.pflip
[3]))
7361 radeon_crtc_handle_flip(rdev
, 3);
7362 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
7363 DRM_DEBUG("IH: D4 vblank\n");
7366 case 1: /* D4 vline */
7367 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
7368 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
7369 DRM_DEBUG("IH: D4 vline\n");
7373 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7377 case 5: /* D5 vblank/vline */
7379 case 0: /* D5 vblank */
7380 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
7381 if (rdev
->irq
.crtc_vblank_int
[4]) {
7382 drm_handle_vblank(rdev
->ddev
, 4);
7383 rdev
->pm
.vblank_sync
= true;
7384 wake_up(&rdev
->irq
.vblank_queue
);
7386 if (atomic_read(&rdev
->irq
.pflip
[4]))
7387 radeon_crtc_handle_flip(rdev
, 4);
7388 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
7389 DRM_DEBUG("IH: D5 vblank\n");
7392 case 1: /* D5 vline */
7393 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
7394 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
7395 DRM_DEBUG("IH: D5 vline\n");
7399 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7403 case 6: /* D6 vblank/vline */
7405 case 0: /* D6 vblank */
7406 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
7407 if (rdev
->irq
.crtc_vblank_int
[5]) {
7408 drm_handle_vblank(rdev
->ddev
, 5);
7409 rdev
->pm
.vblank_sync
= true;
7410 wake_up(&rdev
->irq
.vblank_queue
);
7412 if (atomic_read(&rdev
->irq
.pflip
[5]))
7413 radeon_crtc_handle_flip(rdev
, 5);
7414 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
7415 DRM_DEBUG("IH: D6 vblank\n");
7418 case 1: /* D6 vline */
7419 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
7420 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
7421 DRM_DEBUG("IH: D6 vline\n");
7425 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7429 case 42: /* HPD hotplug */
7432 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
7433 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~DC_HPD1_INTERRUPT
;
7434 queue_hotplug
= true;
7435 DRM_DEBUG("IH: HPD1\n");
7439 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
7440 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
7441 queue_hotplug
= true;
7442 DRM_DEBUG("IH: HPD2\n");
7446 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
7447 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
7448 queue_hotplug
= true;
7449 DRM_DEBUG("IH: HPD3\n");
7453 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
7454 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
7455 queue_hotplug
= true;
7456 DRM_DEBUG("IH: HPD4\n");
7460 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
7461 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
7462 queue_hotplug
= true;
7463 DRM_DEBUG("IH: HPD5\n");
7467 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
7468 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
7469 queue_hotplug
= true;
7470 DRM_DEBUG("IH: HPD6\n");
7474 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7479 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data
);
7480 radeon_fence_process(rdev
, R600_RING_TYPE_UVD_INDEX
);
7484 addr
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
);
7485 status
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
);
7486 mc_client
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT
);
7487 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
7488 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7490 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7492 cik_vm_decode_fault(rdev
, status
, addr
, mc_client
);
7493 /* reset addr and status */
7494 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
7497 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data
);
7500 radeon_fence_process(rdev
, TN_RING_TYPE_VCE1_INDEX
);
7503 radeon_fence_process(rdev
, TN_RING_TYPE_VCE2_INDEX
);
7506 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7510 case 176: /* GFX RB CP_INT */
7511 case 177: /* GFX IB CP_INT */
7512 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7514 case 181: /* CP EOP event */
7515 DRM_DEBUG("IH: CP EOP\n");
7516 /* XXX check the bitfield order! */
7517 me_id
= (ring_id
& 0x60) >> 5;
7518 pipe_id
= (ring_id
& 0x18) >> 3;
7519 queue_id
= (ring_id
& 0x7) >> 0;
7522 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7526 if ((cp1_ring
->me
== me_id
) & (cp1_ring
->pipe
== pipe_id
))
7527 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7528 if ((cp2_ring
->me
== me_id
) & (cp2_ring
->pipe
== pipe_id
))
7529 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7533 case 184: /* CP Privileged reg access */
7534 DRM_ERROR("Illegal register access in command stream\n");
7535 /* XXX check the bitfield order! */
7536 me_id
= (ring_id
& 0x60) >> 5;
7537 pipe_id
= (ring_id
& 0x18) >> 3;
7538 queue_id
= (ring_id
& 0x7) >> 0;
7541 /* This results in a full GPU reset, but all we need to do is soft
7542 * reset the CP for gfx
7556 case 185: /* CP Privileged inst */
7557 DRM_ERROR("Illegal instruction in command stream\n");
7558 /* XXX check the bitfield order! */
7559 me_id
= (ring_id
& 0x60) >> 5;
7560 pipe_id
= (ring_id
& 0x18) >> 3;
7561 queue_id
= (ring_id
& 0x7) >> 0;
7564 /* This results in a full GPU reset, but all we need to do is soft
7565 * reset the CP for gfx
7579 case 224: /* SDMA trap event */
7580 /* XXX check the bitfield order! */
7581 me_id
= (ring_id
& 0x3) >> 0;
7582 queue_id
= (ring_id
& 0xc) >> 2;
7583 DRM_DEBUG("IH: SDMA trap\n");
7588 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
7601 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7613 case 230: /* thermal low to high */
7614 DRM_DEBUG("IH: thermal low to high\n");
7615 rdev
->pm
.dpm
.thermal
.high_to_low
= false;
7616 queue_thermal
= true;
7618 case 231: /* thermal high to low */
7619 DRM_DEBUG("IH: thermal high to low\n");
7620 rdev
->pm
.dpm
.thermal
.high_to_low
= true;
7621 queue_thermal
= true;
7623 case 233: /* GUI IDLE */
7624 DRM_DEBUG("IH: GUI idle\n");
7626 case 241: /* SDMA Privileged inst */
7627 case 247: /* SDMA Privileged inst */
7628 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7629 /* XXX check the bitfield order! */
7630 me_id
= (ring_id
& 0x3) >> 0;
7631 queue_id
= (ring_id
& 0xc) >> 2;
7666 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7670 /* wptr/rptr are in bytes! */
7672 rptr
&= rdev
->ih
.ptr_mask
;
7675 schedule_work(&rdev
->hotplug_work
);
7677 schedule_work(&rdev
->reset_work
);
7679 schedule_work(&rdev
->pm
.dpm
.thermal
.work
);
7680 rdev
->ih
.rptr
= rptr
;
7681 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
7682 atomic_set(&rdev
->ih
.lock
, 0);
7684 /* make sure wptr hasn't changed while processing */
7685 wptr
= cik_get_ih_wptr(rdev
);
7693 * startup/shutdown callbacks
7696 * cik_startup - program the asic to a functional state
7698 * @rdev: radeon_device pointer
7700 * Programs the asic to a functional state (CIK).
7701 * Called by cik_init() and cik_resume().
7702 * Returns 0 for success, error for failure.
7704 static int cik_startup(struct radeon_device
*rdev
)
7706 struct radeon_ring
*ring
;
7709 /* enable pcie gen2/3 link */
7710 cik_pcie_gen3_enable(rdev
);
7712 cik_program_aspm(rdev
);
7714 /* scratch needs to be initialized before MC */
7715 r
= r600_vram_scratch_init(rdev
);
7719 cik_mc_program(rdev
);
7721 if (!(rdev
->flags
& RADEON_IS_IGP
) && !rdev
->pm
.dpm_enabled
) {
7722 r
= ci_mc_load_microcode(rdev
);
7724 DRM_ERROR("Failed to load MC firmware!\n");
7729 r
= cik_pcie_gart_enable(rdev
);
7734 /* allocate rlc buffers */
7735 if (rdev
->flags
& RADEON_IS_IGP
) {
7736 if (rdev
->family
== CHIP_KAVERI
) {
7737 rdev
->rlc
.reg_list
= spectre_rlc_save_restore_register_list
;
7738 rdev
->rlc
.reg_list_size
=
7739 (u32
)ARRAY_SIZE(spectre_rlc_save_restore_register_list
);
7741 rdev
->rlc
.reg_list
= kalindi_rlc_save_restore_register_list
;
7742 rdev
->rlc
.reg_list_size
=
7743 (u32
)ARRAY_SIZE(kalindi_rlc_save_restore_register_list
);
7746 rdev
->rlc
.cs_data
= ci_cs_data
;
7747 rdev
->rlc
.cp_table_size
= CP_ME_TABLE_SIZE
* 5 * 4;
7748 r
= sumo_rlc_init(rdev
);
7750 DRM_ERROR("Failed to init rlc BOs!\n");
7754 /* allocate wb buffer */
7755 r
= radeon_wb_init(rdev
);
7759 /* allocate mec buffers */
7760 r
= cik_mec_init(rdev
);
7762 DRM_ERROR("Failed to init MEC BOs!\n");
7766 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7768 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7772 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7774 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7778 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7780 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7784 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
7786 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7790 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7792 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7796 r
= radeon_uvd_resume(rdev
);
7798 r
= uvd_v4_2_resume(rdev
);
7800 r
= radeon_fence_driver_start_ring(rdev
,
7801 R600_RING_TYPE_UVD_INDEX
);
7803 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
7807 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
7809 r
= radeon_vce_resume(rdev
);
7811 r
= vce_v2_0_resume(rdev
);
7813 r
= radeon_fence_driver_start_ring(rdev
,
7814 TN_RING_TYPE_VCE1_INDEX
);
7816 r
= radeon_fence_driver_start_ring(rdev
,
7817 TN_RING_TYPE_VCE2_INDEX
);
7820 dev_err(rdev
->dev
, "VCE init error (%d).\n", r
);
7821 rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
].ring_size
= 0;
7822 rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
].ring_size
= 0;
7826 if (!rdev
->irq
.installed
) {
7827 r
= radeon_irq_kms_init(rdev
);
7832 r
= cik_irq_init(rdev
);
7834 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
7835 radeon_irq_kms_fini(rdev
);
7840 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
7841 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
7842 PACKET3(PACKET3_NOP
, 0x3FFF));
7846 /* set up the compute queues */
7847 /* type-2 packets are deprecated on MEC, use type-3 instead */
7848 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7849 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
7850 PACKET3(PACKET3_NOP
, 0x3FFF));
7853 ring
->me
= 1; /* first MEC */
7854 ring
->pipe
= 0; /* first pipe */
7855 ring
->queue
= 0; /* first queue */
7856 ring
->wptr_offs
= CIK_WB_CP1_WPTR_OFFSET
;
7858 /* type-2 packets are deprecated on MEC, use type-3 instead */
7859 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7860 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
7861 PACKET3(PACKET3_NOP
, 0x3FFF));
7864 /* dGPU only have 1 MEC */
7865 ring
->me
= 1; /* first MEC */
7866 ring
->pipe
= 0; /* first pipe */
7867 ring
->queue
= 1; /* second queue */
7868 ring
->wptr_offs
= CIK_WB_CP2_WPTR_OFFSET
;
7870 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
7871 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
7872 SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7876 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
7877 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
7878 SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7882 r
= cik_cp_resume(rdev
);
7886 r
= cik_sdma_resume(rdev
);
7890 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
7891 if (ring
->ring_size
) {
7892 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7895 r
= uvd_v1_0_init(rdev
);
7897 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
7902 ring
= &rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
];
7903 if (ring
->ring_size
)
7904 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7907 ring
= &rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
];
7908 if (ring
->ring_size
)
7909 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7913 r
= vce_v1_0_init(rdev
);
7914 else if (r
!= -ENOENT
)
7915 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r
);
7917 r
= radeon_ib_pool_init(rdev
);
7919 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
7923 r
= radeon_vm_manager_init(rdev
);
7925 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
7929 r
= dce6_audio_init(rdev
);
7937 * cik_resume - resume the asic to a functional state
7939 * @rdev: radeon_device pointer
7941 * Programs the asic to a functional state (CIK).
7943 * Returns 0 for success, error for failure.
7945 int cik_resume(struct radeon_device
*rdev
)
7950 atom_asic_init(rdev
->mode_info
.atom_context
);
7952 /* init golden registers */
7953 cik_init_golden_registers(rdev
);
7955 if (rdev
->pm
.pm_method
== PM_METHOD_DPM
)
7956 radeon_pm_resume(rdev
);
7958 rdev
->accel_working
= true;
7959 r
= cik_startup(rdev
);
7961 DRM_ERROR("cik startup failed on resume\n");
7962 rdev
->accel_working
= false;
7971 * cik_suspend - suspend the asic
7973 * @rdev: radeon_device pointer
7975 * Bring the chip into a state suitable for suspend (CIK).
7976 * Called at suspend.
7977 * Returns 0 for success.
7979 int cik_suspend(struct radeon_device
*rdev
)
7981 radeon_pm_suspend(rdev
);
7982 dce6_audio_fini(rdev
);
7983 radeon_vm_manager_fini(rdev
);
7984 cik_cp_enable(rdev
, false);
7985 cik_sdma_enable(rdev
, false);
7986 uvd_v1_0_fini(rdev
);
7987 radeon_uvd_suspend(rdev
);
7988 radeon_vce_suspend(rdev
);
7991 cik_irq_suspend(rdev
);
7992 radeon_wb_disable(rdev
);
7993 cik_pcie_gart_disable(rdev
);
7997 /* Plan is to move initialization in that function and use
7998 * helper function so that radeon_device_init pretty much
7999 * do nothing more than calling asic specific function. This
8000 * should also allow to remove a bunch of callback function
8004 * cik_init - asic specific driver and hw init
8006 * @rdev: radeon_device pointer
8008 * Setup asic specific driver variables and program the hw
8009 * to a functional state (CIK).
8010 * Called at driver startup.
8011 * Returns 0 for success, errors for failure.
8013 int cik_init(struct radeon_device
*rdev
)
8015 struct radeon_ring
*ring
;
8019 if (!radeon_get_bios(rdev
)) {
8020 if (ASIC_IS_AVIVO(rdev
))
8023 /* Must be an ATOMBIOS */
8024 if (!rdev
->is_atom_bios
) {
8025 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
8028 r
= radeon_atombios_init(rdev
);
8032 /* Post card if necessary */
8033 if (!radeon_card_posted(rdev
)) {
8035 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
8038 DRM_INFO("GPU not posted. posting now...\n");
8039 atom_asic_init(rdev
->mode_info
.atom_context
);
8041 /* init golden registers */
8042 cik_init_golden_registers(rdev
);
8043 /* Initialize scratch registers */
8044 cik_scratch_init(rdev
);
8045 /* Initialize surface registers */
8046 radeon_surface_init(rdev
);
8047 /* Initialize clocks */
8048 radeon_get_clock_info(rdev
->ddev
);
8051 r
= radeon_fence_driver_init(rdev
);
8055 /* initialize memory controller */
8056 r
= cik_mc_init(rdev
);
8059 /* Memory manager */
8060 r
= radeon_bo_init(rdev
);
8064 if (rdev
->flags
& RADEON_IS_IGP
) {
8065 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
8066 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
) {
8067 r
= cik_init_microcode(rdev
);
8069 DRM_ERROR("Failed to load firmware!\n");
8074 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
8075 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
||
8077 r
= cik_init_microcode(rdev
);
8079 DRM_ERROR("Failed to load firmware!\n");
8085 /* Initialize power management */
8086 radeon_pm_init(rdev
);
8088 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
8089 ring
->ring_obj
= NULL
;
8090 r600_ring_init(rdev
, ring
, 1024 * 1024);
8092 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
8093 ring
->ring_obj
= NULL
;
8094 r600_ring_init(rdev
, ring
, 1024 * 1024);
8095 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_index
);
8099 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
8100 ring
->ring_obj
= NULL
;
8101 r600_ring_init(rdev
, ring
, 1024 * 1024);
8102 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_index
);
8106 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
8107 ring
->ring_obj
= NULL
;
8108 r600_ring_init(rdev
, ring
, 256 * 1024);
8110 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
8111 ring
->ring_obj
= NULL
;
8112 r600_ring_init(rdev
, ring
, 256 * 1024);
8114 r
= radeon_uvd_init(rdev
);
8116 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
8117 ring
->ring_obj
= NULL
;
8118 r600_ring_init(rdev
, ring
, 4096);
8121 r
= radeon_vce_init(rdev
);
8123 ring
= &rdev
->ring
[TN_RING_TYPE_VCE1_INDEX
];
8124 ring
->ring_obj
= NULL
;
8125 r600_ring_init(rdev
, ring
, 4096);
8127 ring
= &rdev
->ring
[TN_RING_TYPE_VCE2_INDEX
];
8128 ring
->ring_obj
= NULL
;
8129 r600_ring_init(rdev
, ring
, 4096);
8132 rdev
->ih
.ring_obj
= NULL
;
8133 r600_ih_ring_init(rdev
, 64 * 1024);
8135 r
= r600_pcie_gart_init(rdev
);
8139 rdev
->accel_working
= true;
8140 r
= cik_startup(rdev
);
8142 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
8144 cik_sdma_fini(rdev
);
8146 sumo_rlc_fini(rdev
);
8148 radeon_wb_fini(rdev
);
8149 radeon_ib_pool_fini(rdev
);
8150 radeon_vm_manager_fini(rdev
);
8151 radeon_irq_kms_fini(rdev
);
8152 cik_pcie_gart_fini(rdev
);
8153 rdev
->accel_working
= false;
8156 /* Don't start up if the MC ucode is missing.
8157 * The default clocks and voltages before the MC ucode
8158 * is loaded are not suffient for advanced operations.
8160 if (!rdev
->mc_fw
&& !(rdev
->flags
& RADEON_IS_IGP
)) {
8161 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8169 * cik_fini - asic specific driver and hw fini
8171 * @rdev: radeon_device pointer
8173 * Tear down the asic specific driver variables and program the hw
8174 * to an idle state (CIK).
8175 * Called at driver unload.
8177 void cik_fini(struct radeon_device
*rdev
)
8179 radeon_pm_fini(rdev
);
8181 cik_sdma_fini(rdev
);
8185 sumo_rlc_fini(rdev
);
8187 radeon_wb_fini(rdev
);
8188 radeon_vm_manager_fini(rdev
);
8189 radeon_ib_pool_fini(rdev
);
8190 radeon_irq_kms_fini(rdev
);
8191 uvd_v1_0_fini(rdev
);
8192 radeon_uvd_fini(rdev
);
8193 radeon_vce_fini(rdev
);
8194 cik_pcie_gart_fini(rdev
);
8195 r600_vram_scratch_fini(rdev
);
8196 radeon_gem_fini(rdev
);
8197 radeon_fence_driver_fini(rdev
);
8198 radeon_bo_fini(rdev
);
8199 radeon_atombios_fini(rdev
);
8204 void dce8_program_fmt(struct drm_encoder
*encoder
)
8206 struct drm_device
*dev
= encoder
->dev
;
8207 struct radeon_device
*rdev
= dev
->dev_private
;
8208 struct radeon_encoder
*radeon_encoder
= to_radeon_encoder(encoder
);
8209 struct radeon_crtc
*radeon_crtc
= to_radeon_crtc(encoder
->crtc
);
8210 struct drm_connector
*connector
= radeon_get_connector_for_encoder(encoder
);
8213 enum radeon_connector_dither dither
= RADEON_FMT_DITHER_DISABLE
;
8216 struct radeon_connector
*radeon_connector
= to_radeon_connector(connector
);
8217 bpc
= radeon_get_monitor_bpc(connector
);
8218 dither
= radeon_connector
->dither
;
8221 /* LVDS/eDP FMT is set up by atom */
8222 if (radeon_encoder
->devices
& ATOM_DEVICE_LCD_SUPPORT
)
8225 /* not needed for analog */
8226 if ((radeon_encoder
->encoder_id
== ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1
) ||
8227 (radeon_encoder
->encoder_id
== ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2
))
8235 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8236 /* XXX sort out optimal dither settings */
8237 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8238 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(0));
8240 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(0));
8243 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8244 /* XXX sort out optimal dither settings */
8245 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8246 FMT_RGB_RANDOM_ENABLE
|
8247 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(1));
8249 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(1));
8252 if (dither
== RADEON_FMT_DITHER_ENABLE
)
8253 /* XXX sort out optimal dither settings */
8254 tmp
|= (FMT_FRAME_RANDOM_ENABLE
| FMT_HIGHPASS_RANDOM_ENABLE
|
8255 FMT_RGB_RANDOM_ENABLE
|
8256 FMT_SPATIAL_DITHER_EN
| FMT_SPATIAL_DITHER_DEPTH(2));
8258 tmp
|= (FMT_TRUNCATE_EN
| FMT_TRUNCATE_DEPTH(2));
8265 WREG32(FMT_BIT_DEPTH_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8268 /* display watermark setup */
8270 * dce8_line_buffer_adjust - Set up the line buffer
8272 * @rdev: radeon_device pointer
8273 * @radeon_crtc: the selected display controller
8274 * @mode: the current display mode on the selected display
8277 * Setup up the line buffer allocation for
8278 * the selected display controller (CIK).
8279 * Returns the line buffer size in pixels.
8281 static u32
dce8_line_buffer_adjust(struct radeon_device
*rdev
,
8282 struct radeon_crtc
*radeon_crtc
,
8283 struct drm_display_mode
*mode
)
8285 u32 tmp
, buffer_alloc
, i
;
8286 u32 pipe_offset
= radeon_crtc
->crtc_id
* 0x20;
8289 * There are 6 line buffers, one for each display controllers.
8290 * There are 3 partitions per LB. Select the number of partitions
8291 * to enable based on the display width. For display widths larger
8292 * than 4096, you need use to use 2 display controllers and combine
8293 * them using the stereo blender.
8295 if (radeon_crtc
->base
.enabled
&& mode
) {
8296 if (mode
->crtc_hdisplay
< 1920) {
8299 } else if (mode
->crtc_hdisplay
< 2560) {
8302 } else if (mode
->crtc_hdisplay
< 4096) {
8304 buffer_alloc
= (rdev
->flags
& RADEON_IS_IGP
) ? 2 : 4;
8306 DRM_DEBUG_KMS("Mode too big for LB!\n");
8308 buffer_alloc
= (rdev
->flags
& RADEON_IS_IGP
) ? 2 : 4;
8315 WREG32(LB_MEMORY_CTRL
+ radeon_crtc
->crtc_offset
,
8316 LB_MEMORY_CONFIG(tmp
) | LB_MEMORY_SIZE(0x6B0));
8318 WREG32(PIPE0_DMIF_BUFFER_CONTROL
+ pipe_offset
,
8319 DMIF_BUFFERS_ALLOCATED(buffer_alloc
));
8320 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
8321 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL
+ pipe_offset
) &
8322 DMIF_BUFFERS_ALLOCATED_COMPLETED
)
8327 if (radeon_crtc
->base
.enabled
&& mode
) {
8339 /* controller not enabled, so no lb used */
8344 * cik_get_number_of_dram_channels - get the number of dram channels
8346 * @rdev: radeon_device pointer
8348 * Look up the number of video ram channels (CIK).
8349 * Used for display watermark bandwidth calculations
8350 * Returns the number of dram channels
8352 static u32
cik_get_number_of_dram_channels(struct radeon_device
*rdev
)
8354 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
8356 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
8379 struct dce8_wm_params
{
8380 u32 dram_channels
; /* number of dram channels */
8381 u32 yclk
; /* bandwidth per dram data pin in kHz */
8382 u32 sclk
; /* engine clock in kHz */
8383 u32 disp_clk
; /* display clock in kHz */
8384 u32 src_width
; /* viewport width */
8385 u32 active_time
; /* active display time in ns */
8386 u32 blank_time
; /* blank time in ns */
8387 bool interlaced
; /* mode is interlaced */
8388 fixed20_12 vsc
; /* vertical scale ratio */
8389 u32 num_heads
; /* number of active crtcs */
8390 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
8391 u32 lb_size
; /* line buffer allocated to pipe */
8392 u32 vtaps
; /* vertical scaler taps */
8396 * dce8_dram_bandwidth - get the dram bandwidth
8398 * @wm: watermark calculation data
8400 * Calculate the raw dram bandwidth (CIK).
8401 * Used for display watermark bandwidth calculations
8402 * Returns the dram bandwidth in MBytes/s
8404 static u32
dce8_dram_bandwidth(struct dce8_wm_params
*wm
)
8406 /* Calculate raw DRAM Bandwidth */
8407 fixed20_12 dram_efficiency
; /* 0.7 */
8408 fixed20_12 yclk
, dram_channels
, bandwidth
;
8411 a
.full
= dfixed_const(1000);
8412 yclk
.full
= dfixed_const(wm
->yclk
);
8413 yclk
.full
= dfixed_div(yclk
, a
);
8414 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8415 a
.full
= dfixed_const(10);
8416 dram_efficiency
.full
= dfixed_const(7);
8417 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
8418 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8419 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
8421 return dfixed_trunc(bandwidth
);
8425 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8427 * @wm: watermark calculation data
8429 * Calculate the dram bandwidth used for display (CIK).
8430 * Used for display watermark bandwidth calculations
8431 * Returns the dram bandwidth for display in MBytes/s
8433 static u32
dce8_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8435 /* Calculate DRAM Bandwidth and the part allocated to display. */
8436 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
8437 fixed20_12 yclk
, dram_channels
, bandwidth
;
8440 a
.full
= dfixed_const(1000);
8441 yclk
.full
= dfixed_const(wm
->yclk
);
8442 yclk
.full
= dfixed_div(yclk
, a
);
8443 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8444 a
.full
= dfixed_const(10);
8445 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
8446 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
8447 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8448 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
8450 return dfixed_trunc(bandwidth
);
8454 * dce8_data_return_bandwidth - get the data return bandwidth
8456 * @wm: watermark calculation data
8458 * Calculate the data return bandwidth used for display (CIK).
8459 * Used for display watermark bandwidth calculations
8460 * Returns the data return bandwidth in MBytes/s
8462 static u32
dce8_data_return_bandwidth(struct dce8_wm_params
*wm
)
8464 /* Calculate the display Data return Bandwidth */
8465 fixed20_12 return_efficiency
; /* 0.8 */
8466 fixed20_12 sclk
, bandwidth
;
8469 a
.full
= dfixed_const(1000);
8470 sclk
.full
= dfixed_const(wm
->sclk
);
8471 sclk
.full
= dfixed_div(sclk
, a
);
8472 a
.full
= dfixed_const(10);
8473 return_efficiency
.full
= dfixed_const(8);
8474 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
8475 a
.full
= dfixed_const(32);
8476 bandwidth
.full
= dfixed_mul(a
, sclk
);
8477 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
8479 return dfixed_trunc(bandwidth
);
8483 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8485 * @wm: watermark calculation data
8487 * Calculate the dmif bandwidth used for display (CIK).
8488 * Used for display watermark bandwidth calculations
8489 * Returns the dmif bandwidth in MBytes/s
8491 static u32
dce8_dmif_request_bandwidth(struct dce8_wm_params
*wm
)
8493 /* Calculate the DMIF Request Bandwidth */
8494 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
8495 fixed20_12 disp_clk
, bandwidth
;
8498 a
.full
= dfixed_const(1000);
8499 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
8500 disp_clk
.full
= dfixed_div(disp_clk
, a
);
8501 a
.full
= dfixed_const(32);
8502 b
.full
= dfixed_mul(a
, disp_clk
);
8504 a
.full
= dfixed_const(10);
8505 disp_clk_request_efficiency
.full
= dfixed_const(8);
8506 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
8508 bandwidth
.full
= dfixed_mul(b
, disp_clk_request_efficiency
);
8510 return dfixed_trunc(bandwidth
);
8514 * dce8_available_bandwidth - get the min available bandwidth
8516 * @wm: watermark calculation data
8518 * Calculate the min available bandwidth used for display (CIK).
8519 * Used for display watermark bandwidth calculations
8520 * Returns the min available bandwidth in MBytes/s
8522 static u32
dce8_available_bandwidth(struct dce8_wm_params
*wm
)
8524 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8525 u32 dram_bandwidth
= dce8_dram_bandwidth(wm
);
8526 u32 data_return_bandwidth
= dce8_data_return_bandwidth(wm
);
8527 u32 dmif_req_bandwidth
= dce8_dmif_request_bandwidth(wm
);
8529 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
8533 * dce8_average_bandwidth - get the average available bandwidth
8535 * @wm: watermark calculation data
8537 * Calculate the average available bandwidth used for display (CIK).
8538 * Used for display watermark bandwidth calculations
8539 * Returns the average available bandwidth in MBytes/s
8541 static u32
dce8_average_bandwidth(struct dce8_wm_params
*wm
)
8543 /* Calculate the display mode Average Bandwidth
8544 * DisplayMode should contain the source and destination dimensions,
8548 fixed20_12 line_time
;
8549 fixed20_12 src_width
;
8550 fixed20_12 bandwidth
;
8553 a
.full
= dfixed_const(1000);
8554 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
8555 line_time
.full
= dfixed_div(line_time
, a
);
8556 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
8557 src_width
.full
= dfixed_const(wm
->src_width
);
8558 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
8559 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
8560 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
8562 return dfixed_trunc(bandwidth
);
8566 * dce8_latency_watermark - get the latency watermark
8568 * @wm: watermark calculation data
8570 * Calculate the latency watermark (CIK).
8571 * Used for display watermark bandwidth calculations
8572 * Returns the latency watermark in ns
8574 static u32
dce8_latency_watermark(struct dce8_wm_params
*wm
)
8576 /* First calculate the latency in ns */
8577 u32 mc_latency
= 2000; /* 2000 ns. */
8578 u32 available_bandwidth
= dce8_available_bandwidth(wm
);
8579 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
8580 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
8581 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
8582 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
8583 (wm
->num_heads
* cursor_line_pair_return_time
);
8584 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
8585 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
8586 u32 tmp
, dmif_size
= 12288;
8589 if (wm
->num_heads
== 0)
8592 a
.full
= dfixed_const(2);
8593 b
.full
= dfixed_const(1);
8594 if ((wm
->vsc
.full
> a
.full
) ||
8595 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
8597 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
8598 max_src_lines_per_dst_line
= 4;
8600 max_src_lines_per_dst_line
= 2;
8602 a
.full
= dfixed_const(available_bandwidth
);
8603 b
.full
= dfixed_const(wm
->num_heads
);
8604 a
.full
= dfixed_div(a
, b
);
8606 b
.full
= dfixed_const(mc_latency
+ 512);
8607 c
.full
= dfixed_const(wm
->disp_clk
);
8608 b
.full
= dfixed_div(b
, c
);
8610 c
.full
= dfixed_const(dmif_size
);
8611 b
.full
= dfixed_div(c
, b
);
8613 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
8615 b
.full
= dfixed_const(1000);
8616 c
.full
= dfixed_const(wm
->disp_clk
);
8617 b
.full
= dfixed_div(c
, b
);
8618 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
8619 b
.full
= dfixed_mul(b
, c
);
8621 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
8623 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
8624 b
.full
= dfixed_const(1000);
8625 c
.full
= dfixed_const(lb_fill_bw
);
8626 b
.full
= dfixed_div(c
, b
);
8627 a
.full
= dfixed_div(a
, b
);
8628 line_fill_time
= dfixed_trunc(a
);
8630 if (line_fill_time
< wm
->active_time
)
8633 return latency
+ (line_fill_time
- wm
->active_time
);
8638 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8639 * average and available dram bandwidth
8641 * @wm: watermark calculation data
8643 * Check if the display average bandwidth fits in the display
8644 * dram bandwidth (CIK).
8645 * Used for display watermark bandwidth calculations
8646 * Returns true if the display fits, false if not.
8648 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8650 if (dce8_average_bandwidth(wm
) <=
8651 (dce8_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
8658 * dce8_average_bandwidth_vs_available_bandwidth - check
8659 * average and available bandwidth
8661 * @wm: watermark calculation data
8663 * Check if the display average bandwidth fits in the display
8664 * available bandwidth (CIK).
8665 * Used for display watermark bandwidth calculations
8666 * Returns true if the display fits, false if not.
8668 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params
*wm
)
8670 if (dce8_average_bandwidth(wm
) <=
8671 (dce8_available_bandwidth(wm
) / wm
->num_heads
))
8678 * dce8_check_latency_hiding - check latency hiding
8680 * @wm: watermark calculation data
8682 * Check latency hiding (CIK).
8683 * Used for display watermark bandwidth calculations
8684 * Returns true if the display fits, false if not.
8686 static bool dce8_check_latency_hiding(struct dce8_wm_params
*wm
)
8688 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
8689 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
8690 u32 latency_tolerant_lines
;
8694 a
.full
= dfixed_const(1);
8695 if (wm
->vsc
.full
> a
.full
)
8696 latency_tolerant_lines
= 1;
8698 if (lb_partitions
<= (wm
->vtaps
+ 1))
8699 latency_tolerant_lines
= 1;
8701 latency_tolerant_lines
= 2;
8704 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
8706 if (dce8_latency_watermark(wm
) <= latency_hiding
)
8713 * dce8_program_watermarks - program display watermarks
8715 * @rdev: radeon_device pointer
8716 * @radeon_crtc: the selected display controller
8717 * @lb_size: line buffer size
8718 * @num_heads: number of display controllers in use
8720 * Calculate and program the display watermarks for the
8721 * selected display controller (CIK).
8723 static void dce8_program_watermarks(struct radeon_device
*rdev
,
8724 struct radeon_crtc
*radeon_crtc
,
8725 u32 lb_size
, u32 num_heads
)
8727 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
8728 struct dce8_wm_params wm_low
, wm_high
;
8731 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
8734 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
8735 pixel_period
= 1000000 / (u32
)mode
->clock
;
8736 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
8738 /* watermark for high clocks */
8739 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8740 rdev
->pm
.dpm_enabled
) {
8742 radeon_dpm_get_mclk(rdev
, false) * 10;
8744 radeon_dpm_get_sclk(rdev
, false) * 10;
8746 wm_high
.yclk
= rdev
->pm
.current_mclk
* 10;
8747 wm_high
.sclk
= rdev
->pm
.current_sclk
* 10;
8750 wm_high
.disp_clk
= mode
->clock
;
8751 wm_high
.src_width
= mode
->crtc_hdisplay
;
8752 wm_high
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8753 wm_high
.blank_time
= line_time
- wm_high
.active_time
;
8754 wm_high
.interlaced
= false;
8755 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8756 wm_high
.interlaced
= true;
8757 wm_high
.vsc
= radeon_crtc
->vsc
;
8759 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8761 wm_high
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8762 wm_high
.lb_size
= lb_size
;
8763 wm_high
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8764 wm_high
.num_heads
= num_heads
;
8766 /* set for high clocks */
8767 latency_watermark_a
= min(dce8_latency_watermark(&wm_high
), (u32
)65535);
8769 /* possibly force display priority to high */
8770 /* should really do this at mode validation time... */
8771 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high
) ||
8772 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high
) ||
8773 !dce8_check_latency_hiding(&wm_high
) ||
8774 (rdev
->disp_priority
== 2)) {
8775 DRM_DEBUG_KMS("force priority to high\n");
8778 /* watermark for low clocks */
8779 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8780 rdev
->pm
.dpm_enabled
) {
8782 radeon_dpm_get_mclk(rdev
, true) * 10;
8784 radeon_dpm_get_sclk(rdev
, true) * 10;
8786 wm_low
.yclk
= rdev
->pm
.current_mclk
* 10;
8787 wm_low
.sclk
= rdev
->pm
.current_sclk
* 10;
8790 wm_low
.disp_clk
= mode
->clock
;
8791 wm_low
.src_width
= mode
->crtc_hdisplay
;
8792 wm_low
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8793 wm_low
.blank_time
= line_time
- wm_low
.active_time
;
8794 wm_low
.interlaced
= false;
8795 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8796 wm_low
.interlaced
= true;
8797 wm_low
.vsc
= radeon_crtc
->vsc
;
8799 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8801 wm_low
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8802 wm_low
.lb_size
= lb_size
;
8803 wm_low
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8804 wm_low
.num_heads
= num_heads
;
8806 /* set for low clocks */
8807 latency_watermark_b
= min(dce8_latency_watermark(&wm_low
), (u32
)65535);
8809 /* possibly force display priority to high */
8810 /* should really do this at mode validation time... */
8811 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low
) ||
8812 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low
) ||
8813 !dce8_check_latency_hiding(&wm_low
) ||
8814 (rdev
->disp_priority
== 2)) {
8815 DRM_DEBUG_KMS("force priority to high\n");
8820 wm_mask
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8822 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8823 tmp
|= LATENCY_WATERMARK_MASK(1);
8824 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8825 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8826 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
8827 LATENCY_HIGH_WATERMARK(line_time
)));
8829 tmp
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8830 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8831 tmp
|= LATENCY_WATERMARK_MASK(2);
8832 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8833 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8834 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
8835 LATENCY_HIGH_WATERMARK(line_time
)));
8836 /* restore original selection */
8837 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, wm_mask
);
8839 /* save values for DPM */
8840 radeon_crtc
->line_time
= line_time
;
8841 radeon_crtc
->wm_high
= latency_watermark_a
;
8842 radeon_crtc
->wm_low
= latency_watermark_b
;
8846 * dce8_bandwidth_update - program display watermarks
8848 * @rdev: radeon_device pointer
8850 * Calculate and program the display watermarks and line
8851 * buffer allocation (CIK).
8853 void dce8_bandwidth_update(struct radeon_device
*rdev
)
8855 struct drm_display_mode
*mode
= NULL
;
8856 u32 num_heads
= 0, lb_size
;
8859 radeon_update_display_priority(rdev
);
8861 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8862 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
8865 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8866 mode
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
8867 lb_size
= dce8_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode
);
8868 dce8_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
8873 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8875 * @rdev: radeon_device pointer
8877 * Fetches a GPU clock counter snapshot (SI).
8878 * Returns the 64 bit clock counter snapshot.
8880 uint64_t cik_get_gpu_clock_counter(struct radeon_device
*rdev
)
8884 mutex_lock(&rdev
->gpu_clock_mutex
);
8885 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
8886 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
8887 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
8888 mutex_unlock(&rdev
->gpu_clock_mutex
);
8892 static int cik_set_uvd_clock(struct radeon_device
*rdev
, u32 clock
,
8893 u32 cntl_reg
, u32 status_reg
)
8896 struct atom_clock_dividers dividers
;
8899 r
= radeon_atom_get_clock_dividers(rdev
, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK
,
8900 clock
, false, ÷rs
);
8904 tmp
= RREG32_SMC(cntl_reg
);
8905 tmp
&= ~(DCLK_DIR_CNTL_EN
|DCLK_DIVIDER_MASK
);
8906 tmp
|= dividers
.post_divider
;
8907 WREG32_SMC(cntl_reg
, tmp
);
8909 for (i
= 0; i
< 100; i
++) {
8910 if (RREG32_SMC(status_reg
) & DCLK_STATUS
)
8920 int cik_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
8924 r
= cik_set_uvd_clock(rdev
, vclk
, CG_VCLK_CNTL
, CG_VCLK_STATUS
);
8928 r
= cik_set_uvd_clock(rdev
, dclk
, CG_DCLK_CNTL
, CG_DCLK_STATUS
);
8932 int cik_set_vce_clocks(struct radeon_device
*rdev
, u32 evclk
, u32 ecclk
)
8935 struct atom_clock_dividers dividers
;
8938 r
= radeon_atom_get_clock_dividers(rdev
, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK
,
8939 ecclk
, false, ÷rs
);
8943 for (i
= 0; i
< 100; i
++) {
8944 if (RREG32_SMC(CG_ECLK_STATUS
) & ECLK_STATUS
)
8951 tmp
= RREG32_SMC(CG_ECLK_CNTL
);
8952 tmp
&= ~(ECLK_DIR_CNTL_EN
|ECLK_DIVIDER_MASK
);
8953 tmp
|= dividers
.post_divider
;
8954 WREG32_SMC(CG_ECLK_CNTL
, tmp
);
8956 for (i
= 0; i
< 100; i
++) {
8957 if (RREG32_SMC(CG_ECLK_STATUS
) & ECLK_STATUS
)
8967 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
)
8969 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
8970 int bridge_pos
, gpu_pos
;
8971 u32 speed_cntl
, mask
, current_data_rate
;
8975 if (radeon_pcie_gen2
== 0)
8978 if (rdev
->flags
& RADEON_IS_IGP
)
8981 if (!(rdev
->flags
& RADEON_IS_PCIE
))
8984 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
8988 if (!(mask
& (DRM_PCIE_SPEED_50
| DRM_PCIE_SPEED_80
)))
8991 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
8992 current_data_rate
= (speed_cntl
& LC_CURRENT_DATA_RATE_MASK
) >>
8993 LC_CURRENT_DATA_RATE_SHIFT
;
8994 if (mask
& DRM_PCIE_SPEED_80
) {
8995 if (current_data_rate
== 2) {
8996 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8999 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9000 } else if (mask
& DRM_PCIE_SPEED_50
) {
9001 if (current_data_rate
== 1) {
9002 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9005 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9008 bridge_pos
= pci_pcie_cap(root
);
9012 gpu_pos
= pci_pcie_cap(rdev
->pdev
);
9016 if (mask
& DRM_PCIE_SPEED_80
) {
9017 /* re-try equalization if gen3 is not already enabled */
9018 if (current_data_rate
!= 2) {
9019 u16 bridge_cfg
, gpu_cfg
;
9020 u16 bridge_cfg2
, gpu_cfg2
;
9021 u32 max_lw
, current_lw
, tmp
;
9023 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
9024 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
9026 tmp16
= bridge_cfg
| PCI_EXP_LNKCTL_HAWD
;
9027 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9029 tmp16
= gpu_cfg
| PCI_EXP_LNKCTL_HAWD
;
9030 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9032 tmp
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
9033 max_lw
= (tmp
& LC_DETECTED_LINK_WIDTH_MASK
) >> LC_DETECTED_LINK_WIDTH_SHIFT
;
9034 current_lw
= (tmp
& LC_OPERATING_LINK_WIDTH_MASK
) >> LC_OPERATING_LINK_WIDTH_SHIFT
;
9036 if (current_lw
< max_lw
) {
9037 tmp
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
9038 if (tmp
& LC_RENEGOTIATION_SUPPORT
) {
9039 tmp
&= ~(LC_LINK_WIDTH_MASK
| LC_UPCONFIGURE_DIS
);
9040 tmp
|= (max_lw
<< LC_LINK_WIDTH_SHIFT
);
9041 tmp
|= LC_UPCONFIGURE_SUPPORT
| LC_RENEGOTIATE_EN
| LC_RECONFIG_NOW
;
9042 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, tmp
);
9046 for (i
= 0; i
< 10; i
++) {
9048 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_DEVSTA
, &tmp16
);
9049 if (tmp16
& PCI_EXP_DEVSTA_TRPND
)
9052 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
9053 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
9055 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &bridge_cfg2
);
9056 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &gpu_cfg2
);
9058 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9059 tmp
|= LC_SET_QUIESCE
;
9060 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9062 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9064 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9069 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
9070 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
9071 tmp16
|= (bridge_cfg
& PCI_EXP_LNKCTL_HAWD
);
9072 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9074 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
9075 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
9076 tmp16
|= (gpu_cfg
& PCI_EXP_LNKCTL_HAWD
);
9077 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
9080 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9081 tmp16
&= ~((1 << 4) | (7 << 9));
9082 tmp16
|= (bridge_cfg2
& ((1 << 4) | (7 << 9)));
9083 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9085 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9086 tmp16
&= ~((1 << 4) | (7 << 9));
9087 tmp16
|= (gpu_cfg2
& ((1 << 4) | (7 << 9)));
9088 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9090 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
9091 tmp
&= ~LC_SET_QUIESCE
;
9092 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
9097 /* set the link speed */
9098 speed_cntl
|= LC_FORCE_EN_SW_SPEED_CHANGE
| LC_FORCE_DIS_HW_SPEED_CHANGE
;
9099 speed_cntl
&= ~LC_FORCE_DIS_SW_SPEED_CHANGE
;
9100 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
9102 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
9104 if (mask
& DRM_PCIE_SPEED_80
)
9105 tmp16
|= 3; /* gen3 */
9106 else if (mask
& DRM_PCIE_SPEED_50
)
9107 tmp16
|= 2; /* gen2 */
9109 tmp16
|= 1; /* gen1 */
9110 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
9112 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
9113 speed_cntl
|= LC_INITIATE_LINK_SPEED_CHANGE
;
9114 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
9116 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
9117 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
9118 if ((speed_cntl
& LC_INITIATE_LINK_SPEED_CHANGE
) == 0)
9124 static void cik_program_aspm(struct radeon_device
*rdev
)
9127 bool disable_l0s
= false, disable_l1
= false, disable_plloff_in_l1
= false;
9128 bool disable_clkreq
= false;
9130 if (radeon_aspm
== 0)
9133 /* XXX double check IGPs */
9134 if (rdev
->flags
& RADEON_IS_IGP
)
9137 if (!(rdev
->flags
& RADEON_IS_PCIE
))
9140 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
9141 data
&= ~LC_XMIT_N_FTS_MASK
;
9142 data
|= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN
;
9144 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
, data
);
9146 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL3
);
9147 data
|= LC_GO_TO_RECOVERY
;
9149 WREG32_PCIE_PORT(PCIE_LC_CNTL3
, data
);
9151 orig
= data
= RREG32_PCIE_PORT(PCIE_P_CNTL
);
9152 data
|= P_IGNORE_EDB_ERR
;
9154 WREG32_PCIE_PORT(PCIE_P_CNTL
, data
);
9156 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
9157 data
&= ~(LC_L0S_INACTIVITY_MASK
| LC_L1_INACTIVITY_MASK
);
9158 data
|= LC_PMI_TO_L1_DIS
;
9160 data
|= LC_L0S_INACTIVITY(7);
9163 data
|= LC_L1_INACTIVITY(7);
9164 data
&= ~LC_PMI_TO_L1_DIS
;
9166 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
9168 if (!disable_plloff_in_l1
) {
9169 bool clk_req_support
;
9171 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
);
9172 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
9173 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9175 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
, data
);
9177 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
);
9178 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
9179 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9181 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
, data
);
9183 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
);
9184 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
9185 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9187 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
, data
);
9189 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
);
9190 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
9191 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9193 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
, data
);
9195 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
9196 data
&= ~LC_DYN_LANES_PWR_STATE_MASK
;
9197 data
|= LC_DYN_LANES_PWR_STATE(3);
9199 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, data
);
9201 if (!disable_clkreq
) {
9202 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
9205 clk_req_support
= false;
9206 pcie_capability_read_dword(root
, PCI_EXP_LNKCAP
, &lnkcap
);
9207 if (lnkcap
& PCI_EXP_LNKCAP_CLKPM
)
9208 clk_req_support
= true;
9210 clk_req_support
= false;
9213 if (clk_req_support
) {
9214 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL2
);
9215 data
|= LC_ALLOW_PDWN_IN_L1
| LC_ALLOW_PDWN_IN_L23
;
9217 WREG32_PCIE_PORT(PCIE_LC_CNTL2
, data
);
9219 orig
= data
= RREG32_SMC(THM_CLK_CNTL
);
9220 data
&= ~(CMON_CLK_SEL_MASK
| TMON_CLK_SEL_MASK
);
9221 data
|= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9223 WREG32_SMC(THM_CLK_CNTL
, data
);
9225 orig
= data
= RREG32_SMC(MISC_CLK_CTRL
);
9226 data
&= ~(DEEP_SLEEP_CLK_SEL_MASK
| ZCLK_SEL_MASK
);
9227 data
|= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9229 WREG32_SMC(MISC_CLK_CTRL
, data
);
9231 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL
);
9232 data
&= ~BCLK_AS_XCLK
;
9234 WREG32_SMC(CG_CLKPIN_CNTL
, data
);
9236 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL_2
);
9237 data
&= ~FORCE_BIF_REFCLK_EN
;
9239 WREG32_SMC(CG_CLKPIN_CNTL_2
, data
);
9241 orig
= data
= RREG32_SMC(MPLL_BYPASSCLK_SEL
);
9242 data
&= ~MPLL_CLKOUT_SEL_MASK
;
9243 data
|= MPLL_CLKOUT_SEL(4);
9245 WREG32_SMC(MPLL_BYPASSCLK_SEL
, data
);
9250 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
9253 orig
= data
= RREG32_PCIE_PORT(PCIE_CNTL2
);
9254 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
| REPLAY_MEM_LS_EN
;
9256 WREG32_PCIE_PORT(PCIE_CNTL2
, data
);
9259 data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
9260 if((data
& LC_N_FTS_MASK
) == LC_N_FTS_MASK
) {
9261 data
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
9262 if ((data
& LC_REVERSE_XMIT
) && (data
& LC_REVERSE_RCVR
)) {
9263 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
9264 data
&= ~LC_L0S_INACTIVITY_MASK
;
9266 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);