2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
49 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
50 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51 MODULE_FIRMWARE("radeon/KABINI_me.bin");
52 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
55 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
57 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
58 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
59 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
60 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
61 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
62 extern void sumo_rlc_fini(struct radeon_device
*rdev
);
63 extern int sumo_rlc_init(struct radeon_device
*rdev
);
64 extern void si_vram_gtt_location(struct radeon_device
*rdev
, struct radeon_mc
*mc
);
65 extern void si_rlc_reset(struct radeon_device
*rdev
);
66 extern void si_init_uvd_internal_cg(struct radeon_device
*rdev
);
67 static void cik_rlc_stop(struct radeon_device
*rdev
);
68 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
);
69 static void cik_program_aspm(struct radeon_device
*rdev
);
70 static void cik_init_pg(struct radeon_device
*rdev
);
71 static void cik_init_cg(struct radeon_device
*rdev
);
72 void cik_uvd_resume(struct radeon_device
*rdev
);
74 /* get temperature in millidegrees */
75 int ci_get_temp(struct radeon_device
*rdev
)
80 temp
= (RREG32_SMC(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
86 actual_temp
= temp
& 0x1ff;
88 actual_temp
= actual_temp
* 1000;
93 /* get temperature in millidegrees */
94 int kv_get_temp(struct radeon_device
*rdev
)
99 temp
= RREG32_SMC(0xC0300E0C);
102 actual_temp
= (temp
/ 8) - 49;
106 actual_temp
= actual_temp
* 1000;
112 * Indirect registers accessor
114 u32
cik_pciep_rreg(struct radeon_device
*rdev
, u32 reg
)
118 WREG32(PCIE_INDEX
, reg
);
119 (void)RREG32(PCIE_INDEX
);
120 r
= RREG32(PCIE_DATA
);
124 void cik_pciep_wreg(struct radeon_device
*rdev
, u32 reg
, u32 v
)
126 WREG32(PCIE_INDEX
, reg
);
127 (void)RREG32(PCIE_INDEX
);
128 WREG32(PCIE_DATA
, v
);
129 (void)RREG32(PCIE_DATA
);
132 static const u32 spectre_rlc_save_restore_register_list
[] =
134 (0x0e00 << 16) | (0xc12c >> 2),
136 (0x0e00 << 16) | (0xc140 >> 2),
138 (0x0e00 << 16) | (0xc150 >> 2),
140 (0x0e00 << 16) | (0xc15c >> 2),
142 (0x0e00 << 16) | (0xc168 >> 2),
144 (0x0e00 << 16) | (0xc170 >> 2),
146 (0x0e00 << 16) | (0xc178 >> 2),
148 (0x0e00 << 16) | (0xc204 >> 2),
150 (0x0e00 << 16) | (0xc2b4 >> 2),
152 (0x0e00 << 16) | (0xc2b8 >> 2),
154 (0x0e00 << 16) | (0xc2bc >> 2),
156 (0x0e00 << 16) | (0xc2c0 >> 2),
158 (0x0e00 << 16) | (0x8228 >> 2),
160 (0x0e00 << 16) | (0x829c >> 2),
162 (0x0e00 << 16) | (0x869c >> 2),
164 (0x0600 << 16) | (0x98f4 >> 2),
166 (0x0e00 << 16) | (0x98f8 >> 2),
168 (0x0e00 << 16) | (0x9900 >> 2),
170 (0x0e00 << 16) | (0xc260 >> 2),
172 (0x0e00 << 16) | (0x90e8 >> 2),
174 (0x0e00 << 16) | (0x3c000 >> 2),
176 (0x0e00 << 16) | (0x3c00c >> 2),
178 (0x0e00 << 16) | (0x8c1c >> 2),
180 (0x0e00 << 16) | (0x9700 >> 2),
182 (0x0e00 << 16) | (0xcd20 >> 2),
184 (0x4e00 << 16) | (0xcd20 >> 2),
186 (0x5e00 << 16) | (0xcd20 >> 2),
188 (0x6e00 << 16) | (0xcd20 >> 2),
190 (0x7e00 << 16) | (0xcd20 >> 2),
192 (0x8e00 << 16) | (0xcd20 >> 2),
194 (0x9e00 << 16) | (0xcd20 >> 2),
196 (0xae00 << 16) | (0xcd20 >> 2),
198 (0xbe00 << 16) | (0xcd20 >> 2),
200 (0x0e00 << 16) | (0x89bc >> 2),
202 (0x0e00 << 16) | (0x8900 >> 2),
205 (0x0e00 << 16) | (0xc130 >> 2),
207 (0x0e00 << 16) | (0xc134 >> 2),
209 (0x0e00 << 16) | (0xc1fc >> 2),
211 (0x0e00 << 16) | (0xc208 >> 2),
213 (0x0e00 << 16) | (0xc264 >> 2),
215 (0x0e00 << 16) | (0xc268 >> 2),
217 (0x0e00 << 16) | (0xc26c >> 2),
219 (0x0e00 << 16) | (0xc270 >> 2),
221 (0x0e00 << 16) | (0xc274 >> 2),
223 (0x0e00 << 16) | (0xc278 >> 2),
225 (0x0e00 << 16) | (0xc27c >> 2),
227 (0x0e00 << 16) | (0xc280 >> 2),
229 (0x0e00 << 16) | (0xc284 >> 2),
231 (0x0e00 << 16) | (0xc288 >> 2),
233 (0x0e00 << 16) | (0xc28c >> 2),
235 (0x0e00 << 16) | (0xc290 >> 2),
237 (0x0e00 << 16) | (0xc294 >> 2),
239 (0x0e00 << 16) | (0xc298 >> 2),
241 (0x0e00 << 16) | (0xc29c >> 2),
243 (0x0e00 << 16) | (0xc2a0 >> 2),
245 (0x0e00 << 16) | (0xc2a4 >> 2),
247 (0x0e00 << 16) | (0xc2a8 >> 2),
249 (0x0e00 << 16) | (0xc2ac >> 2),
251 (0x0e00 << 16) | (0xc2b0 >> 2),
253 (0x0e00 << 16) | (0x301d0 >> 2),
255 (0x0e00 << 16) | (0x30238 >> 2),
257 (0x0e00 << 16) | (0x30250 >> 2),
259 (0x0e00 << 16) | (0x30254 >> 2),
261 (0x0e00 << 16) | (0x30258 >> 2),
263 (0x0e00 << 16) | (0x3025c >> 2),
265 (0x4e00 << 16) | (0xc900 >> 2),
267 (0x5e00 << 16) | (0xc900 >> 2),
269 (0x6e00 << 16) | (0xc900 >> 2),
271 (0x7e00 << 16) | (0xc900 >> 2),
273 (0x8e00 << 16) | (0xc900 >> 2),
275 (0x9e00 << 16) | (0xc900 >> 2),
277 (0xae00 << 16) | (0xc900 >> 2),
279 (0xbe00 << 16) | (0xc900 >> 2),
281 (0x4e00 << 16) | (0xc904 >> 2),
283 (0x5e00 << 16) | (0xc904 >> 2),
285 (0x6e00 << 16) | (0xc904 >> 2),
287 (0x7e00 << 16) | (0xc904 >> 2),
289 (0x8e00 << 16) | (0xc904 >> 2),
291 (0x9e00 << 16) | (0xc904 >> 2),
293 (0xae00 << 16) | (0xc904 >> 2),
295 (0xbe00 << 16) | (0xc904 >> 2),
297 (0x4e00 << 16) | (0xc908 >> 2),
299 (0x5e00 << 16) | (0xc908 >> 2),
301 (0x6e00 << 16) | (0xc908 >> 2),
303 (0x7e00 << 16) | (0xc908 >> 2),
305 (0x8e00 << 16) | (0xc908 >> 2),
307 (0x9e00 << 16) | (0xc908 >> 2),
309 (0xae00 << 16) | (0xc908 >> 2),
311 (0xbe00 << 16) | (0xc908 >> 2),
313 (0x4e00 << 16) | (0xc90c >> 2),
315 (0x5e00 << 16) | (0xc90c >> 2),
317 (0x6e00 << 16) | (0xc90c >> 2),
319 (0x7e00 << 16) | (0xc90c >> 2),
321 (0x8e00 << 16) | (0xc90c >> 2),
323 (0x9e00 << 16) | (0xc90c >> 2),
325 (0xae00 << 16) | (0xc90c >> 2),
327 (0xbe00 << 16) | (0xc90c >> 2),
329 (0x4e00 << 16) | (0xc910 >> 2),
331 (0x5e00 << 16) | (0xc910 >> 2),
333 (0x6e00 << 16) | (0xc910 >> 2),
335 (0x7e00 << 16) | (0xc910 >> 2),
337 (0x8e00 << 16) | (0xc910 >> 2),
339 (0x9e00 << 16) | (0xc910 >> 2),
341 (0xae00 << 16) | (0xc910 >> 2),
343 (0xbe00 << 16) | (0xc910 >> 2),
345 (0x0e00 << 16) | (0xc99c >> 2),
347 (0x0e00 << 16) | (0x9834 >> 2),
349 (0x0000 << 16) | (0x30f00 >> 2),
351 (0x0001 << 16) | (0x30f00 >> 2),
353 (0x0000 << 16) | (0x30f04 >> 2),
355 (0x0001 << 16) | (0x30f04 >> 2),
357 (0x0000 << 16) | (0x30f08 >> 2),
359 (0x0001 << 16) | (0x30f08 >> 2),
361 (0x0000 << 16) | (0x30f0c >> 2),
363 (0x0001 << 16) | (0x30f0c >> 2),
365 (0x0600 << 16) | (0x9b7c >> 2),
367 (0x0e00 << 16) | (0x8a14 >> 2),
369 (0x0e00 << 16) | (0x8a18 >> 2),
371 (0x0600 << 16) | (0x30a00 >> 2),
373 (0x0e00 << 16) | (0x8bf0 >> 2),
375 (0x0e00 << 16) | (0x8bcc >> 2),
377 (0x0e00 << 16) | (0x8b24 >> 2),
379 (0x0e00 << 16) | (0x30a04 >> 2),
381 (0x0600 << 16) | (0x30a10 >> 2),
383 (0x0600 << 16) | (0x30a14 >> 2),
385 (0x0600 << 16) | (0x30a18 >> 2),
387 (0x0600 << 16) | (0x30a2c >> 2),
389 (0x0e00 << 16) | (0xc700 >> 2),
391 (0x0e00 << 16) | (0xc704 >> 2),
393 (0x0e00 << 16) | (0xc708 >> 2),
395 (0x0e00 << 16) | (0xc768 >> 2),
397 (0x0400 << 16) | (0xc770 >> 2),
399 (0x0400 << 16) | (0xc774 >> 2),
401 (0x0400 << 16) | (0xc778 >> 2),
403 (0x0400 << 16) | (0xc77c >> 2),
405 (0x0400 << 16) | (0xc780 >> 2),
407 (0x0400 << 16) | (0xc784 >> 2),
409 (0x0400 << 16) | (0xc788 >> 2),
411 (0x0400 << 16) | (0xc78c >> 2),
413 (0x0400 << 16) | (0xc798 >> 2),
415 (0x0400 << 16) | (0xc79c >> 2),
417 (0x0400 << 16) | (0xc7a0 >> 2),
419 (0x0400 << 16) | (0xc7a4 >> 2),
421 (0x0400 << 16) | (0xc7a8 >> 2),
423 (0x0400 << 16) | (0xc7ac >> 2),
425 (0x0400 << 16) | (0xc7b0 >> 2),
427 (0x0400 << 16) | (0xc7b4 >> 2),
429 (0x0e00 << 16) | (0x9100 >> 2),
431 (0x0e00 << 16) | (0x3c010 >> 2),
433 (0x0e00 << 16) | (0x92a8 >> 2),
435 (0x0e00 << 16) | (0x92ac >> 2),
437 (0x0e00 << 16) | (0x92b4 >> 2),
439 (0x0e00 << 16) | (0x92b8 >> 2),
441 (0x0e00 << 16) | (0x92bc >> 2),
443 (0x0e00 << 16) | (0x92c0 >> 2),
445 (0x0e00 << 16) | (0x92c4 >> 2),
447 (0x0e00 << 16) | (0x92c8 >> 2),
449 (0x0e00 << 16) | (0x92cc >> 2),
451 (0x0e00 << 16) | (0x92d0 >> 2),
453 (0x0e00 << 16) | (0x8c00 >> 2),
455 (0x0e00 << 16) | (0x8c04 >> 2),
457 (0x0e00 << 16) | (0x8c20 >> 2),
459 (0x0e00 << 16) | (0x8c38 >> 2),
461 (0x0e00 << 16) | (0x8c3c >> 2),
463 (0x0e00 << 16) | (0xae00 >> 2),
465 (0x0e00 << 16) | (0x9604 >> 2),
467 (0x0e00 << 16) | (0xac08 >> 2),
469 (0x0e00 << 16) | (0xac0c >> 2),
471 (0x0e00 << 16) | (0xac10 >> 2),
473 (0x0e00 << 16) | (0xac14 >> 2),
475 (0x0e00 << 16) | (0xac58 >> 2),
477 (0x0e00 << 16) | (0xac68 >> 2),
479 (0x0e00 << 16) | (0xac6c >> 2),
481 (0x0e00 << 16) | (0xac70 >> 2),
483 (0x0e00 << 16) | (0xac74 >> 2),
485 (0x0e00 << 16) | (0xac78 >> 2),
487 (0x0e00 << 16) | (0xac7c >> 2),
489 (0x0e00 << 16) | (0xac80 >> 2),
491 (0x0e00 << 16) | (0xac84 >> 2),
493 (0x0e00 << 16) | (0xac88 >> 2),
495 (0x0e00 << 16) | (0xac8c >> 2),
497 (0x0e00 << 16) | (0x970c >> 2),
499 (0x0e00 << 16) | (0x9714 >> 2),
501 (0x0e00 << 16) | (0x9718 >> 2),
503 (0x0e00 << 16) | (0x971c >> 2),
505 (0x0e00 << 16) | (0x31068 >> 2),
507 (0x4e00 << 16) | (0x31068 >> 2),
509 (0x5e00 << 16) | (0x31068 >> 2),
511 (0x6e00 << 16) | (0x31068 >> 2),
513 (0x7e00 << 16) | (0x31068 >> 2),
515 (0x8e00 << 16) | (0x31068 >> 2),
517 (0x9e00 << 16) | (0x31068 >> 2),
519 (0xae00 << 16) | (0x31068 >> 2),
521 (0xbe00 << 16) | (0x31068 >> 2),
523 (0x0e00 << 16) | (0xcd10 >> 2),
525 (0x0e00 << 16) | (0xcd14 >> 2),
527 (0x0e00 << 16) | (0x88b0 >> 2),
529 (0x0e00 << 16) | (0x88b4 >> 2),
531 (0x0e00 << 16) | (0x88b8 >> 2),
533 (0x0e00 << 16) | (0x88bc >> 2),
535 (0x0400 << 16) | (0x89c0 >> 2),
537 (0x0e00 << 16) | (0x88c4 >> 2),
539 (0x0e00 << 16) | (0x88c8 >> 2),
541 (0x0e00 << 16) | (0x88d0 >> 2),
543 (0x0e00 << 16) | (0x88d4 >> 2),
545 (0x0e00 << 16) | (0x88d8 >> 2),
547 (0x0e00 << 16) | (0x8980 >> 2),
549 (0x0e00 << 16) | (0x30938 >> 2),
551 (0x0e00 << 16) | (0x3093c >> 2),
553 (0x0e00 << 16) | (0x30940 >> 2),
555 (0x0e00 << 16) | (0x89a0 >> 2),
557 (0x0e00 << 16) | (0x30900 >> 2),
559 (0x0e00 << 16) | (0x30904 >> 2),
561 (0x0e00 << 16) | (0x89b4 >> 2),
563 (0x0e00 << 16) | (0x3c210 >> 2),
565 (0x0e00 << 16) | (0x3c214 >> 2),
567 (0x0e00 << 16) | (0x3c218 >> 2),
569 (0x0e00 << 16) | (0x8904 >> 2),
572 (0x0e00 << 16) | (0x8c28 >> 2),
573 (0x0e00 << 16) | (0x8c2c >> 2),
574 (0x0e00 << 16) | (0x8c30 >> 2),
575 (0x0e00 << 16) | (0x8c34 >> 2),
576 (0x0e00 << 16) | (0x9600 >> 2),
579 static const u32 kalindi_rlc_save_restore_register_list
[] =
581 (0x0e00 << 16) | (0xc12c >> 2),
583 (0x0e00 << 16) | (0xc140 >> 2),
585 (0x0e00 << 16) | (0xc150 >> 2),
587 (0x0e00 << 16) | (0xc15c >> 2),
589 (0x0e00 << 16) | (0xc168 >> 2),
591 (0x0e00 << 16) | (0xc170 >> 2),
593 (0x0e00 << 16) | (0xc204 >> 2),
595 (0x0e00 << 16) | (0xc2b4 >> 2),
597 (0x0e00 << 16) | (0xc2b8 >> 2),
599 (0x0e00 << 16) | (0xc2bc >> 2),
601 (0x0e00 << 16) | (0xc2c0 >> 2),
603 (0x0e00 << 16) | (0x8228 >> 2),
605 (0x0e00 << 16) | (0x829c >> 2),
607 (0x0e00 << 16) | (0x869c >> 2),
609 (0x0600 << 16) | (0x98f4 >> 2),
611 (0x0e00 << 16) | (0x98f8 >> 2),
613 (0x0e00 << 16) | (0x9900 >> 2),
615 (0x0e00 << 16) | (0xc260 >> 2),
617 (0x0e00 << 16) | (0x90e8 >> 2),
619 (0x0e00 << 16) | (0x3c000 >> 2),
621 (0x0e00 << 16) | (0x3c00c >> 2),
623 (0x0e00 << 16) | (0x8c1c >> 2),
625 (0x0e00 << 16) | (0x9700 >> 2),
627 (0x0e00 << 16) | (0xcd20 >> 2),
629 (0x4e00 << 16) | (0xcd20 >> 2),
631 (0x5e00 << 16) | (0xcd20 >> 2),
633 (0x6e00 << 16) | (0xcd20 >> 2),
635 (0x7e00 << 16) | (0xcd20 >> 2),
637 (0x0e00 << 16) | (0x89bc >> 2),
639 (0x0e00 << 16) | (0x8900 >> 2),
642 (0x0e00 << 16) | (0xc130 >> 2),
644 (0x0e00 << 16) | (0xc134 >> 2),
646 (0x0e00 << 16) | (0xc1fc >> 2),
648 (0x0e00 << 16) | (0xc208 >> 2),
650 (0x0e00 << 16) | (0xc264 >> 2),
652 (0x0e00 << 16) | (0xc268 >> 2),
654 (0x0e00 << 16) | (0xc26c >> 2),
656 (0x0e00 << 16) | (0xc270 >> 2),
658 (0x0e00 << 16) | (0xc274 >> 2),
660 (0x0e00 << 16) | (0xc28c >> 2),
662 (0x0e00 << 16) | (0xc290 >> 2),
664 (0x0e00 << 16) | (0xc294 >> 2),
666 (0x0e00 << 16) | (0xc298 >> 2),
668 (0x0e00 << 16) | (0xc2a0 >> 2),
670 (0x0e00 << 16) | (0xc2a4 >> 2),
672 (0x0e00 << 16) | (0xc2a8 >> 2),
674 (0x0e00 << 16) | (0xc2ac >> 2),
676 (0x0e00 << 16) | (0x301d0 >> 2),
678 (0x0e00 << 16) | (0x30238 >> 2),
680 (0x0e00 << 16) | (0x30250 >> 2),
682 (0x0e00 << 16) | (0x30254 >> 2),
684 (0x0e00 << 16) | (0x30258 >> 2),
686 (0x0e00 << 16) | (0x3025c >> 2),
688 (0x4e00 << 16) | (0xc900 >> 2),
690 (0x5e00 << 16) | (0xc900 >> 2),
692 (0x6e00 << 16) | (0xc900 >> 2),
694 (0x7e00 << 16) | (0xc900 >> 2),
696 (0x4e00 << 16) | (0xc904 >> 2),
698 (0x5e00 << 16) | (0xc904 >> 2),
700 (0x6e00 << 16) | (0xc904 >> 2),
702 (0x7e00 << 16) | (0xc904 >> 2),
704 (0x4e00 << 16) | (0xc908 >> 2),
706 (0x5e00 << 16) | (0xc908 >> 2),
708 (0x6e00 << 16) | (0xc908 >> 2),
710 (0x7e00 << 16) | (0xc908 >> 2),
712 (0x4e00 << 16) | (0xc90c >> 2),
714 (0x5e00 << 16) | (0xc90c >> 2),
716 (0x6e00 << 16) | (0xc90c >> 2),
718 (0x7e00 << 16) | (0xc90c >> 2),
720 (0x4e00 << 16) | (0xc910 >> 2),
722 (0x5e00 << 16) | (0xc910 >> 2),
724 (0x6e00 << 16) | (0xc910 >> 2),
726 (0x7e00 << 16) | (0xc910 >> 2),
728 (0x0e00 << 16) | (0xc99c >> 2),
730 (0x0e00 << 16) | (0x9834 >> 2),
732 (0x0000 << 16) | (0x30f00 >> 2),
734 (0x0000 << 16) | (0x30f04 >> 2),
736 (0x0000 << 16) | (0x30f08 >> 2),
738 (0x0000 << 16) | (0x30f0c >> 2),
740 (0x0600 << 16) | (0x9b7c >> 2),
742 (0x0e00 << 16) | (0x8a14 >> 2),
744 (0x0e00 << 16) | (0x8a18 >> 2),
746 (0x0600 << 16) | (0x30a00 >> 2),
748 (0x0e00 << 16) | (0x8bf0 >> 2),
750 (0x0e00 << 16) | (0x8bcc >> 2),
752 (0x0e00 << 16) | (0x8b24 >> 2),
754 (0x0e00 << 16) | (0x30a04 >> 2),
756 (0x0600 << 16) | (0x30a10 >> 2),
758 (0x0600 << 16) | (0x30a14 >> 2),
760 (0x0600 << 16) | (0x30a18 >> 2),
762 (0x0600 << 16) | (0x30a2c >> 2),
764 (0x0e00 << 16) | (0xc700 >> 2),
766 (0x0e00 << 16) | (0xc704 >> 2),
768 (0x0e00 << 16) | (0xc708 >> 2),
770 (0x0e00 << 16) | (0xc768 >> 2),
772 (0x0400 << 16) | (0xc770 >> 2),
774 (0x0400 << 16) | (0xc774 >> 2),
776 (0x0400 << 16) | (0xc798 >> 2),
778 (0x0400 << 16) | (0xc79c >> 2),
780 (0x0e00 << 16) | (0x9100 >> 2),
782 (0x0e00 << 16) | (0x3c010 >> 2),
784 (0x0e00 << 16) | (0x8c00 >> 2),
786 (0x0e00 << 16) | (0x8c04 >> 2),
788 (0x0e00 << 16) | (0x8c20 >> 2),
790 (0x0e00 << 16) | (0x8c38 >> 2),
792 (0x0e00 << 16) | (0x8c3c >> 2),
794 (0x0e00 << 16) | (0xae00 >> 2),
796 (0x0e00 << 16) | (0x9604 >> 2),
798 (0x0e00 << 16) | (0xac08 >> 2),
800 (0x0e00 << 16) | (0xac0c >> 2),
802 (0x0e00 << 16) | (0xac10 >> 2),
804 (0x0e00 << 16) | (0xac14 >> 2),
806 (0x0e00 << 16) | (0xac58 >> 2),
808 (0x0e00 << 16) | (0xac68 >> 2),
810 (0x0e00 << 16) | (0xac6c >> 2),
812 (0x0e00 << 16) | (0xac70 >> 2),
814 (0x0e00 << 16) | (0xac74 >> 2),
816 (0x0e00 << 16) | (0xac78 >> 2),
818 (0x0e00 << 16) | (0xac7c >> 2),
820 (0x0e00 << 16) | (0xac80 >> 2),
822 (0x0e00 << 16) | (0xac84 >> 2),
824 (0x0e00 << 16) | (0xac88 >> 2),
826 (0x0e00 << 16) | (0xac8c >> 2),
828 (0x0e00 << 16) | (0x970c >> 2),
830 (0x0e00 << 16) | (0x9714 >> 2),
832 (0x0e00 << 16) | (0x9718 >> 2),
834 (0x0e00 << 16) | (0x971c >> 2),
836 (0x0e00 << 16) | (0x31068 >> 2),
838 (0x4e00 << 16) | (0x31068 >> 2),
840 (0x5e00 << 16) | (0x31068 >> 2),
842 (0x6e00 << 16) | (0x31068 >> 2),
844 (0x7e00 << 16) | (0x31068 >> 2),
846 (0x0e00 << 16) | (0xcd10 >> 2),
848 (0x0e00 << 16) | (0xcd14 >> 2),
850 (0x0e00 << 16) | (0x88b0 >> 2),
852 (0x0e00 << 16) | (0x88b4 >> 2),
854 (0x0e00 << 16) | (0x88b8 >> 2),
856 (0x0e00 << 16) | (0x88bc >> 2),
858 (0x0400 << 16) | (0x89c0 >> 2),
860 (0x0e00 << 16) | (0x88c4 >> 2),
862 (0x0e00 << 16) | (0x88c8 >> 2),
864 (0x0e00 << 16) | (0x88d0 >> 2),
866 (0x0e00 << 16) | (0x88d4 >> 2),
868 (0x0e00 << 16) | (0x88d8 >> 2),
870 (0x0e00 << 16) | (0x8980 >> 2),
872 (0x0e00 << 16) | (0x30938 >> 2),
874 (0x0e00 << 16) | (0x3093c >> 2),
876 (0x0e00 << 16) | (0x30940 >> 2),
878 (0x0e00 << 16) | (0x89a0 >> 2),
880 (0x0e00 << 16) | (0x30900 >> 2),
882 (0x0e00 << 16) | (0x30904 >> 2),
884 (0x0e00 << 16) | (0x89b4 >> 2),
886 (0x0e00 << 16) | (0x3e1fc >> 2),
888 (0x0e00 << 16) | (0x3c210 >> 2),
890 (0x0e00 << 16) | (0x3c214 >> 2),
892 (0x0e00 << 16) | (0x3c218 >> 2),
894 (0x0e00 << 16) | (0x8904 >> 2),
897 (0x0e00 << 16) | (0x8c28 >> 2),
898 (0x0e00 << 16) | (0x8c2c >> 2),
899 (0x0e00 << 16) | (0x8c30 >> 2),
900 (0x0e00 << 16) | (0x8c34 >> 2),
901 (0x0e00 << 16) | (0x9600 >> 2),
904 static const u32 bonaire_golden_spm_registers
[] =
906 0x30800, 0xe0ffffff, 0xe0000000
909 static const u32 bonaire_golden_common_registers
[] =
911 0xc770, 0xffffffff, 0x00000800,
912 0xc774, 0xffffffff, 0x00000800,
913 0xc798, 0xffffffff, 0x00007fbf,
914 0xc79c, 0xffffffff, 0x00007faf
917 static const u32 bonaire_golden_registers
[] =
919 0x3354, 0x00000333, 0x00000333,
920 0x3350, 0x000c0fc0, 0x00040200,
921 0x9a10, 0x00010000, 0x00058208,
922 0x3c000, 0xffff1fff, 0x00140000,
923 0x3c200, 0xfdfc0fff, 0x00000100,
924 0x3c234, 0x40000000, 0x40000200,
925 0x9830, 0xffffffff, 0x00000000,
926 0x9834, 0xf00fffff, 0x00000400,
927 0x9838, 0x0002021c, 0x00020200,
928 0xc78, 0x00000080, 0x00000000,
929 0x5bb0, 0x000000f0, 0x00000070,
930 0x5bc0, 0xf0311fff, 0x80300000,
931 0x98f8, 0x73773777, 0x12010001,
932 0x350c, 0x00810000, 0x408af000,
933 0x7030, 0x31000111, 0x00000011,
934 0x2f48, 0x73773777, 0x12010001,
935 0x220c, 0x00007fb6, 0x0021a1b1,
936 0x2210, 0x00007fb6, 0x002021b1,
937 0x2180, 0x00007fb6, 0x00002191,
938 0x2218, 0x00007fb6, 0x002121b1,
939 0x221c, 0x00007fb6, 0x002021b1,
940 0x21dc, 0x00007fb6, 0x00002191,
941 0x21e0, 0x00007fb6, 0x00002191,
942 0x3628, 0x0000003f, 0x0000000a,
943 0x362c, 0x0000003f, 0x0000000a,
944 0x2ae4, 0x00073ffe, 0x000022a2,
945 0x240c, 0x000007ff, 0x00000000,
946 0x8a14, 0xf000003f, 0x00000007,
947 0x8bf0, 0x00002001, 0x00000001,
948 0x8b24, 0xffffffff, 0x00ffffff,
949 0x30a04, 0x0000ff0f, 0x00000000,
950 0x28a4c, 0x07ffffff, 0x06000000,
951 0x4d8, 0x00000fff, 0x00000100,
952 0x3e78, 0x00000001, 0x00000002,
953 0x9100, 0x03000000, 0x0362c688,
954 0x8c00, 0x000000ff, 0x00000001,
955 0xe40, 0x00001fff, 0x00001fff,
956 0x9060, 0x0000007f, 0x00000020,
957 0x9508, 0x00010000, 0x00010000,
958 0xac14, 0x000003ff, 0x000000f3,
959 0xac0c, 0xffffffff, 0x00001032
962 static const u32 bonaire_mgcg_cgcg_init
[] =
964 0xc420, 0xffffffff, 0xfffffffc,
965 0x30800, 0xffffffff, 0xe0000000,
966 0x3c2a0, 0xffffffff, 0x00000100,
967 0x3c208, 0xffffffff, 0x00000100,
968 0x3c2c0, 0xffffffff, 0xc0000100,
969 0x3c2c8, 0xffffffff, 0xc0000100,
970 0x3c2c4, 0xffffffff, 0xc0000100,
971 0x55e4, 0xffffffff, 0x00600100,
972 0x3c280, 0xffffffff, 0x00000100,
973 0x3c214, 0xffffffff, 0x06000100,
974 0x3c220, 0xffffffff, 0x00000100,
975 0x3c218, 0xffffffff, 0x06000100,
976 0x3c204, 0xffffffff, 0x00000100,
977 0x3c2e0, 0xffffffff, 0x00000100,
978 0x3c224, 0xffffffff, 0x00000100,
979 0x3c200, 0xffffffff, 0x00000100,
980 0x3c230, 0xffffffff, 0x00000100,
981 0x3c234, 0xffffffff, 0x00000100,
982 0x3c250, 0xffffffff, 0x00000100,
983 0x3c254, 0xffffffff, 0x00000100,
984 0x3c258, 0xffffffff, 0x00000100,
985 0x3c25c, 0xffffffff, 0x00000100,
986 0x3c260, 0xffffffff, 0x00000100,
987 0x3c27c, 0xffffffff, 0x00000100,
988 0x3c278, 0xffffffff, 0x00000100,
989 0x3c210, 0xffffffff, 0x06000100,
990 0x3c290, 0xffffffff, 0x00000100,
991 0x3c274, 0xffffffff, 0x00000100,
992 0x3c2b4, 0xffffffff, 0x00000100,
993 0x3c2b0, 0xffffffff, 0x00000100,
994 0x3c270, 0xffffffff, 0x00000100,
995 0x30800, 0xffffffff, 0xe0000000,
996 0x3c020, 0xffffffff, 0x00010000,
997 0x3c024, 0xffffffff, 0x00030002,
998 0x3c028, 0xffffffff, 0x00040007,
999 0x3c02c, 0xffffffff, 0x00060005,
1000 0x3c030, 0xffffffff, 0x00090008,
1001 0x3c034, 0xffffffff, 0x00010000,
1002 0x3c038, 0xffffffff, 0x00030002,
1003 0x3c03c, 0xffffffff, 0x00040007,
1004 0x3c040, 0xffffffff, 0x00060005,
1005 0x3c044, 0xffffffff, 0x00090008,
1006 0x3c048, 0xffffffff, 0x00010000,
1007 0x3c04c, 0xffffffff, 0x00030002,
1008 0x3c050, 0xffffffff, 0x00040007,
1009 0x3c054, 0xffffffff, 0x00060005,
1010 0x3c058, 0xffffffff, 0x00090008,
1011 0x3c05c, 0xffffffff, 0x00010000,
1012 0x3c060, 0xffffffff, 0x00030002,
1013 0x3c064, 0xffffffff, 0x00040007,
1014 0x3c068, 0xffffffff, 0x00060005,
1015 0x3c06c, 0xffffffff, 0x00090008,
1016 0x3c070, 0xffffffff, 0x00010000,
1017 0x3c074, 0xffffffff, 0x00030002,
1018 0x3c078, 0xffffffff, 0x00040007,
1019 0x3c07c, 0xffffffff, 0x00060005,
1020 0x3c080, 0xffffffff, 0x00090008,
1021 0x3c084, 0xffffffff, 0x00010000,
1022 0x3c088, 0xffffffff, 0x00030002,
1023 0x3c08c, 0xffffffff, 0x00040007,
1024 0x3c090, 0xffffffff, 0x00060005,
1025 0x3c094, 0xffffffff, 0x00090008,
1026 0x3c098, 0xffffffff, 0x00010000,
1027 0x3c09c, 0xffffffff, 0x00030002,
1028 0x3c0a0, 0xffffffff, 0x00040007,
1029 0x3c0a4, 0xffffffff, 0x00060005,
1030 0x3c0a8, 0xffffffff, 0x00090008,
1031 0x3c000, 0xffffffff, 0x96e00200,
1032 0x8708, 0xffffffff, 0x00900100,
1033 0xc424, 0xffffffff, 0x0020003f,
1034 0x38, 0xffffffff, 0x0140001c,
1035 0x3c, 0x000f0000, 0x000f0000,
1036 0x220, 0xffffffff, 0xC060000C,
1037 0x224, 0xc0000fff, 0x00000100,
1038 0xf90, 0xffffffff, 0x00000100,
1039 0xf98, 0x00000101, 0x00000000,
1040 0x20a8, 0xffffffff, 0x00000104,
1041 0x55e4, 0xff000fff, 0x00000100,
1042 0x30cc, 0xc0000fff, 0x00000104,
1043 0xc1e4, 0x00000001, 0x00000001,
1044 0xd00c, 0xff000ff0, 0x00000100,
1045 0xd80c, 0xff000ff0, 0x00000100
1048 static const u32 spectre_golden_spm_registers
[] =
1050 0x30800, 0xe0ffffff, 0xe0000000
1053 static const u32 spectre_golden_common_registers
[] =
1055 0xc770, 0xffffffff, 0x00000800,
1056 0xc774, 0xffffffff, 0x00000800,
1057 0xc798, 0xffffffff, 0x00007fbf,
1058 0xc79c, 0xffffffff, 0x00007faf
1061 static const u32 spectre_golden_registers
[] =
1063 0x3c000, 0xffff1fff, 0x96940200,
1064 0x3c00c, 0xffff0001, 0xff000000,
1065 0x3c200, 0xfffc0fff, 0x00000100,
1066 0x6ed8, 0x00010101, 0x00010000,
1067 0x9834, 0xf00fffff, 0x00000400,
1068 0x9838, 0xfffffffc, 0x00020200,
1069 0x5bb0, 0x000000f0, 0x00000070,
1070 0x5bc0, 0xf0311fff, 0x80300000,
1071 0x98f8, 0x73773777, 0x12010001,
1072 0x9b7c, 0x00ff0000, 0x00fc0000,
1073 0x2f48, 0x73773777, 0x12010001,
1074 0x8a14, 0xf000003f, 0x00000007,
1075 0x8b24, 0xffffffff, 0x00ffffff,
1076 0x28350, 0x3f3f3fff, 0x00000082,
1077 0x28355, 0x0000003f, 0x00000000,
1078 0x3e78, 0x00000001, 0x00000002,
1079 0x913c, 0xffff03df, 0x00000004,
1080 0xc768, 0x00000008, 0x00000008,
1081 0x8c00, 0x000008ff, 0x00000800,
1082 0x9508, 0x00010000, 0x00010000,
1083 0xac0c, 0xffffffff, 0x54763210,
1084 0x214f8, 0x01ff01ff, 0x00000002,
1085 0x21498, 0x007ff800, 0x00200000,
1086 0x2015c, 0xffffffff, 0x00000f40,
1087 0x30934, 0xffffffff, 0x00000001
1090 static const u32 spectre_mgcg_cgcg_init
[] =
1092 0xc420, 0xffffffff, 0xfffffffc,
1093 0x30800, 0xffffffff, 0xe0000000,
1094 0x3c2a0, 0xffffffff, 0x00000100,
1095 0x3c208, 0xffffffff, 0x00000100,
1096 0x3c2c0, 0xffffffff, 0x00000100,
1097 0x3c2c8, 0xffffffff, 0x00000100,
1098 0x3c2c4, 0xffffffff, 0x00000100,
1099 0x55e4, 0xffffffff, 0x00600100,
1100 0x3c280, 0xffffffff, 0x00000100,
1101 0x3c214, 0xffffffff, 0x06000100,
1102 0x3c220, 0xffffffff, 0x00000100,
1103 0x3c218, 0xffffffff, 0x06000100,
1104 0x3c204, 0xffffffff, 0x00000100,
1105 0x3c2e0, 0xffffffff, 0x00000100,
1106 0x3c224, 0xffffffff, 0x00000100,
1107 0x3c200, 0xffffffff, 0x00000100,
1108 0x3c230, 0xffffffff, 0x00000100,
1109 0x3c234, 0xffffffff, 0x00000100,
1110 0x3c250, 0xffffffff, 0x00000100,
1111 0x3c254, 0xffffffff, 0x00000100,
1112 0x3c258, 0xffffffff, 0x00000100,
1113 0x3c25c, 0xffffffff, 0x00000100,
1114 0x3c260, 0xffffffff, 0x00000100,
1115 0x3c27c, 0xffffffff, 0x00000100,
1116 0x3c278, 0xffffffff, 0x00000100,
1117 0x3c210, 0xffffffff, 0x06000100,
1118 0x3c290, 0xffffffff, 0x00000100,
1119 0x3c274, 0xffffffff, 0x00000100,
1120 0x3c2b4, 0xffffffff, 0x00000100,
1121 0x3c2b0, 0xffffffff, 0x00000100,
1122 0x3c270, 0xffffffff, 0x00000100,
1123 0x30800, 0xffffffff, 0xe0000000,
1124 0x3c020, 0xffffffff, 0x00010000,
1125 0x3c024, 0xffffffff, 0x00030002,
1126 0x3c028, 0xffffffff, 0x00040007,
1127 0x3c02c, 0xffffffff, 0x00060005,
1128 0x3c030, 0xffffffff, 0x00090008,
1129 0x3c034, 0xffffffff, 0x00010000,
1130 0x3c038, 0xffffffff, 0x00030002,
1131 0x3c03c, 0xffffffff, 0x00040007,
1132 0x3c040, 0xffffffff, 0x00060005,
1133 0x3c044, 0xffffffff, 0x00090008,
1134 0x3c048, 0xffffffff, 0x00010000,
1135 0x3c04c, 0xffffffff, 0x00030002,
1136 0x3c050, 0xffffffff, 0x00040007,
1137 0x3c054, 0xffffffff, 0x00060005,
1138 0x3c058, 0xffffffff, 0x00090008,
1139 0x3c05c, 0xffffffff, 0x00010000,
1140 0x3c060, 0xffffffff, 0x00030002,
1141 0x3c064, 0xffffffff, 0x00040007,
1142 0x3c068, 0xffffffff, 0x00060005,
1143 0x3c06c, 0xffffffff, 0x00090008,
1144 0x3c070, 0xffffffff, 0x00010000,
1145 0x3c074, 0xffffffff, 0x00030002,
1146 0x3c078, 0xffffffff, 0x00040007,
1147 0x3c07c, 0xffffffff, 0x00060005,
1148 0x3c080, 0xffffffff, 0x00090008,
1149 0x3c084, 0xffffffff, 0x00010000,
1150 0x3c088, 0xffffffff, 0x00030002,
1151 0x3c08c, 0xffffffff, 0x00040007,
1152 0x3c090, 0xffffffff, 0x00060005,
1153 0x3c094, 0xffffffff, 0x00090008,
1154 0x3c098, 0xffffffff, 0x00010000,
1155 0x3c09c, 0xffffffff, 0x00030002,
1156 0x3c0a0, 0xffffffff, 0x00040007,
1157 0x3c0a4, 0xffffffff, 0x00060005,
1158 0x3c0a8, 0xffffffff, 0x00090008,
1159 0x3c0ac, 0xffffffff, 0x00010000,
1160 0x3c0b0, 0xffffffff, 0x00030002,
1161 0x3c0b4, 0xffffffff, 0x00040007,
1162 0x3c0b8, 0xffffffff, 0x00060005,
1163 0x3c0bc, 0xffffffff, 0x00090008,
1164 0x3c000, 0xffffffff, 0x96e00200,
1165 0x8708, 0xffffffff, 0x00900100,
1166 0xc424, 0xffffffff, 0x0020003f,
1167 0x38, 0xffffffff, 0x0140001c,
1168 0x3c, 0x000f0000, 0x000f0000,
1169 0x220, 0xffffffff, 0xC060000C,
1170 0x224, 0xc0000fff, 0x00000100,
1171 0xf90, 0xffffffff, 0x00000100,
1172 0xf98, 0x00000101, 0x00000000,
1173 0x20a8, 0xffffffff, 0x00000104,
1174 0x55e4, 0xff000fff, 0x00000100,
1175 0x30cc, 0xc0000fff, 0x00000104,
1176 0xc1e4, 0x00000001, 0x00000001,
1177 0xd00c, 0xff000ff0, 0x00000100,
1178 0xd80c, 0xff000ff0, 0x00000100
1181 static const u32 kalindi_golden_spm_registers
[] =
1183 0x30800, 0xe0ffffff, 0xe0000000
1186 static const u32 kalindi_golden_common_registers
[] =
1188 0xc770, 0xffffffff, 0x00000800,
1189 0xc774, 0xffffffff, 0x00000800,
1190 0xc798, 0xffffffff, 0x00007fbf,
1191 0xc79c, 0xffffffff, 0x00007faf
1194 static const u32 kalindi_golden_registers
[] =
1196 0x3c000, 0xffffdfff, 0x6e944040,
1197 0x55e4, 0xff607fff, 0xfc000100,
1198 0x3c220, 0xff000fff, 0x00000100,
1199 0x3c224, 0xff000fff, 0x00000100,
1200 0x3c200, 0xfffc0fff, 0x00000100,
1201 0x6ed8, 0x00010101, 0x00010000,
1202 0x9830, 0xffffffff, 0x00000000,
1203 0x9834, 0xf00fffff, 0x00000400,
1204 0x5bb0, 0x000000f0, 0x00000070,
1205 0x5bc0, 0xf0311fff, 0x80300000,
1206 0x98f8, 0x73773777, 0x12010001,
1207 0x98fc, 0xffffffff, 0x00000010,
1208 0x9b7c, 0x00ff0000, 0x00fc0000,
1209 0x8030, 0x00001f0f, 0x0000100a,
1210 0x2f48, 0x73773777, 0x12010001,
1211 0x2408, 0x000fffff, 0x000c007f,
1212 0x8a14, 0xf000003f, 0x00000007,
1213 0x8b24, 0x3fff3fff, 0x00ffcfff,
1214 0x30a04, 0x0000ff0f, 0x00000000,
1215 0x28a4c, 0x07ffffff, 0x06000000,
1216 0x4d8, 0x00000fff, 0x00000100,
1217 0x3e78, 0x00000001, 0x00000002,
1218 0xc768, 0x00000008, 0x00000008,
1219 0x8c00, 0x000000ff, 0x00000003,
1220 0x214f8, 0x01ff01ff, 0x00000002,
1221 0x21498, 0x007ff800, 0x00200000,
1222 0x2015c, 0xffffffff, 0x00000f40,
1223 0x88c4, 0x001f3ae3, 0x00000082,
1224 0x88d4, 0x0000001f, 0x00000010,
1225 0x30934, 0xffffffff, 0x00000000
1228 static const u32 kalindi_mgcg_cgcg_init
[] =
1230 0xc420, 0xffffffff, 0xfffffffc,
1231 0x30800, 0xffffffff, 0xe0000000,
1232 0x3c2a0, 0xffffffff, 0x00000100,
1233 0x3c208, 0xffffffff, 0x00000100,
1234 0x3c2c0, 0xffffffff, 0x00000100,
1235 0x3c2c8, 0xffffffff, 0x00000100,
1236 0x3c2c4, 0xffffffff, 0x00000100,
1237 0x55e4, 0xffffffff, 0x00600100,
1238 0x3c280, 0xffffffff, 0x00000100,
1239 0x3c214, 0xffffffff, 0x06000100,
1240 0x3c220, 0xffffffff, 0x00000100,
1241 0x3c218, 0xffffffff, 0x06000100,
1242 0x3c204, 0xffffffff, 0x00000100,
1243 0x3c2e0, 0xffffffff, 0x00000100,
1244 0x3c224, 0xffffffff, 0x00000100,
1245 0x3c200, 0xffffffff, 0x00000100,
1246 0x3c230, 0xffffffff, 0x00000100,
1247 0x3c234, 0xffffffff, 0x00000100,
1248 0x3c250, 0xffffffff, 0x00000100,
1249 0x3c254, 0xffffffff, 0x00000100,
1250 0x3c258, 0xffffffff, 0x00000100,
1251 0x3c25c, 0xffffffff, 0x00000100,
1252 0x3c260, 0xffffffff, 0x00000100,
1253 0x3c27c, 0xffffffff, 0x00000100,
1254 0x3c278, 0xffffffff, 0x00000100,
1255 0x3c210, 0xffffffff, 0x06000100,
1256 0x3c290, 0xffffffff, 0x00000100,
1257 0x3c274, 0xffffffff, 0x00000100,
1258 0x3c2b4, 0xffffffff, 0x00000100,
1259 0x3c2b0, 0xffffffff, 0x00000100,
1260 0x3c270, 0xffffffff, 0x00000100,
1261 0x30800, 0xffffffff, 0xe0000000,
1262 0x3c020, 0xffffffff, 0x00010000,
1263 0x3c024, 0xffffffff, 0x00030002,
1264 0x3c028, 0xffffffff, 0x00040007,
1265 0x3c02c, 0xffffffff, 0x00060005,
1266 0x3c030, 0xffffffff, 0x00090008,
1267 0x3c034, 0xffffffff, 0x00010000,
1268 0x3c038, 0xffffffff, 0x00030002,
1269 0x3c03c, 0xffffffff, 0x00040007,
1270 0x3c040, 0xffffffff, 0x00060005,
1271 0x3c044, 0xffffffff, 0x00090008,
1272 0x3c000, 0xffffffff, 0x96e00200,
1273 0x8708, 0xffffffff, 0x00900100,
1274 0xc424, 0xffffffff, 0x0020003f,
1275 0x38, 0xffffffff, 0x0140001c,
1276 0x3c, 0x000f0000, 0x000f0000,
1277 0x220, 0xffffffff, 0xC060000C,
1278 0x224, 0xc0000fff, 0x00000100,
1279 0x20a8, 0xffffffff, 0x00000104,
1280 0x55e4, 0xff000fff, 0x00000100,
1281 0x30cc, 0xc0000fff, 0x00000104,
1282 0xc1e4, 0x00000001, 0x00000001,
1283 0xd00c, 0xff000ff0, 0x00000100,
1284 0xd80c, 0xff000ff0, 0x00000100
1287 static void cik_init_golden_registers(struct radeon_device
*rdev
)
1289 switch (rdev
->family
) {
1291 radeon_program_register_sequence(rdev
,
1292 bonaire_mgcg_cgcg_init
,
1293 (const u32
)ARRAY_SIZE(bonaire_mgcg_cgcg_init
));
1294 radeon_program_register_sequence(rdev
,
1295 bonaire_golden_registers
,
1296 (const u32
)ARRAY_SIZE(bonaire_golden_registers
));
1297 radeon_program_register_sequence(rdev
,
1298 bonaire_golden_common_registers
,
1299 (const u32
)ARRAY_SIZE(bonaire_golden_common_registers
));
1300 radeon_program_register_sequence(rdev
,
1301 bonaire_golden_spm_registers
,
1302 (const u32
)ARRAY_SIZE(bonaire_golden_spm_registers
));
1305 radeon_program_register_sequence(rdev
,
1306 kalindi_mgcg_cgcg_init
,
1307 (const u32
)ARRAY_SIZE(kalindi_mgcg_cgcg_init
));
1308 radeon_program_register_sequence(rdev
,
1309 kalindi_golden_registers
,
1310 (const u32
)ARRAY_SIZE(kalindi_golden_registers
));
1311 radeon_program_register_sequence(rdev
,
1312 kalindi_golden_common_registers
,
1313 (const u32
)ARRAY_SIZE(kalindi_golden_common_registers
));
1314 radeon_program_register_sequence(rdev
,
1315 kalindi_golden_spm_registers
,
1316 (const u32
)ARRAY_SIZE(kalindi_golden_spm_registers
));
1319 radeon_program_register_sequence(rdev
,
1320 spectre_mgcg_cgcg_init
,
1321 (const u32
)ARRAY_SIZE(spectre_mgcg_cgcg_init
));
1322 radeon_program_register_sequence(rdev
,
1323 spectre_golden_registers
,
1324 (const u32
)ARRAY_SIZE(spectre_golden_registers
));
1325 radeon_program_register_sequence(rdev
,
1326 spectre_golden_common_registers
,
1327 (const u32
)ARRAY_SIZE(spectre_golden_common_registers
));
1328 radeon_program_register_sequence(rdev
,
1329 spectre_golden_spm_registers
,
1330 (const u32
)ARRAY_SIZE(spectre_golden_spm_registers
));
1338 * cik_get_xclk - get the xclk
1340 * @rdev: radeon_device pointer
1342 * Returns the reference clock used by the gfx engine
1345 u32
cik_get_xclk(struct radeon_device
*rdev
)
1347 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1349 if (rdev
->flags
& RADEON_IS_IGP
) {
1350 if (RREG32_SMC(GENERAL_PWRMGT
) & GPU_COUNTER_CLK
)
1351 return reference_clock
/ 2;
1353 if (RREG32_SMC(CG_CLKPIN_CNTL
) & XTALIN_DIVIDE
)
1354 return reference_clock
/ 4;
1356 return reference_clock
;
1360 * cik_mm_rdoorbell - read a doorbell dword
1362 * @rdev: radeon_device pointer
1363 * @offset: byte offset into the aperture
1365 * Returns the value in the doorbell aperture at the
1366 * requested offset (CIK).
1368 u32
cik_mm_rdoorbell(struct radeon_device
*rdev
, u32 offset
)
1370 if (offset
< rdev
->doorbell
.size
) {
1371 return readl(((void __iomem
*)rdev
->doorbell
.ptr
) + offset
);
1373 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset
);
1379 * cik_mm_wdoorbell - write a doorbell dword
1381 * @rdev: radeon_device pointer
1382 * @offset: byte offset into the aperture
1383 * @v: value to write
1385 * Writes @v to the doorbell aperture at the
1386 * requested offset (CIK).
1388 void cik_mm_wdoorbell(struct radeon_device
*rdev
, u32 offset
, u32 v
)
1390 if (offset
< rdev
->doorbell
.size
) {
1391 writel(v
, ((void __iomem
*)rdev
->doorbell
.ptr
) + offset
);
1393 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset
);
1397 #define BONAIRE_IO_MC_REGS_SIZE 36
1399 static const u32 bonaire_io_mc_regs
[BONAIRE_IO_MC_REGS_SIZE
][2] =
1401 {0x00000070, 0x04400000},
1402 {0x00000071, 0x80c01803},
1403 {0x00000072, 0x00004004},
1404 {0x00000073, 0x00000100},
1405 {0x00000074, 0x00ff0000},
1406 {0x00000075, 0x34000000},
1407 {0x00000076, 0x08000014},
1408 {0x00000077, 0x00cc08ec},
1409 {0x00000078, 0x00000400},
1410 {0x00000079, 0x00000000},
1411 {0x0000007a, 0x04090000},
1412 {0x0000007c, 0x00000000},
1413 {0x0000007e, 0x4408a8e8},
1414 {0x0000007f, 0x00000304},
1415 {0x00000080, 0x00000000},
1416 {0x00000082, 0x00000001},
1417 {0x00000083, 0x00000002},
1418 {0x00000084, 0xf3e4f400},
1419 {0x00000085, 0x052024e3},
1420 {0x00000087, 0x00000000},
1421 {0x00000088, 0x01000000},
1422 {0x0000008a, 0x1c0a0000},
1423 {0x0000008b, 0xff010000},
1424 {0x0000008d, 0xffffefff},
1425 {0x0000008e, 0xfff3efff},
1426 {0x0000008f, 0xfff3efbf},
1427 {0x00000092, 0xf7ffffff},
1428 {0x00000093, 0xffffff7f},
1429 {0x00000095, 0x00101101},
1430 {0x00000096, 0x00000fff},
1431 {0x00000097, 0x00116fff},
1432 {0x00000098, 0x60010000},
1433 {0x00000099, 0x10010000},
1434 {0x0000009a, 0x00006000},
1435 {0x0000009b, 0x00001000},
1436 {0x0000009f, 0x00b48000}
1440 * cik_srbm_select - select specific register instances
1442 * @rdev: radeon_device pointer
1443 * @me: selected ME (micro engine)
1448 * Switches the currently active registers instances. Some
1449 * registers are instanced per VMID, others are instanced per
1450 * me/pipe/queue combination.
1452 static void cik_srbm_select(struct radeon_device
*rdev
,
1453 u32 me
, u32 pipe
, u32 queue
, u32 vmid
)
1455 u32 srbm_gfx_cntl
= (PIPEID(pipe
& 0x3) |
1458 QUEUEID(queue
& 0x7));
1459 WREG32(SRBM_GFX_CNTL
, srbm_gfx_cntl
);
1464 * ci_mc_load_microcode - load MC ucode into the hw
1466 * @rdev: radeon_device pointer
1468 * Load the GDDR MC ucode into the hw (CIK).
1469 * Returns 0 on success, error on failure.
1471 static int ci_mc_load_microcode(struct radeon_device
*rdev
)
1473 const __be32
*fw_data
;
1474 u32 running
, blackout
= 0;
1476 int i
, ucode_size
, regs_size
;
1481 switch (rdev
->family
) {
1484 io_mc_regs
= (u32
*)&bonaire_io_mc_regs
;
1485 ucode_size
= CIK_MC_UCODE_SIZE
;
1486 regs_size
= BONAIRE_IO_MC_REGS_SIZE
;
1490 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1494 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1495 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1498 /* reset the engine and set to writable */
1499 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1500 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1502 /* load mc io regs */
1503 for (i
= 0; i
< regs_size
; i
++) {
1504 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1505 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1507 /* load the MC ucode */
1508 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1509 for (i
= 0; i
< ucode_size
; i
++)
1510 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1512 /* put the engine back into the active state */
1513 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1515 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1517 /* wait for training to complete */
1518 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1519 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1523 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1524 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1530 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1537 * cik_init_microcode - load ucode images from disk
1539 * @rdev: radeon_device pointer
1541 * Use the firmware interface to load the ucode images into
1542 * the driver (not loaded into hw).
1543 * Returns 0 on success, error on failure.
1545 static int cik_init_microcode(struct radeon_device
*rdev
)
1547 const char *chip_name
;
1548 size_t pfp_req_size
, me_req_size
, ce_req_size
,
1549 mec_req_size
, rlc_req_size
, mc_req_size
,
1550 sdma_req_size
, smc_req_size
;
1556 switch (rdev
->family
) {
1558 chip_name
= "BONAIRE";
1559 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1560 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1561 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1562 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1563 rlc_req_size
= BONAIRE_RLC_UCODE_SIZE
* 4;
1564 mc_req_size
= CIK_MC_UCODE_SIZE
* 4;
1565 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1566 smc_req_size
= ALIGN(BONAIRE_SMC_UCODE_SIZE
, 4);
1569 chip_name
= "KAVERI";
1570 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1571 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1572 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1573 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1574 rlc_req_size
= KV_RLC_UCODE_SIZE
* 4;
1575 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1578 chip_name
= "KABINI";
1579 pfp_req_size
= CIK_PFP_UCODE_SIZE
* 4;
1580 me_req_size
= CIK_ME_UCODE_SIZE
* 4;
1581 ce_req_size
= CIK_CE_UCODE_SIZE
* 4;
1582 mec_req_size
= CIK_MEC_UCODE_SIZE
* 4;
1583 rlc_req_size
= KB_RLC_UCODE_SIZE
* 4;
1584 sdma_req_size
= CIK_SDMA_UCODE_SIZE
* 4;
1589 DRM_INFO("Loading %s Microcode\n", chip_name
);
1591 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1592 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, rdev
->dev
);
1595 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1597 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1598 rdev
->pfp_fw
->size
, fw_name
);
1603 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1604 err
= request_firmware(&rdev
->me_fw
, fw_name
, rdev
->dev
);
1607 if (rdev
->me_fw
->size
!= me_req_size
) {
1609 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1610 rdev
->me_fw
->size
, fw_name
);
1614 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1615 err
= request_firmware(&rdev
->ce_fw
, fw_name
, rdev
->dev
);
1618 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1620 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1621 rdev
->ce_fw
->size
, fw_name
);
1625 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mec.bin", chip_name
);
1626 err
= request_firmware(&rdev
->mec_fw
, fw_name
, rdev
->dev
);
1629 if (rdev
->mec_fw
->size
!= mec_req_size
) {
1631 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1632 rdev
->mec_fw
->size
, fw_name
);
1636 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", chip_name
);
1637 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, rdev
->dev
);
1640 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1642 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1643 rdev
->rlc_fw
->size
, fw_name
);
1647 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_sdma.bin", chip_name
);
1648 err
= request_firmware(&rdev
->sdma_fw
, fw_name
, rdev
->dev
);
1651 if (rdev
->sdma_fw
->size
!= sdma_req_size
) {
1653 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1654 rdev
->sdma_fw
->size
, fw_name
);
1658 /* No SMC, MC ucode on APUs */
1659 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
1660 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1661 err
= request_firmware(&rdev
->mc_fw
, fw_name
, rdev
->dev
);
1664 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1666 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1667 rdev
->mc_fw
->size
, fw_name
);
1671 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_smc.bin", chip_name
);
1672 err
= request_firmware(&rdev
->smc_fw
, fw_name
, rdev
->dev
);
1675 "smc: error loading firmware \"%s\"\n",
1677 release_firmware(rdev
->smc_fw
);
1678 rdev
->smc_fw
= NULL
;
1679 } else if (rdev
->smc_fw
->size
!= smc_req_size
) {
1681 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1682 rdev
->smc_fw
->size
, fw_name
);
1691 "cik_cp: Failed to load firmware \"%s\"\n",
1693 release_firmware(rdev
->pfp_fw
);
1694 rdev
->pfp_fw
= NULL
;
1695 release_firmware(rdev
->me_fw
);
1697 release_firmware(rdev
->ce_fw
);
1699 release_firmware(rdev
->rlc_fw
);
1700 rdev
->rlc_fw
= NULL
;
1701 release_firmware(rdev
->mc_fw
);
1703 release_firmware(rdev
->smc_fw
);
1704 rdev
->smc_fw
= NULL
;
1713 * cik_tiling_mode_table_init - init the hw tiling table
1715 * @rdev: radeon_device pointer
1717 * Starting with SI, the tiling setup is done globally in a
1718 * set of 32 tiling modes. Rather than selecting each set of
1719 * parameters per surface as on older asics, we just select
1720 * which index in the tiling table we want to use, and the
1721 * surface uses those parameters (CIK).
1723 static void cik_tiling_mode_table_init(struct radeon_device
*rdev
)
1725 const u32 num_tile_mode_states
= 32;
1726 const u32 num_secondary_tile_mode_states
= 16;
1727 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
1728 u32 num_pipe_configs
;
1729 u32 num_rbs
= rdev
->config
.cik
.max_backends_per_se
*
1730 rdev
->config
.cik
.max_shader_engines
;
1732 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
1734 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
1738 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
1741 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
1745 num_pipe_configs
= rdev
->config
.cik
.max_tile_pipes
;
1746 if (num_pipe_configs
> 8)
1747 num_pipe_configs
= 8; /* ??? */
1749 if (num_pipe_configs
== 8) {
1750 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
1751 switch (reg_offset
) {
1753 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1754 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1755 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1756 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
1759 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1760 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1761 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1762 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
1765 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1766 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1767 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1768 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
1771 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1772 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1773 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1774 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
1777 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1778 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1779 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1780 TILE_SPLIT(split_equal_to_row_size
));
1783 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
1784 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
1787 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
1788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1789 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1790 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
1793 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
1794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1795 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1796 TILE_SPLIT(split_equal_to_row_size
));
1799 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
1800 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
));
1803 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
1804 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
1807 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1808 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
1809 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1810 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1813 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
1814 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
1815 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1816 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1819 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
1820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
1821 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1825 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
1826 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
1829 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1830 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1835 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
1836 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
1837 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1841 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
1842 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
1843 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1847 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
1848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
1851 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
1853 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1857 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
1858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
1859 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1863 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
1864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
1865 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16
) |
1866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
1872 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
1873 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
1875 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
1876 switch (reg_offset
) {
1878 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1879 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1880 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
1881 NUM_BANKS(ADDR_SURF_16_BANK
));
1884 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1885 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
1886 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
1887 NUM_BANKS(ADDR_SURF_16_BANK
));
1890 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1891 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1892 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
1893 NUM_BANKS(ADDR_SURF_16_BANK
));
1896 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
1899 NUM_BANKS(ADDR_SURF_16_BANK
));
1902 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1903 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1904 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1905 NUM_BANKS(ADDR_SURF_8_BANK
));
1908 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1911 NUM_BANKS(ADDR_SURF_4_BANK
));
1914 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1917 NUM_BANKS(ADDR_SURF_2_BANK
));
1920 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
1922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
1923 NUM_BANKS(ADDR_SURF_16_BANK
));
1926 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
1929 NUM_BANKS(ADDR_SURF_16_BANK
));
1932 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
1934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
1935 NUM_BANKS(ADDR_SURF_16_BANK
));
1938 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
1941 NUM_BANKS(ADDR_SURF_16_BANK
));
1944 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1947 NUM_BANKS(ADDR_SURF_8_BANK
));
1950 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1953 NUM_BANKS(ADDR_SURF_4_BANK
));
1956 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
1958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
1959 NUM_BANKS(ADDR_SURF_2_BANK
));
1965 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
1967 } else if (num_pipe_configs
== 4) {
1969 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
1970 switch (reg_offset
) {
1972 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1973 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1974 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
1975 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
1978 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1979 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1980 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
1981 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
1984 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1985 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1986 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
1987 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
1990 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1991 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1992 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
1993 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
1996 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1997 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
1998 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
1999 TILE_SPLIT(split_equal_to_row_size
));
2002 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2003 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2006 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2008 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2012 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2014 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2015 TILE_SPLIT(split_equal_to_row_size
));
2018 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2019 PIPE_CONFIG(ADDR_SURF_P4_16x16
));
2022 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2026 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2027 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2028 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2029 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2032 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2033 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2034 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2035 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2038 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2039 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2040 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2041 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2044 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2045 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2048 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2049 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2050 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2051 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2054 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2056 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2060 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2062 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2066 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2070 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2071 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2072 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2073 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2076 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2078 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2082 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2084 PIPE_CONFIG(ADDR_SURF_P4_16x16
) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2091 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2092 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2094 } else if (num_rbs
< 4) {
2095 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2096 switch (reg_offset
) {
2098 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2100 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2101 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2104 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2106 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2107 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2110 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2112 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2113 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2116 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2117 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2118 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2119 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2122 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2123 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2124 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2125 TILE_SPLIT(split_equal_to_row_size
));
2128 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2129 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2132 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2133 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2134 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2135 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2138 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2140 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2141 TILE_SPLIT(split_equal_to_row_size
));
2144 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2145 PIPE_CONFIG(ADDR_SURF_P4_8x16
));
2148 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2149 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2152 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2153 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2154 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2155 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2158 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2159 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2160 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2161 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2164 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2166 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2167 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2170 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2171 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2174 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2175 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2176 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2180 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2182 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2186 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2188 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2189 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2192 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2193 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2196 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2197 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2198 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2202 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2203 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2204 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2208 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2209 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2210 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2217 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2218 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2221 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2222 switch (reg_offset
) {
2224 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2225 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2226 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2227 NUM_BANKS(ADDR_SURF_16_BANK
));
2230 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2231 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2232 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2233 NUM_BANKS(ADDR_SURF_16_BANK
));
2236 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2239 NUM_BANKS(ADDR_SURF_16_BANK
));
2242 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2243 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2244 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2245 NUM_BANKS(ADDR_SURF_16_BANK
));
2248 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2249 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2250 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2251 NUM_BANKS(ADDR_SURF_16_BANK
));
2254 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2255 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2256 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2257 NUM_BANKS(ADDR_SURF_8_BANK
));
2260 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2261 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2262 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2263 NUM_BANKS(ADDR_SURF_4_BANK
));
2266 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2269 NUM_BANKS(ADDR_SURF_16_BANK
));
2272 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2273 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2274 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2275 NUM_BANKS(ADDR_SURF_16_BANK
));
2278 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2279 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2280 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2281 NUM_BANKS(ADDR_SURF_16_BANK
));
2284 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2285 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2286 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2287 NUM_BANKS(ADDR_SURF_16_BANK
));
2290 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2291 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2292 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2293 NUM_BANKS(ADDR_SURF_16_BANK
));
2296 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2299 NUM_BANKS(ADDR_SURF_8_BANK
));
2302 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2303 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2304 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
) |
2305 NUM_BANKS(ADDR_SURF_4_BANK
));
2311 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2313 } else if (num_pipe_configs
== 2) {
2314 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2315 switch (reg_offset
) {
2317 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2319 PIPE_CONFIG(ADDR_SURF_P2
) |
2320 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
));
2323 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2324 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2325 PIPE_CONFIG(ADDR_SURF_P2
) |
2326 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
));
2329 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2331 PIPE_CONFIG(ADDR_SURF_P2
) |
2332 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2335 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2336 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2337 PIPE_CONFIG(ADDR_SURF_P2
) |
2338 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
));
2341 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2342 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2343 PIPE_CONFIG(ADDR_SURF_P2
) |
2344 TILE_SPLIT(split_equal_to_row_size
));
2347 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
));
2351 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2353 PIPE_CONFIG(ADDR_SURF_P2
) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
));
2357 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING
) |
2359 PIPE_CONFIG(ADDR_SURF_P2
) |
2360 TILE_SPLIT(split_equal_to_row_size
));
2363 gb_tile_moden
= ARRAY_MODE(ARRAY_LINEAR_ALIGNED
);
2366 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2367 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
));
2370 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2372 PIPE_CONFIG(ADDR_SURF_P2
) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2376 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2377 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2378 PIPE_CONFIG(ADDR_SURF_P2
) |
2379 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2382 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2384 PIPE_CONFIG(ADDR_SURF_P2
) |
2385 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2388 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2389 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
));
2392 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2394 PIPE_CONFIG(ADDR_SURF_P2
) |
2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2398 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2400 PIPE_CONFIG(ADDR_SURF_P2
) |
2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2404 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING
) |
2406 PIPE_CONFIG(ADDR_SURF_P2
) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2410 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
));
2414 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2416 PIPE_CONFIG(ADDR_SURF_P2
) |
2417 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2420 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_TILED_THIN1
) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2422 PIPE_CONFIG(ADDR_SURF_P2
) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2426 gb_tile_moden
= (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1
) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING
) |
2428 PIPE_CONFIG(ADDR_SURF_P2
) |
2429 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2
));
2435 rdev
->config
.cik
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2436 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2438 for (reg_offset
= 0; reg_offset
< num_secondary_tile_mode_states
; reg_offset
++) {
2439 switch (reg_offset
) {
2441 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2444 NUM_BANKS(ADDR_SURF_16_BANK
));
2447 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2448 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2449 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2450 NUM_BANKS(ADDR_SURF_16_BANK
));
2453 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2456 NUM_BANKS(ADDR_SURF_16_BANK
));
2459 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2460 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2461 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2462 NUM_BANKS(ADDR_SURF_16_BANK
));
2465 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2468 NUM_BANKS(ADDR_SURF_16_BANK
));
2471 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2472 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2473 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2474 NUM_BANKS(ADDR_SURF_16_BANK
));
2477 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2480 NUM_BANKS(ADDR_SURF_8_BANK
));
2483 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8
) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2486 NUM_BANKS(ADDR_SURF_16_BANK
));
2489 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4
) |
2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2492 NUM_BANKS(ADDR_SURF_16_BANK
));
2495 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2498 NUM_BANKS(ADDR_SURF_16_BANK
));
2501 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2504 NUM_BANKS(ADDR_SURF_16_BANK
));
2507 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2510 NUM_BANKS(ADDR_SURF_16_BANK
));
2513 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2514 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2515 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
) |
2516 NUM_BANKS(ADDR_SURF_16_BANK
));
2519 gb_tile_moden
= (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
) |
2522 NUM_BANKS(ADDR_SURF_8_BANK
));
2528 WREG32(GB_MACROTILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2531 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs
);
2535 * cik_select_se_sh - select which SE, SH to address
2537 * @rdev: radeon_device pointer
2538 * @se_num: shader engine to address
2539 * @sh_num: sh block to address
2541 * Select which SE, SH combinations to address. Certain
2542 * registers are instanced per SE or SH. 0xffffffff means
2543 * broadcast to all SEs or SHs (CIK).
2545 static void cik_select_se_sh(struct radeon_device
*rdev
,
2546 u32 se_num
, u32 sh_num
)
2548 u32 data
= INSTANCE_BROADCAST_WRITES
;
2550 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
2551 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
2552 else if (se_num
== 0xffffffff)
2553 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
2554 else if (sh_num
== 0xffffffff)
2555 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
2557 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
2558 WREG32(GRBM_GFX_INDEX
, data
);
2562 * cik_create_bitmask - create a bitmask
2564 * @bit_width: length of the mask
2566 * create a variable length bit mask (CIK).
2567 * Returns the bitmask.
2569 static u32
cik_create_bitmask(u32 bit_width
)
2573 for (i
= 0; i
< bit_width
; i
++) {
2581 * cik_select_se_sh - select which SE, SH to address
2583 * @rdev: radeon_device pointer
2584 * @max_rb_num: max RBs (render backends) for the asic
2585 * @se_num: number of SEs (shader engines) for the asic
2586 * @sh_per_se: number of SH blocks per SE for the asic
2588 * Calculates the bitmask of disabled RBs (CIK).
2589 * Returns the disabled RB bitmask.
2591 static u32
cik_get_rb_disabled(struct radeon_device
*rdev
,
2592 u32 max_rb_num
, u32 se_num
,
2597 data
= RREG32(CC_RB_BACKEND_DISABLE
);
2599 data
&= BACKEND_DISABLE_MASK
;
2602 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
2604 data
>>= BACKEND_DISABLE_SHIFT
;
2606 mask
= cik_create_bitmask(max_rb_num
/ se_num
/ sh_per_se
);
2612 * cik_setup_rb - setup the RBs on the asic
2614 * @rdev: radeon_device pointer
2615 * @se_num: number of SEs (shader engines) for the asic
2616 * @sh_per_se: number of SH blocks per SE for the asic
2617 * @max_rb_num: max RBs (render backends) for the asic
2619 * Configures per-SE/SH RB registers (CIK).
2621 static void cik_setup_rb(struct radeon_device
*rdev
,
2622 u32 se_num
, u32 sh_per_se
,
2627 u32 disabled_rbs
= 0;
2628 u32 enabled_rbs
= 0;
2630 for (i
= 0; i
< se_num
; i
++) {
2631 for (j
= 0; j
< sh_per_se
; j
++) {
2632 cik_select_se_sh(rdev
, i
, j
);
2633 data
= cik_get_rb_disabled(rdev
, max_rb_num
, se_num
, sh_per_se
);
2634 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * CIK_RB_BITMAP_WIDTH_PER_SH
);
2637 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2640 for (i
= 0; i
< max_rb_num
; i
++) {
2641 if (!(disabled_rbs
& mask
))
2642 enabled_rbs
|= mask
;
2646 for (i
= 0; i
< se_num
; i
++) {
2647 cik_select_se_sh(rdev
, i
, 0xffffffff);
2649 for (j
= 0; j
< sh_per_se
; j
++) {
2650 switch (enabled_rbs
& 3) {
2652 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
2655 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
2659 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
2664 WREG32(PA_SC_RASTER_CONFIG
, data
);
2666 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2670 * cik_gpu_init - setup the 3D engine
2672 * @rdev: radeon_device pointer
2674 * Configures the 3D engine and tiling configuration
2675 * registers so that the 3D engine is usable.
2677 static void cik_gpu_init(struct radeon_device
*rdev
)
2679 u32 gb_addr_config
= RREG32(GB_ADDR_CONFIG
);
2680 u32 mc_shared_chmap
, mc_arb_ramcfg
;
2681 u32 hdp_host_path_cntl
;
2685 switch (rdev
->family
) {
2687 rdev
->config
.cik
.max_shader_engines
= 2;
2688 rdev
->config
.cik
.max_tile_pipes
= 4;
2689 rdev
->config
.cik
.max_cu_per_sh
= 7;
2690 rdev
->config
.cik
.max_sh_per_se
= 1;
2691 rdev
->config
.cik
.max_backends_per_se
= 2;
2692 rdev
->config
.cik
.max_texture_channel_caches
= 4;
2693 rdev
->config
.cik
.max_gprs
= 256;
2694 rdev
->config
.cik
.max_gs_threads
= 32;
2695 rdev
->config
.cik
.max_hw_contexts
= 8;
2697 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
2698 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
2699 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
2700 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
2701 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
2708 rdev
->config
.cik
.max_shader_engines
= 1;
2709 rdev
->config
.cik
.max_tile_pipes
= 2;
2710 rdev
->config
.cik
.max_cu_per_sh
= 2;
2711 rdev
->config
.cik
.max_sh_per_se
= 1;
2712 rdev
->config
.cik
.max_backends_per_se
= 1;
2713 rdev
->config
.cik
.max_texture_channel_caches
= 2;
2714 rdev
->config
.cik
.max_gprs
= 256;
2715 rdev
->config
.cik
.max_gs_threads
= 16;
2716 rdev
->config
.cik
.max_hw_contexts
= 8;
2718 rdev
->config
.cik
.sc_prim_fifo_size_frontend
= 0x20;
2719 rdev
->config
.cik
.sc_prim_fifo_size_backend
= 0x100;
2720 rdev
->config
.cik
.sc_hiz_tile_fifo_size
= 0x30;
2721 rdev
->config
.cik
.sc_earlyz_tile_fifo_size
= 0x130;
2722 gb_addr_config
= BONAIRE_GB_ADDR_CONFIG_GOLDEN
;
2726 /* Initialize HDP */
2727 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
2728 WREG32((0x2c14 + j
), 0x00000000);
2729 WREG32((0x2c18 + j
), 0x00000000);
2730 WREG32((0x2c1c + j
), 0x00000000);
2731 WREG32((0x2c20 + j
), 0x00000000);
2732 WREG32((0x2c24 + j
), 0x00000000);
2735 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
2737 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
2739 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
2740 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
2742 rdev
->config
.cik
.num_tile_pipes
= rdev
->config
.cik
.max_tile_pipes
;
2743 rdev
->config
.cik
.mem_max_burst_length_bytes
= 256;
2744 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
2745 rdev
->config
.cik
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
2746 if (rdev
->config
.cik
.mem_row_size_in_kb
> 4)
2747 rdev
->config
.cik
.mem_row_size_in_kb
= 4;
2748 /* XXX use MC settings? */
2749 rdev
->config
.cik
.shader_engine_tile_size
= 32;
2750 rdev
->config
.cik
.num_gpus
= 1;
2751 rdev
->config
.cik
.multi_gpu_tile_size
= 64;
2753 /* fix up row size */
2754 gb_addr_config
&= ~ROW_SIZE_MASK
;
2755 switch (rdev
->config
.cik
.mem_row_size_in_kb
) {
2758 gb_addr_config
|= ROW_SIZE(0);
2761 gb_addr_config
|= ROW_SIZE(1);
2764 gb_addr_config
|= ROW_SIZE(2);
2768 /* setup tiling info dword. gb_addr_config is not adequate since it does
2769 * not have bank info, so create a custom tiling dword.
2770 * bits 3:0 num_pipes
2771 * bits 7:4 num_banks
2772 * bits 11:8 group_size
2773 * bits 15:12 row_size
2775 rdev
->config
.cik
.tile_config
= 0;
2776 switch (rdev
->config
.cik
.num_tile_pipes
) {
2778 rdev
->config
.cik
.tile_config
|= (0 << 0);
2781 rdev
->config
.cik
.tile_config
|= (1 << 0);
2784 rdev
->config
.cik
.tile_config
|= (2 << 0);
2788 /* XXX what about 12? */
2789 rdev
->config
.cik
.tile_config
|= (3 << 0);
2792 if ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
)
2793 rdev
->config
.cik
.tile_config
|= 1 << 4;
2795 rdev
->config
.cik
.tile_config
|= 0 << 4;
2796 rdev
->config
.cik
.tile_config
|=
2797 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
2798 rdev
->config
.cik
.tile_config
|=
2799 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
2801 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
2802 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
2803 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
2804 WREG32(SDMA0_TILING_CONFIG
+ SDMA0_REGISTER_OFFSET
, gb_addr_config
& 0x70);
2805 WREG32(SDMA0_TILING_CONFIG
+ SDMA1_REGISTER_OFFSET
, gb_addr_config
& 0x70);
2806 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
2807 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
2808 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
2810 cik_tiling_mode_table_init(rdev
);
2812 cik_setup_rb(rdev
, rdev
->config
.cik
.max_shader_engines
,
2813 rdev
->config
.cik
.max_sh_per_se
,
2814 rdev
->config
.cik
.max_backends_per_se
);
2816 /* set HW defaults for 3D engine */
2817 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
2819 WREG32(SX_DEBUG_1
, 0x20);
2821 WREG32(TA_CNTL_AUX
, 0x00010000);
2823 tmp
= RREG32(SPI_CONFIG_CNTL
);
2825 WREG32(SPI_CONFIG_CNTL
, tmp
);
2827 WREG32(SQ_CONFIG
, 1);
2829 WREG32(DB_DEBUG
, 0);
2831 tmp
= RREG32(DB_DEBUG2
) & ~0xf00fffff;
2833 WREG32(DB_DEBUG2
, tmp
);
2835 tmp
= RREG32(DB_DEBUG3
) & ~0x0002021c;
2837 WREG32(DB_DEBUG3
, tmp
);
2839 tmp
= RREG32(CB_HW_CONTROL
) & ~0x00010000;
2841 WREG32(CB_HW_CONTROL
, tmp
);
2843 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
2845 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_frontend
) |
2846 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.cik
.sc_prim_fifo_size_backend
) |
2847 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_hiz_tile_fifo_size
) |
2848 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.cik
.sc_earlyz_tile_fifo_size
)));
2850 WREG32(VGT_NUM_INSTANCES
, 1);
2852 WREG32(CP_PERFMON_CNTL
, 0);
2854 WREG32(SQ_CONFIG
, 0);
2856 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
2857 FORCE_EOV_MAX_REZ_CNT(255)));
2859 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
2860 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
2862 WREG32(VGT_GS_VERTEX_REUSE
, 16);
2863 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
2865 tmp
= RREG32(HDP_MISC_CNTL
);
2866 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
2867 WREG32(HDP_MISC_CNTL
, tmp
);
2869 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
2870 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
2872 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
2873 WREG32(PA_SC_ENHANCE
, ENABLE_PA_SC_OUT_OF_ORDER
);
2879 * GPU scratch registers helpers function.
2882 * cik_scratch_init - setup driver info for CP scratch regs
2884 * @rdev: radeon_device pointer
2886 * Set up the number and offset of the CP scratch registers.
2887 * NOTE: use of CP scratch registers is a legacy inferface and
2888 * is not used by default on newer asics (r6xx+). On newer asics,
2889 * memory buffers are used for fences rather than scratch regs.
2891 static void cik_scratch_init(struct radeon_device
*rdev
)
2895 rdev
->scratch
.num_reg
= 7;
2896 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
2897 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
2898 rdev
->scratch
.free
[i
] = true;
2899 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
2904 * cik_ring_test - basic gfx ring test
2906 * @rdev: radeon_device pointer
2907 * @ring: radeon_ring structure holding ring information
2909 * Allocate a scratch register and write to it using the gfx ring (CIK).
2910 * Provides a basic gfx ring test to verify that the ring is working.
2911 * Used by cik_cp_gfx_resume();
2912 * Returns 0 on success, error on failure.
2914 int cik_ring_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
2921 r
= radeon_scratch_get(rdev
, &scratch
);
2923 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r
);
2926 WREG32(scratch
, 0xCAFEDEAD);
2927 r
= radeon_ring_lock(rdev
, ring
, 3);
2929 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring
->idx
, r
);
2930 radeon_scratch_free(rdev
, scratch
);
2933 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
2934 radeon_ring_write(ring
, ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2));
2935 radeon_ring_write(ring
, 0xDEADBEEF);
2936 radeon_ring_unlock_commit(rdev
, ring
);
2938 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
2939 tmp
= RREG32(scratch
);
2940 if (tmp
== 0xDEADBEEF)
2944 if (i
< rdev
->usec_timeout
) {
2945 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring
->idx
, i
);
2947 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2948 ring
->idx
, scratch
, tmp
);
2951 radeon_scratch_free(rdev
, scratch
);
2956 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2958 * @rdev: radeon_device pointer
2959 * @fence: radeon fence object
2961 * Emits a fence sequnce number on the gfx ring and flushes
2964 void cik_fence_gfx_ring_emit(struct radeon_device
*rdev
,
2965 struct radeon_fence
*fence
)
2967 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
2968 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
2970 /* EVENT_WRITE_EOP - flush caches, send int */
2971 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
2972 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
2974 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
2976 radeon_ring_write(ring
, addr
& 0xfffffffc);
2977 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
2978 radeon_ring_write(ring
, fence
->seq
);
2979 radeon_ring_write(ring
, 0);
2981 /* We should be using the new WAIT_REG_MEM special op packet here
2982 * but it causes the CP to hang
2984 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
2985 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
2986 WRITE_DATA_DST_SEL(0)));
2987 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
2988 radeon_ring_write(ring
, 0);
2989 radeon_ring_write(ring
, 0);
2993 * cik_fence_compute_ring_emit - emit a fence on the compute ring
2995 * @rdev: radeon_device pointer
2996 * @fence: radeon fence object
2998 * Emits a fence sequnce number on the compute ring and flushes
3001 void cik_fence_compute_ring_emit(struct radeon_device
*rdev
,
3002 struct radeon_fence
*fence
)
3004 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3005 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3007 /* RELEASE_MEM - flush caches, send int */
3008 radeon_ring_write(ring
, PACKET3(PACKET3_RELEASE_MEM
, 5));
3009 radeon_ring_write(ring
, (EOP_TCL1_ACTION_EN
|
3011 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) |
3013 radeon_ring_write(ring
, DATA_SEL(1) | INT_SEL(2));
3014 radeon_ring_write(ring
, addr
& 0xfffffffc);
3015 radeon_ring_write(ring
, upper_32_bits(addr
));
3016 radeon_ring_write(ring
, fence
->seq
);
3017 radeon_ring_write(ring
, 0);
3019 /* We should be using the new WAIT_REG_MEM special op packet here
3020 * but it causes the CP to hang
3022 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3023 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
3024 WRITE_DATA_DST_SEL(0)));
3025 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
3026 radeon_ring_write(ring
, 0);
3027 radeon_ring_write(ring
, 0);
3030 void cik_semaphore_ring_emit(struct radeon_device
*rdev
,
3031 struct radeon_ring
*ring
,
3032 struct radeon_semaphore
*semaphore
,
3035 uint64_t addr
= semaphore
->gpu_addr
;
3036 unsigned sel
= emit_wait
? PACKET3_SEM_SEL_WAIT
: PACKET3_SEM_SEL_SIGNAL
;
3038 radeon_ring_write(ring
, PACKET3(PACKET3_MEM_SEMAPHORE
, 1));
3039 radeon_ring_write(ring
, addr
& 0xffffffff);
3040 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xffff) | sel
);
3047 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3049 * @rdev: radeon_device pointer
3050 * @ib: radeon indirect buffer object
3052 * Emits an DE (drawing engine) or CE (constant engine) IB
3053 * on the gfx ring. IBs are usually generated by userspace
3054 * acceleration drivers and submitted to the kernel for
3055 * sheduling on the ring. This function schedules the IB
3056 * on the gfx ring for execution by the GPU.
3058 void cik_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
3060 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3061 u32 header
, control
= INDIRECT_BUFFER_VALID
;
3063 if (ib
->is_const_ib
) {
3064 /* set switch buffer packet before const IB */
3065 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
3066 radeon_ring_write(ring
, 0);
3068 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
3071 if (ring
->rptr_save_reg
) {
3072 next_rptr
= ring
->wptr
+ 3 + 4;
3073 radeon_ring_write(ring
, PACKET3(PACKET3_SET_UCONFIG_REG
, 1));
3074 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
3075 PACKET3_SET_UCONFIG_REG_START
) >> 2));
3076 radeon_ring_write(ring
, next_rptr
);
3077 } else if (rdev
->wb
.enabled
) {
3078 next_rptr
= ring
->wptr
+ 5 + 4;
3079 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3080 radeon_ring_write(ring
, WRITE_DATA_DST_SEL(1));
3081 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3082 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
3083 radeon_ring_write(ring
, next_rptr
);
3086 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
3089 control
|= ib
->length_dw
|
3090 (ib
->vm
? (ib
->vm
->id
<< 24) : 0);
3092 radeon_ring_write(ring
, header
);
3093 radeon_ring_write(ring
,
3097 (ib
->gpu_addr
& 0xFFFFFFFC));
3098 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
3099 radeon_ring_write(ring
, control
);
3103 * cik_ib_test - basic gfx ring IB test
3105 * @rdev: radeon_device pointer
3106 * @ring: radeon_ring structure holding ring information
3108 * Allocate an IB and execute it on the gfx ring (CIK).
3109 * Provides a basic gfx ring test to verify that IBs are working.
3110 * Returns 0 on success, error on failure.
3112 int cik_ib_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3114 struct radeon_ib ib
;
3120 r
= radeon_scratch_get(rdev
, &scratch
);
3122 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r
);
3125 WREG32(scratch
, 0xCAFEDEAD);
3126 r
= radeon_ib_get(rdev
, ring
->idx
, &ib
, NULL
, 256);
3128 DRM_ERROR("radeon: failed to get ib (%d).\n", r
);
3131 ib
.ptr
[0] = PACKET3(PACKET3_SET_UCONFIG_REG
, 1);
3132 ib
.ptr
[1] = ((scratch
- PACKET3_SET_UCONFIG_REG_START
) >> 2);
3133 ib
.ptr
[2] = 0xDEADBEEF;
3135 r
= radeon_ib_schedule(rdev
, &ib
, NULL
);
3137 radeon_scratch_free(rdev
, scratch
);
3138 radeon_ib_free(rdev
, &ib
);
3139 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r
);
3142 r
= radeon_fence_wait(ib
.fence
, false);
3144 DRM_ERROR("radeon: fence wait failed (%d).\n", r
);
3147 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3148 tmp
= RREG32(scratch
);
3149 if (tmp
== 0xDEADBEEF)
3153 if (i
< rdev
->usec_timeout
) {
3154 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib
.fence
->ring
, i
);
3156 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3160 radeon_scratch_free(rdev
, scratch
);
3161 radeon_ib_free(rdev
, &ib
);
3167 * On CIK, gfx and compute now have independant command processors.
3170 * Gfx consists of a single ring and can process both gfx jobs and
3171 * compute jobs. The gfx CP consists of three microengines (ME):
3172 * PFP - Pre-Fetch Parser
3174 * CE - Constant Engine
3175 * The PFP and ME make up what is considered the Drawing Engine (DE).
3176 * The CE is an asynchronous engine used for updating buffer desciptors
3177 * used by the DE so that they can be loaded into cache in parallel
3178 * while the DE is processing state update packets.
3181 * The compute CP consists of two microengines (ME):
3182 * MEC1 - Compute MicroEngine 1
3183 * MEC2 - Compute MicroEngine 2
3184 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3185 * The queues are exposed to userspace and are programmed directly
3186 * by the compute runtime.
3189 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3191 * @rdev: radeon_device pointer
3192 * @enable: enable or disable the MEs
3194 * Halts or unhalts the gfx MEs.
3196 static void cik_cp_gfx_enable(struct radeon_device
*rdev
, bool enable
)
3199 WREG32(CP_ME_CNTL
, 0);
3201 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
3202 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3208 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3210 * @rdev: radeon_device pointer
3212 * Loads the gfx PFP, ME, and CE ucode.
3213 * Returns 0 for success, -EINVAL if the ucode is not available.
3215 static int cik_cp_gfx_load_microcode(struct radeon_device
*rdev
)
3217 const __be32
*fw_data
;
3220 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
)
3223 cik_cp_gfx_enable(rdev
, false);
3226 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
3227 WREG32(CP_PFP_UCODE_ADDR
, 0);
3228 for (i
= 0; i
< CIK_PFP_UCODE_SIZE
; i
++)
3229 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
3230 WREG32(CP_PFP_UCODE_ADDR
, 0);
3233 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
3234 WREG32(CP_CE_UCODE_ADDR
, 0);
3235 for (i
= 0; i
< CIK_CE_UCODE_SIZE
; i
++)
3236 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
3237 WREG32(CP_CE_UCODE_ADDR
, 0);
3240 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
3241 WREG32(CP_ME_RAM_WADDR
, 0);
3242 for (i
= 0; i
< CIK_ME_UCODE_SIZE
; i
++)
3243 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
3244 WREG32(CP_ME_RAM_WADDR
, 0);
3246 WREG32(CP_PFP_UCODE_ADDR
, 0);
3247 WREG32(CP_CE_UCODE_ADDR
, 0);
3248 WREG32(CP_ME_RAM_WADDR
, 0);
3249 WREG32(CP_ME_RAM_RADDR
, 0);
3254 * cik_cp_gfx_start - start the gfx ring
3256 * @rdev: radeon_device pointer
3258 * Enables the ring and loads the clear state context and other
3259 * packets required to init the ring.
3260 * Returns 0 for success, error for failure.
3262 static int cik_cp_gfx_start(struct radeon_device
*rdev
)
3264 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3268 WREG32(CP_MAX_CONTEXT
, rdev
->config
.cik
.max_hw_contexts
- 1);
3269 WREG32(CP_ENDIAN_SWAP
, 0);
3270 WREG32(CP_DEVICE_ID
, 1);
3272 cik_cp_gfx_enable(rdev
, true);
3274 r
= radeon_ring_lock(rdev
, ring
, cik_default_size
+ 17);
3276 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3280 /* init the CE partitions. CE only used for gfx on CIK */
3281 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3282 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3283 radeon_ring_write(ring
, 0xc000);
3284 radeon_ring_write(ring
, 0xc000);
3286 /* setup clear context state */
3287 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3288 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3290 radeon_ring_write(ring
, PACKET3(PACKET3_CONTEXT_CONTROL
, 1));
3291 radeon_ring_write(ring
, 0x80000000);
3292 radeon_ring_write(ring
, 0x80000000);
3294 for (i
= 0; i
< cik_default_size
; i
++)
3295 radeon_ring_write(ring
, cik_default_state
[i
]);
3297 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3298 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3300 /* set clear context state */
3301 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3302 radeon_ring_write(ring
, 0);
3304 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3305 radeon_ring_write(ring
, 0x00000316);
3306 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3307 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3309 radeon_ring_unlock_commit(rdev
, ring
);
3315 * cik_cp_gfx_fini - stop the gfx ring
3317 * @rdev: radeon_device pointer
3319 * Stop the gfx ring and tear down the driver ring
3322 static void cik_cp_gfx_fini(struct radeon_device
*rdev
)
3324 cik_cp_gfx_enable(rdev
, false);
3325 radeon_ring_fini(rdev
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3329 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3331 * @rdev: radeon_device pointer
3333 * Program the location and size of the gfx ring buffer
3334 * and test it to make sure it's working.
3335 * Returns 0 for success, error for failure.
3337 static int cik_cp_gfx_resume(struct radeon_device
*rdev
)
3339 struct radeon_ring
*ring
;
3345 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
3346 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
3348 /* Set the write pointer delay */
3349 WREG32(CP_RB_WPTR_DELAY
, 0);
3351 /* set the RB to use vmid 0 */
3352 WREG32(CP_RB_VMID
, 0);
3354 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
3356 /* ring 0 - compute and gfx */
3357 /* Set ring buffer size */
3358 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3359 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3360 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3362 tmp
|= BUF_SWAP_32BIT
;
3364 WREG32(CP_RB0_CNTL
, tmp
);
3366 /* Initialize the ring buffer's read and write pointers */
3367 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3369 WREG32(CP_RB0_WPTR
, ring
->wptr
);
3371 /* set the wb address wether it's enabled or not */
3372 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
3373 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
3375 /* scratch register shadowing is no longer supported */
3376 WREG32(SCRATCH_UMSK
, 0);
3378 if (!rdev
->wb
.enabled
)
3379 tmp
|= RB_NO_UPDATE
;
3382 WREG32(CP_RB0_CNTL
, tmp
);
3384 rb_addr
= ring
->gpu_addr
>> 8;
3385 WREG32(CP_RB0_BASE
, rb_addr
);
3386 WREG32(CP_RB0_BASE_HI
, upper_32_bits(rb_addr
));
3388 ring
->rptr
= RREG32(CP_RB0_RPTR
);
3390 /* start the ring */
3391 cik_cp_gfx_start(rdev
);
3392 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
3393 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3395 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3401 u32
cik_compute_ring_get_rptr(struct radeon_device
*rdev
,
3402 struct radeon_ring
*ring
)
3408 if (rdev
->wb
.enabled
) {
3409 rptr
= le32_to_cpu(rdev
->wb
.wb
[ring
->rptr_offs
/4]);
3411 mutex_lock(&rdev
->srbm_mutex
);
3412 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
3413 rptr
= RREG32(CP_HQD_PQ_RPTR
);
3414 cik_srbm_select(rdev
, 0, 0, 0, 0);
3415 mutex_unlock(&rdev
->srbm_mutex
);
3417 rptr
= (rptr
& ring
->ptr_reg_mask
) >> ring
->ptr_reg_shift
;
3422 u32
cik_compute_ring_get_wptr(struct radeon_device
*rdev
,
3423 struct radeon_ring
*ring
)
3427 if (rdev
->wb
.enabled
) {
3428 wptr
= le32_to_cpu(rdev
->wb
.wb
[ring
->wptr_offs
/4]);
3430 mutex_lock(&rdev
->srbm_mutex
);
3431 cik_srbm_select(rdev
, ring
->me
, ring
->pipe
, ring
->queue
, 0);
3432 wptr
= RREG32(CP_HQD_PQ_WPTR
);
3433 cik_srbm_select(rdev
, 0, 0, 0, 0);
3434 mutex_unlock(&rdev
->srbm_mutex
);
3436 wptr
= (wptr
& ring
->ptr_reg_mask
) >> ring
->ptr_reg_shift
;
3441 void cik_compute_ring_set_wptr(struct radeon_device
*rdev
,
3442 struct radeon_ring
*ring
)
3444 u32 wptr
= (ring
->wptr
<< ring
->ptr_reg_shift
) & ring
->ptr_reg_mask
;
3446 rdev
->wb
.wb
[ring
->wptr_offs
/4] = cpu_to_le32(wptr
);
3447 WDOORBELL32(ring
->doorbell_offset
, wptr
);
3451 * cik_cp_compute_enable - enable/disable the compute CP MEs
3453 * @rdev: radeon_device pointer
3454 * @enable: enable or disable the MEs
3456 * Halts or unhalts the compute MEs.
3458 static void cik_cp_compute_enable(struct radeon_device
*rdev
, bool enable
)
3461 WREG32(CP_MEC_CNTL
, 0);
3463 WREG32(CP_MEC_CNTL
, (MEC_ME1_HALT
| MEC_ME2_HALT
));
3468 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3470 * @rdev: radeon_device pointer
3472 * Loads the compute MEC1&2 ucode.
3473 * Returns 0 for success, -EINVAL if the ucode is not available.
3475 static int cik_cp_compute_load_microcode(struct radeon_device
*rdev
)
3477 const __be32
*fw_data
;
3483 cik_cp_compute_enable(rdev
, false);
3486 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
3487 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
3488 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
3489 WREG32(CP_MEC_ME1_UCODE_DATA
, be32_to_cpup(fw_data
++));
3490 WREG32(CP_MEC_ME1_UCODE_ADDR
, 0);
3492 if (rdev
->family
== CHIP_KAVERI
) {
3494 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
3495 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
3496 for (i
= 0; i
< CIK_MEC_UCODE_SIZE
; i
++)
3497 WREG32(CP_MEC_ME2_UCODE_DATA
, be32_to_cpup(fw_data
++));
3498 WREG32(CP_MEC_ME2_UCODE_ADDR
, 0);
3505 * cik_cp_compute_start - start the compute queues
3507 * @rdev: radeon_device pointer
3509 * Enable the compute queues.
3510 * Returns 0 for success, error for failure.
3512 static int cik_cp_compute_start(struct radeon_device
*rdev
)
3514 cik_cp_compute_enable(rdev
, true);
3520 * cik_cp_compute_fini - stop the compute queues
3522 * @rdev: radeon_device pointer
3524 * Stop the compute queues and tear down the driver queue
3527 static void cik_cp_compute_fini(struct radeon_device
*rdev
)
3531 cik_cp_compute_enable(rdev
, false);
3533 for (i
= 0; i
< 2; i
++) {
3535 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
3537 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
3539 if (rdev
->ring
[idx
].mqd_obj
) {
3540 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
3541 if (unlikely(r
!= 0))
3542 dev_warn(rdev
->dev
, "(%d) reserve MQD bo failed\n", r
);
3544 radeon_bo_unpin(rdev
->ring
[idx
].mqd_obj
);
3545 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
3547 radeon_bo_unref(&rdev
->ring
[idx
].mqd_obj
);
3548 rdev
->ring
[idx
].mqd_obj
= NULL
;
3553 static void cik_mec_fini(struct radeon_device
*rdev
)
3557 if (rdev
->mec
.hpd_eop_obj
) {
3558 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
3559 if (unlikely(r
!= 0))
3560 dev_warn(rdev
->dev
, "(%d) reserve HPD EOP bo failed\n", r
);
3561 radeon_bo_unpin(rdev
->mec
.hpd_eop_obj
);
3562 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
3564 radeon_bo_unref(&rdev
->mec
.hpd_eop_obj
);
3565 rdev
->mec
.hpd_eop_obj
= NULL
;
3569 #define MEC_HPD_SIZE 2048
3571 static int cik_mec_init(struct radeon_device
*rdev
)
3577 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3578 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3580 if (rdev
->family
== CHIP_KAVERI
)
3581 rdev
->mec
.num_mec
= 2;
3583 rdev
->mec
.num_mec
= 1;
3584 rdev
->mec
.num_pipe
= 4;
3585 rdev
->mec
.num_queue
= rdev
->mec
.num_mec
* rdev
->mec
.num_pipe
* 8;
3587 if (rdev
->mec
.hpd_eop_obj
== NULL
) {
3588 r
= radeon_bo_create(rdev
,
3589 rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2,
3591 RADEON_GEM_DOMAIN_GTT
, NULL
,
3592 &rdev
->mec
.hpd_eop_obj
);
3594 dev_warn(rdev
->dev
, "(%d) create HDP EOP bo failed\n", r
);
3599 r
= radeon_bo_reserve(rdev
->mec
.hpd_eop_obj
, false);
3600 if (unlikely(r
!= 0)) {
3604 r
= radeon_bo_pin(rdev
->mec
.hpd_eop_obj
, RADEON_GEM_DOMAIN_GTT
,
3605 &rdev
->mec
.hpd_eop_gpu_addr
);
3607 dev_warn(rdev
->dev
, "(%d) pin HDP EOP bo failed\n", r
);
3611 r
= radeon_bo_kmap(rdev
->mec
.hpd_eop_obj
, (void **)&hpd
);
3613 dev_warn(rdev
->dev
, "(%d) map HDP EOP bo failed\n", r
);
3618 /* clear memory. Not sure if this is required or not */
3619 memset(hpd
, 0, rdev
->mec
.num_mec
*rdev
->mec
.num_pipe
* MEC_HPD_SIZE
* 2);
3621 radeon_bo_kunmap(rdev
->mec
.hpd_eop_obj
);
3622 radeon_bo_unreserve(rdev
->mec
.hpd_eop_obj
);
3627 struct hqd_registers
3629 u32 cp_mqd_base_addr
;
3630 u32 cp_mqd_base_addr_hi
;
3633 u32 cp_hqd_persistent_state
;
3634 u32 cp_hqd_pipe_priority
;
3635 u32 cp_hqd_queue_priority
;
3638 u32 cp_hqd_pq_base_hi
;
3640 u32 cp_hqd_pq_rptr_report_addr
;
3641 u32 cp_hqd_pq_rptr_report_addr_hi
;
3642 u32 cp_hqd_pq_wptr_poll_addr
;
3643 u32 cp_hqd_pq_wptr_poll_addr_hi
;
3644 u32 cp_hqd_pq_doorbell_control
;
3646 u32 cp_hqd_pq_control
;
3647 u32 cp_hqd_ib_base_addr
;
3648 u32 cp_hqd_ib_base_addr_hi
;
3650 u32 cp_hqd_ib_control
;
3651 u32 cp_hqd_iq_timer
;
3653 u32 cp_hqd_dequeue_request
;
3654 u32 cp_hqd_dma_offload
;
3655 u32 cp_hqd_sema_cmd
;
3656 u32 cp_hqd_msg_type
;
3657 u32 cp_hqd_atomic0_preop_lo
;
3658 u32 cp_hqd_atomic0_preop_hi
;
3659 u32 cp_hqd_atomic1_preop_lo
;
3660 u32 cp_hqd_atomic1_preop_hi
;
3661 u32 cp_hqd_hq_scheduler0
;
3662 u32 cp_hqd_hq_scheduler1
;
3669 u32 dispatch_initiator
;
3673 u32 pipeline_stat_enable
;
3674 u32 perf_counter_enable
;
3680 u32 resource_limits
;
3681 u32 static_thread_mgmt01
[2];
3683 u32 static_thread_mgmt23
[2];
3685 u32 thread_trace_enable
;
3688 u32 vgtcs_invoke_count
[2];
3689 struct hqd_registers queue_state
;
3691 u32 interrupt_queue
[64];
3695 * cik_cp_compute_resume - setup the compute queue registers
3697 * @rdev: radeon_device pointer
3699 * Program the compute queues and test them to make sure they
3701 * Returns 0 for success, error for failure.
3703 static int cik_cp_compute_resume(struct radeon_device
*rdev
)
3707 bool use_doorbell
= true;
3713 struct bonaire_mqd
*mqd
;
3715 r
= cik_cp_compute_start(rdev
);
3719 /* fix up chicken bits */
3720 tmp
= RREG32(CP_CPF_DEBUG
);
3722 WREG32(CP_CPF_DEBUG
, tmp
);
3724 /* init the pipes */
3725 mutex_lock(&rdev
->srbm_mutex
);
3726 for (i
= 0; i
< (rdev
->mec
.num_pipe
* rdev
->mec
.num_mec
); i
++) {
3727 int me
= (i
< 4) ? 1 : 2;
3728 int pipe
= (i
< 4) ? i
: (i
- 4);
3730 eop_gpu_addr
= rdev
->mec
.hpd_eop_gpu_addr
+ (i
* MEC_HPD_SIZE
* 2);
3732 cik_srbm_select(rdev
, me
, pipe
, 0, 0);
3734 /* write the EOP addr */
3735 WREG32(CP_HPD_EOP_BASE_ADDR
, eop_gpu_addr
>> 8);
3736 WREG32(CP_HPD_EOP_BASE_ADDR_HI
, upper_32_bits(eop_gpu_addr
) >> 8);
3738 /* set the VMID assigned */
3739 WREG32(CP_HPD_EOP_VMID
, 0);
3741 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3742 tmp
= RREG32(CP_HPD_EOP_CONTROL
);
3743 tmp
&= ~EOP_SIZE_MASK
;
3744 tmp
|= drm_order(MEC_HPD_SIZE
/ 8);
3745 WREG32(CP_HPD_EOP_CONTROL
, tmp
);
3747 cik_srbm_select(rdev
, 0, 0, 0, 0);
3748 mutex_unlock(&rdev
->srbm_mutex
);
3750 /* init the queues. Just two for now. */
3751 for (i
= 0; i
< 2; i
++) {
3753 idx
= CAYMAN_RING_TYPE_CP1_INDEX
;
3755 idx
= CAYMAN_RING_TYPE_CP2_INDEX
;
3757 if (rdev
->ring
[idx
].mqd_obj
== NULL
) {
3758 r
= radeon_bo_create(rdev
,
3759 sizeof(struct bonaire_mqd
),
3761 RADEON_GEM_DOMAIN_GTT
, NULL
,
3762 &rdev
->ring
[idx
].mqd_obj
);
3764 dev_warn(rdev
->dev
, "(%d) create MQD bo failed\n", r
);
3769 r
= radeon_bo_reserve(rdev
->ring
[idx
].mqd_obj
, false);
3770 if (unlikely(r
!= 0)) {
3771 cik_cp_compute_fini(rdev
);
3774 r
= radeon_bo_pin(rdev
->ring
[idx
].mqd_obj
, RADEON_GEM_DOMAIN_GTT
,
3777 dev_warn(rdev
->dev
, "(%d) pin MQD bo failed\n", r
);
3778 cik_cp_compute_fini(rdev
);
3781 r
= radeon_bo_kmap(rdev
->ring
[idx
].mqd_obj
, (void **)&buf
);
3783 dev_warn(rdev
->dev
, "(%d) map MQD bo failed\n", r
);
3784 cik_cp_compute_fini(rdev
);
3788 /* doorbell offset */
3789 rdev
->ring
[idx
].doorbell_offset
=
3790 (rdev
->ring
[idx
].doorbell_page_num
* PAGE_SIZE
) + 0;
3792 /* init the mqd struct */
3793 memset(buf
, 0, sizeof(struct bonaire_mqd
));
3795 mqd
= (struct bonaire_mqd
*)buf
;
3796 mqd
->header
= 0xC0310800;
3797 mqd
->static_thread_mgmt01
[0] = 0xffffffff;
3798 mqd
->static_thread_mgmt01
[1] = 0xffffffff;
3799 mqd
->static_thread_mgmt23
[0] = 0xffffffff;
3800 mqd
->static_thread_mgmt23
[1] = 0xffffffff;
3802 mutex_lock(&rdev
->srbm_mutex
);
3803 cik_srbm_select(rdev
, rdev
->ring
[idx
].me
,
3804 rdev
->ring
[idx
].pipe
,
3805 rdev
->ring
[idx
].queue
, 0);
3807 /* disable wptr polling */
3808 tmp
= RREG32(CP_PQ_WPTR_POLL_CNTL
);
3809 tmp
&= ~WPTR_POLL_EN
;
3810 WREG32(CP_PQ_WPTR_POLL_CNTL
, tmp
);
3812 /* enable doorbell? */
3813 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
3814 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
3816 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
3818 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_EN
;
3819 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
3820 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
3822 /* disable the queue if it's active */
3823 mqd
->queue_state
.cp_hqd_dequeue_request
= 0;
3824 mqd
->queue_state
.cp_hqd_pq_rptr
= 0;
3825 mqd
->queue_state
.cp_hqd_pq_wptr
= 0;
3826 if (RREG32(CP_HQD_ACTIVE
) & 1) {
3827 WREG32(CP_HQD_DEQUEUE_REQUEST
, 1);
3828 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
3829 if (!(RREG32(CP_HQD_ACTIVE
) & 1))
3833 WREG32(CP_HQD_DEQUEUE_REQUEST
, mqd
->queue_state
.cp_hqd_dequeue_request
);
3834 WREG32(CP_HQD_PQ_RPTR
, mqd
->queue_state
.cp_hqd_pq_rptr
);
3835 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
3838 /* set the pointer to the MQD */
3839 mqd
->queue_state
.cp_mqd_base_addr
= mqd_gpu_addr
& 0xfffffffc;
3840 mqd
->queue_state
.cp_mqd_base_addr_hi
= upper_32_bits(mqd_gpu_addr
);
3841 WREG32(CP_MQD_BASE_ADDR
, mqd
->queue_state
.cp_mqd_base_addr
);
3842 WREG32(CP_MQD_BASE_ADDR_HI
, mqd
->queue_state
.cp_mqd_base_addr_hi
);
3843 /* set MQD vmid to 0 */
3844 mqd
->queue_state
.cp_mqd_control
= RREG32(CP_MQD_CONTROL
);
3845 mqd
->queue_state
.cp_mqd_control
&= ~MQD_VMID_MASK
;
3846 WREG32(CP_MQD_CONTROL
, mqd
->queue_state
.cp_mqd_control
);
3848 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3849 hqd_gpu_addr
= rdev
->ring
[idx
].gpu_addr
>> 8;
3850 mqd
->queue_state
.cp_hqd_pq_base
= hqd_gpu_addr
;
3851 mqd
->queue_state
.cp_hqd_pq_base_hi
= upper_32_bits(hqd_gpu_addr
);
3852 WREG32(CP_HQD_PQ_BASE
, mqd
->queue_state
.cp_hqd_pq_base
);
3853 WREG32(CP_HQD_PQ_BASE_HI
, mqd
->queue_state
.cp_hqd_pq_base_hi
);
3855 /* set up the HQD, this is similar to CP_RB0_CNTL */
3856 mqd
->queue_state
.cp_hqd_pq_control
= RREG32(CP_HQD_PQ_CONTROL
);
3857 mqd
->queue_state
.cp_hqd_pq_control
&=
3858 ~(QUEUE_SIZE_MASK
| RPTR_BLOCK_SIZE_MASK
);
3860 mqd
->queue_state
.cp_hqd_pq_control
|=
3861 drm_order(rdev
->ring
[idx
].ring_size
/ 8);
3862 mqd
->queue_state
.cp_hqd_pq_control
|=
3863 (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8);
3865 mqd
->queue_state
.cp_hqd_pq_control
|= BUF_SWAP_32BIT
;
3867 mqd
->queue_state
.cp_hqd_pq_control
&=
3868 ~(UNORD_DISPATCH
| ROQ_PQ_IB_FLIP
| PQ_VOLATILE
);
3869 mqd
->queue_state
.cp_hqd_pq_control
|=
3870 PRIV_STATE
| KMD_QUEUE
; /* assuming kernel queue control */
3871 WREG32(CP_HQD_PQ_CONTROL
, mqd
->queue_state
.cp_hqd_pq_control
);
3873 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3875 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP1_WPTR_OFFSET
;
3877 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ CIK_WB_CP2_WPTR_OFFSET
;
3878 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
= wb_gpu_addr
& 0xfffffffc;
3879 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
= upper_32_bits(wb_gpu_addr
) & 0xffff;
3880 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR
, mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr
);
3881 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI
,
3882 mqd
->queue_state
.cp_hqd_pq_wptr_poll_addr_hi
);
3884 /* set the wb address wether it's enabled or not */
3886 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
;
3888 wb_gpu_addr
= rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
;
3889 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
= wb_gpu_addr
& 0xfffffffc;
3890 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
=
3891 upper_32_bits(wb_gpu_addr
) & 0xffff;
3892 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR
,
3893 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr
);
3894 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI
,
3895 mqd
->queue_state
.cp_hqd_pq_rptr_report_addr_hi
);
3897 /* enable the doorbell if requested */
3899 mqd
->queue_state
.cp_hqd_pq_doorbell_control
=
3900 RREG32(CP_HQD_PQ_DOORBELL_CONTROL
);
3901 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&= ~DOORBELL_OFFSET_MASK
;
3902 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|=
3903 DOORBELL_OFFSET(rdev
->ring
[idx
].doorbell_offset
/ 4);
3904 mqd
->queue_state
.cp_hqd_pq_doorbell_control
|= DOORBELL_EN
;
3905 mqd
->queue_state
.cp_hqd_pq_doorbell_control
&=
3906 ~(DOORBELL_SOURCE
| DOORBELL_HIT
);
3909 mqd
->queue_state
.cp_hqd_pq_doorbell_control
= 0;
3911 WREG32(CP_HQD_PQ_DOORBELL_CONTROL
,
3912 mqd
->queue_state
.cp_hqd_pq_doorbell_control
);
3914 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3915 rdev
->ring
[idx
].wptr
= 0;
3916 mqd
->queue_state
.cp_hqd_pq_wptr
= rdev
->ring
[idx
].wptr
;
3917 WREG32(CP_HQD_PQ_WPTR
, mqd
->queue_state
.cp_hqd_pq_wptr
);
3918 rdev
->ring
[idx
].rptr
= RREG32(CP_HQD_PQ_RPTR
);
3919 mqd
->queue_state
.cp_hqd_pq_rptr
= rdev
->ring
[idx
].rptr
;
3921 /* set the vmid for the queue */
3922 mqd
->queue_state
.cp_hqd_vmid
= 0;
3923 WREG32(CP_HQD_VMID
, mqd
->queue_state
.cp_hqd_vmid
);
3925 /* activate the queue */
3926 mqd
->queue_state
.cp_hqd_active
= 1;
3927 WREG32(CP_HQD_ACTIVE
, mqd
->queue_state
.cp_hqd_active
);
3929 cik_srbm_select(rdev
, 0, 0, 0, 0);
3930 mutex_unlock(&rdev
->srbm_mutex
);
3932 radeon_bo_kunmap(rdev
->ring
[idx
].mqd_obj
);
3933 radeon_bo_unreserve(rdev
->ring
[idx
].mqd_obj
);
3935 rdev
->ring
[idx
].ready
= true;
3936 r
= radeon_ring_test(rdev
, idx
, &rdev
->ring
[idx
]);
3938 rdev
->ring
[idx
].ready
= false;
3944 static void cik_cp_enable(struct radeon_device
*rdev
, bool enable
)
3946 cik_cp_gfx_enable(rdev
, enable
);
3947 cik_cp_compute_enable(rdev
, enable
);
3950 static int cik_cp_load_microcode(struct radeon_device
*rdev
)
3954 r
= cik_cp_gfx_load_microcode(rdev
);
3957 r
= cik_cp_compute_load_microcode(rdev
);
3964 static void cik_cp_fini(struct radeon_device
*rdev
)
3966 cik_cp_gfx_fini(rdev
);
3967 cik_cp_compute_fini(rdev
);
3970 static int cik_cp_resume(struct radeon_device
*rdev
)
3974 /* Reset all cp blocks */
3975 WREG32(GRBM_SOFT_RESET
, SOFT_RESET_CP
);
3976 RREG32(GRBM_SOFT_RESET
);
3978 WREG32(GRBM_SOFT_RESET
, 0);
3979 RREG32(GRBM_SOFT_RESET
);
3981 r
= cik_cp_load_microcode(rdev
);
3985 r
= cik_cp_gfx_resume(rdev
);
3988 r
= cik_cp_compute_resume(rdev
);
3997 * Starting with CIK, the GPU has new asynchronous
3998 * DMA engines. These engines are used for compute
3999 * and gfx. There are two DMA engines (SDMA0, SDMA1)
4000 * and each one supports 1 ring buffer used for gfx
4001 * and 2 queues used for compute.
4003 * The programming model is very similar to the CP
4004 * (ring buffer, IBs, etc.), but sDMA has it's own
4005 * packet format that is different from the PM4 format
4006 * used by the CP. sDMA supports copying data, writing
4007 * embedded data, solid fills, and a number of other
4008 * things. It also has support for tiling/detiling of
4012 * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
4014 * @rdev: radeon_device pointer
4015 * @ib: IB object to schedule
4017 * Schedule an IB in the DMA ring (CIK).
4019 void cik_sdma_ring_ib_execute(struct radeon_device
*rdev
,
4020 struct radeon_ib
*ib
)
4022 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
4023 u32 extra_bits
= (ib
->vm
? ib
->vm
->id
: 0) & 0xf;
4025 if (rdev
->wb
.enabled
) {
4026 u32 next_rptr
= ring
->wptr
+ 5;
4027 while ((next_rptr
& 7) != 4)
4030 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_WRITE
, SDMA_WRITE_SUB_OPCODE_LINEAR
, 0));
4031 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
4032 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
4033 radeon_ring_write(ring
, 1); /* number of DWs to follow */
4034 radeon_ring_write(ring
, next_rptr
);
4037 /* IB packet must end on a 8 DW boundary */
4038 while ((ring
->wptr
& 7) != 4)
4039 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
4040 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER
, 0, extra_bits
));
4041 radeon_ring_write(ring
, ib
->gpu_addr
& 0xffffffe0); /* base must be 32 byte aligned */
4042 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xffffffff);
4043 radeon_ring_write(ring
, ib
->length_dw
);
4048 * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
4050 * @rdev: radeon_device pointer
4051 * @fence: radeon fence object
4053 * Add a DMA fence packet to the ring to write
4054 * the fence seq number and DMA trap packet to generate
4055 * an interrupt if needed (CIK).
4057 void cik_sdma_fence_ring_emit(struct radeon_device
*rdev
,
4058 struct radeon_fence
*fence
)
4060 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
4061 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
4062 u32 extra_bits
= (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
4063 SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
4066 if (fence
->ring
== R600_RING_TYPE_DMA_INDEX
)
4067 ref_and_mask
= SDMA0
;
4069 ref_and_mask
= SDMA1
;
4071 /* write the fence */
4072 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_FENCE
, 0, 0));
4073 radeon_ring_write(ring
, addr
& 0xffffffff);
4074 radeon_ring_write(ring
, upper_32_bits(addr
) & 0xffffffff);
4075 radeon_ring_write(ring
, fence
->seq
);
4076 /* generate an interrupt */
4077 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_TRAP
, 0, 0));
4079 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM
, 0, extra_bits
));
4080 radeon_ring_write(ring
, GPU_HDP_FLUSH_DONE
);
4081 radeon_ring_write(ring
, GPU_HDP_FLUSH_REQ
);
4082 radeon_ring_write(ring
, ref_and_mask
); /* REFERENCE */
4083 radeon_ring_write(ring
, ref_and_mask
); /* MASK */
4084 radeon_ring_write(ring
, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
4088 * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
4090 * @rdev: radeon_device pointer
4091 * @ring: radeon_ring structure holding ring information
4092 * @semaphore: radeon semaphore object
4093 * @emit_wait: wait or signal semaphore
4095 * Add a DMA semaphore packet to the ring wait on or signal
4096 * other rings (CIK).
4098 void cik_sdma_semaphore_ring_emit(struct radeon_device
*rdev
,
4099 struct radeon_ring
*ring
,
4100 struct radeon_semaphore
*semaphore
,
4103 u64 addr
= semaphore
->gpu_addr
;
4104 u32 extra_bits
= emit_wait
? 0 : SDMA_SEMAPHORE_EXTRA_S
;
4106 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE
, 0, extra_bits
));
4107 radeon_ring_write(ring
, addr
& 0xfffffff8);
4108 radeon_ring_write(ring
, upper_32_bits(addr
) & 0xffffffff);
4112 * cik_sdma_gfx_stop - stop the gfx async dma engines
4114 * @rdev: radeon_device pointer
4116 * Stop the gfx async dma ring buffers (CIK).
4118 static void cik_sdma_gfx_stop(struct radeon_device
*rdev
)
4120 u32 rb_cntl
, reg_offset
;
4123 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
4125 for (i
= 0; i
< 2; i
++) {
4127 reg_offset
= SDMA0_REGISTER_OFFSET
;
4129 reg_offset
= SDMA1_REGISTER_OFFSET
;
4130 rb_cntl
= RREG32(SDMA0_GFX_RB_CNTL
+ reg_offset
);
4131 rb_cntl
&= ~SDMA_RB_ENABLE
;
4132 WREG32(SDMA0_GFX_RB_CNTL
+ reg_offset
, rb_cntl
);
4133 WREG32(SDMA0_GFX_IB_CNTL
+ reg_offset
, 0);
4138 * cik_sdma_rlc_stop - stop the compute async dma engines
4140 * @rdev: radeon_device pointer
4142 * Stop the compute async dma queues (CIK).
4144 static void cik_sdma_rlc_stop(struct radeon_device
*rdev
)
4150 * cik_sdma_enable - stop the async dma engines
4152 * @rdev: radeon_device pointer
4153 * @enable: enable/disable the DMA MEs.
4155 * Halt or unhalt the async dma engines (CIK).
4157 static void cik_sdma_enable(struct radeon_device
*rdev
, bool enable
)
4159 u32 me_cntl
, reg_offset
;
4162 for (i
= 0; i
< 2; i
++) {
4164 reg_offset
= SDMA0_REGISTER_OFFSET
;
4166 reg_offset
= SDMA1_REGISTER_OFFSET
;
4167 me_cntl
= RREG32(SDMA0_ME_CNTL
+ reg_offset
);
4169 me_cntl
&= ~SDMA_HALT
;
4171 me_cntl
|= SDMA_HALT
;
4172 WREG32(SDMA0_ME_CNTL
+ reg_offset
, me_cntl
);
4177 * cik_sdma_gfx_resume - setup and start the async dma engines
4179 * @rdev: radeon_device pointer
4181 * Set up the gfx DMA ring buffers and enable them (CIK).
4182 * Returns 0 for success, error for failure.
4184 static int cik_sdma_gfx_resume(struct radeon_device
*rdev
)
4186 struct radeon_ring
*ring
;
4187 u32 rb_cntl
, ib_cntl
;
4189 u32 reg_offset
, wb_offset
;
4192 for (i
= 0; i
< 2; i
++) {
4194 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
4195 reg_offset
= SDMA0_REGISTER_OFFSET
;
4196 wb_offset
= R600_WB_DMA_RPTR_OFFSET
;
4198 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
4199 reg_offset
= SDMA1_REGISTER_OFFSET
;
4200 wb_offset
= CAYMAN_WB_DMA1_RPTR_OFFSET
;
4203 WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL
+ reg_offset
, 0);
4204 WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL
+ reg_offset
, 0);
4206 /* Set ring buffer size in dwords */
4207 rb_bufsz
= drm_order(ring
->ring_size
/ 4);
4208 rb_cntl
= rb_bufsz
<< 1;
4210 rb_cntl
|= SDMA_RB_SWAP_ENABLE
| SDMA_RPTR_WRITEBACK_SWAP_ENABLE
;
4212 WREG32(SDMA0_GFX_RB_CNTL
+ reg_offset
, rb_cntl
);
4214 /* Initialize the ring buffer's read and write pointers */
4215 WREG32(SDMA0_GFX_RB_RPTR
+ reg_offset
, 0);
4216 WREG32(SDMA0_GFX_RB_WPTR
+ reg_offset
, 0);
4218 /* set the wb address whether it's enabled or not */
4219 WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI
+ reg_offset
,
4220 upper_32_bits(rdev
->wb
.gpu_addr
+ wb_offset
) & 0xFFFFFFFF);
4221 WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO
+ reg_offset
,
4222 ((rdev
->wb
.gpu_addr
+ wb_offset
) & 0xFFFFFFFC));
4224 if (rdev
->wb
.enabled
)
4225 rb_cntl
|= SDMA_RPTR_WRITEBACK_ENABLE
;
4227 WREG32(SDMA0_GFX_RB_BASE
+ reg_offset
, ring
->gpu_addr
>> 8);
4228 WREG32(SDMA0_GFX_RB_BASE_HI
+ reg_offset
, ring
->gpu_addr
>> 40);
4231 WREG32(SDMA0_GFX_RB_WPTR
+ reg_offset
, ring
->wptr
<< 2);
4233 ring
->rptr
= RREG32(SDMA0_GFX_RB_RPTR
+ reg_offset
) >> 2;
4236 WREG32(SDMA0_GFX_RB_CNTL
+ reg_offset
, rb_cntl
| SDMA_RB_ENABLE
);
4238 ib_cntl
= SDMA_IB_ENABLE
;
4240 ib_cntl
|= SDMA_IB_SWAP_ENABLE
;
4242 /* enable DMA IBs */
4243 WREG32(SDMA0_GFX_IB_CNTL
+ reg_offset
, ib_cntl
);
4247 r
= radeon_ring_test(rdev
, ring
->idx
, ring
);
4249 ring
->ready
= false;
4254 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.real_vram_size
);
4260 * cik_sdma_rlc_resume - setup and start the async dma engines
4262 * @rdev: radeon_device pointer
4264 * Set up the compute DMA queues and enable them (CIK).
4265 * Returns 0 for success, error for failure.
4267 static int cik_sdma_rlc_resume(struct radeon_device
*rdev
)
4274 * cik_sdma_load_microcode - load the sDMA ME ucode
4276 * @rdev: radeon_device pointer
4278 * Loads the sDMA0/1 ucode.
4279 * Returns 0 for success, -EINVAL if the ucode is not available.
4281 static int cik_sdma_load_microcode(struct radeon_device
*rdev
)
4283 const __be32
*fw_data
;
4289 /* stop the gfx rings and rlc compute queues */
4290 cik_sdma_gfx_stop(rdev
);
4291 cik_sdma_rlc_stop(rdev
);
4294 cik_sdma_enable(rdev
, false);
4297 fw_data
= (const __be32
*)rdev
->sdma_fw
->data
;
4298 WREG32(SDMA0_UCODE_ADDR
+ SDMA0_REGISTER_OFFSET
, 0);
4299 for (i
= 0; i
< CIK_SDMA_UCODE_SIZE
; i
++)
4300 WREG32(SDMA0_UCODE_DATA
+ SDMA0_REGISTER_OFFSET
, be32_to_cpup(fw_data
++));
4301 WREG32(SDMA0_UCODE_DATA
+ SDMA0_REGISTER_OFFSET
, CIK_SDMA_UCODE_VERSION
);
4304 fw_data
= (const __be32
*)rdev
->sdma_fw
->data
;
4305 WREG32(SDMA0_UCODE_ADDR
+ SDMA1_REGISTER_OFFSET
, 0);
4306 for (i
= 0; i
< CIK_SDMA_UCODE_SIZE
; i
++)
4307 WREG32(SDMA0_UCODE_DATA
+ SDMA1_REGISTER_OFFSET
, be32_to_cpup(fw_data
++));
4308 WREG32(SDMA0_UCODE_DATA
+ SDMA1_REGISTER_OFFSET
, CIK_SDMA_UCODE_VERSION
);
4310 WREG32(SDMA0_UCODE_ADDR
+ SDMA0_REGISTER_OFFSET
, 0);
4311 WREG32(SDMA0_UCODE_ADDR
+ SDMA1_REGISTER_OFFSET
, 0);
4316 * cik_sdma_resume - setup and start the async dma engines
4318 * @rdev: radeon_device pointer
4320 * Set up the DMA engines and enable them (CIK).
4321 * Returns 0 for success, error for failure.
4323 static int cik_sdma_resume(struct radeon_device
*rdev
)
4328 WREG32(SRBM_SOFT_RESET
, SOFT_RESET_SDMA
| SOFT_RESET_SDMA1
);
4329 RREG32(SRBM_SOFT_RESET
);
4331 WREG32(SRBM_SOFT_RESET
, 0);
4332 RREG32(SRBM_SOFT_RESET
);
4334 r
= cik_sdma_load_microcode(rdev
);
4338 /* unhalt the MEs */
4339 cik_sdma_enable(rdev
, true);
4341 /* start the gfx rings and rlc compute queues */
4342 r
= cik_sdma_gfx_resume(rdev
);
4345 r
= cik_sdma_rlc_resume(rdev
);
4353 * cik_sdma_fini - tear down the async dma engines
4355 * @rdev: radeon_device pointer
4357 * Stop the async dma engines and free the rings (CIK).
4359 static void cik_sdma_fini(struct radeon_device
*rdev
)
4361 /* stop the gfx rings and rlc compute queues */
4362 cik_sdma_gfx_stop(rdev
);
4363 cik_sdma_rlc_stop(rdev
);
4365 cik_sdma_enable(rdev
, false);
4366 radeon_ring_fini(rdev
, &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
]);
4367 radeon_ring_fini(rdev
, &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
]);
4368 /* XXX - compute dma queue tear down */
4372 * cik_copy_dma - copy pages using the DMA engine
4374 * @rdev: radeon_device pointer
4375 * @src_offset: src GPU address
4376 * @dst_offset: dst GPU address
4377 * @num_gpu_pages: number of GPU pages to xfer
4378 * @fence: radeon fence object
4380 * Copy GPU paging using the DMA engine (CIK).
4381 * Used by the radeon ttm implementation to move pages if
4382 * registered as the asic copy callback.
4384 int cik_copy_dma(struct radeon_device
*rdev
,
4385 uint64_t src_offset
, uint64_t dst_offset
,
4386 unsigned num_gpu_pages
,
4387 struct radeon_fence
**fence
)
4389 struct radeon_semaphore
*sem
= NULL
;
4390 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
4391 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
4392 u32 size_in_bytes
, cur_size_in_bytes
;
4396 r
= radeon_semaphore_create(rdev
, &sem
);
4398 DRM_ERROR("radeon: moving bo (%d).\n", r
);
4402 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
4403 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0x1fffff);
4404 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 7 + 14);
4406 DRM_ERROR("radeon: moving bo (%d).\n", r
);
4407 radeon_semaphore_free(rdev
, &sem
, NULL
);
4411 if (radeon_fence_need_sync(*fence
, ring
->idx
)) {
4412 radeon_semaphore_sync_rings(rdev
, sem
, (*fence
)->ring
,
4414 radeon_fence_note_sync(*fence
, ring
->idx
);
4416 radeon_semaphore_free(rdev
, &sem
, NULL
);
4419 for (i
= 0; i
< num_loops
; i
++) {
4420 cur_size_in_bytes
= size_in_bytes
;
4421 if (cur_size_in_bytes
> 0x1fffff)
4422 cur_size_in_bytes
= 0x1fffff;
4423 size_in_bytes
-= cur_size_in_bytes
;
4424 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_COPY
, SDMA_COPY_SUB_OPCODE_LINEAR
, 0));
4425 radeon_ring_write(ring
, cur_size_in_bytes
);
4426 radeon_ring_write(ring
, 0); /* src/dst endian swap */
4427 radeon_ring_write(ring
, src_offset
& 0xffffffff);
4428 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xffffffff);
4429 radeon_ring_write(ring
, dst_offset
& 0xfffffffc);
4430 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xffffffff);
4431 src_offset
+= cur_size_in_bytes
;
4432 dst_offset
+= cur_size_in_bytes
;
4435 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
4437 radeon_ring_unlock_undo(rdev
, ring
);
4441 radeon_ring_unlock_commit(rdev
, ring
);
4442 radeon_semaphore_free(rdev
, &sem
, *fence
);
4448 * cik_sdma_ring_test - simple async dma engine test
4450 * @rdev: radeon_device pointer
4451 * @ring: radeon_ring structure holding ring information
4453 * Test the DMA engine by writing using it to write an
4454 * value to memory. (CIK).
4455 * Returns 0 for success, error for failure.
4457 int cik_sdma_ring_test(struct radeon_device
*rdev
,
4458 struct radeon_ring
*ring
)
4462 void __iomem
*ptr
= (void *)rdev
->vram_scratch
.ptr
;
4466 DRM_ERROR("invalid vram scratch pointer\n");
4473 r
= radeon_ring_lock(rdev
, ring
, 4);
4475 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring
->idx
, r
);
4478 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_WRITE
, SDMA_WRITE_SUB_OPCODE_LINEAR
, 0));
4479 radeon_ring_write(ring
, rdev
->vram_scratch
.gpu_addr
& 0xfffffffc);
4480 radeon_ring_write(ring
, upper_32_bits(rdev
->vram_scratch
.gpu_addr
) & 0xffffffff);
4481 radeon_ring_write(ring
, 1); /* number of DWs to follow */
4482 radeon_ring_write(ring
, 0xDEADBEEF);
4483 radeon_ring_unlock_commit(rdev
, ring
);
4485 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4487 if (tmp
== 0xDEADBEEF)
4492 if (i
< rdev
->usec_timeout
) {
4493 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring
->idx
, i
);
4495 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
4503 * cik_sdma_ib_test - test an IB on the DMA engine
4505 * @rdev: radeon_device pointer
4506 * @ring: radeon_ring structure holding ring information
4508 * Test a simple IB in the DMA ring (CIK).
4509 * Returns 0 on success, error on failure.
4511 int cik_sdma_ib_test(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
4513 struct radeon_ib ib
;
4516 void __iomem
*ptr
= (void *)rdev
->vram_scratch
.ptr
;
4520 DRM_ERROR("invalid vram scratch pointer\n");
4527 r
= radeon_ib_get(rdev
, ring
->idx
, &ib
, NULL
, 256);
4529 DRM_ERROR("radeon: failed to get ib (%d).\n", r
);
4533 ib
.ptr
[0] = SDMA_PACKET(SDMA_OPCODE_WRITE
, SDMA_WRITE_SUB_OPCODE_LINEAR
, 0);
4534 ib
.ptr
[1] = rdev
->vram_scratch
.gpu_addr
& 0xfffffffc;
4535 ib
.ptr
[2] = upper_32_bits(rdev
->vram_scratch
.gpu_addr
) & 0xffffffff;
4537 ib
.ptr
[4] = 0xDEADBEEF;
4540 r
= radeon_ib_schedule(rdev
, &ib
, NULL
);
4542 radeon_ib_free(rdev
, &ib
);
4543 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r
);
4546 r
= radeon_fence_wait(ib
.fence
, false);
4548 DRM_ERROR("radeon: fence wait failed (%d).\n", r
);
4551 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4553 if (tmp
== 0xDEADBEEF)
4557 if (i
< rdev
->usec_timeout
) {
4558 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib
.fence
->ring
, i
);
4560 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp
);
4563 radeon_ib_free(rdev
, &ib
);
4568 static void cik_print_gpu_status_regs(struct radeon_device
*rdev
)
4570 dev_info(rdev
->dev
, " GRBM_STATUS=0x%08X\n",
4571 RREG32(GRBM_STATUS
));
4572 dev_info(rdev
->dev
, " GRBM_STATUS2=0x%08X\n",
4573 RREG32(GRBM_STATUS2
));
4574 dev_info(rdev
->dev
, " GRBM_STATUS_SE0=0x%08X\n",
4575 RREG32(GRBM_STATUS_SE0
));
4576 dev_info(rdev
->dev
, " GRBM_STATUS_SE1=0x%08X\n",
4577 RREG32(GRBM_STATUS_SE1
));
4578 dev_info(rdev
->dev
, " GRBM_STATUS_SE2=0x%08X\n",
4579 RREG32(GRBM_STATUS_SE2
));
4580 dev_info(rdev
->dev
, " GRBM_STATUS_SE3=0x%08X\n",
4581 RREG32(GRBM_STATUS_SE3
));
4582 dev_info(rdev
->dev
, " SRBM_STATUS=0x%08X\n",
4583 RREG32(SRBM_STATUS
));
4584 dev_info(rdev
->dev
, " SRBM_STATUS2=0x%08X\n",
4585 RREG32(SRBM_STATUS2
));
4586 dev_info(rdev
->dev
, " SDMA0_STATUS_REG = 0x%08X\n",
4587 RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
));
4588 dev_info(rdev
->dev
, " SDMA1_STATUS_REG = 0x%08X\n",
4589 RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
));
4590 dev_info(rdev
->dev
, " CP_STAT = 0x%08x\n", RREG32(CP_STAT
));
4591 dev_info(rdev
->dev
, " CP_STALLED_STAT1 = 0x%08x\n",
4592 RREG32(CP_STALLED_STAT1
));
4593 dev_info(rdev
->dev
, " CP_STALLED_STAT2 = 0x%08x\n",
4594 RREG32(CP_STALLED_STAT2
));
4595 dev_info(rdev
->dev
, " CP_STALLED_STAT3 = 0x%08x\n",
4596 RREG32(CP_STALLED_STAT3
));
4597 dev_info(rdev
->dev
, " CP_CPF_BUSY_STAT = 0x%08x\n",
4598 RREG32(CP_CPF_BUSY_STAT
));
4599 dev_info(rdev
->dev
, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4600 RREG32(CP_CPF_STALLED_STAT1
));
4601 dev_info(rdev
->dev
, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS
));
4602 dev_info(rdev
->dev
, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT
));
4603 dev_info(rdev
->dev
, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4604 RREG32(CP_CPC_STALLED_STAT1
));
4605 dev_info(rdev
->dev
, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS
));
4609 * cik_gpu_check_soft_reset - check which blocks are busy
4611 * @rdev: radeon_device pointer
4613 * Check which blocks are busy and return the relevant reset
4614 * mask to be used by cik_gpu_soft_reset().
4615 * Returns a mask of the blocks to be reset.
4617 static u32
cik_gpu_check_soft_reset(struct radeon_device
*rdev
)
4623 tmp
= RREG32(GRBM_STATUS
);
4624 if (tmp
& (PA_BUSY
| SC_BUSY
|
4625 BCI_BUSY
| SX_BUSY
|
4626 TA_BUSY
| VGT_BUSY
|
4628 GDS_BUSY
| SPI_BUSY
|
4629 IA_BUSY
| IA_BUSY_NO_DMA
))
4630 reset_mask
|= RADEON_RESET_GFX
;
4632 if (tmp
& (CP_BUSY
| CP_COHERENCY_BUSY
))
4633 reset_mask
|= RADEON_RESET_CP
;
4636 tmp
= RREG32(GRBM_STATUS2
);
4638 reset_mask
|= RADEON_RESET_RLC
;
4640 /* SDMA0_STATUS_REG */
4641 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA0_REGISTER_OFFSET
);
4642 if (!(tmp
& SDMA_IDLE
))
4643 reset_mask
|= RADEON_RESET_DMA
;
4645 /* SDMA1_STATUS_REG */
4646 tmp
= RREG32(SDMA0_STATUS_REG
+ SDMA1_REGISTER_OFFSET
);
4647 if (!(tmp
& SDMA_IDLE
))
4648 reset_mask
|= RADEON_RESET_DMA1
;
4651 tmp
= RREG32(SRBM_STATUS2
);
4652 if (tmp
& SDMA_BUSY
)
4653 reset_mask
|= RADEON_RESET_DMA
;
4655 if (tmp
& SDMA1_BUSY
)
4656 reset_mask
|= RADEON_RESET_DMA1
;
4659 tmp
= RREG32(SRBM_STATUS
);
4662 reset_mask
|= RADEON_RESET_IH
;
4665 reset_mask
|= RADEON_RESET_SEM
;
4667 if (tmp
& GRBM_RQ_PENDING
)
4668 reset_mask
|= RADEON_RESET_GRBM
;
4671 reset_mask
|= RADEON_RESET_VMC
;
4673 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
4674 MCC_BUSY
| MCD_BUSY
))
4675 reset_mask
|= RADEON_RESET_MC
;
4677 if (evergreen_is_display_hung(rdev
))
4678 reset_mask
|= RADEON_RESET_DISPLAY
;
4680 /* Skip MC reset as it's mostly likely not hung, just busy */
4681 if (reset_mask
& RADEON_RESET_MC
) {
4682 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
4683 reset_mask
&= ~RADEON_RESET_MC
;
4690 * cik_gpu_soft_reset - soft reset GPU
4692 * @rdev: radeon_device pointer
4693 * @reset_mask: mask of which blocks to reset
4695 * Soft reset the blocks specified in @reset_mask.
4697 static void cik_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
4699 struct evergreen_mc_save save
;
4700 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
4703 if (reset_mask
== 0)
4706 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
4708 cik_print_gpu_status_regs(rdev
);
4709 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4710 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
4711 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4712 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
4717 /* Disable GFX parsing/prefetching */
4718 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
4720 /* Disable MEC parsing/prefetching */
4721 WREG32(CP_MEC_CNTL
, MEC_ME1_HALT
| MEC_ME2_HALT
);
4723 if (reset_mask
& RADEON_RESET_DMA
) {
4725 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
);
4727 WREG32(SDMA0_ME_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
4729 if (reset_mask
& RADEON_RESET_DMA1
) {
4731 tmp
= RREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
);
4733 WREG32(SDMA0_ME_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
4736 evergreen_mc_stop(rdev
, &save
);
4737 if (evergreen_mc_wait_for_idle(rdev
)) {
4738 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
4741 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
))
4742 grbm_soft_reset
= SOFT_RESET_CP
| SOFT_RESET_GFX
;
4744 if (reset_mask
& RADEON_RESET_CP
) {
4745 grbm_soft_reset
|= SOFT_RESET_CP
;
4747 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4750 if (reset_mask
& RADEON_RESET_DMA
)
4751 srbm_soft_reset
|= SOFT_RESET_SDMA
;
4753 if (reset_mask
& RADEON_RESET_DMA1
)
4754 srbm_soft_reset
|= SOFT_RESET_SDMA1
;
4756 if (reset_mask
& RADEON_RESET_DISPLAY
)
4757 srbm_soft_reset
|= SOFT_RESET_DC
;
4759 if (reset_mask
& RADEON_RESET_RLC
)
4760 grbm_soft_reset
|= SOFT_RESET_RLC
;
4762 if (reset_mask
& RADEON_RESET_SEM
)
4763 srbm_soft_reset
|= SOFT_RESET_SEM
;
4765 if (reset_mask
& RADEON_RESET_IH
)
4766 srbm_soft_reset
|= SOFT_RESET_IH
;
4768 if (reset_mask
& RADEON_RESET_GRBM
)
4769 srbm_soft_reset
|= SOFT_RESET_GRBM
;
4771 if (reset_mask
& RADEON_RESET_VMC
)
4772 srbm_soft_reset
|= SOFT_RESET_VMC
;
4774 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
4775 if (reset_mask
& RADEON_RESET_MC
)
4776 srbm_soft_reset
|= SOFT_RESET_MC
;
4779 if (grbm_soft_reset
) {
4780 tmp
= RREG32(GRBM_SOFT_RESET
);
4781 tmp
|= grbm_soft_reset
;
4782 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
4783 WREG32(GRBM_SOFT_RESET
, tmp
);
4784 tmp
= RREG32(GRBM_SOFT_RESET
);
4788 tmp
&= ~grbm_soft_reset
;
4789 WREG32(GRBM_SOFT_RESET
, tmp
);
4790 tmp
= RREG32(GRBM_SOFT_RESET
);
4793 if (srbm_soft_reset
) {
4794 tmp
= RREG32(SRBM_SOFT_RESET
);
4795 tmp
|= srbm_soft_reset
;
4796 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
4797 WREG32(SRBM_SOFT_RESET
, tmp
);
4798 tmp
= RREG32(SRBM_SOFT_RESET
);
4802 tmp
&= ~srbm_soft_reset
;
4803 WREG32(SRBM_SOFT_RESET
, tmp
);
4804 tmp
= RREG32(SRBM_SOFT_RESET
);
4807 /* Wait a little for things to settle down */
4810 evergreen_mc_resume(rdev
, &save
);
4813 cik_print_gpu_status_regs(rdev
);
4817 * cik_asic_reset - soft reset GPU
4819 * @rdev: radeon_device pointer
4821 * Look up which blocks are hung and attempt
4823 * Returns 0 for success.
4825 int cik_asic_reset(struct radeon_device
*rdev
)
4829 reset_mask
= cik_gpu_check_soft_reset(rdev
);
4832 r600_set_bios_scratch_engine_hung(rdev
, true);
4834 cik_gpu_soft_reset(rdev
, reset_mask
);
4836 reset_mask
= cik_gpu_check_soft_reset(rdev
);
4839 r600_set_bios_scratch_engine_hung(rdev
, false);
4845 * cik_gfx_is_lockup - check if the 3D engine is locked up
4847 * @rdev: radeon_device pointer
4848 * @ring: radeon_ring structure holding ring information
4850 * Check if the 3D engine is locked up (CIK).
4851 * Returns true if the engine is locked, false if not.
4853 bool cik_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
4855 u32 reset_mask
= cik_gpu_check_soft_reset(rdev
);
4857 if (!(reset_mask
& (RADEON_RESET_GFX
|
4858 RADEON_RESET_COMPUTE
|
4859 RADEON_RESET_CP
))) {
4860 radeon_ring_lockup_update(ring
);
4863 /* force CP activities */
4864 radeon_ring_force_activity(rdev
, ring
);
4865 return radeon_ring_test_lockup(rdev
, ring
);
4869 * cik_sdma_is_lockup - Check if the DMA engine is locked up
4871 * @rdev: radeon_device pointer
4872 * @ring: radeon_ring structure holding ring information
4874 * Check if the async DMA engine is locked up (CIK).
4875 * Returns true if the engine appears to be locked up, false if not.
4877 bool cik_sdma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
4879 u32 reset_mask
= cik_gpu_check_soft_reset(rdev
);
4882 if (ring
->idx
== R600_RING_TYPE_DMA_INDEX
)
4883 mask
= RADEON_RESET_DMA
;
4885 mask
= RADEON_RESET_DMA1
;
4887 if (!(reset_mask
& mask
)) {
4888 radeon_ring_lockup_update(ring
);
4891 /* force ring activities */
4892 radeon_ring_force_activity(rdev
, ring
);
4893 return radeon_ring_test_lockup(rdev
, ring
);
4898 * cik_mc_program - program the GPU memory controller
4900 * @rdev: radeon_device pointer
4902 * Set the location of vram, gart, and AGP in the GPU's
4903 * physical address space (CIK).
4905 static void cik_mc_program(struct radeon_device
*rdev
)
4907 struct evergreen_mc_save save
;
4911 /* Initialize HDP */
4912 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
4913 WREG32((0x2c14 + j
), 0x00000000);
4914 WREG32((0x2c18 + j
), 0x00000000);
4915 WREG32((0x2c1c + j
), 0x00000000);
4916 WREG32((0x2c20 + j
), 0x00000000);
4917 WREG32((0x2c24 + j
), 0x00000000);
4919 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
4921 evergreen_mc_stop(rdev
, &save
);
4922 if (radeon_mc_wait_for_idle(rdev
)) {
4923 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
4925 /* Lockout access through VGA aperture*/
4926 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
4927 /* Update configuration */
4928 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
4929 rdev
->mc
.vram_start
>> 12);
4930 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
4931 rdev
->mc
.vram_end
>> 12);
4932 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
4933 rdev
->vram_scratch
.gpu_addr
>> 12);
4934 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
4935 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
4936 WREG32(MC_VM_FB_LOCATION
, tmp
);
4937 /* XXX double check these! */
4938 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
4939 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
4940 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
4941 WREG32(MC_VM_AGP_BASE
, 0);
4942 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
4943 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
4944 if (radeon_mc_wait_for_idle(rdev
)) {
4945 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
4947 evergreen_mc_resume(rdev
, &save
);
4948 /* we need to own VRAM, so turn off the VGA renderer here
4949 * to stop it overwriting our objects */
4950 rv515_vga_render_disable(rdev
);
4954 * cik_mc_init - initialize the memory controller driver params
4956 * @rdev: radeon_device pointer
4958 * Look up the amount of vram, vram width, and decide how to place
4959 * vram and gart within the GPU's physical address space (CIK).
4960 * Returns 0 for success.
4962 static int cik_mc_init(struct radeon_device
*rdev
)
4965 int chansize
, numchan
;
4967 /* Get VRAM informations */
4968 rdev
->mc
.vram_is_ddr
= true;
4969 tmp
= RREG32(MC_ARB_RAMCFG
);
4970 if (tmp
& CHANSIZE_MASK
) {
4975 tmp
= RREG32(MC_SHARED_CHMAP
);
4976 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
5006 rdev
->mc
.vram_width
= numchan
* chansize
;
5007 /* Could aper size report 0 ? */
5008 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
5009 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
5010 /* size in MB on si */
5011 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024 * 1024;
5012 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024 * 1024;
5013 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
5014 si_vram_gtt_location(rdev
, &rdev
->mc
);
5015 radeon_update_bandwidth_info(rdev
);
5022 * VMID 0 is the physical GPU addresses as used by the kernel.
5023 * VMIDs 1-15 are used for userspace clients and are handled
5024 * by the radeon vm/hsa code.
5027 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5029 * @rdev: radeon_device pointer
5031 * Flush the TLB for the VMID 0 page table (CIK).
5033 void cik_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
5035 /* flush hdp cache */
5036 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0);
5038 /* bits 0-15 are the VM contexts0-15 */
5039 WREG32(VM_INVALIDATE_REQUEST
, 0x1);
5043 * cik_pcie_gart_enable - gart enable
5045 * @rdev: radeon_device pointer
5047 * This sets up the TLBs, programs the page tables for VMID0,
5048 * sets up the hw for VMIDs 1-15 which are allocated on
5049 * demand, and sets up the global locations for the LDS, GDS,
5050 * and GPUVM for FSA64 clients (CIK).
5051 * Returns 0 for success, errors for failure.
5053 static int cik_pcie_gart_enable(struct radeon_device
*rdev
)
5057 if (rdev
->gart
.robj
== NULL
) {
5058 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
5061 r
= radeon_gart_table_vram_pin(rdev
);
5064 radeon_gart_restore(rdev
);
5065 /* Setup TLB control */
5066 WREG32(MC_VM_MX_L1_TLB_CNTL
,
5069 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5070 ENABLE_ADVANCED_DRIVER_MODEL
|
5071 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5072 /* Setup L2 cache */
5073 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
5074 ENABLE_L2_FRAGMENT_PROCESSING
|
5075 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5076 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5077 EFFECTIVE_L2_QUEUE_SIZE(7) |
5078 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5079 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
5080 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5081 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5082 /* setup context0 */
5083 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
5084 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
5085 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
5086 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
5087 (u32
)(rdev
->dummy_page
.addr
>> 12));
5088 WREG32(VM_CONTEXT0_CNTL2
, 0);
5089 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
5090 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
5096 /* empty context1-15 */
5097 /* FIXME start with 4G, once using 2 level pt switch to full
5100 /* set vm size, must be a multiple of 4 */
5101 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
5102 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
5103 for (i
= 1; i
< 16; i
++) {
5105 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
5106 rdev
->gart
.table_addr
>> 12);
5108 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
5109 rdev
->gart
.table_addr
>> 12);
5112 /* enable context1-15 */
5113 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
5114 (u32
)(rdev
->dummy_page
.addr
>> 12));
5115 WREG32(VM_CONTEXT1_CNTL2
, 4);
5116 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
5117 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5118 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5119 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5120 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
5121 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5122 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
5123 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5124 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
5125 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5126 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
5127 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
5128 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
5130 /* TC cache setup ??? */
5131 WREG32(TC_CFG_L1_LOAD_POLICY0
, 0);
5132 WREG32(TC_CFG_L1_LOAD_POLICY1
, 0);
5133 WREG32(TC_CFG_L1_STORE_POLICY
, 0);
5135 WREG32(TC_CFG_L2_LOAD_POLICY0
, 0);
5136 WREG32(TC_CFG_L2_LOAD_POLICY1
, 0);
5137 WREG32(TC_CFG_L2_STORE_POLICY0
, 0);
5138 WREG32(TC_CFG_L2_STORE_POLICY1
, 0);
5139 WREG32(TC_CFG_L2_ATOMIC_POLICY
, 0);
5141 WREG32(TC_CFG_L1_VOLATILE
, 0);
5142 WREG32(TC_CFG_L2_VOLATILE
, 0);
5144 if (rdev
->family
== CHIP_KAVERI
) {
5145 u32 tmp
= RREG32(CHUB_CONTROL
);
5147 WREG32(CHUB_CONTROL
, tmp
);
5150 /* XXX SH_MEM regs */
5151 /* where to put LDS, scratch, GPUVM in FSA64 space */
5152 mutex_lock(&rdev
->srbm_mutex
);
5153 for (i
= 0; i
< 16; i
++) {
5154 cik_srbm_select(rdev
, 0, 0, 0, i
);
5155 /* CP and shaders */
5156 WREG32(SH_MEM_CONFIG
, 0);
5157 WREG32(SH_MEM_APE1_BASE
, 1);
5158 WREG32(SH_MEM_APE1_LIMIT
, 0);
5159 WREG32(SH_MEM_BASES
, 0);
5161 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA0_REGISTER_OFFSET
, 0);
5162 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA0_REGISTER_OFFSET
, 0);
5163 WREG32(SDMA0_GFX_VIRTUAL_ADDR
+ SDMA1_REGISTER_OFFSET
, 0);
5164 WREG32(SDMA0_GFX_APE1_CNTL
+ SDMA1_REGISTER_OFFSET
, 0);
5165 /* XXX SDMA RLC - todo */
5167 cik_srbm_select(rdev
, 0, 0, 0, 0);
5168 mutex_unlock(&rdev
->srbm_mutex
);
5170 cik_pcie_gart_tlb_flush(rdev
);
5171 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5172 (unsigned)(rdev
->mc
.gtt_size
>> 20),
5173 (unsigned long long)rdev
->gart
.table_addr
);
5174 rdev
->gart
.ready
= true;
5179 * cik_pcie_gart_disable - gart disable
5181 * @rdev: radeon_device pointer
5183 * This disables all VM page table (CIK).
5185 static void cik_pcie_gart_disable(struct radeon_device
*rdev
)
5187 /* Disable all tables */
5188 WREG32(VM_CONTEXT0_CNTL
, 0);
5189 WREG32(VM_CONTEXT1_CNTL
, 0);
5190 /* Setup TLB control */
5191 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
5192 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
5193 /* Setup L2 cache */
5195 ENABLE_L2_FRAGMENT_PROCESSING
|
5196 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
5197 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
5198 EFFECTIVE_L2_QUEUE_SIZE(7) |
5199 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5200 WREG32(VM_L2_CNTL2
, 0);
5201 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
5202 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5203 radeon_gart_table_vram_unpin(rdev
);
5207 * cik_pcie_gart_fini - vm fini callback
5209 * @rdev: radeon_device pointer
5211 * Tears down the driver GART/VM setup (CIK).
5213 static void cik_pcie_gart_fini(struct radeon_device
*rdev
)
5215 cik_pcie_gart_disable(rdev
);
5216 radeon_gart_table_vram_free(rdev
);
5217 radeon_gart_fini(rdev
);
5222 * cik_ib_parse - vm ib_parse callback
5224 * @rdev: radeon_device pointer
5225 * @ib: indirect buffer pointer
5227 * CIK uses hw IB checking so this is a nop (CIK).
5229 int cik_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
5236 * VMID 0 is the physical GPU addresses as used by the kernel.
5237 * VMIDs 1-15 are used for userspace clients and are handled
5238 * by the radeon vm/hsa code.
5241 * cik_vm_init - cik vm init callback
5243 * @rdev: radeon_device pointer
5245 * Inits cik specific vm parameters (number of VMs, base of vram for
5246 * VMIDs 1-15) (CIK).
5247 * Returns 0 for success.
5249 int cik_vm_init(struct radeon_device
*rdev
)
5252 rdev
->vm_manager
.nvm
= 16;
5253 /* base offset of vram pages */
5254 if (rdev
->flags
& RADEON_IS_IGP
) {
5255 u64 tmp
= RREG32(MC_VM_FB_OFFSET
);
5257 rdev
->vm_manager
.vram_base_offset
= tmp
;
5259 rdev
->vm_manager
.vram_base_offset
= 0;
5265 * cik_vm_fini - cik vm fini callback
5267 * @rdev: radeon_device pointer
5269 * Tear down any asic specific VM setup (CIK).
5271 void cik_vm_fini(struct radeon_device
*rdev
)
5276 * cik_vm_decode_fault - print human readable fault info
5278 * @rdev: radeon_device pointer
5279 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5280 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5282 * Print human readable fault information (CIK).
5284 static void cik_vm_decode_fault(struct radeon_device
*rdev
,
5285 u32 status
, u32 addr
, u32 mc_client
)
5287 u32 mc_id
= (status
& MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
5288 u32 vmid
= (status
& FAULT_VMID_MASK
) >> FAULT_VMID_SHIFT
;
5289 u32 protections
= (status
& PROTECTIONS_MASK
) >> PROTECTIONS_SHIFT
;
5290 char *block
= (char *)&mc_client
;
5292 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5293 protections
, vmid
, addr
,
5294 (status
& MEMORY_CLIENT_RW_MASK
) ? "write" : "read",
5299 * cik_vm_flush - cik vm flush using the CP
5301 * @rdev: radeon_device pointer
5303 * Update the page table base and flush the VM TLB
5304 * using the CP (CIK).
5306 void cik_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
5308 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
5313 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5314 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5315 WRITE_DATA_DST_SEL(0)));
5317 radeon_ring_write(ring
,
5318 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
5320 radeon_ring_write(ring
,
5321 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
5323 radeon_ring_write(ring
, 0);
5324 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
5326 /* update SH_MEM_* regs */
5327 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5328 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5329 WRITE_DATA_DST_SEL(0)));
5330 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5331 radeon_ring_write(ring
, 0);
5332 radeon_ring_write(ring
, VMID(vm
->id
));
5334 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 6));
5335 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5336 WRITE_DATA_DST_SEL(0)));
5337 radeon_ring_write(ring
, SH_MEM_BASES
>> 2);
5338 radeon_ring_write(ring
, 0);
5340 radeon_ring_write(ring
, 0); /* SH_MEM_BASES */
5341 radeon_ring_write(ring
, 0); /* SH_MEM_CONFIG */
5342 radeon_ring_write(ring
, 1); /* SH_MEM_APE1_BASE */
5343 radeon_ring_write(ring
, 0); /* SH_MEM_APE1_LIMIT */
5345 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5346 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5347 WRITE_DATA_DST_SEL(0)));
5348 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5349 radeon_ring_write(ring
, 0);
5350 radeon_ring_write(ring
, VMID(0));
5353 /* We should be using the WAIT_REG_MEM packet here like in
5354 * cik_fence_ring_emit(), but it causes the CP to hang in this
5357 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5358 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5359 WRITE_DATA_DST_SEL(0)));
5360 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
5361 radeon_ring_write(ring
, 0);
5362 radeon_ring_write(ring
, 0);
5364 /* bits 0-15 are the VM contexts0-15 */
5365 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
5366 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
5367 WRITE_DATA_DST_SEL(0)));
5368 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
5369 radeon_ring_write(ring
, 0);
5370 radeon_ring_write(ring
, 1 << vm
->id
);
5372 /* compute doesn't have PFP */
5373 if (ridx
== RADEON_RING_TYPE_GFX_INDEX
) {
5374 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5375 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
5376 radeon_ring_write(ring
, 0x0);
5381 * cik_vm_set_page - update the page tables using sDMA
5383 * @rdev: radeon_device pointer
5384 * @ib: indirect buffer to fill with commands
5385 * @pe: addr of the page entry
5386 * @addr: dst addr to write into pe
5387 * @count: number of page entries to update
5388 * @incr: increase next addr by incr bytes
5389 * @flags: access flags
5391 * Update the page tables using CP or sDMA (CIK).
5393 void cik_vm_set_page(struct radeon_device
*rdev
,
5394 struct radeon_ib
*ib
,
5396 uint64_t addr
, unsigned count
,
5397 uint32_t incr
, uint32_t flags
)
5399 uint32_t r600_flags
= cayman_vm_page_flags(rdev
, flags
);
5403 if (rdev
->asic
->vm
.pt_ring_index
== RADEON_RING_TYPE_GFX_INDEX
) {
5406 ndw
= 2 + count
* 2;
5410 ib
->ptr
[ib
->length_dw
++] = PACKET3(PACKET3_WRITE_DATA
, ndw
);
5411 ib
->ptr
[ib
->length_dw
++] = (WRITE_DATA_ENGINE_SEL(0) |
5412 WRITE_DATA_DST_SEL(1));
5413 ib
->ptr
[ib
->length_dw
++] = pe
;
5414 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
5415 for (; ndw
> 2; ndw
-= 2, --count
, pe
+= 8) {
5416 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
5417 value
= radeon_vm_map_gart(rdev
, addr
);
5418 value
&= 0xFFFFFFFFFFFFF000ULL
;
5419 } else if (flags
& RADEON_VM_PAGE_VALID
) {
5425 value
|= r600_flags
;
5426 ib
->ptr
[ib
->length_dw
++] = value
;
5427 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
5432 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
5438 /* for non-physically contiguous pages (system) */
5439 ib
->ptr
[ib
->length_dw
++] = SDMA_PACKET(SDMA_OPCODE_WRITE
, SDMA_WRITE_SUB_OPCODE_LINEAR
, 0);
5440 ib
->ptr
[ib
->length_dw
++] = pe
;
5441 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
5442 ib
->ptr
[ib
->length_dw
++] = ndw
;
5443 for (; ndw
> 0; ndw
-= 2, --count
, pe
+= 8) {
5444 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
5445 value
= radeon_vm_map_gart(rdev
, addr
);
5446 value
&= 0xFFFFFFFFFFFFF000ULL
;
5447 } else if (flags
& RADEON_VM_PAGE_VALID
) {
5453 value
|= r600_flags
;
5454 ib
->ptr
[ib
->length_dw
++] = value
;
5455 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
5464 if (flags
& RADEON_VM_PAGE_VALID
)
5468 /* for physically contiguous pages (vram) */
5469 ib
->ptr
[ib
->length_dw
++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE
, 0, 0);
5470 ib
->ptr
[ib
->length_dw
++] = pe
; /* dst addr */
5471 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
5472 ib
->ptr
[ib
->length_dw
++] = r600_flags
; /* mask */
5473 ib
->ptr
[ib
->length_dw
++] = 0;
5474 ib
->ptr
[ib
->length_dw
++] = value
; /* value */
5475 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
5476 ib
->ptr
[ib
->length_dw
++] = incr
; /* increment size */
5477 ib
->ptr
[ib
->length_dw
++] = 0;
5478 ib
->ptr
[ib
->length_dw
++] = ndw
; /* number of entries */
5484 while (ib
->length_dw
& 0x7)
5485 ib
->ptr
[ib
->length_dw
++] = SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0);
5490 * cik_dma_vm_flush - cik vm flush using sDMA
5492 * @rdev: radeon_device pointer
5494 * Update the page table base and flush the VM TLB
5497 void cik_dma_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
5499 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
5500 u32 extra_bits
= (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
5501 SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
5507 if (ridx
== R600_RING_TYPE_DMA_INDEX
)
5508 ref_and_mask
= SDMA0
;
5510 ref_and_mask
= SDMA1
;
5512 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5514 radeon_ring_write(ring
, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
5516 radeon_ring_write(ring
, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
5518 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
5520 /* update SH_MEM_* regs */
5521 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5522 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5523 radeon_ring_write(ring
, VMID(vm
->id
));
5525 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5526 radeon_ring_write(ring
, SH_MEM_BASES
>> 2);
5527 radeon_ring_write(ring
, 0);
5529 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5530 radeon_ring_write(ring
, SH_MEM_CONFIG
>> 2);
5531 radeon_ring_write(ring
, 0);
5533 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5534 radeon_ring_write(ring
, SH_MEM_APE1_BASE
>> 2);
5535 radeon_ring_write(ring
, 1);
5537 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5538 radeon_ring_write(ring
, SH_MEM_APE1_LIMIT
>> 2);
5539 radeon_ring_write(ring
, 0);
5541 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5542 radeon_ring_write(ring
, SRBM_GFX_CNTL
>> 2);
5543 radeon_ring_write(ring
, VMID(0));
5546 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM
, 0, extra_bits
));
5547 radeon_ring_write(ring
, GPU_HDP_FLUSH_DONE
);
5548 radeon_ring_write(ring
, GPU_HDP_FLUSH_REQ
);
5549 radeon_ring_write(ring
, ref_and_mask
); /* REFERENCE */
5550 radeon_ring_write(ring
, ref_and_mask
); /* MASK */
5551 radeon_ring_write(ring
, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
5554 radeon_ring_write(ring
, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE
, 0, 0xf000));
5555 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
5556 radeon_ring_write(ring
, 1 << vm
->id
);
5561 * The RLC is a multi-purpose microengine that handles a
5562 * variety of functions, the most important of which is
5563 * the interrupt controller.
5565 static void cik_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
5568 u32 tmp
= RREG32(CP_INT_CNTL_RING0
);
5571 tmp
|= (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5573 tmp
&= ~(CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5574 WREG32(CP_INT_CNTL_RING0
, tmp
);
5577 static void cik_enable_lbpw(struct radeon_device
*rdev
, bool enable
)
5581 tmp
= RREG32(RLC_LB_CNTL
);
5583 tmp
|= LOAD_BALANCE_ENABLE
;
5585 tmp
&= ~LOAD_BALANCE_ENABLE
;
5586 WREG32(RLC_LB_CNTL
, tmp
);
5589 static void cik_wait_for_rlc_serdes(struct radeon_device
*rdev
)
5594 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
5595 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
5596 cik_select_se_sh(rdev
, i
, j
);
5597 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5598 if (RREG32(RLC_SERDES_CU_MASTER_BUSY
) == 0)
5604 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5606 mask
= SE_MASTER_BUSY_MASK
| GC_MASTER_BUSY
| TC0_MASTER_BUSY
| TC1_MASTER_BUSY
;
5607 for (k
= 0; k
< rdev
->usec_timeout
; k
++) {
5608 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY
) & mask
) == 0)
5614 static void cik_update_rlc(struct radeon_device
*rdev
, u32 rlc
)
5618 tmp
= RREG32(RLC_CNTL
);
5620 WREG32(RLC_CNTL
, rlc
);
5623 static u32
cik_halt_rlc(struct radeon_device
*rdev
)
5627 orig
= data
= RREG32(RLC_CNTL
);
5629 if (data
& RLC_ENABLE
) {
5632 data
&= ~RLC_ENABLE
;
5633 WREG32(RLC_CNTL
, data
);
5635 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5636 if ((RREG32(RLC_GPM_STAT
) & RLC_GPM_BUSY
) == 0)
5641 cik_wait_for_rlc_serdes(rdev
);
5647 void cik_enter_rlc_safe_mode(struct radeon_device
*rdev
)
5651 tmp
= REQ
| MESSAGE(MSG_ENTER_RLC_SAFE_MODE
);
5652 WREG32(RLC_GPR_REG2
, tmp
);
5654 mask
= GFX_POWER_STATUS
| GFX_CLOCK_STATUS
;
5655 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5656 if ((RREG32(RLC_GPM_STAT
) & mask
) == mask
)
5661 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
5662 if ((RREG32(RLC_GPR_REG2
) & REQ
) == 0)
5668 void cik_exit_rlc_safe_mode(struct radeon_device
*rdev
)
5672 tmp
= REQ
| MESSAGE(MSG_EXIT_RLC_SAFE_MODE
);
5673 WREG32(RLC_GPR_REG2
, tmp
);
5677 * cik_rlc_stop - stop the RLC ME
5679 * @rdev: radeon_device pointer
5681 * Halt the RLC ME (MicroEngine) (CIK).
5683 static void cik_rlc_stop(struct radeon_device
*rdev
)
5685 WREG32(RLC_CNTL
, 0);
5687 cik_enable_gui_idle_interrupt(rdev
, false);
5689 cik_wait_for_rlc_serdes(rdev
);
5693 * cik_rlc_start - start the RLC ME
5695 * @rdev: radeon_device pointer
5697 * Unhalt the RLC ME (MicroEngine) (CIK).
5699 static void cik_rlc_start(struct radeon_device
*rdev
)
5701 WREG32(RLC_CNTL
, RLC_ENABLE
);
5703 cik_enable_gui_idle_interrupt(rdev
, true);
5709 * cik_rlc_resume - setup the RLC hw
5711 * @rdev: radeon_device pointer
5713 * Initialize the RLC registers, load the ucode,
5714 * and start the RLC (CIK).
5715 * Returns 0 for success, -EINVAL if the ucode is not available.
5717 static int cik_rlc_resume(struct radeon_device
*rdev
)
5720 const __be32
*fw_data
;
5725 switch (rdev
->family
) {
5728 size
= BONAIRE_RLC_UCODE_SIZE
;
5731 size
= KV_RLC_UCODE_SIZE
;
5734 size
= KB_RLC_UCODE_SIZE
;
5741 tmp
= RREG32(RLC_CGCG_CGLS_CTRL
) & 0xfffffffc;
5742 WREG32(RLC_CGCG_CGLS_CTRL
, tmp
);
5750 WREG32(RLC_LB_CNTR_INIT
, 0);
5751 WREG32(RLC_LB_CNTR_MAX
, 0x00008000);
5753 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5754 WREG32(RLC_LB_INIT_CU_MASK
, 0xffffffff);
5755 WREG32(RLC_LB_PARAMS
, 0x00600408);
5756 WREG32(RLC_LB_CNTL
, 0x80000004);
5758 WREG32(RLC_MC_CNTL
, 0);
5759 WREG32(RLC_UCODE_CNTL
, 0);
5761 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
5762 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5763 for (i
= 0; i
< size
; i
++)
5764 WREG32(RLC_GPM_UCODE_DATA
, be32_to_cpup(fw_data
++));
5765 WREG32(RLC_GPM_UCODE_ADDR
, 0);
5767 /* XXX - find out what chips support lbpw */
5768 cik_enable_lbpw(rdev
, false);
5770 if (rdev
->family
== CHIP_BONAIRE
)
5771 WREG32(RLC_DRIVER_DMA_STATUS
, 0);
5773 cik_rlc_start(rdev
);
5778 static void cik_enable_cgcg(struct radeon_device
*rdev
, bool enable
)
5780 u32 data
, orig
, tmp
, tmp2
;
5782 orig
= data
= RREG32(RLC_CGCG_CGLS_CTRL
);
5784 cik_enable_gui_idle_interrupt(rdev
, enable
);
5787 tmp
= cik_halt_rlc(rdev
);
5789 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5790 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5791 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5792 tmp2
= BPM_ADDR_MASK
| CGCG_OVERRIDE_0
| CGLS_ENABLE
;
5793 WREG32(RLC_SERDES_WR_CTRL
, tmp2
);
5795 cik_update_rlc(rdev
, tmp
);
5797 data
|= CGCG_EN
| CGLS_EN
;
5799 RREG32(CB_CGTT_SCLK_CTRL
);
5800 RREG32(CB_CGTT_SCLK_CTRL
);
5801 RREG32(CB_CGTT_SCLK_CTRL
);
5802 RREG32(CB_CGTT_SCLK_CTRL
);
5804 data
&= ~(CGCG_EN
| CGLS_EN
);
5808 WREG32(RLC_CGCG_CGLS_CTRL
, data
);
5812 static void cik_enable_mgcg(struct radeon_device
*rdev
, bool enable
)
5814 u32 data
, orig
, tmp
= 0;
5817 orig
= data
= RREG32(CP_MEM_SLP_CNTL
);
5818 data
|= CP_MEM_LS_EN
;
5820 WREG32(CP_MEM_SLP_CNTL
, data
);
5822 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5825 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5827 tmp
= cik_halt_rlc(rdev
);
5829 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5830 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5831 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5832 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_0
;
5833 WREG32(RLC_SERDES_WR_CTRL
, data
);
5835 cik_update_rlc(rdev
, tmp
);
5837 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5838 data
&= ~SM_MODE_MASK
;
5839 data
|= SM_MODE(0x2);
5840 data
|= SM_MODE_ENABLE
;
5841 data
&= ~CGTS_OVERRIDE
;
5842 data
&= ~CGTS_LS_OVERRIDE
;
5843 data
&= ~ON_MONITOR_ADD_MASK
;
5844 data
|= ON_MONITOR_ADD_EN
;
5845 data
|= ON_MONITOR_ADD(0x96);
5847 WREG32(CGTS_SM_CTRL_REG
, data
);
5849 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5852 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5854 data
= RREG32(RLC_MEM_SLP_CNTL
);
5855 if (data
& RLC_MEM_LS_EN
) {
5856 data
&= ~RLC_MEM_LS_EN
;
5857 WREG32(RLC_MEM_SLP_CNTL
, data
);
5860 data
= RREG32(CP_MEM_SLP_CNTL
);
5861 if (data
& CP_MEM_LS_EN
) {
5862 data
&= ~CP_MEM_LS_EN
;
5863 WREG32(CP_MEM_SLP_CNTL
, data
);
5866 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5867 data
|= CGTS_OVERRIDE
| CGTS_LS_OVERRIDE
;
5869 WREG32(CGTS_SM_CTRL_REG
, data
);
5871 tmp
= cik_halt_rlc(rdev
);
5873 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5874 WREG32(RLC_SERDES_WR_CU_MASTER_MASK
, 0xffffffff);
5875 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK
, 0xffffffff);
5876 data
= BPM_ADDR_MASK
| MGCG_OVERRIDE_1
;
5877 WREG32(RLC_SERDES_WR_CTRL
, data
);
5879 cik_update_rlc(rdev
, tmp
);
5883 static const u32 mc_cg_registers
[] =
5896 static void cik_enable_mc_ls(struct radeon_device
*rdev
,
5902 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5903 orig
= data
= RREG32(mc_cg_registers
[i
]);
5905 data
|= MC_LS_ENABLE
;
5907 data
&= ~MC_LS_ENABLE
;
5909 WREG32(mc_cg_registers
[i
], data
);
5913 static void cik_enable_mc_mgcg(struct radeon_device
*rdev
,
5919 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5920 orig
= data
= RREG32(mc_cg_registers
[i
]);
5922 data
|= MC_CG_ENABLE
;
5924 data
&= ~MC_CG_ENABLE
;
5926 WREG32(mc_cg_registers
[i
], data
);
5930 static void cik_enable_sdma_mgcg(struct radeon_device
*rdev
,
5936 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, 0x00000100);
5937 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, 0x00000100);
5939 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
);
5942 WREG32(SDMA0_CLK_CTRL
+ SDMA0_REGISTER_OFFSET
, data
);
5944 orig
= data
= RREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
);
5947 WREG32(SDMA0_CLK_CTRL
+ SDMA1_REGISTER_OFFSET
, data
);
5951 static void cik_enable_sdma_mgls(struct radeon_device
*rdev
,
5957 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
5960 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
5962 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
5965 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
5967 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
);
5970 WREG32(SDMA0_POWER_CNTL
+ SDMA0_REGISTER_OFFSET
, data
);
5972 orig
= data
= RREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
);
5975 WREG32(SDMA0_POWER_CNTL
+ SDMA1_REGISTER_OFFSET
, data
);
5979 static void cik_enable_uvd_mgcg(struct radeon_device
*rdev
,
5985 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5987 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
5989 orig
= data
= RREG32(UVD_CGC_CTRL
);
5992 WREG32(UVD_CGC_CTRL
, data
);
5994 data
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5996 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, data
);
5998 orig
= data
= RREG32(UVD_CGC_CTRL
);
6001 WREG32(UVD_CGC_CTRL
, data
);
6005 static void cik_enable_hdp_mgcg(struct radeon_device
*rdev
,
6010 orig
= data
= RREG32(HDP_HOST_PATH_CNTL
);
6013 data
&= ~CLOCK_GATING_DIS
;
6015 data
|= CLOCK_GATING_DIS
;
6018 WREG32(HDP_HOST_PATH_CNTL
, data
);
6021 static void cik_enable_hdp_ls(struct radeon_device
*rdev
,
6026 orig
= data
= RREG32(HDP_MEM_POWER_LS
);
6029 data
|= HDP_LS_ENABLE
;
6031 data
&= ~HDP_LS_ENABLE
;
6034 WREG32(HDP_MEM_POWER_LS
, data
);
6037 void cik_update_cg(struct radeon_device
*rdev
,
6038 u32 block
, bool enable
)
6040 if (block
& RADEON_CG_BLOCK_GFX
) {
6041 /* order matters! */
6043 cik_enable_mgcg(rdev
, true);
6044 cik_enable_cgcg(rdev
, true);
6046 cik_enable_cgcg(rdev
, false);
6047 cik_enable_mgcg(rdev
, false);
6051 if (block
& RADEON_CG_BLOCK_MC
) {
6052 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
6053 cik_enable_mc_mgcg(rdev
, enable
);
6054 cik_enable_mc_ls(rdev
, enable
);
6058 if (block
& RADEON_CG_BLOCK_SDMA
) {
6059 cik_enable_sdma_mgcg(rdev
, enable
);
6060 cik_enable_sdma_mgls(rdev
, enable
);
6063 if (block
& RADEON_CG_BLOCK_UVD
) {
6065 cik_enable_uvd_mgcg(rdev
, enable
);
6068 if (block
& RADEON_CG_BLOCK_HDP
) {
6069 cik_enable_hdp_mgcg(rdev
, enable
);
6070 cik_enable_hdp_ls(rdev
, enable
);
6074 static void cik_init_cg(struct radeon_device
*rdev
)
6077 cik_update_cg(rdev
, RADEON_CG_BLOCK_GFX
, false); /* XXX true */
6080 si_init_uvd_internal_cg(rdev
);
6082 cik_update_cg(rdev
, (RADEON_CG_BLOCK_MC
|
6083 RADEON_CG_BLOCK_SDMA
|
6084 RADEON_CG_BLOCK_UVD
|
6085 RADEON_CG_BLOCK_HDP
), true);
6088 static void cik_enable_sck_slowdown_on_pu(struct radeon_device
*rdev
,
6093 orig
= data
= RREG32(RLC_PG_CNTL
);
6095 data
|= SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6097 data
&= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE
;
6099 WREG32(RLC_PG_CNTL
, data
);
6102 static void cik_enable_sck_slowdown_on_pd(struct radeon_device
*rdev
,
6107 orig
= data
= RREG32(RLC_PG_CNTL
);
6109 data
|= SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6111 data
&= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE
;
6113 WREG32(RLC_PG_CNTL
, data
);
6116 static void cik_enable_cp_pg(struct radeon_device
*rdev
, bool enable
)
6120 orig
= data
= RREG32(RLC_PG_CNTL
);
6122 data
&= ~DISABLE_CP_PG
;
6124 data
|= DISABLE_CP_PG
;
6126 WREG32(RLC_PG_CNTL
, data
);
6129 static void cik_enable_gds_pg(struct radeon_device
*rdev
, bool enable
)
6133 orig
= data
= RREG32(RLC_PG_CNTL
);
6135 data
&= ~DISABLE_GDS_PG
;
6137 data
|= DISABLE_GDS_PG
;
6139 WREG32(RLC_PG_CNTL
, data
);
6142 #define CP_ME_TABLE_SIZE 96
6143 #define CP_ME_TABLE_OFFSET 2048
6144 #define CP_MEC_TABLE_OFFSET 4096
6146 void cik_init_cp_pg_table(struct radeon_device
*rdev
)
6148 const __be32
*fw_data
;
6149 volatile u32
*dst_ptr
;
6150 int me
, i
, max_me
= 4;
6154 if (rdev
->family
== CHIP_KAVERI
)
6157 if (rdev
->rlc
.cp_table_ptr
== NULL
)
6160 /* write the cp table buffer */
6161 dst_ptr
= rdev
->rlc
.cp_table_ptr
;
6162 for (me
= 0; me
< max_me
; me
++) {
6164 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
6165 table_offset
= CP_ME_TABLE_OFFSET
;
6166 } else if (me
== 1) {
6167 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
6168 table_offset
= CP_ME_TABLE_OFFSET
;
6169 } else if (me
== 2) {
6170 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
6171 table_offset
= CP_ME_TABLE_OFFSET
;
6173 fw_data
= (const __be32
*)rdev
->mec_fw
->data
;
6174 table_offset
= CP_MEC_TABLE_OFFSET
;
6177 for (i
= 0; i
< CP_ME_TABLE_SIZE
; i
++) {
6178 dst_ptr
[bo_offset
+ i
] = be32_to_cpu(fw_data
[table_offset
+ i
]);
6180 bo_offset
+= CP_ME_TABLE_SIZE
;
6184 static void cik_enable_gfx_cgpg(struct radeon_device
*rdev
,
6190 orig
= data
= RREG32(RLC_PG_CNTL
);
6191 data
|= GFX_PG_ENABLE
;
6193 WREG32(RLC_PG_CNTL
, data
);
6195 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6198 WREG32(RLC_AUTO_PG_CTRL
, data
);
6200 orig
= data
= RREG32(RLC_PG_CNTL
);
6201 data
&= ~GFX_PG_ENABLE
;
6203 WREG32(RLC_PG_CNTL
, data
);
6205 orig
= data
= RREG32(RLC_AUTO_PG_CTRL
);
6206 data
&= ~AUTO_PG_EN
;
6208 WREG32(RLC_AUTO_PG_CTRL
, data
);
6210 data
= RREG32(DB_RENDER_CONTROL
);
6214 static u32
cik_get_cu_active_bitmap(struct radeon_device
*rdev
, u32 se
, u32 sh
)
6216 u32 mask
= 0, tmp
, tmp1
;
6219 cik_select_se_sh(rdev
, se
, sh
);
6220 tmp
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
6221 tmp1
= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
6222 cik_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
6229 for (i
= 0; i
< rdev
->config
.cik
.max_cu_per_sh
; i
++) {
6234 return (~tmp
) & mask
;
6237 static void cik_init_ao_cu_mask(struct radeon_device
*rdev
)
6239 u32 i
, j
, k
, active_cu_number
= 0;
6240 u32 mask
, counter
, cu_bitmap
;
6243 for (i
= 0; i
< rdev
->config
.cik
.max_shader_engines
; i
++) {
6244 for (j
= 0; j
< rdev
->config
.cik
.max_sh_per_se
; j
++) {
6248 for (k
= 0; k
< rdev
->config
.cik
.max_cu_per_sh
; k
++) {
6249 if (cik_get_cu_active_bitmap(rdev
, i
, j
) & mask
) {
6257 active_cu_number
+= counter
;
6258 tmp
|= (cu_bitmap
<< (i
* 16 + j
* 8));
6262 WREG32(RLC_PG_AO_CU_MASK
, tmp
);
6264 tmp
= RREG32(RLC_MAX_PG_CU
);
6265 tmp
&= ~MAX_PU_CU_MASK
;
6266 tmp
|= MAX_PU_CU(active_cu_number
);
6267 WREG32(RLC_MAX_PG_CU
, tmp
);
6270 static void cik_enable_gfx_static_mgpg(struct radeon_device
*rdev
,
6275 orig
= data
= RREG32(RLC_PG_CNTL
);
6277 data
|= STATIC_PER_CU_PG_ENABLE
;
6279 data
&= ~STATIC_PER_CU_PG_ENABLE
;
6281 WREG32(RLC_PG_CNTL
, data
);
6284 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device
*rdev
,
6289 orig
= data
= RREG32(RLC_PG_CNTL
);
6291 data
|= DYN_PER_CU_PG_ENABLE
;
6293 data
&= ~DYN_PER_CU_PG_ENABLE
;
6295 WREG32(RLC_PG_CNTL
, data
);
6298 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6299 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6301 static void cik_init_gfx_cgpg(struct radeon_device
*rdev
)
6306 if (rdev
->rlc
.cs_data
) {
6307 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6308 WREG32(RLC_GPM_SCRATCH_DATA
, upper_32_bits(rdev
->rlc
.clear_state_gpu_addr
));
6309 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.clear_state_gpu_addr
);
6310 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.clear_state_size
);
6312 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET
);
6313 for (i
= 0; i
< 3; i
++)
6314 WREG32(RLC_GPM_SCRATCH_DATA
, 0);
6316 if (rdev
->rlc
.reg_list
) {
6317 WREG32(RLC_GPM_SCRATCH_ADDR
, RLC_SAVE_AND_RESTORE_STARTING_OFFSET
);
6318 for (i
= 0; i
< rdev
->rlc
.reg_list_size
; i
++)
6319 WREG32(RLC_GPM_SCRATCH_DATA
, rdev
->rlc
.reg_list
[i
]);
6322 orig
= data
= RREG32(RLC_PG_CNTL
);
6325 WREG32(RLC_PG_CNTL
, data
);
6327 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
6328 WREG32(RLC_CP_TABLE_RESTORE
, rdev
->rlc
.cp_table_gpu_addr
>> 8);
6330 data
= RREG32(CP_RB_WPTR_POLL_CNTL
);
6331 data
&= ~IDLE_POLL_COUNT_MASK
;
6332 data
|= IDLE_POLL_COUNT(0x60);
6333 WREG32(CP_RB_WPTR_POLL_CNTL
, data
);
6336 WREG32(RLC_PG_DELAY
, data
);
6338 data
= RREG32(RLC_PG_DELAY_2
);
6341 WREG32(RLC_PG_DELAY_2
, data
);
6343 data
= RREG32(RLC_AUTO_PG_CTRL
);
6344 data
&= ~GRBM_REG_SGIT_MASK
;
6345 data
|= GRBM_REG_SGIT(0x700);
6346 WREG32(RLC_AUTO_PG_CTRL
, data
);
6350 static void cik_update_gfx_pg(struct radeon_device
*rdev
, bool enable
)
6352 bool has_pg
= false;
6353 bool has_dyn_mgpg
= false;
6354 bool has_static_mgpg
= false;
6356 /* only APUs have PG */
6357 if (rdev
->flags
& RADEON_IS_IGP
) {
6359 has_static_mgpg
= true;
6360 if (rdev
->family
== CHIP_KAVERI
)
6361 has_dyn_mgpg
= true;
6365 cik_enable_gfx_cgpg(rdev
, enable
);
6367 cik_enable_gfx_static_mgpg(rdev
, has_static_mgpg
);
6368 cik_enable_gfx_dynamic_mgpg(rdev
, has_dyn_mgpg
);
6370 cik_enable_gfx_static_mgpg(rdev
, false);
6371 cik_enable_gfx_dynamic_mgpg(rdev
, false);
6377 void cik_init_pg(struct radeon_device
*rdev
)
6379 bool has_pg
= false;
6381 /* only APUs have PG */
6382 if (rdev
->flags
& RADEON_IS_IGP
) {
6383 /* XXX disable this for now */
6384 /* has_pg = true; */
6388 cik_enable_sck_slowdown_on_pu(rdev
, true);
6389 cik_enable_sck_slowdown_on_pd(rdev
, true);
6390 cik_init_gfx_cgpg(rdev
);
6391 cik_enable_cp_pg(rdev
, true);
6392 cik_enable_gds_pg(rdev
, true);
6393 cik_init_ao_cu_mask(rdev
);
6394 cik_update_gfx_pg(rdev
, true);
6400 * Starting with r6xx, interrupts are handled via a ring buffer.
6401 * Ring buffers are areas of GPU accessible memory that the GPU
6402 * writes interrupt vectors into and the host reads vectors out of.
6403 * There is a rptr (read pointer) that determines where the
6404 * host is currently reading, and a wptr (write pointer)
6405 * which determines where the GPU has written. When the
6406 * pointers are equal, the ring is idle. When the GPU
6407 * writes vectors to the ring buffer, it increments the
6408 * wptr. When there is an interrupt, the host then starts
6409 * fetching commands and processing them until the pointers are
6410 * equal again at which point it updates the rptr.
6414 * cik_enable_interrupts - Enable the interrupt ring buffer
6416 * @rdev: radeon_device pointer
6418 * Enable the interrupt ring buffer (CIK).
6420 static void cik_enable_interrupts(struct radeon_device
*rdev
)
6422 u32 ih_cntl
= RREG32(IH_CNTL
);
6423 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6425 ih_cntl
|= ENABLE_INTR
;
6426 ih_rb_cntl
|= IH_RB_ENABLE
;
6427 WREG32(IH_CNTL
, ih_cntl
);
6428 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6429 rdev
->ih
.enabled
= true;
6433 * cik_disable_interrupts - Disable the interrupt ring buffer
6435 * @rdev: radeon_device pointer
6437 * Disable the interrupt ring buffer (CIK).
6439 static void cik_disable_interrupts(struct radeon_device
*rdev
)
6441 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
6442 u32 ih_cntl
= RREG32(IH_CNTL
);
6444 ih_rb_cntl
&= ~IH_RB_ENABLE
;
6445 ih_cntl
&= ~ENABLE_INTR
;
6446 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6447 WREG32(IH_CNTL
, ih_cntl
);
6448 /* set rptr, wptr to 0 */
6449 WREG32(IH_RB_RPTR
, 0);
6450 WREG32(IH_RB_WPTR
, 0);
6451 rdev
->ih
.enabled
= false;
6456 * cik_disable_interrupt_state - Disable all interrupt sources
6458 * @rdev: radeon_device pointer
6460 * Clear all interrupt enable bits used by the driver (CIK).
6462 static void cik_disable_interrupt_state(struct radeon_device
*rdev
)
6467 WREG32(CP_INT_CNTL_RING0
, CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
6469 tmp
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6470 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, tmp
);
6471 tmp
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6472 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, tmp
);
6473 /* compute queues */
6474 WREG32(CP_ME1_PIPE0_INT_CNTL
, 0);
6475 WREG32(CP_ME1_PIPE1_INT_CNTL
, 0);
6476 WREG32(CP_ME1_PIPE2_INT_CNTL
, 0);
6477 WREG32(CP_ME1_PIPE3_INT_CNTL
, 0);
6478 WREG32(CP_ME2_PIPE0_INT_CNTL
, 0);
6479 WREG32(CP_ME2_PIPE1_INT_CNTL
, 0);
6480 WREG32(CP_ME2_PIPE2_INT_CNTL
, 0);
6481 WREG32(CP_ME2_PIPE3_INT_CNTL
, 0);
6483 WREG32(GRBM_INT_CNTL
, 0);
6484 /* vline/vblank, etc. */
6485 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
6486 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
6487 if (rdev
->num_crtc
>= 4) {
6488 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
6489 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
6491 if (rdev
->num_crtc
>= 6) {
6492 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
6493 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
6497 WREG32(DAC_AUTODETECT_INT_CONTROL
, 0);
6499 /* digital hotplug */
6500 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6501 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
6502 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6503 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
6504 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6505 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
6506 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6507 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
6508 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6509 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
6510 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
6511 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
6516 * cik_irq_init - init and enable the interrupt ring
6518 * @rdev: radeon_device pointer
6520 * Allocate a ring buffer for the interrupt controller,
6521 * enable the RLC, disable interrupts, enable the IH
6522 * ring buffer and enable it (CIK).
6523 * Called at device load and reume.
6524 * Returns 0 for success, errors for failure.
6526 static int cik_irq_init(struct radeon_device
*rdev
)
6530 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
6533 ret
= r600_ih_ring_alloc(rdev
);
6538 cik_disable_interrupts(rdev
);
6541 ret
= cik_rlc_resume(rdev
);
6543 r600_ih_ring_fini(rdev
);
6547 /* setup interrupt control */
6548 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6549 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
6550 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
6551 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6552 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6554 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
6555 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6556 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
6557 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
6559 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
6560 rb_bufsz
= drm_order(rdev
->ih
.ring_size
/ 4);
6562 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
6563 IH_WPTR_OVERFLOW_CLEAR
|
6566 if (rdev
->wb
.enabled
)
6567 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
6569 /* set the writeback address whether it's enabled or not */
6570 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
6571 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
6573 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
6575 /* set rptr, wptr to 0 */
6576 WREG32(IH_RB_RPTR
, 0);
6577 WREG32(IH_RB_WPTR
, 0);
6579 /* Default settings for IH_CNTL (disabled at first) */
6580 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6581 /* RPTR_REARM only works if msi's are enabled */
6582 if (rdev
->msi_enabled
)
6583 ih_cntl
|= RPTR_REARM
;
6584 WREG32(IH_CNTL
, ih_cntl
);
6586 /* force the active interrupt state to all disabled */
6587 cik_disable_interrupt_state(rdev
);
6589 pci_set_master(rdev
->pdev
);
6592 cik_enable_interrupts(rdev
);
6598 * cik_irq_set - enable/disable interrupt sources
6600 * @rdev: radeon_device pointer
6602 * Enable interrupt sources on the GPU (vblanks, hpd,
6604 * Returns 0 for success, errors for failure.
6606 int cik_irq_set(struct radeon_device
*rdev
)
6608 u32 cp_int_cntl
= CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
|
6609 PRIV_INSTR_INT_ENABLE
| PRIV_REG_INT_ENABLE
;
6610 u32 cp_m1p0
, cp_m1p1
, cp_m1p2
, cp_m1p3
;
6611 u32 cp_m2p0
, cp_m2p1
, cp_m2p2
, cp_m2p3
;
6612 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
6613 u32 hpd1
, hpd2
, hpd3
, hpd4
, hpd5
, hpd6
;
6614 u32 grbm_int_cntl
= 0;
6615 u32 dma_cntl
, dma_cntl1
;
6618 if (!rdev
->irq
.installed
) {
6619 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6622 /* don't enable anything if the ih is disabled */
6623 if (!rdev
->ih
.enabled
) {
6624 cik_disable_interrupts(rdev
);
6625 /* force the active interrupt state to all disabled */
6626 cik_disable_interrupt_state(rdev
);
6630 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6631 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6632 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6633 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6634 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6635 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
6637 dma_cntl
= RREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6638 dma_cntl1
= RREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
6640 cp_m1p0
= RREG32(CP_ME1_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6641 cp_m1p1
= RREG32(CP_ME1_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6642 cp_m1p2
= RREG32(CP_ME1_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6643 cp_m1p3
= RREG32(CP_ME1_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6644 cp_m2p0
= RREG32(CP_ME2_PIPE0_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6645 cp_m2p1
= RREG32(CP_ME2_PIPE1_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6646 cp_m2p2
= RREG32(CP_ME2_PIPE2_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6647 cp_m2p3
= RREG32(CP_ME2_PIPE3_INT_CNTL
) & ~TIME_STAMP_INT_ENABLE
;
6649 if (rdev
->flags
& RADEON_IS_IGP
)
6650 thermal_int
= RREG32_SMC(CG_THERMAL_INT_CTRL
) &
6651 ~(THERM_INTH_MASK
| THERM_INTL_MASK
);
6653 thermal_int
= RREG32_SMC(CG_THERMAL_INT
) &
6654 ~(THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
);
6656 /* enable CP interrupts on all rings */
6657 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
6658 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6659 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
6661 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
6662 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6663 DRM_DEBUG("si_irq_set: sw int cp1\n");
6664 if (ring
->me
== 1) {
6665 switch (ring
->pipe
) {
6667 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6670 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6673 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6676 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6679 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6682 } else if (ring
->me
== 2) {
6683 switch (ring
->pipe
) {
6685 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6688 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6691 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6694 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6697 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring
->pipe
);
6701 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring
->me
);
6704 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
6705 struct radeon_ring
*ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6706 DRM_DEBUG("si_irq_set: sw int cp2\n");
6707 if (ring
->me
== 1) {
6708 switch (ring
->pipe
) {
6710 cp_m1p0
|= TIME_STAMP_INT_ENABLE
;
6713 cp_m1p1
|= TIME_STAMP_INT_ENABLE
;
6716 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6719 cp_m1p2
|= TIME_STAMP_INT_ENABLE
;
6722 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6725 } else if (ring
->me
== 2) {
6726 switch (ring
->pipe
) {
6728 cp_m2p0
|= TIME_STAMP_INT_ENABLE
;
6731 cp_m2p1
|= TIME_STAMP_INT_ENABLE
;
6734 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6737 cp_m2p2
|= TIME_STAMP_INT_ENABLE
;
6740 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring
->pipe
);
6744 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring
->me
);
6748 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
6749 DRM_DEBUG("cik_irq_set: sw int dma\n");
6750 dma_cntl
|= TRAP_ENABLE
;
6753 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
6754 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6755 dma_cntl1
|= TRAP_ENABLE
;
6758 if (rdev
->irq
.crtc_vblank_int
[0] ||
6759 atomic_read(&rdev
->irq
.pflip
[0])) {
6760 DRM_DEBUG("cik_irq_set: vblank 0\n");
6761 crtc1
|= VBLANK_INTERRUPT_MASK
;
6763 if (rdev
->irq
.crtc_vblank_int
[1] ||
6764 atomic_read(&rdev
->irq
.pflip
[1])) {
6765 DRM_DEBUG("cik_irq_set: vblank 1\n");
6766 crtc2
|= VBLANK_INTERRUPT_MASK
;
6768 if (rdev
->irq
.crtc_vblank_int
[2] ||
6769 atomic_read(&rdev
->irq
.pflip
[2])) {
6770 DRM_DEBUG("cik_irq_set: vblank 2\n");
6771 crtc3
|= VBLANK_INTERRUPT_MASK
;
6773 if (rdev
->irq
.crtc_vblank_int
[3] ||
6774 atomic_read(&rdev
->irq
.pflip
[3])) {
6775 DRM_DEBUG("cik_irq_set: vblank 3\n");
6776 crtc4
|= VBLANK_INTERRUPT_MASK
;
6778 if (rdev
->irq
.crtc_vblank_int
[4] ||
6779 atomic_read(&rdev
->irq
.pflip
[4])) {
6780 DRM_DEBUG("cik_irq_set: vblank 4\n");
6781 crtc5
|= VBLANK_INTERRUPT_MASK
;
6783 if (rdev
->irq
.crtc_vblank_int
[5] ||
6784 atomic_read(&rdev
->irq
.pflip
[5])) {
6785 DRM_DEBUG("cik_irq_set: vblank 5\n");
6786 crtc6
|= VBLANK_INTERRUPT_MASK
;
6788 if (rdev
->irq
.hpd
[0]) {
6789 DRM_DEBUG("cik_irq_set: hpd 1\n");
6790 hpd1
|= DC_HPDx_INT_EN
;
6792 if (rdev
->irq
.hpd
[1]) {
6793 DRM_DEBUG("cik_irq_set: hpd 2\n");
6794 hpd2
|= DC_HPDx_INT_EN
;
6796 if (rdev
->irq
.hpd
[2]) {
6797 DRM_DEBUG("cik_irq_set: hpd 3\n");
6798 hpd3
|= DC_HPDx_INT_EN
;
6800 if (rdev
->irq
.hpd
[3]) {
6801 DRM_DEBUG("cik_irq_set: hpd 4\n");
6802 hpd4
|= DC_HPDx_INT_EN
;
6804 if (rdev
->irq
.hpd
[4]) {
6805 DRM_DEBUG("cik_irq_set: hpd 5\n");
6806 hpd5
|= DC_HPDx_INT_EN
;
6808 if (rdev
->irq
.hpd
[5]) {
6809 DRM_DEBUG("cik_irq_set: hpd 6\n");
6810 hpd6
|= DC_HPDx_INT_EN
;
6813 if (rdev
->irq
.dpm_thermal
) {
6814 DRM_DEBUG("dpm thermal\n");
6815 if (rdev
->flags
& RADEON_IS_IGP
)
6816 thermal_int
|= THERM_INTH_MASK
| THERM_INTL_MASK
;
6818 thermal_int
|= THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
;
6821 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
6823 WREG32(SDMA0_CNTL
+ SDMA0_REGISTER_OFFSET
, dma_cntl
);
6824 WREG32(SDMA0_CNTL
+ SDMA1_REGISTER_OFFSET
, dma_cntl1
);
6826 WREG32(CP_ME1_PIPE0_INT_CNTL
, cp_m1p0
);
6827 WREG32(CP_ME1_PIPE1_INT_CNTL
, cp_m1p1
);
6828 WREG32(CP_ME1_PIPE2_INT_CNTL
, cp_m1p2
);
6829 WREG32(CP_ME1_PIPE3_INT_CNTL
, cp_m1p3
);
6830 WREG32(CP_ME2_PIPE0_INT_CNTL
, cp_m2p0
);
6831 WREG32(CP_ME2_PIPE1_INT_CNTL
, cp_m2p1
);
6832 WREG32(CP_ME2_PIPE2_INT_CNTL
, cp_m2p2
);
6833 WREG32(CP_ME2_PIPE3_INT_CNTL
, cp_m2p3
);
6835 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
6837 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
6838 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
6839 if (rdev
->num_crtc
>= 4) {
6840 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
6841 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
6843 if (rdev
->num_crtc
>= 6) {
6844 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
6845 WREG32(LB_INTERRUPT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
6848 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
6849 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
6850 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
6851 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
6852 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
6853 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
6855 if (rdev
->flags
& RADEON_IS_IGP
)
6856 WREG32_SMC(CG_THERMAL_INT_CTRL
, thermal_int
);
6858 WREG32_SMC(CG_THERMAL_INT
, thermal_int
);
6864 * cik_irq_ack - ack interrupt sources
6866 * @rdev: radeon_device pointer
6868 * Ack interrupt sources on the GPU (vblanks, hpd,
6869 * etc.) (CIK). Certain interrupts sources are sw
6870 * generated and do not require an explicit ack.
6872 static inline void cik_irq_ack(struct radeon_device
*rdev
)
6876 rdev
->irq
.stat_regs
.cik
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
6877 rdev
->irq
.stat_regs
.cik
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
6878 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
6879 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
6880 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
6881 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
6882 rdev
->irq
.stat_regs
.cik
.disp_int_cont6
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE6
);
6884 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
6885 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
6886 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
)
6887 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
6888 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
6889 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
6890 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
6891 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
6893 if (rdev
->num_crtc
>= 4) {
6894 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
6895 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
6896 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
6897 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
6898 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
6899 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
6900 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
6901 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
6904 if (rdev
->num_crtc
>= 6) {
6905 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
6906 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
6907 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
6908 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
6909 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
6910 WREG32(LB_VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
6911 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
6912 WREG32(LB_VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
6915 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
6916 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
6917 tmp
|= DC_HPDx_INT_ACK
;
6918 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
6920 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
6921 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
6922 tmp
|= DC_HPDx_INT_ACK
;
6923 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
6925 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
6926 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
6927 tmp
|= DC_HPDx_INT_ACK
;
6928 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
6930 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
6931 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
6932 tmp
|= DC_HPDx_INT_ACK
;
6933 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
6935 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
6936 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
6937 tmp
|= DC_HPDx_INT_ACK
;
6938 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
6940 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
6941 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
6942 tmp
|= DC_HPDx_INT_ACK
;
6943 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
6948 * cik_irq_disable - disable interrupts
6950 * @rdev: radeon_device pointer
6952 * Disable interrupts on the hw (CIK).
6954 static void cik_irq_disable(struct radeon_device
*rdev
)
6956 cik_disable_interrupts(rdev
);
6957 /* Wait and acknowledge irq */
6960 cik_disable_interrupt_state(rdev
);
6964 * cik_irq_disable - disable interrupts for suspend
6966 * @rdev: radeon_device pointer
6968 * Disable interrupts and stop the RLC (CIK).
6971 static void cik_irq_suspend(struct radeon_device
*rdev
)
6973 cik_irq_disable(rdev
);
6978 * cik_irq_fini - tear down interrupt support
6980 * @rdev: radeon_device pointer
6982 * Disable interrupts on the hw and free the IH ring
6984 * Used for driver unload.
6986 static void cik_irq_fini(struct radeon_device
*rdev
)
6988 cik_irq_suspend(rdev
);
6989 r600_ih_ring_fini(rdev
);
6993 * cik_get_ih_wptr - get the IH ring buffer wptr
6995 * @rdev: radeon_device pointer
6997 * Get the IH ring buffer wptr from either the register
6998 * or the writeback memory buffer (CIK). Also check for
6999 * ring buffer overflow and deal with it.
7000 * Used by cik_irq_process().
7001 * Returns the value of the wptr.
7003 static inline u32
cik_get_ih_wptr(struct radeon_device
*rdev
)
7007 if (rdev
->wb
.enabled
)
7008 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
7010 wptr
= RREG32(IH_RB_WPTR
);
7012 if (wptr
& RB_OVERFLOW
) {
7013 /* When a ring buffer overflow happen start parsing interrupt
7014 * from the last not overwritten vector (wptr + 16). Hopefully
7015 * this should allow us to catchup.
7017 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7018 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
7019 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
7020 tmp
= RREG32(IH_RB_CNTL
);
7021 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
7022 WREG32(IH_RB_CNTL
, tmp
);
7024 return (wptr
& rdev
->ih
.ptr_mask
);
7028 * Each IV ring entry is 128 bits:
7029 * [7:0] - interrupt source id
7031 * [59:32] - interrupt source data
7032 * [63:60] - reserved
7035 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7036 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7037 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7038 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7039 * PIPE_ID - ME0 0=3D
7040 * - ME1&2 compute dispatcher (4 pipes each)
7042 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7043 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7044 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7047 * [127:96] - reserved
7050 * cik_irq_process - interrupt handler
7052 * @rdev: radeon_device pointer
7054 * Interrupt hander (CIK). Walk the IH ring,
7055 * ack interrupts and schedule work to handle
7057 * Returns irq process return code.
7059 int cik_irq_process(struct radeon_device
*rdev
)
7061 struct radeon_ring
*cp1_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7062 struct radeon_ring
*cp2_ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7065 u32 src_id
, src_data
, ring_id
;
7066 u8 me_id
, pipe_id
, queue_id
;
7068 bool queue_hotplug
= false;
7069 bool queue_reset
= false;
7070 u32 addr
, status
, mc_client
;
7071 bool queue_thermal
= false;
7073 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
7076 wptr
= cik_get_ih_wptr(rdev
);
7079 /* is somebody else already processing irqs? */
7080 if (atomic_xchg(&rdev
->ih
.lock
, 1))
7083 rptr
= rdev
->ih
.rptr
;
7084 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
7086 /* Order reading of wptr vs. reading of IH ring data */
7089 /* display interrupts */
7092 while (rptr
!= wptr
) {
7093 /* wptr/rptr are in bytes! */
7094 ring_index
= rptr
/ 4;
7095 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
7096 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
7097 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
7100 case 1: /* D1 vblank/vline */
7102 case 0: /* D1 vblank */
7103 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
7104 if (rdev
->irq
.crtc_vblank_int
[0]) {
7105 drm_handle_vblank(rdev
->ddev
, 0);
7106 rdev
->pm
.vblank_sync
= true;
7107 wake_up(&rdev
->irq
.vblank_queue
);
7109 if (atomic_read(&rdev
->irq
.pflip
[0]))
7110 radeon_crtc_handle_flip(rdev
, 0);
7111 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
7112 DRM_DEBUG("IH: D1 vblank\n");
7115 case 1: /* D1 vline */
7116 if (rdev
->irq
.stat_regs
.cik
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
7117 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
7118 DRM_DEBUG("IH: D1 vline\n");
7122 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7126 case 2: /* D2 vblank/vline */
7128 case 0: /* D2 vblank */
7129 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
7130 if (rdev
->irq
.crtc_vblank_int
[1]) {
7131 drm_handle_vblank(rdev
->ddev
, 1);
7132 rdev
->pm
.vblank_sync
= true;
7133 wake_up(&rdev
->irq
.vblank_queue
);
7135 if (atomic_read(&rdev
->irq
.pflip
[1]))
7136 radeon_crtc_handle_flip(rdev
, 1);
7137 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
7138 DRM_DEBUG("IH: D2 vblank\n");
7141 case 1: /* D2 vline */
7142 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
7143 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
7144 DRM_DEBUG("IH: D2 vline\n");
7148 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7152 case 3: /* D3 vblank/vline */
7154 case 0: /* D3 vblank */
7155 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
7156 if (rdev
->irq
.crtc_vblank_int
[2]) {
7157 drm_handle_vblank(rdev
->ddev
, 2);
7158 rdev
->pm
.vblank_sync
= true;
7159 wake_up(&rdev
->irq
.vblank_queue
);
7161 if (atomic_read(&rdev
->irq
.pflip
[2]))
7162 radeon_crtc_handle_flip(rdev
, 2);
7163 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
7164 DRM_DEBUG("IH: D3 vblank\n");
7167 case 1: /* D3 vline */
7168 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
7169 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
7170 DRM_DEBUG("IH: D3 vline\n");
7174 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7178 case 4: /* D4 vblank/vline */
7180 case 0: /* D4 vblank */
7181 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
7182 if (rdev
->irq
.crtc_vblank_int
[3]) {
7183 drm_handle_vblank(rdev
->ddev
, 3);
7184 rdev
->pm
.vblank_sync
= true;
7185 wake_up(&rdev
->irq
.vblank_queue
);
7187 if (atomic_read(&rdev
->irq
.pflip
[3]))
7188 radeon_crtc_handle_flip(rdev
, 3);
7189 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
7190 DRM_DEBUG("IH: D4 vblank\n");
7193 case 1: /* D4 vline */
7194 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
7195 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
7196 DRM_DEBUG("IH: D4 vline\n");
7200 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7204 case 5: /* D5 vblank/vline */
7206 case 0: /* D5 vblank */
7207 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
7208 if (rdev
->irq
.crtc_vblank_int
[4]) {
7209 drm_handle_vblank(rdev
->ddev
, 4);
7210 rdev
->pm
.vblank_sync
= true;
7211 wake_up(&rdev
->irq
.vblank_queue
);
7213 if (atomic_read(&rdev
->irq
.pflip
[4]))
7214 radeon_crtc_handle_flip(rdev
, 4);
7215 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
7216 DRM_DEBUG("IH: D5 vblank\n");
7219 case 1: /* D5 vline */
7220 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
7221 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
7222 DRM_DEBUG("IH: D5 vline\n");
7226 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7230 case 6: /* D6 vblank/vline */
7232 case 0: /* D6 vblank */
7233 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
7234 if (rdev
->irq
.crtc_vblank_int
[5]) {
7235 drm_handle_vblank(rdev
->ddev
, 5);
7236 rdev
->pm
.vblank_sync
= true;
7237 wake_up(&rdev
->irq
.vblank_queue
);
7239 if (atomic_read(&rdev
->irq
.pflip
[5]))
7240 radeon_crtc_handle_flip(rdev
, 5);
7241 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
7242 DRM_DEBUG("IH: D6 vblank\n");
7245 case 1: /* D6 vline */
7246 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
7247 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
7248 DRM_DEBUG("IH: D6 vline\n");
7252 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7256 case 42: /* HPD hotplug */
7259 if (rdev
->irq
.stat_regs
.cik
.disp_int
& DC_HPD1_INTERRUPT
) {
7260 rdev
->irq
.stat_regs
.cik
.disp_int
&= ~DC_HPD1_INTERRUPT
;
7261 queue_hotplug
= true;
7262 DRM_DEBUG("IH: HPD1\n");
7266 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
7267 rdev
->irq
.stat_regs
.cik
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
7268 queue_hotplug
= true;
7269 DRM_DEBUG("IH: HPD2\n");
7273 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
7274 rdev
->irq
.stat_regs
.cik
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
7275 queue_hotplug
= true;
7276 DRM_DEBUG("IH: HPD3\n");
7280 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
7281 rdev
->irq
.stat_regs
.cik
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
7282 queue_hotplug
= true;
7283 DRM_DEBUG("IH: HPD4\n");
7287 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
7288 rdev
->irq
.stat_regs
.cik
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
7289 queue_hotplug
= true;
7290 DRM_DEBUG("IH: HPD5\n");
7294 if (rdev
->irq
.stat_regs
.cik
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
7295 rdev
->irq
.stat_regs
.cik
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
7296 queue_hotplug
= true;
7297 DRM_DEBUG("IH: HPD6\n");
7301 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7307 addr
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
);
7308 status
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
);
7309 mc_client
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT
);
7310 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
7311 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7313 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7315 cik_vm_decode_fault(rdev
, status
, addr
, mc_client
);
7316 /* reset addr and status */
7317 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
7319 case 176: /* GFX RB CP_INT */
7320 case 177: /* GFX IB CP_INT */
7321 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7323 case 181: /* CP EOP event */
7324 DRM_DEBUG("IH: CP EOP\n");
7325 /* XXX check the bitfield order! */
7326 me_id
= (ring_id
& 0x60) >> 5;
7327 pipe_id
= (ring_id
& 0x18) >> 3;
7328 queue_id
= (ring_id
& 0x7) >> 0;
7331 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7335 if ((cp1_ring
->me
== me_id
) & (cp1_ring
->pipe
== pipe_id
))
7336 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7337 if ((cp2_ring
->me
== me_id
) & (cp2_ring
->pipe
== pipe_id
))
7338 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7342 case 184: /* CP Privileged reg access */
7343 DRM_ERROR("Illegal register access in command stream\n");
7344 /* XXX check the bitfield order! */
7345 me_id
= (ring_id
& 0x60) >> 5;
7346 pipe_id
= (ring_id
& 0x18) >> 3;
7347 queue_id
= (ring_id
& 0x7) >> 0;
7350 /* This results in a full GPU reset, but all we need to do is soft
7351 * reset the CP for gfx
7365 case 185: /* CP Privileged inst */
7366 DRM_ERROR("Illegal instruction in command stream\n");
7367 /* XXX check the bitfield order! */
7368 me_id
= (ring_id
& 0x60) >> 5;
7369 pipe_id
= (ring_id
& 0x18) >> 3;
7370 queue_id
= (ring_id
& 0x7) >> 0;
7373 /* This results in a full GPU reset, but all we need to do is soft
7374 * reset the CP for gfx
7388 case 224: /* SDMA trap event */
7389 /* XXX check the bitfield order! */
7390 me_id
= (ring_id
& 0x3) >> 0;
7391 queue_id
= (ring_id
& 0xc) >> 2;
7392 DRM_DEBUG("IH: SDMA trap\n");
7397 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
7410 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7422 case 230: /* thermal low to high */
7423 DRM_DEBUG("IH: thermal low to high\n");
7424 rdev
->pm
.dpm
.thermal
.high_to_low
= false;
7425 queue_thermal
= true;
7427 case 231: /* thermal high to low */
7428 DRM_DEBUG("IH: thermal high to low\n");
7429 rdev
->pm
.dpm
.thermal
.high_to_low
= true;
7430 queue_thermal
= true;
7432 case 233: /* GUI IDLE */
7433 DRM_DEBUG("IH: GUI idle\n");
7435 case 241: /* SDMA Privileged inst */
7436 case 247: /* SDMA Privileged inst */
7437 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7438 /* XXX check the bitfield order! */
7439 me_id
= (ring_id
& 0x3) >> 0;
7440 queue_id
= (ring_id
& 0xc) >> 2;
7475 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
7479 /* wptr/rptr are in bytes! */
7481 rptr
&= rdev
->ih
.ptr_mask
;
7484 schedule_work(&rdev
->hotplug_work
);
7486 schedule_work(&rdev
->reset_work
);
7488 schedule_work(&rdev
->pm
.dpm
.thermal
.work
);
7489 rdev
->ih
.rptr
= rptr
;
7490 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
7491 atomic_set(&rdev
->ih
.lock
, 0);
7493 /* make sure wptr hasn't changed while processing */
7494 wptr
= cik_get_ih_wptr(rdev
);
7502 * startup/shutdown callbacks
7505 * cik_startup - program the asic to a functional state
7507 * @rdev: radeon_device pointer
7509 * Programs the asic to a functional state (CIK).
7510 * Called by cik_init() and cik_resume().
7511 * Returns 0 for success, error for failure.
7513 static int cik_startup(struct radeon_device
*rdev
)
7515 struct radeon_ring
*ring
;
7518 /* enable pcie gen2/3 link */
7519 cik_pcie_gen3_enable(rdev
);
7521 cik_program_aspm(rdev
);
7523 cik_mc_program(rdev
);
7525 if (rdev
->flags
& RADEON_IS_IGP
) {
7526 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
7527 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
) {
7528 r
= cik_init_microcode(rdev
);
7530 DRM_ERROR("Failed to load firmware!\n");
7535 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
7536 !rdev
->mec_fw
|| !rdev
->sdma_fw
|| !rdev
->rlc_fw
||
7538 r
= cik_init_microcode(rdev
);
7540 DRM_ERROR("Failed to load firmware!\n");
7545 r
= ci_mc_load_microcode(rdev
);
7547 DRM_ERROR("Failed to load MC firmware!\n");
7552 r
= r600_vram_scratch_init(rdev
);
7556 r
= cik_pcie_gart_enable(rdev
);
7561 /* allocate rlc buffers */
7562 if (rdev
->flags
& RADEON_IS_IGP
) {
7563 if (rdev
->family
== CHIP_KAVERI
) {
7564 rdev
->rlc
.reg_list
= spectre_rlc_save_restore_register_list
;
7565 rdev
->rlc
.reg_list_size
=
7566 (u32
)ARRAY_SIZE(spectre_rlc_save_restore_register_list
);
7568 rdev
->rlc
.reg_list
= kalindi_rlc_save_restore_register_list
;
7569 rdev
->rlc
.reg_list_size
=
7570 (u32
)ARRAY_SIZE(kalindi_rlc_save_restore_register_list
);
7573 rdev
->rlc
.cs_data
= ci_cs_data
;
7574 rdev
->rlc
.cp_table_size
= CP_ME_TABLE_SIZE
* 5 * 4;
7575 r
= sumo_rlc_init(rdev
);
7577 DRM_ERROR("Failed to init rlc BOs!\n");
7581 /* allocate wb buffer */
7582 r
= radeon_wb_init(rdev
);
7586 /* allocate mec buffers */
7587 r
= cik_mec_init(rdev
);
7589 DRM_ERROR("Failed to init MEC BOs!\n");
7593 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
7595 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7599 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
7601 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7605 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
7607 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
7611 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
7613 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7617 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
7619 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
7623 r
= radeon_uvd_resume(rdev
);
7625 cik_uvd_resume(rdev
);
7626 r
= radeon_fence_driver_start_ring(rdev
,
7627 R600_RING_TYPE_UVD_INDEX
);
7629 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
7632 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
7635 if (!rdev
->irq
.installed
) {
7636 r
= radeon_irq_kms_init(rdev
);
7641 r
= cik_irq_init(rdev
);
7643 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
7644 radeon_irq_kms_fini(rdev
);
7649 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
7650 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
7651 CP_RB0_RPTR
, CP_RB0_WPTR
,
7652 0, 0xfffff, RADEON_CP_PACKET2
);
7656 /* set up the compute queues */
7657 /* type-2 packets are deprecated on MEC, use type-3 instead */
7658 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7659 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
7660 CP_HQD_PQ_RPTR
, CP_HQD_PQ_WPTR
,
7661 0, 0xfffff, PACKET3(PACKET3_NOP
, 0x3FFF));
7664 ring
->me
= 1; /* first MEC */
7665 ring
->pipe
= 0; /* first pipe */
7666 ring
->queue
= 0; /* first queue */
7667 ring
->wptr_offs
= CIK_WB_CP1_WPTR_OFFSET
;
7669 /* type-2 packets are deprecated on MEC, use type-3 instead */
7670 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7671 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
7672 CP_HQD_PQ_RPTR
, CP_HQD_PQ_WPTR
,
7673 0, 0xffffffff, PACKET3(PACKET3_NOP
, 0x3FFF));
7676 /* dGPU only have 1 MEC */
7677 ring
->me
= 1; /* first MEC */
7678 ring
->pipe
= 0; /* first pipe */
7679 ring
->queue
= 1; /* second queue */
7680 ring
->wptr_offs
= CIK_WB_CP2_WPTR_OFFSET
;
7682 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
7683 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
7684 SDMA0_GFX_RB_RPTR
+ SDMA0_REGISTER_OFFSET
,
7685 SDMA0_GFX_RB_WPTR
+ SDMA0_REGISTER_OFFSET
,
7686 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7690 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
7691 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
7692 SDMA0_GFX_RB_RPTR
+ SDMA1_REGISTER_OFFSET
,
7693 SDMA0_GFX_RB_WPTR
+ SDMA1_REGISTER_OFFSET
,
7694 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP
, 0, 0));
7698 r
= cik_cp_resume(rdev
);
7702 r
= cik_sdma_resume(rdev
);
7706 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
7707 if (ring
->ring_size
) {
7708 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
7709 UVD_RBC_RB_RPTR
, UVD_RBC_RB_WPTR
,
7710 0, 0xfffff, RADEON_CP_PACKET2
);
7712 r
= r600_uvd_init(rdev
, true);
7714 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
7717 r
= radeon_ib_pool_init(rdev
);
7719 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
7723 r
= radeon_vm_manager_init(rdev
);
7725 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
7733 * cik_resume - resume the asic to a functional state
7735 * @rdev: radeon_device pointer
7737 * Programs the asic to a functional state (CIK).
7739 * Returns 0 for success, error for failure.
7741 int cik_resume(struct radeon_device
*rdev
)
7746 atom_asic_init(rdev
->mode_info
.atom_context
);
7748 /* init golden registers */
7749 cik_init_golden_registers(rdev
);
7751 rdev
->accel_working
= true;
7752 r
= cik_startup(rdev
);
7754 DRM_ERROR("cik startup failed on resume\n");
7755 rdev
->accel_working
= false;
7764 * cik_suspend - suspend the asic
7766 * @rdev: radeon_device pointer
7768 * Bring the chip into a state suitable for suspend (CIK).
7769 * Called at suspend.
7770 * Returns 0 for success.
7772 int cik_suspend(struct radeon_device
*rdev
)
7774 radeon_vm_manager_fini(rdev
);
7775 cik_cp_enable(rdev
, false);
7776 cik_sdma_enable(rdev
, false);
7777 r600_uvd_stop(rdev
);
7778 radeon_uvd_suspend(rdev
);
7779 cik_irq_suspend(rdev
);
7780 radeon_wb_disable(rdev
);
7781 cik_pcie_gart_disable(rdev
);
7785 /* Plan is to move initialization in that function and use
7786 * helper function so that radeon_device_init pretty much
7787 * do nothing more than calling asic specific function. This
7788 * should also allow to remove a bunch of callback function
7792 * cik_init - asic specific driver and hw init
7794 * @rdev: radeon_device pointer
7796 * Setup asic specific driver variables and program the hw
7797 * to a functional state (CIK).
7798 * Called at driver startup.
7799 * Returns 0 for success, errors for failure.
7801 int cik_init(struct radeon_device
*rdev
)
7803 struct radeon_ring
*ring
;
7807 if (!radeon_get_bios(rdev
)) {
7808 if (ASIC_IS_AVIVO(rdev
))
7811 /* Must be an ATOMBIOS */
7812 if (!rdev
->is_atom_bios
) {
7813 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
7816 r
= radeon_atombios_init(rdev
);
7820 /* Post card if necessary */
7821 if (!radeon_card_posted(rdev
)) {
7823 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
7826 DRM_INFO("GPU not posted. posting now...\n");
7827 atom_asic_init(rdev
->mode_info
.atom_context
);
7829 /* init golden registers */
7830 cik_init_golden_registers(rdev
);
7831 /* Initialize scratch registers */
7832 cik_scratch_init(rdev
);
7833 /* Initialize surface registers */
7834 radeon_surface_init(rdev
);
7835 /* Initialize clocks */
7836 radeon_get_clock_info(rdev
->ddev
);
7839 r
= radeon_fence_driver_init(rdev
);
7843 /* initialize memory controller */
7844 r
= cik_mc_init(rdev
);
7847 /* Memory manager */
7848 r
= radeon_bo_init(rdev
);
7852 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
7853 ring
->ring_obj
= NULL
;
7854 r600_ring_init(rdev
, ring
, 1024 * 1024);
7856 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
7857 ring
->ring_obj
= NULL
;
7858 r600_ring_init(rdev
, ring
, 1024 * 1024);
7859 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_page_num
);
7863 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
7864 ring
->ring_obj
= NULL
;
7865 r600_ring_init(rdev
, ring
, 1024 * 1024);
7866 r
= radeon_doorbell_get(rdev
, &ring
->doorbell_page_num
);
7870 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
7871 ring
->ring_obj
= NULL
;
7872 r600_ring_init(rdev
, ring
, 256 * 1024);
7874 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
7875 ring
->ring_obj
= NULL
;
7876 r600_ring_init(rdev
, ring
, 256 * 1024);
7878 r
= radeon_uvd_init(rdev
);
7880 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
7881 ring
->ring_obj
= NULL
;
7882 r600_ring_init(rdev
, ring
, 4096);
7885 rdev
->ih
.ring_obj
= NULL
;
7886 r600_ih_ring_init(rdev
, 64 * 1024);
7888 r
= r600_pcie_gart_init(rdev
);
7892 rdev
->accel_working
= true;
7893 r
= cik_startup(rdev
);
7895 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
7897 cik_sdma_fini(rdev
);
7899 sumo_rlc_fini(rdev
);
7901 radeon_wb_fini(rdev
);
7902 radeon_ib_pool_fini(rdev
);
7903 radeon_vm_manager_fini(rdev
);
7904 radeon_irq_kms_fini(rdev
);
7905 cik_pcie_gart_fini(rdev
);
7906 rdev
->accel_working
= false;
7909 /* Don't start up if the MC ucode is missing.
7910 * The default clocks and voltages before the MC ucode
7911 * is loaded are not suffient for advanced operations.
7913 if (!rdev
->mc_fw
&& !(rdev
->flags
& RADEON_IS_IGP
)) {
7914 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7922 * cik_fini - asic specific driver and hw fini
7924 * @rdev: radeon_device pointer
7926 * Tear down the asic specific driver variables and program the hw
7927 * to an idle state (CIK).
7928 * Called at driver unload.
7930 void cik_fini(struct radeon_device
*rdev
)
7933 cik_sdma_fini(rdev
);
7935 sumo_rlc_fini(rdev
);
7937 radeon_wb_fini(rdev
);
7938 radeon_vm_manager_fini(rdev
);
7939 radeon_ib_pool_fini(rdev
);
7940 radeon_irq_kms_fini(rdev
);
7941 r600_uvd_stop(rdev
);
7942 radeon_uvd_fini(rdev
);
7943 cik_pcie_gart_fini(rdev
);
7944 r600_vram_scratch_fini(rdev
);
7945 radeon_gem_fini(rdev
);
7946 radeon_fence_driver_fini(rdev
);
7947 radeon_bo_fini(rdev
);
7948 radeon_atombios_fini(rdev
);
7953 /* display watermark setup */
7955 * dce8_line_buffer_adjust - Set up the line buffer
7957 * @rdev: radeon_device pointer
7958 * @radeon_crtc: the selected display controller
7959 * @mode: the current display mode on the selected display
7962 * Setup up the line buffer allocation for
7963 * the selected display controller (CIK).
7964 * Returns the line buffer size in pixels.
7966 static u32
dce8_line_buffer_adjust(struct radeon_device
*rdev
,
7967 struct radeon_crtc
*radeon_crtc
,
7968 struct drm_display_mode
*mode
)
7974 * There are 6 line buffers, one for each display controllers.
7975 * There are 3 partitions per LB. Select the number of partitions
7976 * to enable based on the display width. For display widths larger
7977 * than 4096, you need use to use 2 display controllers and combine
7978 * them using the stereo blender.
7980 if (radeon_crtc
->base
.enabled
&& mode
) {
7981 if (mode
->crtc_hdisplay
< 1920)
7983 else if (mode
->crtc_hdisplay
< 2560)
7985 else if (mode
->crtc_hdisplay
< 4096)
7988 DRM_DEBUG_KMS("Mode too big for LB!\n");
7994 WREG32(LB_MEMORY_CTRL
+ radeon_crtc
->crtc_offset
,
7995 LB_MEMORY_CONFIG(tmp
) | LB_MEMORY_SIZE(0x6B0));
7997 if (radeon_crtc
->base
.enabled
&& mode
) {
8009 /* controller not enabled, so no lb used */
8014 * cik_get_number_of_dram_channels - get the number of dram channels
8016 * @rdev: radeon_device pointer
8018 * Look up the number of video ram channels (CIK).
8019 * Used for display watermark bandwidth calculations
8020 * Returns the number of dram channels
8022 static u32
cik_get_number_of_dram_channels(struct radeon_device
*rdev
)
8024 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
8026 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
8049 struct dce8_wm_params
{
8050 u32 dram_channels
; /* number of dram channels */
8051 u32 yclk
; /* bandwidth per dram data pin in kHz */
8052 u32 sclk
; /* engine clock in kHz */
8053 u32 disp_clk
; /* display clock in kHz */
8054 u32 src_width
; /* viewport width */
8055 u32 active_time
; /* active display time in ns */
8056 u32 blank_time
; /* blank time in ns */
8057 bool interlaced
; /* mode is interlaced */
8058 fixed20_12 vsc
; /* vertical scale ratio */
8059 u32 num_heads
; /* number of active crtcs */
8060 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
8061 u32 lb_size
; /* line buffer allocated to pipe */
8062 u32 vtaps
; /* vertical scaler taps */
8066 * dce8_dram_bandwidth - get the dram bandwidth
8068 * @wm: watermark calculation data
8070 * Calculate the raw dram bandwidth (CIK).
8071 * Used for display watermark bandwidth calculations
8072 * Returns the dram bandwidth in MBytes/s
8074 static u32
dce8_dram_bandwidth(struct dce8_wm_params
*wm
)
8076 /* Calculate raw DRAM Bandwidth */
8077 fixed20_12 dram_efficiency
; /* 0.7 */
8078 fixed20_12 yclk
, dram_channels
, bandwidth
;
8081 a
.full
= dfixed_const(1000);
8082 yclk
.full
= dfixed_const(wm
->yclk
);
8083 yclk
.full
= dfixed_div(yclk
, a
);
8084 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8085 a
.full
= dfixed_const(10);
8086 dram_efficiency
.full
= dfixed_const(7);
8087 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
8088 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8089 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
8091 return dfixed_trunc(bandwidth
);
8095 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8097 * @wm: watermark calculation data
8099 * Calculate the dram bandwidth used for display (CIK).
8100 * Used for display watermark bandwidth calculations
8101 * Returns the dram bandwidth for display in MBytes/s
8103 static u32
dce8_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8105 /* Calculate DRAM Bandwidth and the part allocated to display. */
8106 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
8107 fixed20_12 yclk
, dram_channels
, bandwidth
;
8110 a
.full
= dfixed_const(1000);
8111 yclk
.full
= dfixed_const(wm
->yclk
);
8112 yclk
.full
= dfixed_div(yclk
, a
);
8113 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
8114 a
.full
= dfixed_const(10);
8115 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
8116 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
8117 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
8118 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
8120 return dfixed_trunc(bandwidth
);
8124 * dce8_data_return_bandwidth - get the data return bandwidth
8126 * @wm: watermark calculation data
8128 * Calculate the data return bandwidth used for display (CIK).
8129 * Used for display watermark bandwidth calculations
8130 * Returns the data return bandwidth in MBytes/s
8132 static u32
dce8_data_return_bandwidth(struct dce8_wm_params
*wm
)
8134 /* Calculate the display Data return Bandwidth */
8135 fixed20_12 return_efficiency
; /* 0.8 */
8136 fixed20_12 sclk
, bandwidth
;
8139 a
.full
= dfixed_const(1000);
8140 sclk
.full
= dfixed_const(wm
->sclk
);
8141 sclk
.full
= dfixed_div(sclk
, a
);
8142 a
.full
= dfixed_const(10);
8143 return_efficiency
.full
= dfixed_const(8);
8144 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
8145 a
.full
= dfixed_const(32);
8146 bandwidth
.full
= dfixed_mul(a
, sclk
);
8147 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
8149 return dfixed_trunc(bandwidth
);
8153 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8155 * @wm: watermark calculation data
8157 * Calculate the dmif bandwidth used for display (CIK).
8158 * Used for display watermark bandwidth calculations
8159 * Returns the dmif bandwidth in MBytes/s
8161 static u32
dce8_dmif_request_bandwidth(struct dce8_wm_params
*wm
)
8163 /* Calculate the DMIF Request Bandwidth */
8164 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
8165 fixed20_12 disp_clk
, bandwidth
;
8168 a
.full
= dfixed_const(1000);
8169 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
8170 disp_clk
.full
= dfixed_div(disp_clk
, a
);
8171 a
.full
= dfixed_const(32);
8172 b
.full
= dfixed_mul(a
, disp_clk
);
8174 a
.full
= dfixed_const(10);
8175 disp_clk_request_efficiency
.full
= dfixed_const(8);
8176 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
8178 bandwidth
.full
= dfixed_mul(b
, disp_clk_request_efficiency
);
8180 return dfixed_trunc(bandwidth
);
8184 * dce8_available_bandwidth - get the min available bandwidth
8186 * @wm: watermark calculation data
8188 * Calculate the min available bandwidth used for display (CIK).
8189 * Used for display watermark bandwidth calculations
8190 * Returns the min available bandwidth in MBytes/s
8192 static u32
dce8_available_bandwidth(struct dce8_wm_params
*wm
)
8194 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8195 u32 dram_bandwidth
= dce8_dram_bandwidth(wm
);
8196 u32 data_return_bandwidth
= dce8_data_return_bandwidth(wm
);
8197 u32 dmif_req_bandwidth
= dce8_dmif_request_bandwidth(wm
);
8199 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
8203 * dce8_average_bandwidth - get the average available bandwidth
8205 * @wm: watermark calculation data
8207 * Calculate the average available bandwidth used for display (CIK).
8208 * Used for display watermark bandwidth calculations
8209 * Returns the average available bandwidth in MBytes/s
8211 static u32
dce8_average_bandwidth(struct dce8_wm_params
*wm
)
8213 /* Calculate the display mode Average Bandwidth
8214 * DisplayMode should contain the source and destination dimensions,
8218 fixed20_12 line_time
;
8219 fixed20_12 src_width
;
8220 fixed20_12 bandwidth
;
8223 a
.full
= dfixed_const(1000);
8224 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
8225 line_time
.full
= dfixed_div(line_time
, a
);
8226 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
8227 src_width
.full
= dfixed_const(wm
->src_width
);
8228 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
8229 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
8230 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
8232 return dfixed_trunc(bandwidth
);
8236 * dce8_latency_watermark - get the latency watermark
8238 * @wm: watermark calculation data
8240 * Calculate the latency watermark (CIK).
8241 * Used for display watermark bandwidth calculations
8242 * Returns the latency watermark in ns
8244 static u32
dce8_latency_watermark(struct dce8_wm_params
*wm
)
8246 /* First calculate the latency in ns */
8247 u32 mc_latency
= 2000; /* 2000 ns. */
8248 u32 available_bandwidth
= dce8_available_bandwidth(wm
);
8249 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
8250 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
8251 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
8252 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
8253 (wm
->num_heads
* cursor_line_pair_return_time
);
8254 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
8255 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
8256 u32 tmp
, dmif_size
= 12288;
8259 if (wm
->num_heads
== 0)
8262 a
.full
= dfixed_const(2);
8263 b
.full
= dfixed_const(1);
8264 if ((wm
->vsc
.full
> a
.full
) ||
8265 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
8267 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
8268 max_src_lines_per_dst_line
= 4;
8270 max_src_lines_per_dst_line
= 2;
8272 a
.full
= dfixed_const(available_bandwidth
);
8273 b
.full
= dfixed_const(wm
->num_heads
);
8274 a
.full
= dfixed_div(a
, b
);
8276 b
.full
= dfixed_const(mc_latency
+ 512);
8277 c
.full
= dfixed_const(wm
->disp_clk
);
8278 b
.full
= dfixed_div(b
, c
);
8280 c
.full
= dfixed_const(dmif_size
);
8281 b
.full
= dfixed_div(c
, b
);
8283 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
8285 b
.full
= dfixed_const(1000);
8286 c
.full
= dfixed_const(wm
->disp_clk
);
8287 b
.full
= dfixed_div(c
, b
);
8288 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
8289 b
.full
= dfixed_mul(b
, c
);
8291 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
8293 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
8294 b
.full
= dfixed_const(1000);
8295 c
.full
= dfixed_const(lb_fill_bw
);
8296 b
.full
= dfixed_div(c
, b
);
8297 a
.full
= dfixed_div(a
, b
);
8298 line_fill_time
= dfixed_trunc(a
);
8300 if (line_fill_time
< wm
->active_time
)
8303 return latency
+ (line_fill_time
- wm
->active_time
);
8308 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8309 * average and available dram bandwidth
8311 * @wm: watermark calculation data
8313 * Check if the display average bandwidth fits in the display
8314 * dram bandwidth (CIK).
8315 * Used for display watermark bandwidth calculations
8316 * Returns true if the display fits, false if not.
8318 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params
*wm
)
8320 if (dce8_average_bandwidth(wm
) <=
8321 (dce8_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
8328 * dce8_average_bandwidth_vs_available_bandwidth - check
8329 * average and available bandwidth
8331 * @wm: watermark calculation data
8333 * Check if the display average bandwidth fits in the display
8334 * available bandwidth (CIK).
8335 * Used for display watermark bandwidth calculations
8336 * Returns true if the display fits, false if not.
8338 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params
*wm
)
8340 if (dce8_average_bandwidth(wm
) <=
8341 (dce8_available_bandwidth(wm
) / wm
->num_heads
))
8348 * dce8_check_latency_hiding - check latency hiding
8350 * @wm: watermark calculation data
8352 * Check latency hiding (CIK).
8353 * Used for display watermark bandwidth calculations
8354 * Returns true if the display fits, false if not.
8356 static bool dce8_check_latency_hiding(struct dce8_wm_params
*wm
)
8358 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
8359 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
8360 u32 latency_tolerant_lines
;
8364 a
.full
= dfixed_const(1);
8365 if (wm
->vsc
.full
> a
.full
)
8366 latency_tolerant_lines
= 1;
8368 if (lb_partitions
<= (wm
->vtaps
+ 1))
8369 latency_tolerant_lines
= 1;
8371 latency_tolerant_lines
= 2;
8374 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
8376 if (dce8_latency_watermark(wm
) <= latency_hiding
)
8383 * dce8_program_watermarks - program display watermarks
8385 * @rdev: radeon_device pointer
8386 * @radeon_crtc: the selected display controller
8387 * @lb_size: line buffer size
8388 * @num_heads: number of display controllers in use
8390 * Calculate and program the display watermarks for the
8391 * selected display controller (CIK).
8393 static void dce8_program_watermarks(struct radeon_device
*rdev
,
8394 struct radeon_crtc
*radeon_crtc
,
8395 u32 lb_size
, u32 num_heads
)
8397 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
8398 struct dce8_wm_params wm_low
, wm_high
;
8401 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
8404 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
8405 pixel_period
= 1000000 / (u32
)mode
->clock
;
8406 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
8408 /* watermark for high clocks */
8409 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8410 rdev
->pm
.dpm_enabled
) {
8412 radeon_dpm_get_mclk(rdev
, false) * 10;
8414 radeon_dpm_get_sclk(rdev
, false) * 10;
8416 wm_high
.yclk
= rdev
->pm
.current_mclk
* 10;
8417 wm_high
.sclk
= rdev
->pm
.current_sclk
* 10;
8420 wm_high
.disp_clk
= mode
->clock
;
8421 wm_high
.src_width
= mode
->crtc_hdisplay
;
8422 wm_high
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8423 wm_high
.blank_time
= line_time
- wm_high
.active_time
;
8424 wm_high
.interlaced
= false;
8425 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8426 wm_high
.interlaced
= true;
8427 wm_high
.vsc
= radeon_crtc
->vsc
;
8429 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8431 wm_high
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8432 wm_high
.lb_size
= lb_size
;
8433 wm_high
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8434 wm_high
.num_heads
= num_heads
;
8436 /* set for high clocks */
8437 latency_watermark_a
= min(dce8_latency_watermark(&wm_high
), (u32
)65535);
8439 /* possibly force display priority to high */
8440 /* should really do this at mode validation time... */
8441 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high
) ||
8442 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high
) ||
8443 !dce8_check_latency_hiding(&wm_high
) ||
8444 (rdev
->disp_priority
== 2)) {
8445 DRM_DEBUG_KMS("force priority to high\n");
8448 /* watermark for low clocks */
8449 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) &&
8450 rdev
->pm
.dpm_enabled
) {
8452 radeon_dpm_get_mclk(rdev
, true) * 10;
8454 radeon_dpm_get_sclk(rdev
, true) * 10;
8456 wm_low
.yclk
= rdev
->pm
.current_mclk
* 10;
8457 wm_low
.sclk
= rdev
->pm
.current_sclk
* 10;
8460 wm_low
.disp_clk
= mode
->clock
;
8461 wm_low
.src_width
= mode
->crtc_hdisplay
;
8462 wm_low
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
8463 wm_low
.blank_time
= line_time
- wm_low
.active_time
;
8464 wm_low
.interlaced
= false;
8465 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
8466 wm_low
.interlaced
= true;
8467 wm_low
.vsc
= radeon_crtc
->vsc
;
8469 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
8471 wm_low
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
8472 wm_low
.lb_size
= lb_size
;
8473 wm_low
.dram_channels
= cik_get_number_of_dram_channels(rdev
);
8474 wm_low
.num_heads
= num_heads
;
8476 /* set for low clocks */
8477 latency_watermark_b
= min(dce8_latency_watermark(&wm_low
), (u32
)65535);
8479 /* possibly force display priority to high */
8480 /* should really do this at mode validation time... */
8481 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low
) ||
8482 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low
) ||
8483 !dce8_check_latency_hiding(&wm_low
) ||
8484 (rdev
->disp_priority
== 2)) {
8485 DRM_DEBUG_KMS("force priority to high\n");
8490 wm_mask
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8492 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8493 tmp
|= LATENCY_WATERMARK_MASK(1);
8494 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8495 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8496 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
8497 LATENCY_HIGH_WATERMARK(line_time
)));
8499 tmp
= RREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
);
8500 tmp
&= ~LATENCY_WATERMARK_MASK(3);
8501 tmp
|= LATENCY_WATERMARK_MASK(2);
8502 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
8503 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
8504 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
8505 LATENCY_HIGH_WATERMARK(line_time
)));
8506 /* restore original selection */
8507 WREG32(DPG_WATERMARK_MASK_CONTROL
+ radeon_crtc
->crtc_offset
, wm_mask
);
8509 /* save values for DPM */
8510 radeon_crtc
->line_time
= line_time
;
8511 radeon_crtc
->wm_high
= latency_watermark_a
;
8512 radeon_crtc
->wm_low
= latency_watermark_b
;
8516 * dce8_bandwidth_update - program display watermarks
8518 * @rdev: radeon_device pointer
8520 * Calculate and program the display watermarks and line
8521 * buffer allocation (CIK).
8523 void dce8_bandwidth_update(struct radeon_device
*rdev
)
8525 struct drm_display_mode
*mode
= NULL
;
8526 u32 num_heads
= 0, lb_size
;
8529 radeon_update_display_priority(rdev
);
8531 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8532 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
8535 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
8536 mode
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
8537 lb_size
= dce8_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode
);
8538 dce8_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
8543 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8545 * @rdev: radeon_device pointer
8547 * Fetches a GPU clock counter snapshot (SI).
8548 * Returns the 64 bit clock counter snapshot.
8550 uint64_t cik_get_gpu_clock_counter(struct radeon_device
*rdev
)
8554 mutex_lock(&rdev
->gpu_clock_mutex
);
8555 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
8556 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
8557 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
8558 mutex_unlock(&rdev
->gpu_clock_mutex
);
8562 static int cik_set_uvd_clock(struct radeon_device
*rdev
, u32 clock
,
8563 u32 cntl_reg
, u32 status_reg
)
8566 struct atom_clock_dividers dividers
;
8569 r
= radeon_atom_get_clock_dividers(rdev
, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK
,
8570 clock
, false, ÷rs
);
8574 tmp
= RREG32_SMC(cntl_reg
);
8575 tmp
&= ~(DCLK_DIR_CNTL_EN
|DCLK_DIVIDER_MASK
);
8576 tmp
|= dividers
.post_divider
;
8577 WREG32_SMC(cntl_reg
, tmp
);
8579 for (i
= 0; i
< 100; i
++) {
8580 if (RREG32_SMC(status_reg
) & DCLK_STATUS
)
8590 int cik_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
8594 r
= cik_set_uvd_clock(rdev
, vclk
, CG_VCLK_CNTL
, CG_VCLK_STATUS
);
8598 r
= cik_set_uvd_clock(rdev
, dclk
, CG_DCLK_CNTL
, CG_DCLK_STATUS
);
8602 void cik_uvd_resume(struct radeon_device
*rdev
)
8607 /* programm the VCPU memory controller bits 0-27 */
8608 addr
= rdev
->uvd
.gpu_addr
>> 3;
8609 size
= RADEON_GPU_PAGE_ALIGN(rdev
->uvd_fw
->size
+ 4) >> 3;
8610 WREG32(UVD_VCPU_CACHE_OFFSET0
, addr
);
8611 WREG32(UVD_VCPU_CACHE_SIZE0
, size
);
8614 size
= RADEON_UVD_STACK_SIZE
>> 3;
8615 WREG32(UVD_VCPU_CACHE_OFFSET1
, addr
);
8616 WREG32(UVD_VCPU_CACHE_SIZE1
, size
);
8619 size
= RADEON_UVD_HEAP_SIZE
>> 3;
8620 WREG32(UVD_VCPU_CACHE_OFFSET2
, addr
);
8621 WREG32(UVD_VCPU_CACHE_SIZE2
, size
);
8624 addr
= (rdev
->uvd
.gpu_addr
>> 28) & 0xF;
8625 WREG32(UVD_LMI_ADDR_EXT
, (addr
<< 12) | (addr
<< 0));
8628 addr
= (rdev
->uvd
.gpu_addr
>> 32) & 0xFF;
8629 WREG32(UVD_LMI_EXT40_ADDR
, addr
| (0x9 << 16) | (0x1 << 31));
8633 static void cik_pcie_gen3_enable(struct radeon_device
*rdev
)
8635 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
8636 int bridge_pos
, gpu_pos
;
8637 u32 speed_cntl
, mask
, current_data_rate
;
8641 if (radeon_pcie_gen2
== 0)
8644 if (rdev
->flags
& RADEON_IS_IGP
)
8647 if (!(rdev
->flags
& RADEON_IS_PCIE
))
8650 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
8654 if (!(mask
& (DRM_PCIE_SPEED_50
| DRM_PCIE_SPEED_80
)))
8657 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
8658 current_data_rate
= (speed_cntl
& LC_CURRENT_DATA_RATE_MASK
) >>
8659 LC_CURRENT_DATA_RATE_SHIFT
;
8660 if (mask
& DRM_PCIE_SPEED_80
) {
8661 if (current_data_rate
== 2) {
8662 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8665 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8666 } else if (mask
& DRM_PCIE_SPEED_50
) {
8667 if (current_data_rate
== 1) {
8668 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8671 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8674 bridge_pos
= pci_pcie_cap(root
);
8678 gpu_pos
= pci_pcie_cap(rdev
->pdev
);
8682 if (mask
& DRM_PCIE_SPEED_80
) {
8683 /* re-try equalization if gen3 is not already enabled */
8684 if (current_data_rate
!= 2) {
8685 u16 bridge_cfg
, gpu_cfg
;
8686 u16 bridge_cfg2
, gpu_cfg2
;
8687 u32 max_lw
, current_lw
, tmp
;
8689 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
8690 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
8692 tmp16
= bridge_cfg
| PCI_EXP_LNKCTL_HAWD
;
8693 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
8695 tmp16
= gpu_cfg
| PCI_EXP_LNKCTL_HAWD
;
8696 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
8698 tmp
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
8699 max_lw
= (tmp
& LC_DETECTED_LINK_WIDTH_MASK
) >> LC_DETECTED_LINK_WIDTH_SHIFT
;
8700 current_lw
= (tmp
& LC_OPERATING_LINK_WIDTH_MASK
) >> LC_OPERATING_LINK_WIDTH_SHIFT
;
8702 if (current_lw
< max_lw
) {
8703 tmp
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
8704 if (tmp
& LC_RENEGOTIATION_SUPPORT
) {
8705 tmp
&= ~(LC_LINK_WIDTH_MASK
| LC_UPCONFIGURE_DIS
);
8706 tmp
|= (max_lw
<< LC_LINK_WIDTH_SHIFT
);
8707 tmp
|= LC_UPCONFIGURE_SUPPORT
| LC_RENEGOTIATE_EN
| LC_RECONFIG_NOW
;
8708 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, tmp
);
8712 for (i
= 0; i
< 10; i
++) {
8714 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_DEVSTA
, &tmp16
);
8715 if (tmp16
& PCI_EXP_DEVSTA_TRPND
)
8718 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
8719 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
8721 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &bridge_cfg2
);
8722 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &gpu_cfg2
);
8724 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
8725 tmp
|= LC_SET_QUIESCE
;
8726 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
8728 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
8730 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
8735 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
8736 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
8737 tmp16
|= (bridge_cfg
& PCI_EXP_LNKCTL_HAWD
);
8738 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
8740 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
8741 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
8742 tmp16
|= (gpu_cfg
& PCI_EXP_LNKCTL_HAWD
);
8743 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
8746 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
8747 tmp16
&= ~((1 << 4) | (7 << 9));
8748 tmp16
|= (bridge_cfg2
& ((1 << 4) | (7 << 9)));
8749 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
8751 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
8752 tmp16
&= ~((1 << 4) | (7 << 9));
8753 tmp16
|= (gpu_cfg2
& ((1 << 4) | (7 << 9)));
8754 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
8756 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
8757 tmp
&= ~LC_SET_QUIESCE
;
8758 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
8763 /* set the link speed */
8764 speed_cntl
|= LC_FORCE_EN_SW_SPEED_CHANGE
| LC_FORCE_DIS_HW_SPEED_CHANGE
;
8765 speed_cntl
&= ~LC_FORCE_DIS_SW_SPEED_CHANGE
;
8766 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
8768 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
8770 if (mask
& DRM_PCIE_SPEED_80
)
8771 tmp16
|= 3; /* gen3 */
8772 else if (mask
& DRM_PCIE_SPEED_50
)
8773 tmp16
|= 2; /* gen2 */
8775 tmp16
|= 1; /* gen1 */
8776 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
8778 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
8779 speed_cntl
|= LC_INITIATE_LINK_SPEED_CHANGE
;
8780 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
8782 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
8783 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
8784 if ((speed_cntl
& LC_INITIATE_LINK_SPEED_CHANGE
) == 0)
8790 static void cik_program_aspm(struct radeon_device
*rdev
)
8793 bool disable_l0s
= false, disable_l1
= false, disable_plloff_in_l1
= false;
8794 bool disable_clkreq
= false;
8796 if (radeon_aspm
== 0)
8799 /* XXX double check IGPs */
8800 if (rdev
->flags
& RADEON_IS_IGP
)
8803 if (!(rdev
->flags
& RADEON_IS_PCIE
))
8806 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
8807 data
&= ~LC_XMIT_N_FTS_MASK
;
8808 data
|= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN
;
8810 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
, data
);
8812 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL3
);
8813 data
|= LC_GO_TO_RECOVERY
;
8815 WREG32_PCIE_PORT(PCIE_LC_CNTL3
, data
);
8817 orig
= data
= RREG32_PCIE_PORT(PCIE_P_CNTL
);
8818 data
|= P_IGNORE_EDB_ERR
;
8820 WREG32_PCIE_PORT(PCIE_P_CNTL
, data
);
8822 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
8823 data
&= ~(LC_L0S_INACTIVITY_MASK
| LC_L1_INACTIVITY_MASK
);
8824 data
|= LC_PMI_TO_L1_DIS
;
8826 data
|= LC_L0S_INACTIVITY(7);
8829 data
|= LC_L1_INACTIVITY(7);
8830 data
&= ~LC_PMI_TO_L1_DIS
;
8832 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
8834 if (!disable_plloff_in_l1
) {
8835 bool clk_req_support
;
8837 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
);
8838 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
8839 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8841 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0
, data
);
8843 orig
= data
= RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
);
8844 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
8845 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8847 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1
, data
);
8849 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
);
8850 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
8851 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8853 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0
, data
);
8855 orig
= data
= RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
);
8856 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
8857 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8859 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1
, data
);
8861 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
8862 data
&= ~LC_DYN_LANES_PWR_STATE_MASK
;
8863 data
|= LC_DYN_LANES_PWR_STATE(3);
8865 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, data
);
8867 if (!disable_clkreq
) {
8868 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
8871 clk_req_support
= false;
8872 pcie_capability_read_dword(root
, PCI_EXP_LNKCAP
, &lnkcap
);
8873 if (lnkcap
& PCI_EXP_LNKCAP_CLKPM
)
8874 clk_req_support
= true;
8876 clk_req_support
= false;
8879 if (clk_req_support
) {
8880 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL2
);
8881 data
|= LC_ALLOW_PDWN_IN_L1
| LC_ALLOW_PDWN_IN_L23
;
8883 WREG32_PCIE_PORT(PCIE_LC_CNTL2
, data
);
8885 orig
= data
= RREG32_SMC(THM_CLK_CNTL
);
8886 data
&= ~(CMON_CLK_SEL_MASK
| TMON_CLK_SEL_MASK
);
8887 data
|= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8889 WREG32_SMC(THM_CLK_CNTL
, data
);
8891 orig
= data
= RREG32_SMC(MISC_CLK_CTRL
);
8892 data
&= ~(DEEP_SLEEP_CLK_SEL_MASK
| ZCLK_SEL_MASK
);
8893 data
|= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8895 WREG32_SMC(MISC_CLK_CTRL
, data
);
8897 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL
);
8898 data
&= ~BCLK_AS_XCLK
;
8900 WREG32_SMC(CG_CLKPIN_CNTL
, data
);
8902 orig
= data
= RREG32_SMC(CG_CLKPIN_CNTL_2
);
8903 data
&= ~FORCE_BIF_REFCLK_EN
;
8905 WREG32_SMC(CG_CLKPIN_CNTL_2
, data
);
8907 orig
= data
= RREG32_SMC(MPLL_BYPASSCLK_SEL
);
8908 data
&= ~MPLL_CLKOUT_SEL_MASK
;
8909 data
|= MPLL_CLKOUT_SEL(4);
8911 WREG32_SMC(MPLL_BYPASSCLK_SEL
, data
);
8916 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
8919 orig
= data
= RREG32_PCIE_PORT(PCIE_CNTL2
);
8920 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
| REPLAY_MEM_LS_EN
;
8922 WREG32_PCIE_PORT(PCIE_CNTL2
, data
);
8925 data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
8926 if((data
& LC_N_FTS_MASK
) == LC_N_FTS_MASK
) {
8927 data
= RREG32_PCIE_PORT(PCIE_LC_STATUS1
);
8928 if ((data
& LC_REVERSE_XMIT
) && (data
& LC_REVERSE_RCVR
)) {
8929 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
8930 data
&= ~LC_L0S_INACTIVITY_MASK
;
8932 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);