projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/radeon: Sync ME and PFP after CP semaphore waits v4
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
35
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45
46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47 MODULE_FIRMWARE("radeon/bonaire_me.bin");
48 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
54
55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
64
65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66 MODULE_FIRMWARE("radeon/hawaii_me.bin");
67 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
73
74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
80
81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82 MODULE_FIRMWARE("radeon/kaveri_me.bin");
83 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
88
89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90 MODULE_FIRMWARE("radeon/KABINI_me.bin");
91 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
95
96 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97 MODULE_FIRMWARE("radeon/kabini_me.bin");
98 MODULE_FIRMWARE("radeon/kabini_ce.bin");
99 MODULE_FIRMWARE("radeon/kabini_mec.bin");
100 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
102
103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
109
110 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111 MODULE_FIRMWARE("radeon/mullins_me.bin");
112 MODULE_FIRMWARE("radeon/mullins_ce.bin");
113 MODULE_FIRMWARE("radeon/mullins_mec.bin");
114 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
116
117 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118 extern void r600_ih_ring_fini(struct radeon_device *rdev);
119 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
121 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
122 extern void sumo_rlc_fini(struct radeon_device *rdev);
123 extern int sumo_rlc_init(struct radeon_device *rdev);
124 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
125 extern void si_rlc_reset(struct radeon_device *rdev);
126 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
127 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
128 extern int cik_sdma_resume(struct radeon_device *rdev);
129 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130 extern void cik_sdma_fini(struct radeon_device *rdev);
131 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
132 static void cik_rlc_stop(struct radeon_device *rdev);
133 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
134 static void cik_program_aspm(struct radeon_device *rdev);
135 static void cik_init_pg(struct radeon_device *rdev);
136 static void cik_init_cg(struct radeon_device *rdev);
137 static void cik_fini_pg(struct radeon_device *rdev);
138 static void cik_fini_cg(struct radeon_device *rdev);
139 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
140 bool enable);
141
142 /* get temperature in millidegrees */
143 int ci_get_temp(struct radeon_device *rdev)
144 {
145 u32 temp;
146 int actual_temp = 0;
147
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
149 CTF_TEMP_SHIFT;
150
151 if (temp & 0x200)
152 actual_temp = 255;
153 else
154 actual_temp = temp & 0x1ff;
155
156 actual_temp = actual_temp * 1000;
157
158 return actual_temp;
159 }
160
161 /* get temperature in millidegrees */
162 int kv_get_temp(struct radeon_device *rdev)
163 {
164 u32 temp;
165 int actual_temp = 0;
166
167 temp = RREG32_SMC(0xC0300E0C);
168
169 if (temp)
170 actual_temp = (temp / 8) - 49;
171 else
172 actual_temp = 0;
173
174 actual_temp = actual_temp * 1000;
175
176 return actual_temp;
177 }
178
179 /*
180 * Indirect registers accessor
181 */
182 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
183 {
184 unsigned long flags;
185 u32 r;
186
187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
192 return r;
193 }
194
195 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
196 {
197 unsigned long flags;
198
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
205 }
206
207 static const u32 spectre_rlc_save_restore_register_list[] =
208 {
209 (0x0e00 << 16) | (0xc12c >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0xc140 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc150 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc15c >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc168 >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc170 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc178 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc204 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc2b4 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b8 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2bc >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2c0 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x8228 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x829c >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x869c >> 2),
238 0x00000000,
239 (0x0600 << 16) | (0x98f4 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0x98f8 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x9900 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc260 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0x90e8 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x3c000 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c00c >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x8c1c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x9700 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xcd20 >> 2),
258 0x00000000,
259 (0x4e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x5e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x6e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x7e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x8e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x9e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0xae00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xbe00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x89bc >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x8900 >> 2),
278 0x00000000,
279 0x3,
280 (0x0e00 << 16) | (0xc130 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc134 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc1fc >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc208 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc264 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc268 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc26c >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc270 >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc274 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc278 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc27c >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc280 >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc284 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc288 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc28c >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc290 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc294 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc298 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc29c >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc2a0 >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a4 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a8 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2ac >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2b0 >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0x301d0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x30238 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30250 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30254 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30258 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x3025c >> 2),
339 0x00000000,
340 (0x4e00 << 16) | (0xc900 >> 2),
341 0x00000000,
342 (0x5e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x6e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x7e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x8e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x9e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0xae00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xbe00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0x4e00 << 16) | (0xc904 >> 2),
357 0x00000000,
358 (0x5e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x6e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x7e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x8e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x9e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0xae00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xbe00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0x4e00 << 16) | (0xc908 >> 2),
373 0x00000000,
374 (0x5e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x6e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x7e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x8e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x9e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0xae00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xbe00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0x4e00 << 16) | (0xc90c >> 2),
389 0x00000000,
390 (0x5e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x6e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x7e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x8e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x9e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0xae00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xbe00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc910 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0x0e00 << 16) | (0xc99c >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0x9834 >> 2),
423 0x00000000,
424 (0x0000 << 16) | (0x30f00 >> 2),
425 0x00000000,
426 (0x0001 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0000 << 16) | (0x30f04 >> 2),
429 0x00000000,
430 (0x0001 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0000 << 16) | (0x30f08 >> 2),
433 0x00000000,
434 (0x0001 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0000 << 16) | (0x30f0c >> 2),
437 0x00000000,
438 (0x0001 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0600 << 16) | (0x9b7c >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x8a14 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a18 >> 2),
445 0x00000000,
446 (0x0600 << 16) | (0x30a00 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x8bf0 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bcc >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8b24 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x30a04 >> 2),
455 0x00000000,
456 (0x0600 << 16) | (0x30a10 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a14 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a18 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a2c >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0xc700 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc704 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc708 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc768 >> 2),
471 0x00000000,
472 (0x0400 << 16) | (0xc770 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc774 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc778 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc77c >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc780 >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc784 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc788 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc78c >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc798 >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc79c >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc7a0 >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a4 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a8 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7ac >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7b0 >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b4 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x9100 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x3c010 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x92a8 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92ac >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92b4 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b8 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92bc >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92c0 >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c4 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c8 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92cc >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92d0 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x8c00 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c04 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c20 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c38 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c3c >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xae00 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x9604 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0xac08 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac0c >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac10 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac14 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac58 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac68 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac6c >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac70 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac74 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac78 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac7c >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac80 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac84 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac88 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac8c >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x970c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x9714 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9718 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x971c >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x31068 >> 2),
581 0x00000000,
582 (0x4e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x5e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x6e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x7e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x8e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x9e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0xae00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xbe00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xcd10 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd14 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x88b0 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b4 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b8 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88bc >> 2),
609 0x00000000,
610 (0x0400 << 16) | (0x89c0 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x88c4 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c8 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88d0 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d4 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d8 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x8980 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x30938 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x3093c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x30940 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x89a0 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x30900 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30904 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x89b4 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x3c210 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c214 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c218 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8904 >> 2),
645 0x00000000,
646 0x5,
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
652 };
653
654 static const u32 kalindi_rlc_save_restore_register_list[] =
655 {
656 (0x0e00 << 16) | (0xc12c >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xc140 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc150 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc15c >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc168 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc170 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc204 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc2b4 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b8 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2bc >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2c0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x8228 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x829c >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x869c >> 2),
683 0x00000000,
684 (0x0600 << 16) | (0x98f4 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x98f8 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x9900 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc260 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x90e8 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x3c000 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c00c >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x8c1c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x9700 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0xcd20 >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x0e00 << 16) | (0x89bc >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x8900 >> 2),
715 0x00000000,
716 0x3,
717 (0x0e00 << 16) | (0xc130 >> 2),
718 0x00000000,
719 (0x0e00 << 16) | (0xc134 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc1fc >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc208 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc264 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc268 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc26c >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc270 >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc274 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc28c >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc290 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc294 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc298 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc2a0 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a4 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a8 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2ac >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x301d0 >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x30238 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30250 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30254 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30258 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x3025c >> 2),
762 0x00000000,
763 (0x4e00 << 16) | (0xc900 >> 2),
764 0x00000000,
765 (0x5e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x6e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x7e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x4e00 << 16) | (0xc904 >> 2),
772 0x00000000,
773 (0x5e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x6e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x7e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x4e00 << 16) | (0xc908 >> 2),
780 0x00000000,
781 (0x5e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x6e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x7e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x4e00 << 16) | (0xc90c >> 2),
788 0x00000000,
789 (0x5e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x6e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x7e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x4e00 << 16) | (0xc910 >> 2),
796 0x00000000,
797 (0x5e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x6e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x7e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc99c >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x9834 >> 2),
806 0x00000000,
807 (0x0000 << 16) | (0x30f00 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f04 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f08 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f0c >> 2),
814 0x00000000,
815 (0x0600 << 16) | (0x9b7c >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x8a14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a18 >> 2),
820 0x00000000,
821 (0x0600 << 16) | (0x30a00 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x8bf0 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bcc >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8b24 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x30a04 >> 2),
830 0x00000000,
831 (0x0600 << 16) | (0x30a10 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a14 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a18 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a2c >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xc700 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc704 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc708 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc768 >> 2),
846 0x00000000,
847 (0x0400 << 16) | (0xc770 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc774 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc798 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc79c >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x9100 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x3c010 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x8c00 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c04 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c20 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c38 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c3c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xae00 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x9604 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0xac08 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac0c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac10 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac58 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac68 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac6c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac70 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac74 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac78 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac7c >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac80 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac84 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac88 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac8c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x970c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x9714 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9718 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x971c >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x31068 >> 2),
912 0x00000000,
913 (0x4e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x5e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x6e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x7e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0xcd10 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd14 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x88b0 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b4 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b8 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88bc >> 2),
932 0x00000000,
933 (0x0400 << 16) | (0x89c0 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x88c4 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c8 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88d0 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d4 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d8 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x8980 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x30938 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x3093c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x30940 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x89a0 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x30900 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30904 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x89b4 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x3e1fc >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3c210 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c214 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c218 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x8904 >> 2),
970 0x00000000,
971 0x5,
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
977 };
978
979 static const u32 bonaire_golden_spm_registers[] =
980 {
981 0x30800, 0xe0ffffff, 0xe0000000
982 };
983
984 static const u32 bonaire_golden_common_registers[] =
985 {
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
990 };
991
992 static const u32 bonaire_golden_registers[] =
993 {
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1035 };
1036
1037 static const u32 bonaire_mgcg_cgcg_init[] =
1038 {
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1121 };
1122
1123 static const u32 spectre_golden_spm_registers[] =
1124 {
1125 0x30800, 0xe0ffffff, 0xe0000000
1126 };
1127
1128 static const u32 spectre_golden_common_registers[] =
1129 {
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1134 };
1135
1136 static const u32 spectre_golden_registers[] =
1137 {
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
1152 0x28354, 0x0000003f, 0x00000000,
1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1163 };
1164
1165 static const u32 spectre_mgcg_cgcg_init[] =
1166 {
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1254 };
1255
1256 static const u32 kalindi_golden_spm_registers[] =
1257 {
1258 0x30800, 0xe0ffffff, 0xe0000000
1259 };
1260
1261 static const u32 kalindi_golden_common_registers[] =
1262 {
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1267 };
1268
1269 static const u32 kalindi_golden_registers[] =
1270 {
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1301 };
1302
1303 static const u32 kalindi_mgcg_cgcg_init[] =
1304 {
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1360 };
1361
1362 static const u32 hawaii_golden_spm_registers[] =
1363 {
1364 0x30800, 0xe0ffffff, 0xe0000000
1365 };
1366
1367 static const u32 hawaii_golden_common_registers[] =
1368 {
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1374 };
1375
1376 static const u32 hawaii_golden_registers[] =
1377 {
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1414 };
1415
1416 static const u32 hawaii_mgcg_cgcg_init[] =
1417 {
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1525 };
1526
1527 static const u32 godavari_golden_registers[] =
1528 {
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1561 };
1562
1563
1564 static void cik_init_golden_registers(struct radeon_device *rdev)
1565 {
1566 switch (rdev->family) {
1567 case CHIP_BONAIRE:
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580 break;
1581 case CHIP_KABINI:
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594 break;
1595 case CHIP_MULLINS:
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608 break;
1609 case CHIP_KAVERI:
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622 break;
1623 case CHIP_HAWAII:
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636 break;
1637 default:
1638 break;
1639 }
1640 }
1641
1642 /**
1643 * cik_get_xclk - get the xclk
1644 *
1645 * @rdev: radeon_device pointer
1646 *
1647 * Returns the reference clock used by the gfx engine
1648 * (CIK).
1649 */
1650 u32 cik_get_xclk(struct radeon_device *rdev)
1651 {
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1657 } else {
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1660 }
1661 return reference_clock;
1662 }
1663
1664 /**
1665 * cik_mm_rdoorbell - read a doorbell dword
1666 *
1667 * @rdev: radeon_device pointer
1668 * @index: doorbell index
1669 *
1670 * Returns the value in the doorbell aperture at the
1671 * requested doorbell index (CIK).
1672 */
1673 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1674 {
1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
1677 } else {
1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1679 return 0;
1680 }
1681 }
1682
1683 /**
1684 * cik_mm_wdoorbell - write a doorbell dword
1685 *
1686 * @rdev: radeon_device pointer
1687 * @index: doorbell index
1688 * @v: value to write
1689 *
1690 * Writes @v to the doorbell aperture at the
1691 * requested doorbell index (CIK).
1692 */
1693 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1694 {
1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
1697 } else {
1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1699 }
1700 }
1701
1702 #define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705 {
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1742 };
1743
1744 #define HAWAII_IO_MC_REGS_SIZE 22
1745
1746 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747 {
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1770 };
1771
1772
1773 /**
1774 * cik_srbm_select - select specific register instances
1775 *
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1778 * @pipe: pipe
1779 * @queue: queue
1780 * @vmid: VMID
1781 *
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1785 */
1786 static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1788 {
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790 MEID(me & 0x3) |
1791 VMID(vmid & 0xf) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794 }
1795
1796 /* ucode loading */
1797 /**
1798 * ci_mc_load_microcode - load MC ucode into the hw
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1804 */
1805 int ci_mc_load_microcode(struct radeon_device *rdev)
1806 {
1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
1809 u32 running, blackout = 0;
1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
1812 int i, regs_size, ucode_size;
1813
1814 if (!rdev->mc_fw)
1815 return -EINVAL;
1816
1817 if (rdev->new_fw) {
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1820
1821 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829 } else {
1830 ucode_size = rdev->mc_fw->size / 4;
1831
1832 switch (rdev->family) {
1833 case CHIP_BONAIRE:
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836 break;
1837 case CHIP_HAWAII:
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1840 break;
1841 default:
1842 return -EINVAL;
1843 }
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
1845 }
1846
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849 if (running == 0) {
1850 if (running) {
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853 }
1854
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
1861 if (rdev->new_fw) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864 } else {
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867 }
1868 }
1869 /* load the MC ucode */
1870 for (i = 0; i < ucode_size; i++) {
1871 if (rdev->new_fw)
1872 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873 else
1874 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875 }
1876
1877 /* put the engine back into the active state */
1878 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881
1882 /* wait for training to complete */
1883 for (i = 0; i < rdev->usec_timeout; i++) {
1884 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885 break;
1886 udelay(1);
1887 }
1888 for (i = 0; i < rdev->usec_timeout; i++) {
1889 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890 break;
1891 udelay(1);
1892 }
1893
1894 if (running)
1895 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896 }
1897
1898 return 0;
1899 }
1900
1901 /**
1902 * cik_init_microcode - load ucode images from disk
1903 *
1904 * @rdev: radeon_device pointer
1905 *
1906 * Use the firmware interface to load the ucode images into
1907 * the driver (not loaded into hw).
1908 * Returns 0 on success, error on failure.
1909 */
1910 static int cik_init_microcode(struct radeon_device *rdev)
1911 {
1912 const char *chip_name;
1913 const char *new_chip_name;
1914 size_t pfp_req_size, me_req_size, ce_req_size,
1915 mec_req_size, rlc_req_size, mc_req_size = 0,
1916 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1917 char fw_name[30];
1918 int new_fw = 0;
1919 int err;
1920 int num_fw;
1921
1922 DRM_DEBUG("\n");
1923
1924 switch (rdev->family) {
1925 case CHIP_BONAIRE:
1926 chip_name = "BONAIRE";
1927 new_chip_name = "bonaire";
1928 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1937 num_fw = 8;
1938 break;
1939 case CHIP_HAWAII:
1940 chip_name = "HAWAII";
1941 new_chip_name = "hawaii";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1951 num_fw = 8;
1952 break;
1953 case CHIP_KAVERI:
1954 chip_name = "KAVERI";
1955 new_chip_name = "kaveri";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1962 num_fw = 7;
1963 break;
1964 case CHIP_KABINI:
1965 chip_name = "KABINI";
1966 new_chip_name = "kabini";
1967 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1973 num_fw = 6;
1974 break;
1975 case CHIP_MULLINS:
1976 chip_name = "MULLINS";
1977 new_chip_name = "mullins";
1978 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1984 num_fw = 6;
1985 break;
1986 default: BUG();
1987 }
1988
1989 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1990
1991 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1993 if (err) {
1994 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996 if (err)
1997 goto out;
1998 if (rdev->pfp_fw->size != pfp_req_size) {
1999 printk(KERN_ERR
2000 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 rdev->pfp_fw->size, fw_name);
2002 err = -EINVAL;
2003 goto out;
2004 }
2005 } else {
2006 err = radeon_ucode_validate(rdev->pfp_fw);
2007 if (err) {
2008 printk(KERN_ERR
2009 "cik_fw: validation failed for firmware \"%s\"\n",
2010 fw_name);
2011 goto out;
2012 } else {
2013 new_fw++;
2014 }
2015 }
2016
2017 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2019 if (err) {
2020 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022 if (err)
2023 goto out;
2024 if (rdev->me_fw->size != me_req_size) {
2025 printk(KERN_ERR
2026 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 rdev->me_fw->size, fw_name);
2028 err = -EINVAL;
2029 }
2030 } else {
2031 err = radeon_ucode_validate(rdev->me_fw);
2032 if (err) {
2033 printk(KERN_ERR
2034 "cik_fw: validation failed for firmware \"%s\"\n",
2035 fw_name);
2036 goto out;
2037 } else {
2038 new_fw++;
2039 }
2040 }
2041
2042 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2044 if (err) {
2045 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047 if (err)
2048 goto out;
2049 if (rdev->ce_fw->size != ce_req_size) {
2050 printk(KERN_ERR
2051 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 rdev->ce_fw->size, fw_name);
2053 err = -EINVAL;
2054 }
2055 } else {
2056 err = radeon_ucode_validate(rdev->ce_fw);
2057 if (err) {
2058 printk(KERN_ERR
2059 "cik_fw: validation failed for firmware \"%s\"\n",
2060 fw_name);
2061 goto out;
2062 } else {
2063 new_fw++;
2064 }
2065 }
2066
2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2069 if (err) {
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072 if (err)
2073 goto out;
2074 if (rdev->mec_fw->size != mec_req_size) {
2075 printk(KERN_ERR
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->mec_fw->size, fw_name);
2078 err = -EINVAL;
2079 }
2080 } else {
2081 err = radeon_ucode_validate(rdev->mec_fw);
2082 if (err) {
2083 printk(KERN_ERR
2084 "cik_fw: validation failed for firmware \"%s\"\n",
2085 fw_name);
2086 goto out;
2087 } else {
2088 new_fw++;
2089 }
2090 }
2091
2092 if (rdev->family == CHIP_KAVERI) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095 if (err) {
2096 goto out;
2097 } else {
2098 err = radeon_ucode_validate(rdev->mec2_fw);
2099 if (err) {
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
2104 }
2105 }
2106
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2109 if (err) {
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112 if (err)
2113 goto out;
2114 if (rdev->rlc_fw->size != rlc_req_size) {
2115 printk(KERN_ERR
2116 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 rdev->rlc_fw->size, fw_name);
2118 err = -EINVAL;
2119 }
2120 } else {
2121 err = radeon_ucode_validate(rdev->rlc_fw);
2122 if (err) {
2123 printk(KERN_ERR
2124 "cik_fw: validation failed for firmware \"%s\"\n",
2125 fw_name);
2126 goto out;
2127 } else {
2128 new_fw++;
2129 }
2130 }
2131
2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2134 if (err) {
2135 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137 if (err)
2138 goto out;
2139 if (rdev->sdma_fw->size != sdma_req_size) {
2140 printk(KERN_ERR
2141 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->sdma_fw->size, fw_name);
2143 err = -EINVAL;
2144 }
2145 } else {
2146 err = radeon_ucode_validate(rdev->sdma_fw);
2147 if (err) {
2148 printk(KERN_ERR
2149 "cik_fw: validation failed for firmware \"%s\"\n",
2150 fw_name);
2151 goto out;
2152 } else {
2153 new_fw++;
2154 }
2155 }
2156
2157 /* No SMC, MC ucode on APUs */
2158 if (!(rdev->flags & RADEON_IS_IGP)) {
2159 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2161 if (err) {
2162 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2164 if (err) {
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167 if (err)
2168 goto out;
2169 }
2170 if ((rdev->mc_fw->size != mc_req_size) &&
2171 (rdev->mc_fw->size != mc2_req_size)){
2172 printk(KERN_ERR
2173 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 rdev->mc_fw->size, fw_name);
2175 err = -EINVAL;
2176 }
2177 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178 } else {
2179 err = radeon_ucode_validate(rdev->mc_fw);
2180 if (err) {
2181 printk(KERN_ERR
2182 "cik_fw: validation failed for firmware \"%s\"\n",
2183 fw_name);
2184 goto out;
2185 } else {
2186 new_fw++;
2187 }
2188 }
2189
2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192 if (err) {
2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195 if (err) {
2196 printk(KERN_ERR
2197 "smc: error loading firmware \"%s\"\n",
2198 fw_name);
2199 release_firmware(rdev->smc_fw);
2200 rdev->smc_fw = NULL;
2201 err = 0;
2202 } else if (rdev->smc_fw->size != smc_req_size) {
2203 printk(KERN_ERR
2204 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 rdev->smc_fw->size, fw_name);
2206 err = -EINVAL;
2207 }
2208 } else {
2209 err = radeon_ucode_validate(rdev->smc_fw);
2210 if (err) {
2211 printk(KERN_ERR
2212 "cik_fw: validation failed for firmware \"%s\"\n",
2213 fw_name);
2214 goto out;
2215 } else {
2216 new_fw++;
2217 }
2218 }
2219 }
2220
2221 if (new_fw == 0) {
2222 rdev->new_fw = false;
2223 } else if (new_fw < num_fw) {
2224 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225 err = -EINVAL;
2226 } else {
2227 rdev->new_fw = true;
2228 }
2229
2230 out:
2231 if (err) {
2232 if (err != -EINVAL)
2233 printk(KERN_ERR
2234 "cik_cp: Failed to load firmware \"%s\"\n",
2235 fw_name);
2236 release_firmware(rdev->pfp_fw);
2237 rdev->pfp_fw = NULL;
2238 release_firmware(rdev->me_fw);
2239 rdev->me_fw = NULL;
2240 release_firmware(rdev->ce_fw);
2241 rdev->ce_fw = NULL;
2242 release_firmware(rdev->mec_fw);
2243 rdev->mec_fw = NULL;
2244 release_firmware(rdev->mec2_fw);
2245 rdev->mec2_fw = NULL;
2246 release_firmware(rdev->rlc_fw);
2247 rdev->rlc_fw = NULL;
2248 release_firmware(rdev->sdma_fw);
2249 rdev->sdma_fw = NULL;
2250 release_firmware(rdev->mc_fw);
2251 rdev->mc_fw = NULL;
2252 release_firmware(rdev->smc_fw);
2253 rdev->smc_fw = NULL;
2254 }
2255 return err;
2256 }
2257
2258 /*
2259 * Core functions
2260 */
2261 /**
2262 * cik_tiling_mode_table_init - init the hw tiling table
2263 *
2264 * @rdev: radeon_device pointer
2265 *
2266 * Starting with SI, the tiling setup is done globally in a
2267 * set of 32 tiling modes. Rather than selecting each set of
2268 * parameters per surface as on older asics, we just select
2269 * which index in the tiling table we want to use, and the
2270 * surface uses those parameters (CIK).
2271 */
2272 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273 {
2274 const u32 num_tile_mode_states = 32;
2275 const u32 num_secondary_tile_mode_states = 16;
2276 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 u32 num_pipe_configs;
2278 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 rdev->config.cik.max_shader_engines;
2280
2281 switch (rdev->config.cik.mem_row_size_in_kb) {
2282 case 1:
2283 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284 break;
2285 case 2:
2286 default:
2287 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288 break;
2289 case 4:
2290 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291 break;
2292 }
2293
2294 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 if (num_pipe_configs > 8)
2296 num_pipe_configs = 16;
2297
2298 if (num_pipe_configs == 16) {
2299 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 switch (reg_offset) {
2301 case 0:
2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306 break;
2307 case 1:
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312 break;
2313 case 2:
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318 break;
2319 case 3:
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324 break;
2325 case 4:
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(split_equal_to_row_size));
2330 break;
2331 case 5:
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335 break;
2336 case 6:
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341 break;
2342 case 7:
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 TILE_SPLIT(split_equal_to_row_size));
2347 break;
2348 case 8:
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351 break;
2352 case 9:
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356 break;
2357 case 10:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362 break;
2363 case 11:
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 break;
2369 case 12:
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 break;
2375 case 13:
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379 break;
2380 case 14:
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 break;
2386 case 16:
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 break;
2392 case 17:
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 break;
2398 case 27:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402 break;
2403 case 28:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408 break;
2409 case 29:
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 break;
2415 case 30:
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 break;
2421 default:
2422 gb_tile_moden = 0;
2423 break;
2424 }
2425 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427 }
2428 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 switch (reg_offset) {
2430 case 0:
2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 NUM_BANKS(ADDR_SURF_16_BANK));
2435 break;
2436 case 1:
2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 NUM_BANKS(ADDR_SURF_16_BANK));
2441 break;
2442 case 2:
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2447 break;
2448 case 3:
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 break;
2454 case 4:
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_8_BANK));
2459 break;
2460 case 5:
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2465 break;
2466 case 6:
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_2_BANK));
2471 break;
2472 case 8:
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2477 break;
2478 case 9:
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2483 break;
2484 case 10:
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2489 break;
2490 case 11:
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_8_BANK));
2495 break;
2496 case 12:
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_4_BANK));
2501 break;
2502 case 13:
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_2_BANK));
2507 break;
2508 case 14:
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_2_BANK));
2513 break;
2514 default:
2515 gb_tile_moden = 0;
2516 break;
2517 }
2518 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520 }
2521 } else if (num_pipe_configs == 8) {
2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2524 case 0:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529 break;
2530 case 1:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535 break;
2536 case 2:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541 break;
2542 case 3:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 break;
2548 case 4:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 5:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558 break;
2559 case 6:
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564 break;
2565 case 7:
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2570 break;
2571 case 8:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574 break;
2575 case 9:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579 break;
2580 case 10:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 break;
2586 case 11:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 break;
2592 case 12:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 break;
2598 case 13:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602 break;
2603 case 14:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608 break;
2609 case 16:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614 break;
2615 case 17:
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 break;
2621 case 27:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625 break;
2626 case 28:
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631 break;
2632 case 29:
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637 break;
2638 case 30:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 default:
2645 gb_tile_moden = 0;
2646 break;
2647 }
2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 }
2651 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2653 case 0:
2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 NUM_BANKS(ADDR_SURF_16_BANK));
2658 break;
2659 case 1:
2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 NUM_BANKS(ADDR_SURF_16_BANK));
2664 break;
2665 case 2:
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2670 break;
2671 case 3:
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2676 break;
2677 case 4:
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 NUM_BANKS(ADDR_SURF_8_BANK));
2682 break;
2683 case 5:
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 NUM_BANKS(ADDR_SURF_4_BANK));
2688 break;
2689 case 6:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_2_BANK));
2694 break;
2695 case 8:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 9:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 10:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 11:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 12:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 13:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 14:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_2_BANK));
2736 break;
2737 default:
2738 gb_tile_moden = 0;
2739 break;
2740 }
2741 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743 }
2744 } else if (num_pipe_configs == 4) {
2745 if (num_rbs == 4) {
2746 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 switch (reg_offset) {
2748 case 0:
2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753 break;
2754 case 1:
2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759 break;
2760 case 2:
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765 break;
2766 case 3:
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771 break;
2772 case 4:
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(split_equal_to_row_size));
2777 break;
2778 case 5:
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782 break;
2783 case 6:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788 break;
2789 case 7:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 TILE_SPLIT(split_equal_to_row_size));
2794 break;
2795 case 8:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798 break;
2799 case 9:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803 break;
2804 case 10:
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809 break;
2810 case 11:
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 break;
2816 case 12:
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821 break;
2822 case 13:
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826 break;
2827 case 14:
2828 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832 break;
2833 case 16:
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838 break;
2839 case 17:
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844 break;
2845 case 27:
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849 break;
2850 case 28:
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855 break;
2856 case 29:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 30:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 default:
2869 gb_tile_moden = 0;
2870 break;
2871 }
2872 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 }
2875 } else if (num_rbs < 4) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2878 case 0:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883 break;
2884 case 1:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889 break;
2890 case 2:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895 break;
2896 case 3:
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901 break;
2902 case 4:
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(split_equal_to_row_size));
2907 break;
2908 case 5:
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912 break;
2913 case 6:
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918 break;
2919 case 7:
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 break;
2925 case 8:
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928 break;
2929 case 9:
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933 break;
2934 case 10:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 break;
2940 case 11:
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 break;
2946 case 12:
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 break;
2952 case 13:
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956 break;
2957 case 14:
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 break;
2963 case 16:
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968 break;
2969 case 17:
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 break;
2975 case 27:
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979 break;
2980 case 28:
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985 break;
2986 case 29:
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991 break;
2992 case 30:
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 break;
2998 default:
2999 gb_tile_moden = 0;
3000 break;
3001 }
3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 }
3005 }
3006 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 switch (reg_offset) {
3008 case 0:
3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 NUM_BANKS(ADDR_SURF_16_BANK));
3013 break;
3014 case 1:
3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3019 break;
3020 case 2:
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3025 break;
3026 case 3:
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 break;
3032 case 4:
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3037 break;
3038 case 5:
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_8_BANK));
3043 break;
3044 case 6:
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 NUM_BANKS(ADDR_SURF_4_BANK));
3049 break;
3050 case 8:
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 NUM_BANKS(ADDR_SURF_16_BANK));
3055 break;
3056 case 9:
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 NUM_BANKS(ADDR_SURF_16_BANK));
3061 break;
3062 case 10:
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3067 break;
3068 case 11:
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3073 break;
3074 case 12:
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3079 break;
3080 case 13:
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 NUM_BANKS(ADDR_SURF_8_BANK));
3085 break;
3086 case 14:
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 NUM_BANKS(ADDR_SURF_4_BANK));
3091 break;
3092 default:
3093 gb_tile_moden = 0;
3094 break;
3095 }
3096 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098 }
3099 } else if (num_pipe_configs == 2) {
3100 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 switch (reg_offset) {
3102 case 0:
3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 PIPE_CONFIG(ADDR_SURF_P2) |
3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107 break;
3108 case 1:
3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 PIPE_CONFIG(ADDR_SURF_P2) |
3112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113 break;
3114 case 2:
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119 break;
3120 case 3:
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125 break;
3126 case 4:
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(split_equal_to_row_size));
3131 break;
3132 case 5:
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134 PIPE_CONFIG(ADDR_SURF_P2) |
3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136 break;
3137 case 6:
3138 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 PIPE_CONFIG(ADDR_SURF_P2) |
3141 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142 break;
3143 case 7:
3144 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 TILE_SPLIT(split_equal_to_row_size));
3148 break;
3149 case 8:
3150 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 PIPE_CONFIG(ADDR_SURF_P2);
3152 break;
3153 case 9:
3154 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 PIPE_CONFIG(ADDR_SURF_P2));
3157 break;
3158 case 10:
3159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 PIPE_CONFIG(ADDR_SURF_P2) |
3162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163 break;
3164 case 11:
3165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 PIPE_CONFIG(ADDR_SURF_P2) |
3168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169 break;
3170 case 12:
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175 break;
3176 case 13:
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178 PIPE_CONFIG(ADDR_SURF_P2) |
3179 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180 break;
3181 case 14:
3182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 PIPE_CONFIG(ADDR_SURF_P2) |
3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186 break;
3187 case 16:
3188 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192 break;
3193 case 17:
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198 break;
3199 case 27:
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2));
3203 break;
3204 case 28:
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 PIPE_CONFIG(ADDR_SURF_P2) |
3208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209 break;
3210 case 29:
3211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 PIPE_CONFIG(ADDR_SURF_P2) |
3214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215 break;
3216 case 30:
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221 break;
3222 default:
3223 gb_tile_moden = 0;
3224 break;
3225 }
3226 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228 }
3229 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 switch (reg_offset) {
3231 case 0:
3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 NUM_BANKS(ADDR_SURF_16_BANK));
3236 break;
3237 case 1:
3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 NUM_BANKS(ADDR_SURF_16_BANK));
3242 break;
3243 case 2:
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3248 break;
3249 case 3:
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3254 break;
3255 case 4:
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3260 break;
3261 case 5:
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3266 break;
3267 case 6:
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 NUM_BANKS(ADDR_SURF_8_BANK));
3272 break;
3273 case 8:
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3278 break;
3279 case 9:
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 NUM_BANKS(ADDR_SURF_16_BANK));
3284 break;
3285 case 10:
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3290 break;
3291 case 11:
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3296 break;
3297 case 12:
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3302 break;
3303 case 13:
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3308 break;
3309 case 14:
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3314 break;
3315 default:
3316 gb_tile_moden = 0;
3317 break;
3318 }
3319 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321 }
3322 } else
3323 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324 }
3325
3326 /**
3327 * cik_select_se_sh - select which SE, SH to address
3328 *
3329 * @rdev: radeon_device pointer
3330 * @se_num: shader engine to address
3331 * @sh_num: sh block to address
3332 *
3333 * Select which SE, SH combinations to address. Certain
3334 * registers are instanced per SE or SH. 0xffffffff means
3335 * broadcast to all SEs or SHs (CIK).
3336 */
3337 static void cik_select_se_sh(struct radeon_device *rdev,
3338 u32 se_num, u32 sh_num)
3339 {
3340 u32 data = INSTANCE_BROADCAST_WRITES;
3341
3342 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344 else if (se_num == 0xffffffff)
3345 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 else if (sh_num == 0xffffffff)
3347 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348 else
3349 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 WREG32(GRBM_GFX_INDEX, data);
3351 }
3352
3353 /**
3354 * cik_create_bitmask - create a bitmask
3355 *
3356 * @bit_width: length of the mask
3357 *
3358 * create a variable length bit mask (CIK).
3359 * Returns the bitmask.
3360 */
3361 static u32 cik_create_bitmask(u32 bit_width)
3362 {
3363 u32 i, mask = 0;
3364
3365 for (i = 0; i < bit_width; i++) {
3366 mask <<= 1;
3367 mask |= 1;
3368 }
3369 return mask;
3370 }
3371
3372 /**
3373 * cik_get_rb_disabled - computes the mask of disabled RBs
3374 *
3375 * @rdev: radeon_device pointer
3376 * @max_rb_num: max RBs (render backends) for the asic
3377 * @se_num: number of SEs (shader engines) for the asic
3378 * @sh_per_se: number of SH blocks per SE for the asic
3379 *
3380 * Calculates the bitmask of disabled RBs (CIK).
3381 * Returns the disabled RB bitmask.
3382 */
3383 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384 u32 max_rb_num_per_se,
3385 u32 sh_per_se)
3386 {
3387 u32 data, mask;
3388
3389 data = RREG32(CC_RB_BACKEND_DISABLE);
3390 if (data & 1)
3391 data &= BACKEND_DISABLE_MASK;
3392 else
3393 data = 0;
3394 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395
3396 data >>= BACKEND_DISABLE_SHIFT;
3397
3398 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3399
3400 return data & mask;
3401 }
3402
3403 /**
3404 * cik_setup_rb - setup the RBs on the asic
3405 *
3406 * @rdev: radeon_device pointer
3407 * @se_num: number of SEs (shader engines) for the asic
3408 * @sh_per_se: number of SH blocks per SE for the asic
3409 * @max_rb_num: max RBs (render backends) for the asic
3410 *
3411 * Configures per-SE/SH RB registers (CIK).
3412 */
3413 static void cik_setup_rb(struct radeon_device *rdev,
3414 u32 se_num, u32 sh_per_se,
3415 u32 max_rb_num_per_se)
3416 {
3417 int i, j;
3418 u32 data, mask;
3419 u32 disabled_rbs = 0;
3420 u32 enabled_rbs = 0;
3421
3422 for (i = 0; i < se_num; i++) {
3423 for (j = 0; j < sh_per_se; j++) {
3424 cik_select_se_sh(rdev, i, j);
3425 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426 if (rdev->family == CHIP_HAWAII)
3427 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428 else
3429 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3430 }
3431 }
3432 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433
3434 mask = 1;
3435 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436 if (!(disabled_rbs & mask))
3437 enabled_rbs |= mask;
3438 mask <<= 1;
3439 }
3440
3441 rdev->config.cik.backend_enable_mask = enabled_rbs;
3442
3443 for (i = 0; i < se_num; i++) {
3444 cik_select_se_sh(rdev, i, 0xffffffff);
3445 data = 0;
3446 for (j = 0; j < sh_per_se; j++) {
3447 switch (enabled_rbs & 3) {
3448 case 0:
3449 if (j == 0)
3450 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451 else
3452 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453 break;
3454 case 1:
3455 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456 break;
3457 case 2:
3458 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459 break;
3460 case 3:
3461 default:
3462 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463 break;
3464 }
3465 enabled_rbs >>= 2;
3466 }
3467 WREG32(PA_SC_RASTER_CONFIG, data);
3468 }
3469 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470 }
3471
3472 /**
3473 * cik_gpu_init - setup the 3D engine
3474 *
3475 * @rdev: radeon_device pointer
3476 *
3477 * Configures the 3D engine and tiling configuration
3478 * registers so that the 3D engine is usable.
3479 */
3480 static void cik_gpu_init(struct radeon_device *rdev)
3481 {
3482 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 u32 mc_shared_chmap, mc_arb_ramcfg;
3484 u32 hdp_host_path_cntl;
3485 u32 tmp;
3486 int i, j, k;
3487
3488 switch (rdev->family) {
3489 case CHIP_BONAIRE:
3490 rdev->config.cik.max_shader_engines = 2;
3491 rdev->config.cik.max_tile_pipes = 4;
3492 rdev->config.cik.max_cu_per_sh = 7;
3493 rdev->config.cik.max_sh_per_se = 1;
3494 rdev->config.cik.max_backends_per_se = 2;
3495 rdev->config.cik.max_texture_channel_caches = 4;
3496 rdev->config.cik.max_gprs = 256;
3497 rdev->config.cik.max_gs_threads = 32;
3498 rdev->config.cik.max_hw_contexts = 8;
3499
3500 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505 break;
3506 case CHIP_HAWAII:
3507 rdev->config.cik.max_shader_engines = 4;
3508 rdev->config.cik.max_tile_pipes = 16;
3509 rdev->config.cik.max_cu_per_sh = 11;
3510 rdev->config.cik.max_sh_per_se = 1;
3511 rdev->config.cik.max_backends_per_se = 4;
3512 rdev->config.cik.max_texture_channel_caches = 16;
3513 rdev->config.cik.max_gprs = 256;
3514 rdev->config.cik.max_gs_threads = 32;
3515 rdev->config.cik.max_hw_contexts = 8;
3516
3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522 break;
3523 case CHIP_KAVERI:
3524 rdev->config.cik.max_shader_engines = 1;
3525 rdev->config.cik.max_tile_pipes = 4;
3526 if ((rdev->pdev->device == 0x1304) ||
3527 (rdev->pdev->device == 0x1305) ||
3528 (rdev->pdev->device == 0x130C) ||
3529 (rdev->pdev->device == 0x130F) ||
3530 (rdev->pdev->device == 0x1310) ||
3531 (rdev->pdev->device == 0x1311) ||
3532 (rdev->pdev->device == 0x131C)) {
3533 rdev->config.cik.max_cu_per_sh = 8;
3534 rdev->config.cik.max_backends_per_se = 2;
3535 } else if ((rdev->pdev->device == 0x1309) ||
3536 (rdev->pdev->device == 0x130A) ||
3537 (rdev->pdev->device == 0x130D) ||
3538 (rdev->pdev->device == 0x1313) ||
3539 (rdev->pdev->device == 0x131D)) {
3540 rdev->config.cik.max_cu_per_sh = 6;
3541 rdev->config.cik.max_backends_per_se = 2;
3542 } else if ((rdev->pdev->device == 0x1306) ||
3543 (rdev->pdev->device == 0x1307) ||
3544 (rdev->pdev->device == 0x130B) ||
3545 (rdev->pdev->device == 0x130E) ||
3546 (rdev->pdev->device == 0x1315) ||
3547 (rdev->pdev->device == 0x131B)) {
3548 rdev->config.cik.max_cu_per_sh = 4;
3549 rdev->config.cik.max_backends_per_se = 1;
3550 } else {
3551 rdev->config.cik.max_cu_per_sh = 3;
3552 rdev->config.cik.max_backends_per_se = 1;
3553 }
3554 rdev->config.cik.max_sh_per_se = 1;
3555 rdev->config.cik.max_texture_channel_caches = 4;
3556 rdev->config.cik.max_gprs = 256;
3557 rdev->config.cik.max_gs_threads = 16;
3558 rdev->config.cik.max_hw_contexts = 8;
3559
3560 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3561 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3562 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3563 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3564 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3565 break;
3566 case CHIP_KABINI:
3567 case CHIP_MULLINS:
3568 default:
3569 rdev->config.cik.max_shader_engines = 1;
3570 rdev->config.cik.max_tile_pipes = 2;
3571 rdev->config.cik.max_cu_per_sh = 2;
3572 rdev->config.cik.max_sh_per_se = 1;
3573 rdev->config.cik.max_backends_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 2;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3578
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3584 break;
3585 }
3586
3587 /* Initialize HDP */
3588 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3589 WREG32((0x2c14 + j), 0x00000000);
3590 WREG32((0x2c18 + j), 0x00000000);
3591 WREG32((0x2c1c + j), 0x00000000);
3592 WREG32((0x2c20 + j), 0x00000000);
3593 WREG32((0x2c24 + j), 0x00000000);
3594 }
3595
3596 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3597
3598 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3599
3600 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3601 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3602
3603 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3604 rdev->config.cik.mem_max_burst_length_bytes = 256;
3605 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3606 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3607 if (rdev->config.cik.mem_row_size_in_kb > 4)
3608 rdev->config.cik.mem_row_size_in_kb = 4;
3609 /* XXX use MC settings? */
3610 rdev->config.cik.shader_engine_tile_size = 32;
3611 rdev->config.cik.num_gpus = 1;
3612 rdev->config.cik.multi_gpu_tile_size = 64;
3613
3614 /* fix up row size */
3615 gb_addr_config &= ~ROW_SIZE_MASK;
3616 switch (rdev->config.cik.mem_row_size_in_kb) {
3617 case 1:
3618 default:
3619 gb_addr_config |= ROW_SIZE(0);
3620 break;
3621 case 2:
3622 gb_addr_config |= ROW_SIZE(1);
3623 break;
3624 case 4:
3625 gb_addr_config |= ROW_SIZE(2);
3626 break;
3627 }
3628
3629 /* setup tiling info dword. gb_addr_config is not adequate since it does
3630 * not have bank info, so create a custom tiling dword.
3631 * bits 3:0 num_pipes
3632 * bits 7:4 num_banks
3633 * bits 11:8 group_size
3634 * bits 15:12 row_size
3635 */
3636 rdev->config.cik.tile_config = 0;
3637 switch (rdev->config.cik.num_tile_pipes) {
3638 case 1:
3639 rdev->config.cik.tile_config |= (0 << 0);
3640 break;
3641 case 2:
3642 rdev->config.cik.tile_config |= (1 << 0);
3643 break;
3644 case 4:
3645 rdev->config.cik.tile_config |= (2 << 0);
3646 break;
3647 case 8:
3648 default:
3649 /* XXX what about 12? */
3650 rdev->config.cik.tile_config |= (3 << 0);
3651 break;
3652 }
3653 rdev->config.cik.tile_config |=
3654 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3655 rdev->config.cik.tile_config |=
3656 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3657 rdev->config.cik.tile_config |=
3658 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3659
3660 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3661 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3662 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3663 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3664 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3665 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3666 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3667 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3668
3669 cik_tiling_mode_table_init(rdev);
3670
3671 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3672 rdev->config.cik.max_sh_per_se,
3673 rdev->config.cik.max_backends_per_se);
3674
3675 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3676 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3677 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3678 rdev->config.cik.active_cus +=
3679 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3680 }
3681 }
3682 }
3683
3684 /* set HW defaults for 3D engine */
3685 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3686
3687 WREG32(SX_DEBUG_1, 0x20);
3688
3689 WREG32(TA_CNTL_AUX, 0x00010000);
3690
3691 tmp = RREG32(SPI_CONFIG_CNTL);
3692 tmp |= 0x03000000;
3693 WREG32(SPI_CONFIG_CNTL, tmp);
3694
3695 WREG32(SQ_CONFIG, 1);
3696
3697 WREG32(DB_DEBUG, 0);
3698
3699 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3700 tmp |= 0x00000400;
3701 WREG32(DB_DEBUG2, tmp);
3702
3703 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3704 tmp |= 0x00020200;
3705 WREG32(DB_DEBUG3, tmp);
3706
3707 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3708 tmp |= 0x00018208;
3709 WREG32(CB_HW_CONTROL, tmp);
3710
3711 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3712
3713 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3717
3718 WREG32(VGT_NUM_INSTANCES, 1);
3719
3720 WREG32(CP_PERFMON_CNTL, 0);
3721
3722 WREG32(SQ_CONFIG, 0);
3723
3724 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725 FORCE_EOV_MAX_REZ_CNT(255)));
3726
3727 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3729
3730 WREG32(VGT_GS_VERTEX_REUSE, 16);
3731 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3732
3733 tmp = RREG32(HDP_MISC_CNTL);
3734 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735 WREG32(HDP_MISC_CNTL, tmp);
3736
3737 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3739
3740 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3742
3743 udelay(50);
3744 }
3745
3746 /*
3747 * GPU scratch registers helpers function.
3748 */
3749 /**
3750 * cik_scratch_init - setup driver info for CP scratch regs
3751 *
3752 * @rdev: radeon_device pointer
3753 *
3754 * Set up the number and offset of the CP scratch registers.
3755 * NOTE: use of CP scratch registers is a legacy inferface and
3756 * is not used by default on newer asics (r6xx+). On newer asics,
3757 * memory buffers are used for fences rather than scratch regs.
3758 */
3759 static void cik_scratch_init(struct radeon_device *rdev)
3760 {
3761 int i;
3762
3763 rdev->scratch.num_reg = 7;
3764 rdev->scratch.reg_base = SCRATCH_REG0;
3765 for (i = 0; i < rdev->scratch.num_reg; i++) {
3766 rdev->scratch.free[i] = true;
3767 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3768 }
3769 }
3770
3771 /**
3772 * cik_ring_test - basic gfx ring test
3773 *
3774 * @rdev: radeon_device pointer
3775 * @ring: radeon_ring structure holding ring information
3776 *
3777 * Allocate a scratch register and write to it using the gfx ring (CIK).
3778 * Provides a basic gfx ring test to verify that the ring is working.
3779 * Used by cik_cp_gfx_resume();
3780 * Returns 0 on success, error on failure.
3781 */
3782 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3783 {
3784 uint32_t scratch;
3785 uint32_t tmp = 0;
3786 unsigned i;
3787 int r;
3788
3789 r = radeon_scratch_get(rdev, &scratch);
3790 if (r) {
3791 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3792 return r;
3793 }
3794 WREG32(scratch, 0xCAFEDEAD);
3795 r = radeon_ring_lock(rdev, ring, 3);
3796 if (r) {
3797 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798 radeon_scratch_free(rdev, scratch);
3799 return r;
3800 }
3801 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803 radeon_ring_write(ring, 0xDEADBEEF);
3804 radeon_ring_unlock_commit(rdev, ring, false);
3805
3806 for (i = 0; i < rdev->usec_timeout; i++) {
3807 tmp = RREG32(scratch);
3808 if (tmp == 0xDEADBEEF)
3809 break;
3810 DRM_UDELAY(1);
3811 }
3812 if (i < rdev->usec_timeout) {
3813 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3814 } else {
3815 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816 ring->idx, scratch, tmp);
3817 r = -EINVAL;
3818 }
3819 radeon_scratch_free(rdev, scratch);
3820 return r;
3821 }
3822
3823 /**
3824 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3825 *
3826 * @rdev: radeon_device pointer
3827 * @ridx: radeon ring index
3828 *
3829 * Emits an hdp flush on the cp.
3830 */
3831 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3832 int ridx)
3833 {
3834 struct radeon_ring *ring = &rdev->ring[ridx];
3835 u32 ref_and_mask;
3836
3837 switch (ring->idx) {
3838 case CAYMAN_RING_TYPE_CP1_INDEX:
3839 case CAYMAN_RING_TYPE_CP2_INDEX:
3840 default:
3841 switch (ring->me) {
3842 case 0:
3843 ref_and_mask = CP2 << ring->pipe;
3844 break;
3845 case 1:
3846 ref_and_mask = CP6 << ring->pipe;
3847 break;
3848 default:
3849 return;
3850 }
3851 break;
3852 case RADEON_RING_TYPE_GFX_INDEX:
3853 ref_and_mask = CP0;
3854 break;
3855 }
3856
3857 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859 WAIT_REG_MEM_FUNCTION(3) | /* == */
3860 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3861 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863 radeon_ring_write(ring, ref_and_mask);
3864 radeon_ring_write(ring, ref_and_mask);
3865 radeon_ring_write(ring, 0x20); /* poll interval */
3866 }
3867
3868 /**
3869 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3870 *
3871 * @rdev: radeon_device pointer
3872 * @fence: radeon fence object
3873 *
3874 * Emits a fence sequnce number on the gfx ring and flushes
3875 * GPU caches.
3876 */
3877 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878 struct radeon_fence *fence)
3879 {
3880 struct radeon_ring *ring = &rdev->ring[fence->ring];
3881 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3882
3883 /* EVENT_WRITE_EOP - flush caches, send int */
3884 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3886 EOP_TC_ACTION_EN |
3887 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3888 EVENT_INDEX(5)));
3889 radeon_ring_write(ring, addr & 0xfffffffc);
3890 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891 radeon_ring_write(ring, fence->seq);
3892 radeon_ring_write(ring, 0);
3893 }
3894
3895 /**
3896 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3897 *
3898 * @rdev: radeon_device pointer
3899 * @fence: radeon fence object
3900 *
3901 * Emits a fence sequnce number on the compute ring and flushes
3902 * GPU caches.
3903 */
3904 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905 struct radeon_fence *fence)
3906 {
3907 struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3909
3910 /* RELEASE_MEM - flush caches, send int */
3911 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3913 EOP_TC_ACTION_EN |
3914 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3915 EVENT_INDEX(5)));
3916 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917 radeon_ring_write(ring, addr & 0xfffffffc);
3918 radeon_ring_write(ring, upper_32_bits(addr));
3919 radeon_ring_write(ring, fence->seq);
3920 radeon_ring_write(ring, 0);
3921 }
3922
3923 /**
3924 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3925 *
3926 * @rdev: radeon_device pointer
3927 * @ring: radeon ring buffer object
3928 * @semaphore: radeon semaphore object
3929 * @emit_wait: Is this a sempahore wait?
3930 *
3931 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3932 * from running ahead of semaphore waits.
3933 */
3934 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3935 struct radeon_ring *ring,
3936 struct radeon_semaphore *semaphore,
3937 bool emit_wait)
3938 {
3939 uint64_t addr = semaphore->gpu_addr;
3940 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3941
3942 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3943 radeon_ring_write(ring, lower_32_bits(addr));
3944 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3945
3946 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3947 /* Prevent the PFP from running ahead of the semaphore wait */
3948 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3949 radeon_ring_write(ring, 0x0);
3950 }
3951
3952 return true;
3953 }
3954
3955 /**
3956 * cik_copy_cpdma - copy pages using the CP DMA engine
3957 *
3958 * @rdev: radeon_device pointer
3959 * @src_offset: src GPU address
3960 * @dst_offset: dst GPU address
3961 * @num_gpu_pages: number of GPU pages to xfer
3962 * @fence: radeon fence object
3963 *
3964 * Copy GPU paging using the CP DMA engine (CIK+).
3965 * Used by the radeon ttm implementation to move pages if
3966 * registered as the asic copy callback.
3967 */
3968 int cik_copy_cpdma(struct radeon_device *rdev,
3969 uint64_t src_offset, uint64_t dst_offset,
3970 unsigned num_gpu_pages,
3971 struct radeon_fence **fence)
3972 {
3973 struct radeon_semaphore *sem = NULL;
3974 int ring_index = rdev->asic->copy.blit_ring_index;
3975 struct radeon_ring *ring = &rdev->ring[ring_index];
3976 u32 size_in_bytes, cur_size_in_bytes, control;
3977 int i, num_loops;
3978 int r = 0;
3979
3980 r = radeon_semaphore_create(rdev, &sem);
3981 if (r) {
3982 DRM_ERROR("radeon: moving bo (%d).\n", r);
3983 return r;
3984 }
3985
3986 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3987 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3988 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3989 if (r) {
3990 DRM_ERROR("radeon: moving bo (%d).\n", r);
3991 radeon_semaphore_free(rdev, &sem, NULL);
3992 return r;
3993 }
3994
3995 radeon_semaphore_sync_to(sem, *fence);
3996 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3997
3998 for (i = 0; i < num_loops; i++) {
3999 cur_size_in_bytes = size_in_bytes;
4000 if (cur_size_in_bytes > 0x1fffff)
4001 cur_size_in_bytes = 0x1fffff;
4002 size_in_bytes -= cur_size_in_bytes;
4003 control = 0;
4004 if (size_in_bytes == 0)
4005 control |= PACKET3_DMA_DATA_CP_SYNC;
4006 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4007 radeon_ring_write(ring, control);
4008 radeon_ring_write(ring, lower_32_bits(src_offset));
4009 radeon_ring_write(ring, upper_32_bits(src_offset));
4010 radeon_ring_write(ring, lower_32_bits(dst_offset));
4011 radeon_ring_write(ring, upper_32_bits(dst_offset));
4012 radeon_ring_write(ring, cur_size_in_bytes);
4013 src_offset += cur_size_in_bytes;
4014 dst_offset += cur_size_in_bytes;
4015 }
4016
4017 r = radeon_fence_emit(rdev, fence, ring->idx);
4018 if (r) {
4019 radeon_ring_unlock_undo(rdev, ring);
4020 radeon_semaphore_free(rdev, &sem, NULL);
4021 return r;
4022 }
4023
4024 radeon_ring_unlock_commit(rdev, ring, false);
4025 radeon_semaphore_free(rdev, &sem, *fence);
4026
4027 return r;
4028 }
4029
4030 /*
4031 * IB stuff
4032 */
4033 /**
4034 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4035 *
4036 * @rdev: radeon_device pointer
4037 * @ib: radeon indirect buffer object
4038 *
4039 * Emits an DE (drawing engine) or CE (constant engine) IB
4040 * on the gfx ring. IBs are usually generated by userspace
4041 * acceleration drivers and submitted to the kernel for
4042 * sheduling on the ring. This function schedules the IB
4043 * on the gfx ring for execution by the GPU.
4044 */
4045 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4046 {
4047 struct radeon_ring *ring = &rdev->ring[ib->ring];
4048 u32 header, control = INDIRECT_BUFFER_VALID;
4049
4050 if (ib->is_const_ib) {
4051 /* set switch buffer packet before const IB */
4052 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4053 radeon_ring_write(ring, 0);
4054
4055 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4056 } else {
4057 u32 next_rptr;
4058 if (ring->rptr_save_reg) {
4059 next_rptr = ring->wptr + 3 + 4;
4060 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4061 radeon_ring_write(ring, ((ring->rptr_save_reg -
4062 PACKET3_SET_UCONFIG_REG_START) >> 2));
4063 radeon_ring_write(ring, next_rptr);
4064 } else if (rdev->wb.enabled) {
4065 next_rptr = ring->wptr + 5 + 4;
4066 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4067 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4068 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4069 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4070 radeon_ring_write(ring, next_rptr);
4071 }
4072
4073 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4074 }
4075
4076 control |= ib->length_dw |
4077 (ib->vm ? (ib->vm->id << 24) : 0);
4078
4079 radeon_ring_write(ring, header);
4080 radeon_ring_write(ring,
4081 #ifdef __BIG_ENDIAN
4082 (2 << 0) |
4083 #endif
4084 (ib->gpu_addr & 0xFFFFFFFC));
4085 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4086 radeon_ring_write(ring, control);
4087 }
4088
4089 /**
4090 * cik_ib_test - basic gfx ring IB test
4091 *
4092 * @rdev: radeon_device pointer
4093 * @ring: radeon_ring structure holding ring information
4094 *
4095 * Allocate an IB and execute it on the gfx ring (CIK).
4096 * Provides a basic gfx ring test to verify that IBs are working.
4097 * Returns 0 on success, error on failure.
4098 */
4099 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4100 {
4101 struct radeon_ib ib;
4102 uint32_t scratch;
4103 uint32_t tmp = 0;
4104 unsigned i;
4105 int r;
4106
4107 r = radeon_scratch_get(rdev, &scratch);
4108 if (r) {
4109 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4110 return r;
4111 }
4112 WREG32(scratch, 0xCAFEDEAD);
4113 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4114 if (r) {
4115 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4116 radeon_scratch_free(rdev, scratch);
4117 return r;
4118 }
4119 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4120 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4121 ib.ptr[2] = 0xDEADBEEF;
4122 ib.length_dw = 3;
4123 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4124 if (r) {
4125 radeon_scratch_free(rdev, scratch);
4126 radeon_ib_free(rdev, &ib);
4127 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4128 return r;
4129 }
4130 r = radeon_fence_wait(ib.fence, false);
4131 if (r) {
4132 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4133 radeon_scratch_free(rdev, scratch);
4134 radeon_ib_free(rdev, &ib);
4135 return r;
4136 }
4137 for (i = 0; i < rdev->usec_timeout; i++) {
4138 tmp = RREG32(scratch);
4139 if (tmp == 0xDEADBEEF)
4140 break;
4141 DRM_UDELAY(1);
4142 }
4143 if (i < rdev->usec_timeout) {
4144 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4145 } else {
4146 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4147 scratch, tmp);
4148 r = -EINVAL;
4149 }
4150 radeon_scratch_free(rdev, scratch);
4151 radeon_ib_free(rdev, &ib);
4152 return r;
4153 }
4154
4155 /*
4156 * CP.
4157 * On CIK, gfx and compute now have independant command processors.
4158 *
4159 * GFX
4160 * Gfx consists of a single ring and can process both gfx jobs and
4161 * compute jobs. The gfx CP consists of three microengines (ME):
4162 * PFP - Pre-Fetch Parser
4163 * ME - Micro Engine
4164 * CE - Constant Engine
4165 * The PFP and ME make up what is considered the Drawing Engine (DE).
4166 * The CE is an asynchronous engine used for updating buffer desciptors
4167 * used by the DE so that they can be loaded into cache in parallel
4168 * while the DE is processing state update packets.
4169 *
4170 * Compute
4171 * The compute CP consists of two microengines (ME):
4172 * MEC1 - Compute MicroEngine 1
4173 * MEC2 - Compute MicroEngine 2
4174 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4175 * The queues are exposed to userspace and are programmed directly
4176 * by the compute runtime.
4177 */
4178 /**
4179 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4180 *
4181 * @rdev: radeon_device pointer
4182 * @enable: enable or disable the MEs
4183 *
4184 * Halts or unhalts the gfx MEs.
4185 */
4186 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4187 {
4188 if (enable)
4189 WREG32(CP_ME_CNTL, 0);
4190 else {
4191 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4192 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4193 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4194 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4195 }
4196 udelay(50);
4197 }
4198
4199 /**
4200 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4201 *
4202 * @rdev: radeon_device pointer
4203 *
4204 * Loads the gfx PFP, ME, and CE ucode.
4205 * Returns 0 for success, -EINVAL if the ucode is not available.
4206 */
4207 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4208 {
4209 int i;
4210
4211 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4212 return -EINVAL;
4213
4214 cik_cp_gfx_enable(rdev, false);
4215
4216 if (rdev->new_fw) {
4217 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4218 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4219 const struct gfx_firmware_header_v1_0 *ce_hdr =
4220 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4221 const struct gfx_firmware_header_v1_0 *me_hdr =
4222 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4223 const __le32 *fw_data;
4224 u32 fw_size;
4225
4226 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4227 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4228 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4229
4230 /* PFP */
4231 fw_data = (const __le32 *)
4232 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4233 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4234 WREG32(CP_PFP_UCODE_ADDR, 0);
4235 for (i = 0; i < fw_size; i++)
4236 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4237 WREG32(CP_PFP_UCODE_ADDR, 0);
4238
4239 /* CE */
4240 fw_data = (const __le32 *)
4241 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4242 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4243 WREG32(CP_CE_UCODE_ADDR, 0);
4244 for (i = 0; i < fw_size; i++)
4245 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4246 WREG32(CP_CE_UCODE_ADDR, 0);
4247
4248 /* ME */
4249 fw_data = (const __be32 *)
4250 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4251 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4252 WREG32(CP_ME_RAM_WADDR, 0);
4253 for (i = 0; i < fw_size; i++)
4254 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4255 WREG32(CP_ME_RAM_WADDR, 0);
4256 } else {
4257 const __be32 *fw_data;
4258
4259 /* PFP */
4260 fw_data = (const __be32 *)rdev->pfp_fw->data;
4261 WREG32(CP_PFP_UCODE_ADDR, 0);
4262 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4263 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4264 WREG32(CP_PFP_UCODE_ADDR, 0);
4265
4266 /* CE */
4267 fw_data = (const __be32 *)rdev->ce_fw->data;
4268 WREG32(CP_CE_UCODE_ADDR, 0);
4269 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4270 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4271 WREG32(CP_CE_UCODE_ADDR, 0);
4272
4273 /* ME */
4274 fw_data = (const __be32 *)rdev->me_fw->data;
4275 WREG32(CP_ME_RAM_WADDR, 0);
4276 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4277 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4278 WREG32(CP_ME_RAM_WADDR, 0);
4279 }
4280
4281 WREG32(CP_PFP_UCODE_ADDR, 0);
4282 WREG32(CP_CE_UCODE_ADDR, 0);
4283 WREG32(CP_ME_RAM_WADDR, 0);
4284 WREG32(CP_ME_RAM_RADDR, 0);
4285 return 0;
4286 }
4287
4288 /**
4289 * cik_cp_gfx_start - start the gfx ring
4290 *
4291 * @rdev: radeon_device pointer
4292 *
4293 * Enables the ring and loads the clear state context and other
4294 * packets required to init the ring.
4295 * Returns 0 for success, error for failure.
4296 */
4297 static int cik_cp_gfx_start(struct radeon_device *rdev)
4298 {
4299 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4300 int r, i;
4301
4302 /* init the CP */
4303 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4304 WREG32(CP_ENDIAN_SWAP, 0);
4305 WREG32(CP_DEVICE_ID, 1);
4306
4307 cik_cp_gfx_enable(rdev, true);
4308
4309 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4310 if (r) {
4311 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4312 return r;
4313 }
4314
4315 /* init the CE partitions. CE only used for gfx on CIK */
4316 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4317 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4318 radeon_ring_write(ring, 0xc000);
4319 radeon_ring_write(ring, 0xc000);
4320
4321 /* setup clear context state */
4322 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4323 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4324
4325 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4326 radeon_ring_write(ring, 0x80000000);
4327 radeon_ring_write(ring, 0x80000000);
4328
4329 for (i = 0; i < cik_default_size; i++)
4330 radeon_ring_write(ring, cik_default_state[i]);
4331
4332 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4333 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4334
4335 /* set clear context state */
4336 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4337 radeon_ring_write(ring, 0);
4338
4339 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4340 radeon_ring_write(ring, 0x00000316);
4341 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4342 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4343
4344 radeon_ring_unlock_commit(rdev, ring, false);
4345
4346 return 0;
4347 }
4348
4349 /**
4350 * cik_cp_gfx_fini - stop the gfx ring
4351 *
4352 * @rdev: radeon_device pointer
4353 *
4354 * Stop the gfx ring and tear down the driver ring
4355 * info.
4356 */
4357 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4358 {
4359 cik_cp_gfx_enable(rdev, false);
4360 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4361 }
4362
4363 /**
4364 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4365 *
4366 * @rdev: radeon_device pointer
4367 *
4368 * Program the location and size of the gfx ring buffer
4369 * and test it to make sure it's working.
4370 * Returns 0 for success, error for failure.
4371 */
4372 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4373 {
4374 struct radeon_ring *ring;
4375 u32 tmp;
4376 u32 rb_bufsz;
4377 u64 rb_addr;
4378 int r;
4379
4380 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4381 if (rdev->family != CHIP_HAWAII)
4382 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4383
4384 /* Set the write pointer delay */
4385 WREG32(CP_RB_WPTR_DELAY, 0);
4386
4387 /* set the RB to use vmid 0 */
4388 WREG32(CP_RB_VMID, 0);
4389
4390 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4391
4392 /* ring 0 - compute and gfx */
4393 /* Set ring buffer size */
4394 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4395 rb_bufsz = order_base_2(ring->ring_size / 8);
4396 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4397 #ifdef __BIG_ENDIAN
4398 tmp |= BUF_SWAP_32BIT;
4399 #endif
4400 WREG32(CP_RB0_CNTL, tmp);
4401
4402 /* Initialize the ring buffer's read and write pointers */
4403 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4404 ring->wptr = 0;
4405 WREG32(CP_RB0_WPTR, ring->wptr);
4406
4407 /* set the wb address wether it's enabled or not */
4408 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4409 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4410
4411 /* scratch register shadowing is no longer supported */
4412 WREG32(SCRATCH_UMSK, 0);
4413
4414 if (!rdev->wb.enabled)
4415 tmp |= RB_NO_UPDATE;
4416
4417 mdelay(1);
4418 WREG32(CP_RB0_CNTL, tmp);
4419
4420 rb_addr = ring->gpu_addr >> 8;
4421 WREG32(CP_RB0_BASE, rb_addr);
4422 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4423
4424 /* start the ring */
4425 cik_cp_gfx_start(rdev);
4426 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4427 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4428 if (r) {
4429 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4430 return r;
4431 }
4432
4433 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4434 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4435
4436 return 0;
4437 }
4438
4439 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4440 struct radeon_ring *ring)
4441 {
4442 u32 rptr;
4443
4444 if (rdev->wb.enabled)
4445 rptr = rdev->wb.wb[ring->rptr_offs/4];
4446 else
4447 rptr = RREG32(CP_RB0_RPTR);
4448
4449 return rptr;
4450 }
4451
4452 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4453 struct radeon_ring *ring)
4454 {
4455 u32 wptr;
4456
4457 wptr = RREG32(CP_RB0_WPTR);
4458
4459 return wptr;
4460 }
4461
4462 void cik_gfx_set_wptr(struct radeon_device *rdev,
4463 struct radeon_ring *ring)
4464 {
4465 WREG32(CP_RB0_WPTR, ring->wptr);
4466 (void)RREG32(CP_RB0_WPTR);
4467 }
4468
4469 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4470 struct radeon_ring *ring)
4471 {
4472 u32 rptr;
4473
4474 if (rdev->wb.enabled) {
4475 rptr = rdev->wb.wb[ring->rptr_offs/4];
4476 } else {
4477 mutex_lock(&rdev->srbm_mutex);
4478 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4479 rptr = RREG32(CP_HQD_PQ_RPTR);
4480 cik_srbm_select(rdev, 0, 0, 0, 0);
4481 mutex_unlock(&rdev->srbm_mutex);
4482 }
4483
4484 return rptr;
4485 }
4486
4487 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4488 struct radeon_ring *ring)
4489 {
4490 u32 wptr;
4491
4492 if (rdev->wb.enabled) {
4493 /* XXX check if swapping is necessary on BE */
4494 wptr = rdev->wb.wb[ring->wptr_offs/4];
4495 } else {
4496 mutex_lock(&rdev->srbm_mutex);
4497 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4498 wptr = RREG32(CP_HQD_PQ_WPTR);
4499 cik_srbm_select(rdev, 0, 0, 0, 0);
4500 mutex_unlock(&rdev->srbm_mutex);
4501 }
4502
4503 return wptr;
4504 }
4505
4506 void cik_compute_set_wptr(struct radeon_device *rdev,
4507 struct radeon_ring *ring)
4508 {
4509 /* XXX check if swapping is necessary on BE */
4510 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4511 WDOORBELL32(ring->doorbell_index, ring->wptr);
4512 }
4513
4514 /**
4515 * cik_cp_compute_enable - enable/disable the compute CP MEs
4516 *
4517 * @rdev: radeon_device pointer
4518 * @enable: enable or disable the MEs
4519 *
4520 * Halts or unhalts the compute MEs.
4521 */
4522 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4523 {
4524 if (enable)
4525 WREG32(CP_MEC_CNTL, 0);
4526 else {
4527 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4528 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4529 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4530 }
4531 udelay(50);
4532 }
4533
4534 /**
4535 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4536 *
4537 * @rdev: radeon_device pointer
4538 *
4539 * Loads the compute MEC1&2 ucode.
4540 * Returns 0 for success, -EINVAL if the ucode is not available.
4541 */
4542 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4543 {
4544 int i;
4545
4546 if (!rdev->mec_fw)
4547 return -EINVAL;
4548
4549 cik_cp_compute_enable(rdev, false);
4550
4551 if (rdev->new_fw) {
4552 const struct gfx_firmware_header_v1_0 *mec_hdr =
4553 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4554 const __le32 *fw_data;
4555 u32 fw_size;
4556
4557 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4558
4559 /* MEC1 */
4560 fw_data = (const __le32 *)
4561 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4562 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4563 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4564 for (i = 0; i < fw_size; i++)
4565 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4566 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4567
4568 /* MEC2 */
4569 if (rdev->family == CHIP_KAVERI) {
4570 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4571 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4572
4573 fw_data = (const __le32 *)
4574 (rdev->mec2_fw->data +
4575 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4576 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4577 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4578 for (i = 0; i < fw_size; i++)
4579 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4580 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4581 }
4582 } else {
4583 const __be32 *fw_data;
4584
4585 /* MEC1 */
4586 fw_data = (const __be32 *)rdev->mec_fw->data;
4587 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4588 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4589 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4590 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4591
4592 if (rdev->family == CHIP_KAVERI) {
4593 /* MEC2 */
4594 fw_data = (const __be32 *)rdev->mec_fw->data;
4595 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4596 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4597 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4598 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4599 }
4600 }
4601
4602 return 0;
4603 }
4604
4605 /**
4606 * cik_cp_compute_start - start the compute queues
4607 *
4608 * @rdev: radeon_device pointer
4609 *
4610 * Enable the compute queues.
4611 * Returns 0 for success, error for failure.
4612 */
4613 static int cik_cp_compute_start(struct radeon_device *rdev)
4614 {
4615 cik_cp_compute_enable(rdev, true);
4616
4617 return 0;
4618 }
4619
4620 /**
4621 * cik_cp_compute_fini - stop the compute queues
4622 *
4623 * @rdev: radeon_device pointer
4624 *
4625 * Stop the compute queues and tear down the driver queue
4626 * info.
4627 */
4628 static void cik_cp_compute_fini(struct radeon_device *rdev)
4629 {
4630 int i, idx, r;
4631
4632 cik_cp_compute_enable(rdev, false);
4633
4634 for (i = 0; i < 2; i++) {
4635 if (i == 0)
4636 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4637 else
4638 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4639
4640 if (rdev->ring[idx].mqd_obj) {
4641 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4642 if (unlikely(r != 0))
4643 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4644
4645 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4646 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4647
4648 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4649 rdev->ring[idx].mqd_obj = NULL;
4650 }
4651 }
4652 }
4653
4654 static void cik_mec_fini(struct radeon_device *rdev)
4655 {
4656 int r;
4657
4658 if (rdev->mec.hpd_eop_obj) {
4659 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4660 if (unlikely(r != 0))
4661 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4662 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4663 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4664
4665 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4666 rdev->mec.hpd_eop_obj = NULL;
4667 }
4668 }
4669
4670 #define MEC_HPD_SIZE 2048
4671
4672 static int cik_mec_init(struct radeon_device *rdev)
4673 {
4674 int r;
4675 u32 *hpd;
4676
4677 /*
4678 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4679 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4680 */
4681 if (rdev->family == CHIP_KAVERI)
4682 rdev->mec.num_mec = 2;
4683 else
4684 rdev->mec.num_mec = 1;
4685 rdev->mec.num_pipe = 4;
4686 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4687
4688 if (rdev->mec.hpd_eop_obj == NULL) {
4689 r = radeon_bo_create(rdev,
4690 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4691 PAGE_SIZE, true,
4692 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4693 &rdev->mec.hpd_eop_obj);
4694 if (r) {
4695 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4696 return r;
4697 }
4698 }
4699
4700 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4701 if (unlikely(r != 0)) {
4702 cik_mec_fini(rdev);
4703 return r;
4704 }
4705 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4706 &rdev->mec.hpd_eop_gpu_addr);
4707 if (r) {
4708 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4709 cik_mec_fini(rdev);
4710 return r;
4711 }
4712 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4713 if (r) {
4714 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4715 cik_mec_fini(rdev);
4716 return r;
4717 }
4718
4719 /* clear memory. Not sure if this is required or not */
4720 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4721
4722 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4723 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4724
4725 return 0;
4726 }
4727
4728 struct hqd_registers
4729 {
4730 u32 cp_mqd_base_addr;
4731 u32 cp_mqd_base_addr_hi;
4732 u32 cp_hqd_active;
4733 u32 cp_hqd_vmid;
4734 u32 cp_hqd_persistent_state;
4735 u32 cp_hqd_pipe_priority;
4736 u32 cp_hqd_queue_priority;
4737 u32 cp_hqd_quantum;
4738 u32 cp_hqd_pq_base;
4739 u32 cp_hqd_pq_base_hi;
4740 u32 cp_hqd_pq_rptr;
4741 u32 cp_hqd_pq_rptr_report_addr;
4742 u32 cp_hqd_pq_rptr_report_addr_hi;
4743 u32 cp_hqd_pq_wptr_poll_addr;
4744 u32 cp_hqd_pq_wptr_poll_addr_hi;
4745 u32 cp_hqd_pq_doorbell_control;
4746 u32 cp_hqd_pq_wptr;
4747 u32 cp_hqd_pq_control;
4748 u32 cp_hqd_ib_base_addr;
4749 u32 cp_hqd_ib_base_addr_hi;
4750 u32 cp_hqd_ib_rptr;
4751 u32 cp_hqd_ib_control;
4752 u32 cp_hqd_iq_timer;
4753 u32 cp_hqd_iq_rptr;
4754 u32 cp_hqd_dequeue_request;
4755 u32 cp_hqd_dma_offload;
4756 u32 cp_hqd_sema_cmd;
4757 u32 cp_hqd_msg_type;
4758 u32 cp_hqd_atomic0_preop_lo;
4759 u32 cp_hqd_atomic0_preop_hi;
4760 u32 cp_hqd_atomic1_preop_lo;
4761 u32 cp_hqd_atomic1_preop_hi;
4762 u32 cp_hqd_hq_scheduler0;
4763 u32 cp_hqd_hq_scheduler1;
4764 u32 cp_mqd_control;
4765 };
4766
4767 struct bonaire_mqd
4768 {
4769 u32 header;
4770 u32 dispatch_initiator;
4771 u32 dimensions[3];
4772 u32 start_idx[3];
4773 u32 num_threads[3];
4774 u32 pipeline_stat_enable;
4775 u32 perf_counter_enable;
4776 u32 pgm[2];
4777 u32 tba[2];
4778 u32 tma[2];
4779 u32 pgm_rsrc[2];
4780 u32 vmid;
4781 u32 resource_limits;
4782 u32 static_thread_mgmt01[2];
4783 u32 tmp_ring_size;
4784 u32 static_thread_mgmt23[2];
4785 u32 restart[3];
4786 u32 thread_trace_enable;
4787 u32 reserved1;
4788 u32 user_data[16];
4789 u32 vgtcs_invoke_count[2];
4790 struct hqd_registers queue_state;
4791 u32 dequeue_cntr;
4792 u32 interrupt_queue[64];
4793 };
4794
4795 /**
4796 * cik_cp_compute_resume - setup the compute queue registers
4797 *
4798 * @rdev: radeon_device pointer
4799 *
4800 * Program the compute queues and test them to make sure they
4801 * are working.
4802 * Returns 0 for success, error for failure.
4803 */
4804 static int cik_cp_compute_resume(struct radeon_device *rdev)
4805 {
4806 int r, i, idx;
4807 u32 tmp;
4808 bool use_doorbell = true;
4809 u64 hqd_gpu_addr;
4810 u64 mqd_gpu_addr;
4811 u64 eop_gpu_addr;
4812 u64 wb_gpu_addr;
4813 u32 *buf;
4814 struct bonaire_mqd *mqd;
4815
4816 r = cik_cp_compute_start(rdev);
4817 if (r)
4818 return r;
4819
4820 /* fix up chicken bits */
4821 tmp = RREG32(CP_CPF_DEBUG);
4822 tmp |= (1 << 23);
4823 WREG32(CP_CPF_DEBUG, tmp);
4824
4825 /* init the pipes */
4826 mutex_lock(&rdev->srbm_mutex);
4827 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4828 int me = (i < 4) ? 1 : 2;
4829 int pipe = (i < 4) ? i : (i - 4);
4830
4831 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4832
4833 cik_srbm_select(rdev, me, pipe, 0, 0);
4834
4835 /* write the EOP addr */
4836 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4837 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4838
4839 /* set the VMID assigned */
4840 WREG32(CP_HPD_EOP_VMID, 0);
4841
4842 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4843 tmp = RREG32(CP_HPD_EOP_CONTROL);
4844 tmp &= ~EOP_SIZE_MASK;
4845 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4846 WREG32(CP_HPD_EOP_CONTROL, tmp);
4847 }
4848 cik_srbm_select(rdev, 0, 0, 0, 0);
4849 mutex_unlock(&rdev->srbm_mutex);
4850
4851 /* init the queues. Just two for now. */
4852 for (i = 0; i < 2; i++) {
4853 if (i == 0)
4854 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4855 else
4856 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4857
4858 if (rdev->ring[idx].mqd_obj == NULL) {
4859 r = radeon_bo_create(rdev,
4860 sizeof(struct bonaire_mqd),
4861 PAGE_SIZE, true,
4862 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4863 &rdev->ring[idx].mqd_obj);
4864 if (r) {
4865 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4866 return r;
4867 }
4868 }
4869
4870 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4871 if (unlikely(r != 0)) {
4872 cik_cp_compute_fini(rdev);
4873 return r;
4874 }
4875 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4876 &mqd_gpu_addr);
4877 if (r) {
4878 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4879 cik_cp_compute_fini(rdev);
4880 return r;
4881 }
4882 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4883 if (r) {
4884 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4885 cik_cp_compute_fini(rdev);
4886 return r;
4887 }
4888
4889 /* init the mqd struct */
4890 memset(buf, 0, sizeof(struct bonaire_mqd));
4891
4892 mqd = (struct bonaire_mqd *)buf;
4893 mqd->header = 0xC0310800;
4894 mqd->static_thread_mgmt01[0] = 0xffffffff;
4895 mqd->static_thread_mgmt01[1] = 0xffffffff;
4896 mqd->static_thread_mgmt23[0] = 0xffffffff;
4897 mqd->static_thread_mgmt23[1] = 0xffffffff;
4898
4899 mutex_lock(&rdev->srbm_mutex);
4900 cik_srbm_select(rdev, rdev->ring[idx].me,
4901 rdev->ring[idx].pipe,
4902 rdev->ring[idx].queue, 0);
4903
4904 /* disable wptr polling */
4905 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4906 tmp &= ~WPTR_POLL_EN;
4907 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4908
4909 /* enable doorbell? */
4910 mqd->queue_state.cp_hqd_pq_doorbell_control =
4911 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4912 if (use_doorbell)
4913 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4914 else
4915 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4916 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4917 mqd->queue_state.cp_hqd_pq_doorbell_control);
4918
4919 /* disable the queue if it's active */
4920 mqd->queue_state.cp_hqd_dequeue_request = 0;
4921 mqd->queue_state.cp_hqd_pq_rptr = 0;
4922 mqd->queue_state.cp_hqd_pq_wptr= 0;
4923 if (RREG32(CP_HQD_ACTIVE) & 1) {
4924 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4925 for (i = 0; i < rdev->usec_timeout; i++) {
4926 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4927 break;
4928 udelay(1);
4929 }
4930 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4931 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4932 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4933 }
4934
4935 /* set the pointer to the MQD */
4936 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4937 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4938 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4939 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4940 /* set MQD vmid to 0 */
4941 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4942 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4943 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4944
4945 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4946 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4947 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4948 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4949 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4950 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4951
4952 /* set up the HQD, this is similar to CP_RB0_CNTL */
4953 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4954 mqd->queue_state.cp_hqd_pq_control &=
4955 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4956
4957 mqd->queue_state.cp_hqd_pq_control |=
4958 order_base_2(rdev->ring[idx].ring_size / 8);
4959 mqd->queue_state.cp_hqd_pq_control |=
4960 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4961 #ifdef __BIG_ENDIAN
4962 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4963 #endif
4964 mqd->queue_state.cp_hqd_pq_control &=
4965 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4966 mqd->queue_state.cp_hqd_pq_control |=
4967 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4968 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4969
4970 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4971 if (i == 0)
4972 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4973 else
4974 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4975 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4976 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4977 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4978 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4979 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4980
4981 /* set the wb address wether it's enabled or not */
4982 if (i == 0)
4983 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4984 else
4985 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4986 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4987 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4988 upper_32_bits(wb_gpu_addr) & 0xffff;
4989 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4990 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4991 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4992 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4993
4994 /* enable the doorbell if requested */
4995 if (use_doorbell) {
4996 mqd->queue_state.cp_hqd_pq_doorbell_control =
4997 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4998 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4999 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5000 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5001 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5002 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5003 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5004
5005 } else {
5006 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5007 }
5008 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5009 mqd->queue_state.cp_hqd_pq_doorbell_control);
5010
5011 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5012 rdev->ring[idx].wptr = 0;
5013 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5014 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5015 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5016
5017 /* set the vmid for the queue */
5018 mqd->queue_state.cp_hqd_vmid = 0;
5019 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5020
5021 /* activate the queue */
5022 mqd->queue_state.cp_hqd_active = 1;
5023 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5024
5025 cik_srbm_select(rdev, 0, 0, 0, 0);
5026 mutex_unlock(&rdev->srbm_mutex);
5027
5028 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5029 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5030
5031 rdev->ring[idx].ready = true;
5032 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5033 if (r)
5034 rdev->ring[idx].ready = false;
5035 }
5036
5037 return 0;
5038 }
5039
5040 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5041 {
5042 cik_cp_gfx_enable(rdev, enable);
5043 cik_cp_compute_enable(rdev, enable);
5044 }
5045
5046 static int cik_cp_load_microcode(struct radeon_device *rdev)
5047 {
5048 int r;
5049
5050 r = cik_cp_gfx_load_microcode(rdev);
5051 if (r)
5052 return r;
5053 r = cik_cp_compute_load_microcode(rdev);
5054 if (r)
5055 return r;
5056
5057 return 0;
5058 }
5059
5060 static void cik_cp_fini(struct radeon_device *rdev)
5061 {
5062 cik_cp_gfx_fini(rdev);
5063 cik_cp_compute_fini(rdev);
5064 }
5065
5066 static int cik_cp_resume(struct radeon_device *rdev)
5067 {
5068 int r;
5069
5070 cik_enable_gui_idle_interrupt(rdev, false);
5071
5072 r = cik_cp_load_microcode(rdev);
5073 if (r)
5074 return r;
5075
5076 r = cik_cp_gfx_resume(rdev);
5077 if (r)
5078 return r;
5079 r = cik_cp_compute_resume(rdev);
5080 if (r)
5081 return r;
5082
5083 cik_enable_gui_idle_interrupt(rdev, true);
5084
5085 return 0;
5086 }
5087
5088 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5089 {
5090 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5091 RREG32(GRBM_STATUS));
5092 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5093 RREG32(GRBM_STATUS2));
5094 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5095 RREG32(GRBM_STATUS_SE0));
5096 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5097 RREG32(GRBM_STATUS_SE1));
5098 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5099 RREG32(GRBM_STATUS_SE2));
5100 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5101 RREG32(GRBM_STATUS_SE3));
5102 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5103 RREG32(SRBM_STATUS));
5104 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5105 RREG32(SRBM_STATUS2));
5106 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5107 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5108 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5109 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5110 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5111 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5112 RREG32(CP_STALLED_STAT1));
5113 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5114 RREG32(CP_STALLED_STAT2));
5115 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5116 RREG32(CP_STALLED_STAT3));
5117 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5118 RREG32(CP_CPF_BUSY_STAT));
5119 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5120 RREG32(CP_CPF_STALLED_STAT1));
5121 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5122 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5123 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5124 RREG32(CP_CPC_STALLED_STAT1));
5125 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5126 }
5127
5128 /**
5129 * cik_gpu_check_soft_reset - check which blocks are busy
5130 *
5131 * @rdev: radeon_device pointer
5132 *
5133 * Check which blocks are busy and return the relevant reset
5134 * mask to be used by cik_gpu_soft_reset().
5135 * Returns a mask of the blocks to be reset.
5136 */
5137 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5138 {
5139 u32 reset_mask = 0;
5140 u32 tmp;
5141
5142 /* GRBM_STATUS */
5143 tmp = RREG32(GRBM_STATUS);
5144 if (tmp & (PA_BUSY | SC_BUSY |
5145 BCI_BUSY | SX_BUSY |
5146 TA_BUSY | VGT_BUSY |
5147 DB_BUSY | CB_BUSY |
5148 GDS_BUSY | SPI_BUSY |
5149 IA_BUSY | IA_BUSY_NO_DMA))
5150 reset_mask |= RADEON_RESET_GFX;
5151
5152 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5153 reset_mask |= RADEON_RESET_CP;
5154
5155 /* GRBM_STATUS2 */
5156 tmp = RREG32(GRBM_STATUS2);
5157 if (tmp & RLC_BUSY)
5158 reset_mask |= RADEON_RESET_RLC;
5159
5160 /* SDMA0_STATUS_REG */
5161 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5162 if (!(tmp & SDMA_IDLE))
5163 reset_mask |= RADEON_RESET_DMA;
5164
5165 /* SDMA1_STATUS_REG */
5166 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5167 if (!(tmp & SDMA_IDLE))
5168 reset_mask |= RADEON_RESET_DMA1;
5169
5170 /* SRBM_STATUS2 */
5171 tmp = RREG32(SRBM_STATUS2);
5172 if (tmp & SDMA_BUSY)
5173 reset_mask |= RADEON_RESET_DMA;
5174
5175 if (tmp & SDMA1_BUSY)
5176 reset_mask |= RADEON_RESET_DMA1;
5177
5178 /* SRBM_STATUS */
5179 tmp = RREG32(SRBM_STATUS);
5180
5181 if (tmp & IH_BUSY)
5182 reset_mask |= RADEON_RESET_IH;
5183
5184 if (tmp & SEM_BUSY)
5185 reset_mask |= RADEON_RESET_SEM;
5186
5187 if (tmp & GRBM_RQ_PENDING)
5188 reset_mask |= RADEON_RESET_GRBM;
5189
5190 if (tmp & VMC_BUSY)
5191 reset_mask |= RADEON_RESET_VMC;
5192
5193 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5194 MCC_BUSY | MCD_BUSY))
5195 reset_mask |= RADEON_RESET_MC;
5196
5197 if (evergreen_is_display_hung(rdev))
5198 reset_mask |= RADEON_RESET_DISPLAY;
5199
5200 /* Skip MC reset as it's mostly likely not hung, just busy */
5201 if (reset_mask & RADEON_RESET_MC) {
5202 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5203 reset_mask &= ~RADEON_RESET_MC;
5204 }
5205
5206 return reset_mask;
5207 }
5208
5209 /**
5210 * cik_gpu_soft_reset - soft reset GPU
5211 *
5212 * @rdev: radeon_device pointer
5213 * @reset_mask: mask of which blocks to reset
5214 *
5215 * Soft reset the blocks specified in @reset_mask.
5216 */
5217 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5218 {
5219 struct evergreen_mc_save save;
5220 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5221 u32 tmp;
5222
5223 if (reset_mask == 0)
5224 return;
5225
5226 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5227
5228 cik_print_gpu_status_regs(rdev);
5229 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5230 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5231 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5232 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5233
5234 /* disable CG/PG */
5235 cik_fini_pg(rdev);
5236 cik_fini_cg(rdev);
5237
5238 /* stop the rlc */
5239 cik_rlc_stop(rdev);
5240
5241 /* Disable GFX parsing/prefetching */
5242 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5243
5244 /* Disable MEC parsing/prefetching */
5245 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5246
5247 if (reset_mask & RADEON_RESET_DMA) {
5248 /* sdma0 */
5249 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5250 tmp |= SDMA_HALT;
5251 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5252 }
5253 if (reset_mask & RADEON_RESET_DMA1) {
5254 /* sdma1 */
5255 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5256 tmp |= SDMA_HALT;
5257 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5258 }
5259
5260 evergreen_mc_stop(rdev, &save);
5261 if (evergreen_mc_wait_for_idle(rdev)) {
5262 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5263 }
5264
5265 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5266 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5267
5268 if (reset_mask & RADEON_RESET_CP) {
5269 grbm_soft_reset |= SOFT_RESET_CP;
5270
5271 srbm_soft_reset |= SOFT_RESET_GRBM;
5272 }
5273
5274 if (reset_mask & RADEON_RESET_DMA)
5275 srbm_soft_reset |= SOFT_RESET_SDMA;
5276
5277 if (reset_mask & RADEON_RESET_DMA1)
5278 srbm_soft_reset |= SOFT_RESET_SDMA1;
5279
5280 if (reset_mask & RADEON_RESET_DISPLAY)
5281 srbm_soft_reset |= SOFT_RESET_DC;
5282
5283 if (reset_mask & RADEON_RESET_RLC)
5284 grbm_soft_reset |= SOFT_RESET_RLC;
5285
5286 if (reset_mask & RADEON_RESET_SEM)
5287 srbm_soft_reset |= SOFT_RESET_SEM;
5288
5289 if (reset_mask & RADEON_RESET_IH)
5290 srbm_soft_reset |= SOFT_RESET_IH;
5291
5292 if (reset_mask & RADEON_RESET_GRBM)
5293 srbm_soft_reset |= SOFT_RESET_GRBM;
5294
5295 if (reset_mask & RADEON_RESET_VMC)
5296 srbm_soft_reset |= SOFT_RESET_VMC;
5297
5298 if (!(rdev->flags & RADEON_IS_IGP)) {
5299 if (reset_mask & RADEON_RESET_MC)
5300 srbm_soft_reset |= SOFT_RESET_MC;
5301 }
5302
5303 if (grbm_soft_reset) {
5304 tmp = RREG32(GRBM_SOFT_RESET);
5305 tmp |= grbm_soft_reset;
5306 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5307 WREG32(GRBM_SOFT_RESET, tmp);
5308 tmp = RREG32(GRBM_SOFT_RESET);
5309
5310 udelay(50);
5311
5312 tmp &= ~grbm_soft_reset;
5313 WREG32(GRBM_SOFT_RESET, tmp);
5314 tmp = RREG32(GRBM_SOFT_RESET);
5315 }
5316
5317 if (srbm_soft_reset) {
5318 tmp = RREG32(SRBM_SOFT_RESET);
5319 tmp |= srbm_soft_reset;
5320 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5321 WREG32(SRBM_SOFT_RESET, tmp);
5322 tmp = RREG32(SRBM_SOFT_RESET);
5323
5324 udelay(50);
5325
5326 tmp &= ~srbm_soft_reset;
5327 WREG32(SRBM_SOFT_RESET, tmp);
5328 tmp = RREG32(SRBM_SOFT_RESET);
5329 }
5330
5331 /* Wait a little for things to settle down */
5332 udelay(50);
5333
5334 evergreen_mc_resume(rdev, &save);
5335 udelay(50);
5336
5337 cik_print_gpu_status_regs(rdev);
5338 }
5339
5340 struct kv_reset_save_regs {
5341 u32 gmcon_reng_execute;
5342 u32 gmcon_misc;
5343 u32 gmcon_misc3;
5344 };
5345
5346 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5347 struct kv_reset_save_regs *save)
5348 {
5349 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5350 save->gmcon_misc = RREG32(GMCON_MISC);
5351 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5352
5353 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5354 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5355 STCTRL_STUTTER_EN));
5356 }
5357
5358 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5359 struct kv_reset_save_regs *save)
5360 {
5361 int i;
5362
5363 WREG32(GMCON_PGFSM_WRITE, 0);
5364 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5365
5366 for (i = 0; i < 5; i++)
5367 WREG32(GMCON_PGFSM_WRITE, 0);
5368
5369 WREG32(GMCON_PGFSM_WRITE, 0);
5370 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5371
5372 for (i = 0; i < 5; i++)
5373 WREG32(GMCON_PGFSM_WRITE, 0);
5374
5375 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5376 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5377
5378 for (i = 0; i < 5; i++)
5379 WREG32(GMCON_PGFSM_WRITE, 0);
5380
5381 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5382 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5383
5384 for (i = 0; i < 5; i++)
5385 WREG32(GMCON_PGFSM_WRITE, 0);
5386
5387 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5388 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5389
5390 for (i = 0; i < 5; i++)
5391 WREG32(GMCON_PGFSM_WRITE, 0);
5392
5393 WREG32(GMCON_PGFSM_WRITE, 0);
5394 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5395
5396 for (i = 0; i < 5; i++)
5397 WREG32(GMCON_PGFSM_WRITE, 0);
5398
5399 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5400 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5401
5402 for (i = 0; i < 5; i++)
5403 WREG32(GMCON_PGFSM_WRITE, 0);
5404
5405 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5406 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5407
5408 for (i = 0; i < 5; i++)
5409 WREG32(GMCON_PGFSM_WRITE, 0);
5410
5411 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5412 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5413
5414 for (i = 0; i < 5; i++)
5415 WREG32(GMCON_PGFSM_WRITE, 0);
5416
5417 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5418 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5419
5420 for (i = 0; i < 5; i++)
5421 WREG32(GMCON_PGFSM_WRITE, 0);
5422
5423 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5424 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5425
5426 WREG32(GMCON_MISC3, save->gmcon_misc3);
5427 WREG32(GMCON_MISC, save->gmcon_misc);
5428 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5429 }
5430
5431 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5432 {
5433 struct evergreen_mc_save save;
5434 struct kv_reset_save_regs kv_save = { 0 };
5435 u32 tmp, i;
5436
5437 dev_info(rdev->dev, "GPU pci config reset\n");
5438
5439 /* disable dpm? */
5440
5441 /* disable cg/pg */
5442 cik_fini_pg(rdev);
5443 cik_fini_cg(rdev);
5444
5445 /* Disable GFX parsing/prefetching */
5446 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5447
5448 /* Disable MEC parsing/prefetching */
5449 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5450
5451 /* sdma0 */
5452 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5453 tmp |= SDMA_HALT;
5454 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5455 /* sdma1 */
5456 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5457 tmp |= SDMA_HALT;
5458 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5459 /* XXX other engines? */
5460
5461 /* halt the rlc, disable cp internal ints */
5462 cik_rlc_stop(rdev);
5463
5464 udelay(50);
5465
5466 /* disable mem access */
5467 evergreen_mc_stop(rdev, &save);
5468 if (evergreen_mc_wait_for_idle(rdev)) {
5469 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5470 }
5471
5472 if (rdev->flags & RADEON_IS_IGP)
5473 kv_save_regs_for_reset(rdev, &kv_save);
5474
5475 /* disable BM */
5476 pci_clear_master(rdev->pdev);
5477 /* reset */
5478 radeon_pci_config_reset(rdev);
5479
5480 udelay(100);
5481
5482 /* wait for asic to come out of reset */
5483 for (i = 0; i < rdev->usec_timeout; i++) {
5484 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5485 break;
5486 udelay(1);
5487 }
5488
5489 /* does asic init need to be run first??? */
5490 if (rdev->flags & RADEON_IS_IGP)
5491 kv_restore_regs_for_reset(rdev, &kv_save);
5492 }
5493
5494 /**
5495 * cik_asic_reset - soft reset GPU
5496 *
5497 * @rdev: radeon_device pointer
5498 *
5499 * Look up which blocks are hung and attempt
5500 * to reset them.
5501 * Returns 0 for success.
5502 */
5503 int cik_asic_reset(struct radeon_device *rdev)
5504 {
5505 u32 reset_mask;
5506
5507 reset_mask = cik_gpu_check_soft_reset(rdev);
5508
5509 if (reset_mask)
5510 r600_set_bios_scratch_engine_hung(rdev, true);
5511
5512 /* try soft reset */
5513 cik_gpu_soft_reset(rdev, reset_mask);
5514
5515 reset_mask = cik_gpu_check_soft_reset(rdev);
5516
5517 /* try pci config reset */
5518 if (reset_mask && radeon_hard_reset)
5519 cik_gpu_pci_config_reset(rdev);
5520
5521 reset_mask = cik_gpu_check_soft_reset(rdev);
5522
5523 if (!reset_mask)
5524 r600_set_bios_scratch_engine_hung(rdev, false);
5525
5526 return 0;
5527 }
5528
5529 /**
5530 * cik_gfx_is_lockup - check if the 3D engine is locked up
5531 *
5532 * @rdev: radeon_device pointer
5533 * @ring: radeon_ring structure holding ring information
5534 *
5535 * Check if the 3D engine is locked up (CIK).
5536 * Returns true if the engine is locked, false if not.
5537 */
5538 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5539 {
5540 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5541
5542 if (!(reset_mask & (RADEON_RESET_GFX |
5543 RADEON_RESET_COMPUTE |
5544 RADEON_RESET_CP))) {
5545 radeon_ring_lockup_update(rdev, ring);
5546 return false;
5547 }
5548 return radeon_ring_test_lockup(rdev, ring);
5549 }
5550
5551 /* MC */
5552 /**
5553 * cik_mc_program - program the GPU memory controller
5554 *
5555 * @rdev: radeon_device pointer
5556 *
5557 * Set the location of vram, gart, and AGP in the GPU's
5558 * physical address space (CIK).
5559 */
5560 static void cik_mc_program(struct radeon_device *rdev)
5561 {
5562 struct evergreen_mc_save save;
5563 u32 tmp;
5564 int i, j;
5565
5566 /* Initialize HDP */
5567 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5568 WREG32((0x2c14 + j), 0x00000000);
5569 WREG32((0x2c18 + j), 0x00000000);
5570 WREG32((0x2c1c + j), 0x00000000);
5571 WREG32((0x2c20 + j), 0x00000000);
5572 WREG32((0x2c24 + j), 0x00000000);
5573 }
5574 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5575
5576 evergreen_mc_stop(rdev, &save);
5577 if (radeon_mc_wait_for_idle(rdev)) {
5578 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5579 }
5580 /* Lockout access through VGA aperture*/
5581 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5582 /* Update configuration */
5583 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5584 rdev->mc.vram_start >> 12);
5585 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5586 rdev->mc.vram_end >> 12);
5587 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5588 rdev->vram_scratch.gpu_addr >> 12);
5589 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5590 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5591 WREG32(MC_VM_FB_LOCATION, tmp);
5592 /* XXX double check these! */
5593 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5594 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5595 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5596 WREG32(MC_VM_AGP_BASE, 0);
5597 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5598 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5599 if (radeon_mc_wait_for_idle(rdev)) {
5600 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5601 }
5602 evergreen_mc_resume(rdev, &save);
5603 /* we need to own VRAM, so turn off the VGA renderer here
5604 * to stop it overwriting our objects */
5605 rv515_vga_render_disable(rdev);
5606 }
5607
5608 /**
5609 * cik_mc_init - initialize the memory controller driver params
5610 *
5611 * @rdev: radeon_device pointer
5612 *
5613 * Look up the amount of vram, vram width, and decide how to place
5614 * vram and gart within the GPU's physical address space (CIK).
5615 * Returns 0 for success.
5616 */
5617 static int cik_mc_init(struct radeon_device *rdev)
5618 {
5619 u32 tmp;
5620 int chansize, numchan;
5621
5622 /* Get VRAM informations */
5623 rdev->mc.vram_is_ddr = true;
5624 tmp = RREG32(MC_ARB_RAMCFG);
5625 if (tmp & CHANSIZE_MASK) {
5626 chansize = 64;
5627 } else {
5628 chansize = 32;
5629 }
5630 tmp = RREG32(MC_SHARED_CHMAP);
5631 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5632 case 0:
5633 default:
5634 numchan = 1;
5635 break;
5636 case 1:
5637 numchan = 2;
5638 break;
5639 case 2:
5640 numchan = 4;
5641 break;
5642 case 3:
5643 numchan = 8;
5644 break;
5645 case 4:
5646 numchan = 3;
5647 break;
5648 case 5:
5649 numchan = 6;
5650 break;
5651 case 6:
5652 numchan = 10;
5653 break;
5654 case 7:
5655 numchan = 12;
5656 break;
5657 case 8:
5658 numchan = 16;
5659 break;
5660 }
5661 rdev->mc.vram_width = numchan * chansize;
5662 /* Could aper size report 0 ? */
5663 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5664 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5665 /* size in MB on si */
5666 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5667 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5668 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5669 si_vram_gtt_location(rdev, &rdev->mc);
5670 radeon_update_bandwidth_info(rdev);
5671
5672 return 0;
5673 }
5674
5675 /*
5676 * GART
5677 * VMID 0 is the physical GPU addresses as used by the kernel.
5678 * VMIDs 1-15 are used for userspace clients and are handled
5679 * by the radeon vm/hsa code.
5680 */
5681 /**
5682 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5683 *
5684 * @rdev: radeon_device pointer
5685 *
5686 * Flush the TLB for the VMID 0 page table (CIK).
5687 */
5688 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5689 {
5690 /* flush hdp cache */
5691 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5692
5693 /* bits 0-15 are the VM contexts0-15 */
5694 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5695 }
5696
5697 /**
5698 * cik_pcie_gart_enable - gart enable
5699 *
5700 * @rdev: radeon_device pointer
5701 *
5702 * This sets up the TLBs, programs the page tables for VMID0,
5703 * sets up the hw for VMIDs 1-15 which are allocated on
5704 * demand, and sets up the global locations for the LDS, GDS,
5705 * and GPUVM for FSA64 clients (CIK).
5706 * Returns 0 for success, errors for failure.
5707 */
5708 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5709 {
5710 int r, i;
5711
5712 if (rdev->gart.robj == NULL) {
5713 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5714 return -EINVAL;
5715 }
5716 r = radeon_gart_table_vram_pin(rdev);
5717 if (r)
5718 return r;
5719 /* Setup TLB control */
5720 WREG32(MC_VM_MX_L1_TLB_CNTL,
5721 (0xA << 7) |
5722 ENABLE_L1_TLB |
5723 ENABLE_L1_FRAGMENT_PROCESSING |
5724 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5725 ENABLE_ADVANCED_DRIVER_MODEL |
5726 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5727 /* Setup L2 cache */
5728 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5729 ENABLE_L2_FRAGMENT_PROCESSING |
5730 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5731 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5732 EFFECTIVE_L2_QUEUE_SIZE(7) |
5733 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5734 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5735 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5736 BANK_SELECT(4) |
5737 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5738 /* setup context0 */
5739 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5740 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5741 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5742 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5743 (u32)(rdev->dummy_page.addr >> 12));
5744 WREG32(VM_CONTEXT0_CNTL2, 0);
5745 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5746 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5747
5748 WREG32(0x15D4, 0);
5749 WREG32(0x15D8, 0);
5750 WREG32(0x15DC, 0);
5751
5752 /* empty context1-15 */
5753 /* FIXME start with 4G, once using 2 level pt switch to full
5754 * vm size space
5755 */
5756 /* set vm size, must be a multiple of 4 */
5757 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5758 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5759 for (i = 1; i < 16; i++) {
5760 if (i < 8)
5761 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5762 rdev->gart.table_addr >> 12);
5763 else
5764 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5765 rdev->gart.table_addr >> 12);
5766 }
5767
5768 /* enable context1-15 */
5769 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5770 (u32)(rdev->dummy_page.addr >> 12));
5771 WREG32(VM_CONTEXT1_CNTL2, 4);
5772 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5773 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5774 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5775 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5776 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5777 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5778 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5779 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5780 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5781 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5782 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5783 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5784 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5785 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5786
5787 if (rdev->family == CHIP_KAVERI) {
5788 u32 tmp = RREG32(CHUB_CONTROL);
5789 tmp &= ~BYPASS_VM;
5790 WREG32(CHUB_CONTROL, tmp);
5791 }
5792
5793 /* XXX SH_MEM regs */
5794 /* where to put LDS, scratch, GPUVM in FSA64 space */
5795 mutex_lock(&rdev->srbm_mutex);
5796 for (i = 0; i < 16; i++) {
5797 cik_srbm_select(rdev, 0, 0, 0, i);
5798 /* CP and shaders */
5799 WREG32(SH_MEM_CONFIG, 0);
5800 WREG32(SH_MEM_APE1_BASE, 1);
5801 WREG32(SH_MEM_APE1_LIMIT, 0);
5802 WREG32(SH_MEM_BASES, 0);
5803 /* SDMA GFX */
5804 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5805 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5806 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5807 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5808 /* XXX SDMA RLC - todo */
5809 }
5810 cik_srbm_select(rdev, 0, 0, 0, 0);
5811 mutex_unlock(&rdev->srbm_mutex);
5812
5813 cik_pcie_gart_tlb_flush(rdev);
5814 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5815 (unsigned)(rdev->mc.gtt_size >> 20),
5816 (unsigned long long)rdev->gart.table_addr);
5817 rdev->gart.ready = true;
5818 return 0;
5819 }
5820
5821 /**
5822 * cik_pcie_gart_disable - gart disable
5823 *
5824 * @rdev: radeon_device pointer
5825 *
5826 * This disables all VM page table (CIK).
5827 */
5828 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5829 {
5830 /* Disable all tables */
5831 WREG32(VM_CONTEXT0_CNTL, 0);
5832 WREG32(VM_CONTEXT1_CNTL, 0);
5833 /* Setup TLB control */
5834 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5835 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5836 /* Setup L2 cache */
5837 WREG32(VM_L2_CNTL,
5838 ENABLE_L2_FRAGMENT_PROCESSING |
5839 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5840 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5841 EFFECTIVE_L2_QUEUE_SIZE(7) |
5842 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5843 WREG32(VM_L2_CNTL2, 0);
5844 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5845 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5846 radeon_gart_table_vram_unpin(rdev);
5847 }
5848
5849 /**
5850 * cik_pcie_gart_fini - vm fini callback
5851 *
5852 * @rdev: radeon_device pointer
5853 *
5854 * Tears down the driver GART/VM setup (CIK).
5855 */
5856 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5857 {
5858 cik_pcie_gart_disable(rdev);
5859 radeon_gart_table_vram_free(rdev);
5860 radeon_gart_fini(rdev);
5861 }
5862
5863 /* vm parser */
5864 /**
5865 * cik_ib_parse - vm ib_parse callback
5866 *
5867 * @rdev: radeon_device pointer
5868 * @ib: indirect buffer pointer
5869 *
5870 * CIK uses hw IB checking so this is a nop (CIK).
5871 */
5872 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5873 {
5874 return 0;
5875 }
5876
5877 /*
5878 * vm
5879 * VMID 0 is the physical GPU addresses as used by the kernel.
5880 * VMIDs 1-15 are used for userspace clients and are handled
5881 * by the radeon vm/hsa code.
5882 */
5883 /**
5884 * cik_vm_init - cik vm init callback
5885 *
5886 * @rdev: radeon_device pointer
5887 *
5888 * Inits cik specific vm parameters (number of VMs, base of vram for
5889 * VMIDs 1-15) (CIK).
5890 * Returns 0 for success.
5891 */
5892 int cik_vm_init(struct radeon_device *rdev)
5893 {
5894 /* number of VMs */
5895 rdev->vm_manager.nvm = 16;
5896 /* base offset of vram pages */
5897 if (rdev->flags & RADEON_IS_IGP) {
5898 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5899 tmp <<= 22;
5900 rdev->vm_manager.vram_base_offset = tmp;
5901 } else
5902 rdev->vm_manager.vram_base_offset = 0;
5903
5904 return 0;
5905 }
5906
5907 /**
5908 * cik_vm_fini - cik vm fini callback
5909 *
5910 * @rdev: radeon_device pointer
5911 *
5912 * Tear down any asic specific VM setup (CIK).
5913 */
5914 void cik_vm_fini(struct radeon_device *rdev)
5915 {
5916 }
5917
5918 /**
5919 * cik_vm_decode_fault - print human readable fault info
5920 *
5921 * @rdev: radeon_device pointer
5922 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5923 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5924 *
5925 * Print human readable fault information (CIK).
5926 */
5927 static void cik_vm_decode_fault(struct radeon_device *rdev,
5928 u32 status, u32 addr, u32 mc_client)
5929 {
5930 u32 mc_id;
5931 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5932 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5933 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5934 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5935
5936 if (rdev->family == CHIP_HAWAII)
5937 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5938 else
5939 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5940
5941 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5942 protections, vmid, addr,
5943 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5944 block, mc_client, mc_id);
5945 }
5946
5947 /**
5948 * cik_vm_flush - cik vm flush using the CP
5949 *
5950 * @rdev: radeon_device pointer
5951 *
5952 * Update the page table base and flush the VM TLB
5953 * using the CP (CIK).
5954 */
5955 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5956 {
5957 struct radeon_ring *ring = &rdev->ring[ridx];
5958 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5959
5960 if (vm == NULL)
5961 return;
5962
5963 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5964 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5965 WRITE_DATA_DST_SEL(0)));
5966 if (vm->id < 8) {
5967 radeon_ring_write(ring,
5968 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5969 } else {
5970 radeon_ring_write(ring,
5971 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5972 }
5973 radeon_ring_write(ring, 0);
5974 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5975
5976 /* update SH_MEM_* regs */
5977 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5978 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5979 WRITE_DATA_DST_SEL(0)));
5980 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5981 radeon_ring_write(ring, 0);
5982 radeon_ring_write(ring, VMID(vm->id));
5983
5984 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5985 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5986 WRITE_DATA_DST_SEL(0)));
5987 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5988 radeon_ring_write(ring, 0);
5989
5990 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5991 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5992 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5993 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5994
5995 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5996 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5997 WRITE_DATA_DST_SEL(0)));
5998 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5999 radeon_ring_write(ring, 0);
6000 radeon_ring_write(ring, VMID(0));
6001
6002 /* HDP flush */
6003 cik_hdp_flush_cp_ring_emit(rdev, ridx);
6004
6005 /* bits 0-15 are the VM contexts0-15 */
6006 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6007 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6008 WRITE_DATA_DST_SEL(0)));
6009 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6010 radeon_ring_write(ring, 0);
6011 radeon_ring_write(ring, 1 << vm->id);
6012
6013 /* compute doesn't have PFP */
6014 if (usepfp) {
6015 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6016 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6017 radeon_ring_write(ring, 0x0);
6018 }
6019 }
6020
6021 /*
6022 * RLC
6023 * The RLC is a multi-purpose microengine that handles a
6024 * variety of functions, the most important of which is
6025 * the interrupt controller.
6026 */
6027 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6028 bool enable)
6029 {
6030 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6031
6032 if (enable)
6033 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6034 else
6035 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6036 WREG32(CP_INT_CNTL_RING0, tmp);
6037 }
6038
6039 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6040 {
6041 u32 tmp;
6042
6043 tmp = RREG32(RLC_LB_CNTL);
6044 if (enable)
6045 tmp |= LOAD_BALANCE_ENABLE;
6046 else
6047 tmp &= ~LOAD_BALANCE_ENABLE;
6048 WREG32(RLC_LB_CNTL, tmp);
6049 }
6050
6051 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6052 {
6053 u32 i, j, k;
6054 u32 mask;
6055
6056 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6057 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6058 cik_select_se_sh(rdev, i, j);
6059 for (k = 0; k < rdev->usec_timeout; k++) {
6060 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6061 break;
6062 udelay(1);
6063 }
6064 }
6065 }
6066 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6067
6068 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6069 for (k = 0; k < rdev->usec_timeout; k++) {
6070 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6071 break;
6072 udelay(1);
6073 }
6074 }
6075
6076 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6077 {
6078 u32 tmp;
6079
6080 tmp = RREG32(RLC_CNTL);
6081 if (tmp != rlc)
6082 WREG32(RLC_CNTL, rlc);
6083 }
6084
6085 static u32 cik_halt_rlc(struct radeon_device *rdev)
6086 {
6087 u32 data, orig;
6088
6089 orig = data = RREG32(RLC_CNTL);
6090
6091 if (data & RLC_ENABLE) {
6092 u32 i;
6093
6094 data &= ~RLC_ENABLE;
6095 WREG32(RLC_CNTL, data);
6096
6097 for (i = 0; i < rdev->usec_timeout; i++) {
6098 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6099 break;
6100 udelay(1);
6101 }
6102
6103 cik_wait_for_rlc_serdes(rdev);
6104 }
6105
6106 return orig;
6107 }
6108
6109 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6110 {
6111 u32 tmp, i, mask;
6112
6113 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6114 WREG32(RLC_GPR_REG2, tmp);
6115
6116 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6117 for (i = 0; i < rdev->usec_timeout; i++) {
6118 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6119 break;
6120 udelay(1);
6121 }
6122
6123 for (i = 0; i < rdev->usec_timeout; i++) {
6124 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6125 break;
6126 udelay(1);
6127 }
6128 }
6129
6130 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6131 {
6132 u32 tmp;
6133
6134 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6135 WREG32(RLC_GPR_REG2, tmp);
6136 }
6137
6138 /**
6139 * cik_rlc_stop - stop the RLC ME
6140 *
6141 * @rdev: radeon_device pointer
6142 *
6143 * Halt the RLC ME (MicroEngine) (CIK).
6144 */
6145 static void cik_rlc_stop(struct radeon_device *rdev)
6146 {
6147 WREG32(RLC_CNTL, 0);
6148
6149 cik_enable_gui_idle_interrupt(rdev, false);
6150
6151 cik_wait_for_rlc_serdes(rdev);
6152 }
6153
6154 /**
6155 * cik_rlc_start - start the RLC ME
6156 *
6157 * @rdev: radeon_device pointer
6158 *
6159 * Unhalt the RLC ME (MicroEngine) (CIK).
6160 */
6161 static void cik_rlc_start(struct radeon_device *rdev)
6162 {
6163 WREG32(RLC_CNTL, RLC_ENABLE);
6164
6165 cik_enable_gui_idle_interrupt(rdev, true);
6166
6167 udelay(50);
6168 }
6169
6170 /**
6171 * cik_rlc_resume - setup the RLC hw
6172 *
6173 * @rdev: radeon_device pointer
6174 *
6175 * Initialize the RLC registers, load the ucode,
6176 * and start the RLC (CIK).
6177 * Returns 0 for success, -EINVAL if the ucode is not available.
6178 */
6179 static int cik_rlc_resume(struct radeon_device *rdev)
6180 {
6181 u32 i, size, tmp;
6182
6183 if (!rdev->rlc_fw)
6184 return -EINVAL;
6185
6186 cik_rlc_stop(rdev);
6187
6188 /* disable CG */
6189 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6190 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6191
6192 si_rlc_reset(rdev);
6193
6194 cik_init_pg(rdev);
6195
6196 cik_init_cg(rdev);
6197
6198 WREG32(RLC_LB_CNTR_INIT, 0);
6199 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6200
6201 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6202 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6203 WREG32(RLC_LB_PARAMS, 0x00600408);
6204 WREG32(RLC_LB_CNTL, 0x80000004);
6205
6206 WREG32(RLC_MC_CNTL, 0);
6207 WREG32(RLC_UCODE_CNTL, 0);
6208
6209 if (rdev->new_fw) {
6210 const struct rlc_firmware_header_v1_0 *hdr =
6211 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6212 const __le32 *fw_data = (const __le32 *)
6213 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6214
6215 radeon_ucode_print_rlc_hdr(&hdr->header);
6216
6217 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6218 WREG32(RLC_GPM_UCODE_ADDR, 0);
6219 for (i = 0; i < size; i++)
6220 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6221 WREG32(RLC_GPM_UCODE_ADDR, 0);
6222 } else {
6223 const __be32 *fw_data;
6224
6225 switch (rdev->family) {
6226 case CHIP_BONAIRE:
6227 case CHIP_HAWAII:
6228 default:
6229 size = BONAIRE_RLC_UCODE_SIZE;
6230 break;
6231 case CHIP_KAVERI:
6232 size = KV_RLC_UCODE_SIZE;
6233 break;
6234 case CHIP_KABINI:
6235 size = KB_RLC_UCODE_SIZE;
6236 break;
6237 case CHIP_MULLINS:
6238 size = ML_RLC_UCODE_SIZE;
6239 break;
6240 }
6241
6242 fw_data = (const __be32 *)rdev->rlc_fw->data;
6243 WREG32(RLC_GPM_UCODE_ADDR, 0);
6244 for (i = 0; i < size; i++)
6245 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6246 WREG32(RLC_GPM_UCODE_ADDR, 0);
6247 }
6248
6249 /* XXX - find out what chips support lbpw */
6250 cik_enable_lbpw(rdev, false);
6251
6252 if (rdev->family == CHIP_BONAIRE)
6253 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6254
6255 cik_rlc_start(rdev);
6256
6257 return 0;
6258 }
6259
6260 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6261 {
6262 u32 data, orig, tmp, tmp2;
6263
6264 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6265
6266 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6267 cik_enable_gui_idle_interrupt(rdev, true);
6268
6269 tmp = cik_halt_rlc(rdev);
6270
6271 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6272 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6273 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6274 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6275 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6276
6277 cik_update_rlc(rdev, tmp);
6278
6279 data |= CGCG_EN | CGLS_EN;
6280 } else {
6281 cik_enable_gui_idle_interrupt(rdev, false);
6282
6283 RREG32(CB_CGTT_SCLK_CTRL);
6284 RREG32(CB_CGTT_SCLK_CTRL);
6285 RREG32(CB_CGTT_SCLK_CTRL);
6286 RREG32(CB_CGTT_SCLK_CTRL);
6287
6288 data &= ~(CGCG_EN | CGLS_EN);
6289 }
6290
6291 if (orig != data)
6292 WREG32(RLC_CGCG_CGLS_CTRL, data);
6293
6294 }
6295
6296 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6297 {
6298 u32 data, orig, tmp = 0;
6299
6300 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6301 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6302 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6303 orig = data = RREG32(CP_MEM_SLP_CNTL);
6304 data |= CP_MEM_LS_EN;
6305 if (orig != data)
6306 WREG32(CP_MEM_SLP_CNTL, data);
6307 }
6308 }
6309
6310 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6311 data &= 0xfffffffd;
6312 if (orig != data)
6313 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6314
6315 tmp = cik_halt_rlc(rdev);
6316
6317 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6318 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6319 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6320 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6321 WREG32(RLC_SERDES_WR_CTRL, data);
6322
6323 cik_update_rlc(rdev, tmp);
6324
6325 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6326 orig = data = RREG32(CGTS_SM_CTRL_REG);
6327 data &= ~SM_MODE_MASK;
6328 data |= SM_MODE(0x2);
6329 data |= SM_MODE_ENABLE;
6330 data &= ~CGTS_OVERRIDE;
6331 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6332 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6333 data &= ~CGTS_LS_OVERRIDE;
6334 data &= ~ON_MONITOR_ADD_MASK;
6335 data |= ON_MONITOR_ADD_EN;
6336 data |= ON_MONITOR_ADD(0x96);
6337 if (orig != data)
6338 WREG32(CGTS_SM_CTRL_REG, data);
6339 }
6340 } else {
6341 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6342 data |= 0x00000002;
6343 if (orig != data)
6344 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6345
6346 data = RREG32(RLC_MEM_SLP_CNTL);
6347 if (data & RLC_MEM_LS_EN) {
6348 data &= ~RLC_MEM_LS_EN;
6349 WREG32(RLC_MEM_SLP_CNTL, data);
6350 }
6351
6352 data = RREG32(CP_MEM_SLP_CNTL);
6353 if (data & CP_MEM_LS_EN) {
6354 data &= ~CP_MEM_LS_EN;
6355 WREG32(CP_MEM_SLP_CNTL, data);
6356 }
6357
6358 orig = data = RREG32(CGTS_SM_CTRL_REG);
6359 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6360 if (orig != data)
6361 WREG32(CGTS_SM_CTRL_REG, data);
6362
6363 tmp = cik_halt_rlc(rdev);
6364
6365 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6366 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6367 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6368 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6369 WREG32(RLC_SERDES_WR_CTRL, data);
6370
6371 cik_update_rlc(rdev, tmp);
6372 }
6373 }
6374
6375 static const u32 mc_cg_registers[] =
6376 {
6377 MC_HUB_MISC_HUB_CG,
6378 MC_HUB_MISC_SIP_CG,
6379 MC_HUB_MISC_VM_CG,
6380 MC_XPB_CLK_GAT,
6381 ATC_MISC_CG,
6382 MC_CITF_MISC_WR_CG,
6383 MC_CITF_MISC_RD_CG,
6384 MC_CITF_MISC_VM_CG,
6385 VM_L2_CG,
6386 };
6387
6388 static void cik_enable_mc_ls(struct radeon_device *rdev,
6389 bool enable)
6390 {
6391 int i;
6392 u32 orig, data;
6393
6394 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6395 orig = data = RREG32(mc_cg_registers[i]);
6396 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6397 data |= MC_LS_ENABLE;
6398 else
6399 data &= ~MC_LS_ENABLE;
6400 if (data != orig)
6401 WREG32(mc_cg_registers[i], data);
6402 }
6403 }
6404
6405 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6406 bool enable)
6407 {
6408 int i;
6409 u32 orig, data;
6410
6411 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6412 orig = data = RREG32(mc_cg_registers[i]);
6413 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6414 data |= MC_CG_ENABLE;
6415 else
6416 data &= ~MC_CG_ENABLE;
6417 if (data != orig)
6418 WREG32(mc_cg_registers[i], data);
6419 }
6420 }
6421
6422 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6423 bool enable)
6424 {
6425 u32 orig, data;
6426
6427 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6428 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6429 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6430 } else {
6431 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6432 data |= 0xff000000;
6433 if (data != orig)
6434 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6435
6436 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6437 data |= 0xff000000;
6438 if (data != orig)
6439 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6440 }
6441 }
6442
6443 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6444 bool enable)
6445 {
6446 u32 orig, data;
6447
6448 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6449 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6450 data |= 0x100;
6451 if (orig != data)
6452 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6453
6454 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6455 data |= 0x100;
6456 if (orig != data)
6457 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6458 } else {
6459 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6460 data &= ~0x100;
6461 if (orig != data)
6462 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6463
6464 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6465 data &= ~0x100;
6466 if (orig != data)
6467 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6468 }
6469 }
6470
6471 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6472 bool enable)
6473 {
6474 u32 orig, data;
6475
6476 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6477 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6478 data = 0xfff;
6479 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6480
6481 orig = data = RREG32(UVD_CGC_CTRL);
6482 data |= DCM;
6483 if (orig != data)
6484 WREG32(UVD_CGC_CTRL, data);
6485 } else {
6486 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6487 data &= ~0xfff;
6488 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6489
6490 orig = data = RREG32(UVD_CGC_CTRL);
6491 data &= ~DCM;
6492 if (orig != data)
6493 WREG32(UVD_CGC_CTRL, data);
6494 }
6495 }
6496
6497 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6498 bool enable)
6499 {
6500 u32 orig, data;
6501
6502 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6503
6504 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6505 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6506 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6507 else
6508 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6509 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6510
6511 if (orig != data)
6512 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6513 }
6514
6515 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6516 bool enable)
6517 {
6518 u32 orig, data;
6519
6520 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6521
6522 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6523 data &= ~CLOCK_GATING_DIS;
6524 else
6525 data |= CLOCK_GATING_DIS;
6526
6527 if (orig != data)
6528 WREG32(HDP_HOST_PATH_CNTL, data);
6529 }
6530
6531 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6532 bool enable)
6533 {
6534 u32 orig, data;
6535
6536 orig = data = RREG32(HDP_MEM_POWER_LS);
6537
6538 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6539 data |= HDP_LS_ENABLE;
6540 else
6541 data &= ~HDP_LS_ENABLE;
6542
6543 if (orig != data)
6544 WREG32(HDP_MEM_POWER_LS, data);
6545 }
6546
6547 void cik_update_cg(struct radeon_device *rdev,
6548 u32 block, bool enable)
6549 {
6550
6551 if (block & RADEON_CG_BLOCK_GFX) {
6552 cik_enable_gui_idle_interrupt(rdev, false);
6553 /* order matters! */
6554 if (enable) {
6555 cik_enable_mgcg(rdev, true);
6556 cik_enable_cgcg(rdev, true);
6557 } else {
6558 cik_enable_cgcg(rdev, false);
6559 cik_enable_mgcg(rdev, false);
6560 }
6561 cik_enable_gui_idle_interrupt(rdev, true);
6562 }
6563
6564 if (block & RADEON_CG_BLOCK_MC) {
6565 if (!(rdev->flags & RADEON_IS_IGP)) {
6566 cik_enable_mc_mgcg(rdev, enable);
6567 cik_enable_mc_ls(rdev, enable);
6568 }
6569 }
6570
6571 if (block & RADEON_CG_BLOCK_SDMA) {
6572 cik_enable_sdma_mgcg(rdev, enable);
6573 cik_enable_sdma_mgls(rdev, enable);
6574 }
6575
6576 if (block & RADEON_CG_BLOCK_BIF) {
6577 cik_enable_bif_mgls(rdev, enable);
6578 }
6579
6580 if (block & RADEON_CG_BLOCK_UVD) {
6581 if (rdev->has_uvd)
6582 cik_enable_uvd_mgcg(rdev, enable);
6583 }
6584
6585 if (block & RADEON_CG_BLOCK_HDP) {
6586 cik_enable_hdp_mgcg(rdev, enable);
6587 cik_enable_hdp_ls(rdev, enable);
6588 }
6589
6590 if (block & RADEON_CG_BLOCK_VCE) {
6591 vce_v2_0_enable_mgcg(rdev, enable);
6592 }
6593 }
6594
6595 static void cik_init_cg(struct radeon_device *rdev)
6596 {
6597
6598 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6599
6600 if (rdev->has_uvd)
6601 si_init_uvd_internal_cg(rdev);
6602
6603 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6604 RADEON_CG_BLOCK_SDMA |
6605 RADEON_CG_BLOCK_BIF |
6606 RADEON_CG_BLOCK_UVD |
6607 RADEON_CG_BLOCK_HDP), true);
6608 }
6609
6610 static void cik_fini_cg(struct radeon_device *rdev)
6611 {
6612 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6613 RADEON_CG_BLOCK_SDMA |
6614 RADEON_CG_BLOCK_BIF |
6615 RADEON_CG_BLOCK_UVD |
6616 RADEON_CG_BLOCK_HDP), false);
6617
6618 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6619 }
6620
6621 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6622 bool enable)
6623 {
6624 u32 data, orig;
6625
6626 orig = data = RREG32(RLC_PG_CNTL);
6627 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6628 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6629 else
6630 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6631 if (orig != data)
6632 WREG32(RLC_PG_CNTL, data);
6633 }
6634
6635 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6636 bool enable)
6637 {
6638 u32 data, orig;
6639
6640 orig = data = RREG32(RLC_PG_CNTL);
6641 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6642 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6643 else
6644 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6645 if (orig != data)
6646 WREG32(RLC_PG_CNTL, data);
6647 }
6648
6649 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6650 {
6651 u32 data, orig;
6652
6653 orig = data = RREG32(RLC_PG_CNTL);
6654 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6655 data &= ~DISABLE_CP_PG;
6656 else
6657 data |= DISABLE_CP_PG;
6658 if (orig != data)
6659 WREG32(RLC_PG_CNTL, data);
6660 }
6661
6662 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6663 {
6664 u32 data, orig;
6665
6666 orig = data = RREG32(RLC_PG_CNTL);
6667 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6668 data &= ~DISABLE_GDS_PG;
6669 else
6670 data |= DISABLE_GDS_PG;
6671 if (orig != data)
6672 WREG32(RLC_PG_CNTL, data);
6673 }
6674
6675 #define CP_ME_TABLE_SIZE 96
6676 #define CP_ME_TABLE_OFFSET 2048
6677 #define CP_MEC_TABLE_OFFSET 4096
6678
6679 void cik_init_cp_pg_table(struct radeon_device *rdev)
6680 {
6681 volatile u32 *dst_ptr;
6682 int me, i, max_me = 4;
6683 u32 bo_offset = 0;
6684 u32 table_offset, table_size;
6685
6686 if (rdev->family == CHIP_KAVERI)
6687 max_me = 5;
6688
6689 if (rdev->rlc.cp_table_ptr == NULL)
6690 return;
6691
6692 /* write the cp table buffer */
6693 dst_ptr = rdev->rlc.cp_table_ptr;
6694 for (me = 0; me < max_me; me++) {
6695 if (rdev->new_fw) {
6696 const __le32 *fw_data;
6697 const struct gfx_firmware_header_v1_0 *hdr;
6698
6699 if (me == 0) {
6700 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6701 fw_data = (const __le32 *)
6702 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6703 table_offset = le32_to_cpu(hdr->jt_offset);
6704 table_size = le32_to_cpu(hdr->jt_size);
6705 } else if (me == 1) {
6706 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6707 fw_data = (const __le32 *)
6708 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6709 table_offset = le32_to_cpu(hdr->jt_offset);
6710 table_size = le32_to_cpu(hdr->jt_size);
6711 } else if (me == 2) {
6712 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6713 fw_data = (const __le32 *)
6714 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6715 table_offset = le32_to_cpu(hdr->jt_offset);
6716 table_size = le32_to_cpu(hdr->jt_size);
6717 } else if (me == 3) {
6718 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6719 fw_data = (const __le32 *)
6720 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6721 table_offset = le32_to_cpu(hdr->jt_offset);
6722 table_size = le32_to_cpu(hdr->jt_size);
6723 } else {
6724 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6725 fw_data = (const __le32 *)
6726 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6727 table_offset = le32_to_cpu(hdr->jt_offset);
6728 table_size = le32_to_cpu(hdr->jt_size);
6729 }
6730
6731 for (i = 0; i < table_size; i ++) {
6732 dst_ptr[bo_offset + i] =
6733 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6734 }
6735 bo_offset += table_size;
6736 } else {
6737 const __be32 *fw_data;
6738 table_size = CP_ME_TABLE_SIZE;
6739
6740 if (me == 0) {
6741 fw_data = (const __be32 *)rdev->ce_fw->data;
6742 table_offset = CP_ME_TABLE_OFFSET;
6743 } else if (me == 1) {
6744 fw_data = (const __be32 *)rdev->pfp_fw->data;
6745 table_offset = CP_ME_TABLE_OFFSET;
6746 } else if (me == 2) {
6747 fw_data = (const __be32 *)rdev->me_fw->data;
6748 table_offset = CP_ME_TABLE_OFFSET;
6749 } else {
6750 fw_data = (const __be32 *)rdev->mec_fw->data;
6751 table_offset = CP_MEC_TABLE_OFFSET;
6752 }
6753
6754 for (i = 0; i < table_size; i ++) {
6755 dst_ptr[bo_offset + i] =
6756 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6757 }
6758 bo_offset += table_size;
6759 }
6760 }
6761 }
6762
6763 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6764 bool enable)
6765 {
6766 u32 data, orig;
6767
6768 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6769 orig = data = RREG32(RLC_PG_CNTL);
6770 data |= GFX_PG_ENABLE;
6771 if (orig != data)
6772 WREG32(RLC_PG_CNTL, data);
6773
6774 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6775 data |= AUTO_PG_EN;
6776 if (orig != data)
6777 WREG32(RLC_AUTO_PG_CTRL, data);
6778 } else {
6779 orig = data = RREG32(RLC_PG_CNTL);
6780 data &= ~GFX_PG_ENABLE;
6781 if (orig != data)
6782 WREG32(RLC_PG_CNTL, data);
6783
6784 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6785 data &= ~AUTO_PG_EN;
6786 if (orig != data)
6787 WREG32(RLC_AUTO_PG_CTRL, data);
6788
6789 data = RREG32(DB_RENDER_CONTROL);
6790 }
6791 }
6792
6793 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6794 {
6795 u32 mask = 0, tmp, tmp1;
6796 int i;
6797
6798 cik_select_se_sh(rdev, se, sh);
6799 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6800 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6801 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6802
6803 tmp &= 0xffff0000;
6804
6805 tmp |= tmp1;
6806 tmp >>= 16;
6807
6808 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6809 mask <<= 1;
6810 mask |= 1;
6811 }
6812
6813 return (~tmp) & mask;
6814 }
6815
6816 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6817 {
6818 u32 i, j, k, active_cu_number = 0;
6819 u32 mask, counter, cu_bitmap;
6820 u32 tmp = 0;
6821
6822 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6823 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6824 mask = 1;
6825 cu_bitmap = 0;
6826 counter = 0;
6827 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6828 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6829 if (counter < 2)
6830 cu_bitmap |= mask;
6831 counter ++;
6832 }
6833 mask <<= 1;
6834 }
6835
6836 active_cu_number += counter;
6837 tmp |= (cu_bitmap << (i * 16 + j * 8));
6838 }
6839 }
6840
6841 WREG32(RLC_PG_AO_CU_MASK, tmp);
6842
6843 tmp = RREG32(RLC_MAX_PG_CU);
6844 tmp &= ~MAX_PU_CU_MASK;
6845 tmp |= MAX_PU_CU(active_cu_number);
6846 WREG32(RLC_MAX_PG_CU, tmp);
6847 }
6848
6849 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6850 bool enable)
6851 {
6852 u32 data, orig;
6853
6854 orig = data = RREG32(RLC_PG_CNTL);
6855 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6856 data |= STATIC_PER_CU_PG_ENABLE;
6857 else
6858 data &= ~STATIC_PER_CU_PG_ENABLE;
6859 if (orig != data)
6860 WREG32(RLC_PG_CNTL, data);
6861 }
6862
6863 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6864 bool enable)
6865 {
6866 u32 data, orig;
6867
6868 orig = data = RREG32(RLC_PG_CNTL);
6869 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6870 data |= DYN_PER_CU_PG_ENABLE;
6871 else
6872 data &= ~DYN_PER_CU_PG_ENABLE;
6873 if (orig != data)
6874 WREG32(RLC_PG_CNTL, data);
6875 }
6876
6877 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6878 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6879
6880 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6881 {
6882 u32 data, orig;
6883 u32 i;
6884
6885 if (rdev->rlc.cs_data) {
6886 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6887 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6888 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6889 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6890 } else {
6891 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6892 for (i = 0; i < 3; i++)
6893 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6894 }
6895 if (rdev->rlc.reg_list) {
6896 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6897 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6898 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6899 }
6900
6901 orig = data = RREG32(RLC_PG_CNTL);
6902 data |= GFX_PG_SRC;
6903 if (orig != data)
6904 WREG32(RLC_PG_CNTL, data);
6905
6906 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6907 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6908
6909 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6910 data &= ~IDLE_POLL_COUNT_MASK;
6911 data |= IDLE_POLL_COUNT(0x60);
6912 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6913
6914 data = 0x10101010;
6915 WREG32(RLC_PG_DELAY, data);
6916
6917 data = RREG32(RLC_PG_DELAY_2);
6918 data &= ~0xff;
6919 data |= 0x3;
6920 WREG32(RLC_PG_DELAY_2, data);
6921
6922 data = RREG32(RLC_AUTO_PG_CTRL);
6923 data &= ~GRBM_REG_SGIT_MASK;
6924 data |= GRBM_REG_SGIT(0x700);
6925 WREG32(RLC_AUTO_PG_CTRL, data);
6926
6927 }
6928
6929 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6930 {
6931 cik_enable_gfx_cgpg(rdev, enable);
6932 cik_enable_gfx_static_mgpg(rdev, enable);
6933 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6934 }
6935
6936 u32 cik_get_csb_size(struct radeon_device *rdev)
6937 {
6938 u32 count = 0;
6939 const struct cs_section_def *sect = NULL;
6940 const struct cs_extent_def *ext = NULL;
6941
6942 if (rdev->rlc.cs_data == NULL)
6943 return 0;
6944
6945 /* begin clear state */
6946 count += 2;
6947 /* context control state */
6948 count += 3;
6949
6950 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6951 for (ext = sect->section; ext->extent != NULL; ++ext) {
6952 if (sect->id == SECT_CONTEXT)
6953 count += 2 + ext->reg_count;
6954 else
6955 return 0;
6956 }
6957 }
6958 /* pa_sc_raster_config/pa_sc_raster_config1 */
6959 count += 4;
6960 /* end clear state */
6961 count += 2;
6962 /* clear state */
6963 count += 2;
6964
6965 return count;
6966 }
6967
6968 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6969 {
6970 u32 count = 0, i;
6971 const struct cs_section_def *sect = NULL;
6972 const struct cs_extent_def *ext = NULL;
6973
6974 if (rdev->rlc.cs_data == NULL)
6975 return;
6976 if (buffer == NULL)
6977 return;
6978
6979 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6980 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6981
6982 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6983 buffer[count++] = cpu_to_le32(0x80000000);
6984 buffer[count++] = cpu_to_le32(0x80000000);
6985
6986 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6987 for (ext = sect->section; ext->extent != NULL; ++ext) {
6988 if (sect->id == SECT_CONTEXT) {
6989 buffer[count++] =
6990 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6991 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6992 for (i = 0; i < ext->reg_count; i++)
6993 buffer[count++] = cpu_to_le32(ext->extent[i]);
6994 } else {
6995 return;
6996 }
6997 }
6998 }
6999
7000 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7001 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7002 switch (rdev->family) {
7003 case CHIP_BONAIRE:
7004 buffer[count++] = cpu_to_le32(0x16000012);
7005 buffer[count++] = cpu_to_le32(0x00000000);
7006 break;
7007 case CHIP_KAVERI:
7008 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7009 buffer[count++] = cpu_to_le32(0x00000000);
7010 break;
7011 case CHIP_KABINI:
7012 case CHIP_MULLINS:
7013 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7014 buffer[count++] = cpu_to_le32(0x00000000);
7015 break;
7016 case CHIP_HAWAII:
7017 buffer[count++] = cpu_to_le32(0x3a00161a);
7018 buffer[count++] = cpu_to_le32(0x0000002e);
7019 break;
7020 default:
7021 buffer[count++] = cpu_to_le32(0x00000000);
7022 buffer[count++] = cpu_to_le32(0x00000000);
7023 break;
7024 }
7025
7026 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7027 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7028
7029 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7030 buffer[count++] = cpu_to_le32(0);
7031 }
7032
7033 static void cik_init_pg(struct radeon_device *rdev)
7034 {
7035 if (rdev->pg_flags) {
7036 cik_enable_sck_slowdown_on_pu(rdev, true);
7037 cik_enable_sck_slowdown_on_pd(rdev, true);
7038 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7039 cik_init_gfx_cgpg(rdev);
7040 cik_enable_cp_pg(rdev, true);
7041 cik_enable_gds_pg(rdev, true);
7042 }
7043 cik_init_ao_cu_mask(rdev);
7044 cik_update_gfx_pg(rdev, true);
7045 }
7046 }
7047
7048 static void cik_fini_pg(struct radeon_device *rdev)
7049 {
7050 if (rdev->pg_flags) {
7051 cik_update_gfx_pg(rdev, false);
7052 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7053 cik_enable_cp_pg(rdev, false);
7054 cik_enable_gds_pg(rdev, false);
7055 }
7056 }
7057 }
7058
7059 /*
7060 * Interrupts
7061 * Starting with r6xx, interrupts are handled via a ring buffer.
7062 * Ring buffers are areas of GPU accessible memory that the GPU
7063 * writes interrupt vectors into and the host reads vectors out of.
7064 * There is a rptr (read pointer) that determines where the
7065 * host is currently reading, and a wptr (write pointer)
7066 * which determines where the GPU has written. When the
7067 * pointers are equal, the ring is idle. When the GPU
7068 * writes vectors to the ring buffer, it increments the
7069 * wptr. When there is an interrupt, the host then starts
7070 * fetching commands and processing them until the pointers are
7071 * equal again at which point it updates the rptr.
7072 */
7073
7074 /**
7075 * cik_enable_interrupts - Enable the interrupt ring buffer
7076 *
7077 * @rdev: radeon_device pointer
7078 *
7079 * Enable the interrupt ring buffer (CIK).
7080 */
7081 static void cik_enable_interrupts(struct radeon_device *rdev)
7082 {
7083 u32 ih_cntl = RREG32(IH_CNTL);
7084 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7085
7086 ih_cntl |= ENABLE_INTR;
7087 ih_rb_cntl |= IH_RB_ENABLE;
7088 WREG32(IH_CNTL, ih_cntl);
7089 WREG32(IH_RB_CNTL, ih_rb_cntl);
7090 rdev->ih.enabled = true;
7091 }
7092
7093 /**
7094 * cik_disable_interrupts - Disable the interrupt ring buffer
7095 *
7096 * @rdev: radeon_device pointer
7097 *
7098 * Disable the interrupt ring buffer (CIK).
7099 */
7100 static void cik_disable_interrupts(struct radeon_device *rdev)
7101 {
7102 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7103 u32 ih_cntl = RREG32(IH_CNTL);
7104
7105 ih_rb_cntl &= ~IH_RB_ENABLE;
7106 ih_cntl &= ~ENABLE_INTR;
7107 WREG32(IH_RB_CNTL, ih_rb_cntl);
7108 WREG32(IH_CNTL, ih_cntl);
7109 /* set rptr, wptr to 0 */
7110 WREG32(IH_RB_RPTR, 0);
7111 WREG32(IH_RB_WPTR, 0);
7112 rdev->ih.enabled = false;
7113 rdev->ih.rptr = 0;
7114 }
7115
7116 /**
7117 * cik_disable_interrupt_state - Disable all interrupt sources
7118 *
7119 * @rdev: radeon_device pointer
7120 *
7121 * Clear all interrupt enable bits used by the driver (CIK).
7122 */
7123 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7124 {
7125 u32 tmp;
7126
7127 /* gfx ring */
7128 tmp = RREG32(CP_INT_CNTL_RING0) &
7129 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7130 WREG32(CP_INT_CNTL_RING0, tmp);
7131 /* sdma */
7132 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7133 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7134 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7135 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7136 /* compute queues */
7137 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7138 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7139 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7140 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7141 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7142 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7143 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7144 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7145 /* grbm */
7146 WREG32(GRBM_INT_CNTL, 0);
7147 /* vline/vblank, etc. */
7148 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7149 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7150 if (rdev->num_crtc >= 4) {
7151 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7152 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7153 }
7154 if (rdev->num_crtc >= 6) {
7155 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7156 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7157 }
7158 /* pflip */
7159 if (rdev->num_crtc >= 2) {
7160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7161 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7162 }
7163 if (rdev->num_crtc >= 4) {
7164 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7165 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7166 }
7167 if (rdev->num_crtc >= 6) {
7168 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7169 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7170 }
7171
7172 /* dac hotplug */
7173 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7174
7175 /* digital hotplug */
7176 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7177 WREG32(DC_HPD1_INT_CONTROL, tmp);
7178 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7179 WREG32(DC_HPD2_INT_CONTROL, tmp);
7180 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7181 WREG32(DC_HPD3_INT_CONTROL, tmp);
7182 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7183 WREG32(DC_HPD4_INT_CONTROL, tmp);
7184 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7185 WREG32(DC_HPD5_INT_CONTROL, tmp);
7186 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7187 WREG32(DC_HPD6_INT_CONTROL, tmp);
7188
7189 }
7190
7191 /**
7192 * cik_irq_init - init and enable the interrupt ring
7193 *
7194 * @rdev: radeon_device pointer
7195 *
7196 * Allocate a ring buffer for the interrupt controller,
7197 * enable the RLC, disable interrupts, enable the IH
7198 * ring buffer and enable it (CIK).
7199 * Called at device load and reume.
7200 * Returns 0 for success, errors for failure.
7201 */
7202 static int cik_irq_init(struct radeon_device *rdev)
7203 {
7204 int ret = 0;
7205 int rb_bufsz;
7206 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7207
7208 /* allocate ring */
7209 ret = r600_ih_ring_alloc(rdev);
7210 if (ret)
7211 return ret;
7212
7213 /* disable irqs */
7214 cik_disable_interrupts(rdev);
7215
7216 /* init rlc */
7217 ret = cik_rlc_resume(rdev);
7218 if (ret) {
7219 r600_ih_ring_fini(rdev);
7220 return ret;
7221 }
7222
7223 /* setup interrupt control */
7224 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7225 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7226 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7227 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7228 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7229 */
7230 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7231 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7232 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7233 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7234
7235 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7236 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7237
7238 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7239 IH_WPTR_OVERFLOW_CLEAR |
7240 (rb_bufsz << 1));
7241
7242 if (rdev->wb.enabled)
7243 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7244
7245 /* set the writeback address whether it's enabled or not */
7246 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7247 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7248
7249 WREG32(IH_RB_CNTL, ih_rb_cntl);
7250
7251 /* set rptr, wptr to 0 */
7252 WREG32(IH_RB_RPTR, 0);
7253 WREG32(IH_RB_WPTR, 0);
7254
7255 /* Default settings for IH_CNTL (disabled at first) */
7256 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7257 /* RPTR_REARM only works if msi's are enabled */
7258 if (rdev->msi_enabled)
7259 ih_cntl |= RPTR_REARM;
7260 WREG32(IH_CNTL, ih_cntl);
7261
7262 /* force the active interrupt state to all disabled */
7263 cik_disable_interrupt_state(rdev);
7264
7265 pci_set_master(rdev->pdev);
7266
7267 /* enable irqs */
7268 cik_enable_interrupts(rdev);
7269
7270 return ret;
7271 }
7272
7273 /**
7274 * cik_irq_set - enable/disable interrupt sources
7275 *
7276 * @rdev: radeon_device pointer
7277 *
7278 * Enable interrupt sources on the GPU (vblanks, hpd,
7279 * etc.) (CIK).
7280 * Returns 0 for success, errors for failure.
7281 */
7282 int cik_irq_set(struct radeon_device *rdev)
7283 {
7284 u32 cp_int_cntl;
7285 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7286 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7287 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7288 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7289 u32 grbm_int_cntl = 0;
7290 u32 dma_cntl, dma_cntl1;
7291 u32 thermal_int;
7292
7293 if (!rdev->irq.installed) {
7294 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7295 return -EINVAL;
7296 }
7297 /* don't enable anything if the ih is disabled */
7298 if (!rdev->ih.enabled) {
7299 cik_disable_interrupts(rdev);
7300 /* force the active interrupt state to all disabled */
7301 cik_disable_interrupt_state(rdev);
7302 return 0;
7303 }
7304
7305 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7306 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7307 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7308
7309 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7310 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7311 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7312 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7313 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7314 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7315
7316 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7317 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7318
7319 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7320 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7321 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7322 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7323 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7324 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7325 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7326 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7327
7328 if (rdev->flags & RADEON_IS_IGP)
7329 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7330 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7331 else
7332 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7333 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7334
7335 /* enable CP interrupts on all rings */
7336 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7337 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7338 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7339 }
7340 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7341 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7342 DRM_DEBUG("si_irq_set: sw int cp1\n");
7343 if (ring->me == 1) {
7344 switch (ring->pipe) {
7345 case 0:
7346 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7347 break;
7348 case 1:
7349 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7350 break;
7351 case 2:
7352 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7353 break;
7354 case 3:
7355 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7356 break;
7357 default:
7358 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7359 break;
7360 }
7361 } else if (ring->me == 2) {
7362 switch (ring->pipe) {
7363 case 0:
7364 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7365 break;
7366 case 1:
7367 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7368 break;
7369 case 2:
7370 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7371 break;
7372 case 3:
7373 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7374 break;
7375 default:
7376 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7377 break;
7378 }
7379 } else {
7380 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7381 }
7382 }
7383 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7384 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7385 DRM_DEBUG("si_irq_set: sw int cp2\n");
7386 if (ring->me == 1) {
7387 switch (ring->pipe) {
7388 case 0:
7389 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7390 break;
7391 case 1:
7392 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7393 break;
7394 case 2:
7395 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7396 break;
7397 case 3:
7398 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7399 break;
7400 default:
7401 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7402 break;
7403 }
7404 } else if (ring->me == 2) {
7405 switch (ring->pipe) {
7406 case 0:
7407 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7408 break;
7409 case 1:
7410 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7411 break;
7412 case 2:
7413 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7414 break;
7415 case 3:
7416 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7417 break;
7418 default:
7419 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7420 break;
7421 }
7422 } else {
7423 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7424 }
7425 }
7426
7427 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7428 DRM_DEBUG("cik_irq_set: sw int dma\n");
7429 dma_cntl |= TRAP_ENABLE;
7430 }
7431
7432 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7433 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7434 dma_cntl1 |= TRAP_ENABLE;
7435 }
7436
7437 if (rdev->irq.crtc_vblank_int[0] ||
7438 atomic_read(&rdev->irq.pflip[0])) {
7439 DRM_DEBUG("cik_irq_set: vblank 0\n");
7440 crtc1 |= VBLANK_INTERRUPT_MASK;
7441 }
7442 if (rdev->irq.crtc_vblank_int[1] ||
7443 atomic_read(&rdev->irq.pflip[1])) {
7444 DRM_DEBUG("cik_irq_set: vblank 1\n");
7445 crtc2 |= VBLANK_INTERRUPT_MASK;
7446 }
7447 if (rdev->irq.crtc_vblank_int[2] ||
7448 atomic_read(&rdev->irq.pflip[2])) {
7449 DRM_DEBUG("cik_irq_set: vblank 2\n");
7450 crtc3 |= VBLANK_INTERRUPT_MASK;
7451 }
7452 if (rdev->irq.crtc_vblank_int[3] ||
7453 atomic_read(&rdev->irq.pflip[3])) {
7454 DRM_DEBUG("cik_irq_set: vblank 3\n");
7455 crtc4 |= VBLANK_INTERRUPT_MASK;
7456 }
7457 if (rdev->irq.crtc_vblank_int[4] ||
7458 atomic_read(&rdev->irq.pflip[4])) {
7459 DRM_DEBUG("cik_irq_set: vblank 4\n");
7460 crtc5 |= VBLANK_INTERRUPT_MASK;
7461 }
7462 if (rdev->irq.crtc_vblank_int[5] ||
7463 atomic_read(&rdev->irq.pflip[5])) {
7464 DRM_DEBUG("cik_irq_set: vblank 5\n");
7465 crtc6 |= VBLANK_INTERRUPT_MASK;
7466 }
7467 if (rdev->irq.hpd[0]) {
7468 DRM_DEBUG("cik_irq_set: hpd 1\n");
7469 hpd1 |= DC_HPDx_INT_EN;
7470 }
7471 if (rdev->irq.hpd[1]) {
7472 DRM_DEBUG("cik_irq_set: hpd 2\n");
7473 hpd2 |= DC_HPDx_INT_EN;
7474 }
7475 if (rdev->irq.hpd[2]) {
7476 DRM_DEBUG("cik_irq_set: hpd 3\n");
7477 hpd3 |= DC_HPDx_INT_EN;
7478 }
7479 if (rdev->irq.hpd[3]) {
7480 DRM_DEBUG("cik_irq_set: hpd 4\n");
7481 hpd4 |= DC_HPDx_INT_EN;
7482 }
7483 if (rdev->irq.hpd[4]) {
7484 DRM_DEBUG("cik_irq_set: hpd 5\n");
7485 hpd5 |= DC_HPDx_INT_EN;
7486 }
7487 if (rdev->irq.hpd[5]) {
7488 DRM_DEBUG("cik_irq_set: hpd 6\n");
7489 hpd6 |= DC_HPDx_INT_EN;
7490 }
7491
7492 if (rdev->irq.dpm_thermal) {
7493 DRM_DEBUG("dpm thermal\n");
7494 if (rdev->flags & RADEON_IS_IGP)
7495 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7496 else
7497 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7498 }
7499
7500 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7501
7502 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7503 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7504
7505 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7506 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7507 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7508 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7509 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7510 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7511 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7512 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7513
7514 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7515
7516 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7517 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7518 if (rdev->num_crtc >= 4) {
7519 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7520 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7521 }
7522 if (rdev->num_crtc >= 6) {
7523 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7525 }
7526
7527 if (rdev->num_crtc >= 2) {
7528 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7529 GRPH_PFLIP_INT_MASK);
7530 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7531 GRPH_PFLIP_INT_MASK);
7532 }
7533 if (rdev->num_crtc >= 4) {
7534 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7535 GRPH_PFLIP_INT_MASK);
7536 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7537 GRPH_PFLIP_INT_MASK);
7538 }
7539 if (rdev->num_crtc >= 6) {
7540 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7541 GRPH_PFLIP_INT_MASK);
7542 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7543 GRPH_PFLIP_INT_MASK);
7544 }
7545
7546 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7547 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7548 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7549 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7550 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7551 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7552
7553 if (rdev->flags & RADEON_IS_IGP)
7554 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7555 else
7556 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7557
7558 return 0;
7559 }
7560
7561 /**
7562 * cik_irq_ack - ack interrupt sources
7563 *
7564 * @rdev: radeon_device pointer
7565 *
7566 * Ack interrupt sources on the GPU (vblanks, hpd,
7567 * etc.) (CIK). Certain interrupts sources are sw
7568 * generated and do not require an explicit ack.
7569 */
7570 static inline void cik_irq_ack(struct radeon_device *rdev)
7571 {
7572 u32 tmp;
7573
7574 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7575 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7576 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7577 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7578 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7579 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7580 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7581
7582 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7583 EVERGREEN_CRTC0_REGISTER_OFFSET);
7584 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7585 EVERGREEN_CRTC1_REGISTER_OFFSET);
7586 if (rdev->num_crtc >= 4) {
7587 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7588 EVERGREEN_CRTC2_REGISTER_OFFSET);
7589 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7590 EVERGREEN_CRTC3_REGISTER_OFFSET);
7591 }
7592 if (rdev->num_crtc >= 6) {
7593 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7594 EVERGREEN_CRTC4_REGISTER_OFFSET);
7595 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7596 EVERGREEN_CRTC5_REGISTER_OFFSET);
7597 }
7598
7599 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7600 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7601 GRPH_PFLIP_INT_CLEAR);
7602 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7604 GRPH_PFLIP_INT_CLEAR);
7605 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7606 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7607 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7608 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7609 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7610 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7611 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7612 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7613
7614 if (rdev->num_crtc >= 4) {
7615 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7616 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7617 GRPH_PFLIP_INT_CLEAR);
7618 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7622 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7623 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7624 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7625 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7626 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7627 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7628 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7629 }
7630
7631 if (rdev->num_crtc >= 6) {
7632 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7634 GRPH_PFLIP_INT_CLEAR);
7635 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7639 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7640 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7641 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7642 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7643 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7644 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7645 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7646 }
7647
7648 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7649 tmp = RREG32(DC_HPD1_INT_CONTROL);
7650 tmp |= DC_HPDx_INT_ACK;
7651 WREG32(DC_HPD1_INT_CONTROL, tmp);
7652 }
7653 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7654 tmp = RREG32(DC_HPD2_INT_CONTROL);
7655 tmp |= DC_HPDx_INT_ACK;
7656 WREG32(DC_HPD2_INT_CONTROL, tmp);
7657 }
7658 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7659 tmp = RREG32(DC_HPD3_INT_CONTROL);
7660 tmp |= DC_HPDx_INT_ACK;
7661 WREG32(DC_HPD3_INT_CONTROL, tmp);
7662 }
7663 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7664 tmp = RREG32(DC_HPD4_INT_CONTROL);
7665 tmp |= DC_HPDx_INT_ACK;
7666 WREG32(DC_HPD4_INT_CONTROL, tmp);
7667 }
7668 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7669 tmp = RREG32(DC_HPD5_INT_CONTROL);
7670 tmp |= DC_HPDx_INT_ACK;
7671 WREG32(DC_HPD5_INT_CONTROL, tmp);
7672 }
7673 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7674 tmp = RREG32(DC_HPD5_INT_CONTROL);
7675 tmp |= DC_HPDx_INT_ACK;
7676 WREG32(DC_HPD6_INT_CONTROL, tmp);
7677 }
7678 }
7679
7680 /**
7681 * cik_irq_disable - disable interrupts
7682 *
7683 * @rdev: radeon_device pointer
7684 *
7685 * Disable interrupts on the hw (CIK).
7686 */
7687 static void cik_irq_disable(struct radeon_device *rdev)
7688 {
7689 cik_disable_interrupts(rdev);
7690 /* Wait and acknowledge irq */
7691 mdelay(1);
7692 cik_irq_ack(rdev);
7693 cik_disable_interrupt_state(rdev);
7694 }
7695
7696 /**
7697 * cik_irq_disable - disable interrupts for suspend
7698 *
7699 * @rdev: radeon_device pointer
7700 *
7701 * Disable interrupts and stop the RLC (CIK).
7702 * Used for suspend.
7703 */
7704 static void cik_irq_suspend(struct radeon_device *rdev)
7705 {
7706 cik_irq_disable(rdev);
7707 cik_rlc_stop(rdev);
7708 }
7709
7710 /**
7711 * cik_irq_fini - tear down interrupt support
7712 *
7713 * @rdev: radeon_device pointer
7714 *
7715 * Disable interrupts on the hw and free the IH ring
7716 * buffer (CIK).
7717 * Used for driver unload.
7718 */
7719 static void cik_irq_fini(struct radeon_device *rdev)
7720 {
7721 cik_irq_suspend(rdev);
7722 r600_ih_ring_fini(rdev);
7723 }
7724
7725 /**
7726 * cik_get_ih_wptr - get the IH ring buffer wptr
7727 *
7728 * @rdev: radeon_device pointer
7729 *
7730 * Get the IH ring buffer wptr from either the register
7731 * or the writeback memory buffer (CIK). Also check for
7732 * ring buffer overflow and deal with it.
7733 * Used by cik_irq_process().
7734 * Returns the value of the wptr.
7735 */
7736 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7737 {
7738 u32 wptr, tmp;
7739
7740 if (rdev->wb.enabled)
7741 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7742 else
7743 wptr = RREG32(IH_RB_WPTR);
7744
7745 if (wptr & RB_OVERFLOW) {
7746 /* When a ring buffer overflow happen start parsing interrupt
7747 * from the last not overwritten vector (wptr + 16). Hopefully
7748 * this should allow us to catchup.
7749 */
7750 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7751 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7752 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7753 tmp = RREG32(IH_RB_CNTL);
7754 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7755 WREG32(IH_RB_CNTL, tmp);
7756 wptr &= ~RB_OVERFLOW;
7757 }
7758 return (wptr & rdev->ih.ptr_mask);
7759 }
7760
7761 /* CIK IV Ring
7762 * Each IV ring entry is 128 bits:
7763 * [7:0] - interrupt source id
7764 * [31:8] - reserved
7765 * [59:32] - interrupt source data
7766 * [63:60] - reserved
7767 * [71:64] - RINGID
7768 * CP:
7769 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7770 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7771 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7772 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7773 * PIPE_ID - ME0 0=3D
7774 * - ME1&2 compute dispatcher (4 pipes each)
7775 * SDMA:
7776 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7777 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7778 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7779 * [79:72] - VMID
7780 * [95:80] - PASID
7781 * [127:96] - reserved
7782 */
7783 /**
7784 * cik_irq_process - interrupt handler
7785 *
7786 * @rdev: radeon_device pointer
7787 *
7788 * Interrupt hander (CIK). Walk the IH ring,
7789 * ack interrupts and schedule work to handle
7790 * interrupt events.
7791 * Returns irq process return code.
7792 */
7793 int cik_irq_process(struct radeon_device *rdev)
7794 {
7795 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7796 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7797 u32 wptr;
7798 u32 rptr;
7799 u32 src_id, src_data, ring_id;
7800 u8 me_id, pipe_id, queue_id;
7801 u32 ring_index;
7802 bool queue_hotplug = false;
7803 bool queue_reset = false;
7804 u32 addr, status, mc_client;
7805 bool queue_thermal = false;
7806
7807 if (!rdev->ih.enabled || rdev->shutdown)
7808 return IRQ_NONE;
7809
7810 wptr = cik_get_ih_wptr(rdev);
7811
7812 restart_ih:
7813 /* is somebody else already processing irqs? */
7814 if (atomic_xchg(&rdev->ih.lock, 1))
7815 return IRQ_NONE;
7816
7817 rptr = rdev->ih.rptr;
7818 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7819
7820 /* Order reading of wptr vs. reading of IH ring data */
7821 rmb();
7822
7823 /* display interrupts */
7824 cik_irq_ack(rdev);
7825
7826 while (rptr != wptr) {
7827 /* wptr/rptr are in bytes! */
7828 ring_index = rptr / 4;
7829 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7830 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7831 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7832
7833 switch (src_id) {
7834 case 1: /* D1 vblank/vline */
7835 switch (src_data) {
7836 case 0: /* D1 vblank */
7837 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7838 if (rdev->irq.crtc_vblank_int[0]) {
7839 drm_handle_vblank(rdev->ddev, 0);
7840 rdev->pm.vblank_sync = true;
7841 wake_up(&rdev->irq.vblank_queue);
7842 }
7843 if (atomic_read(&rdev->irq.pflip[0]))
7844 radeon_crtc_handle_vblank(rdev, 0);
7845 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7846 DRM_DEBUG("IH: D1 vblank\n");
7847 }
7848 break;
7849 case 1: /* D1 vline */
7850 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7851 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7852 DRM_DEBUG("IH: D1 vline\n");
7853 }
7854 break;
7855 default:
7856 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7857 break;
7858 }
7859 break;
7860 case 2: /* D2 vblank/vline */
7861 switch (src_data) {
7862 case 0: /* D2 vblank */
7863 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7864 if (rdev->irq.crtc_vblank_int[1]) {
7865 drm_handle_vblank(rdev->ddev, 1);
7866 rdev->pm.vblank_sync = true;
7867 wake_up(&rdev->irq.vblank_queue);
7868 }
7869 if (atomic_read(&rdev->irq.pflip[1]))
7870 radeon_crtc_handle_vblank(rdev, 1);
7871 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7872 DRM_DEBUG("IH: D2 vblank\n");
7873 }
7874 break;
7875 case 1: /* D2 vline */
7876 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7877 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7878 DRM_DEBUG("IH: D2 vline\n");
7879 }
7880 break;
7881 default:
7882 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7883 break;
7884 }
7885 break;
7886 case 3: /* D3 vblank/vline */
7887 switch (src_data) {
7888 case 0: /* D3 vblank */
7889 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7890 if (rdev->irq.crtc_vblank_int[2]) {
7891 drm_handle_vblank(rdev->ddev, 2);
7892 rdev->pm.vblank_sync = true;
7893 wake_up(&rdev->irq.vblank_queue);
7894 }
7895 if (atomic_read(&rdev->irq.pflip[2]))
7896 radeon_crtc_handle_vblank(rdev, 2);
7897 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7898 DRM_DEBUG("IH: D3 vblank\n");
7899 }
7900 break;
7901 case 1: /* D3 vline */
7902 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7903 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7904 DRM_DEBUG("IH: D3 vline\n");
7905 }
7906 break;
7907 default:
7908 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7909 break;
7910 }
7911 break;
7912 case 4: /* D4 vblank/vline */
7913 switch (src_data) {
7914 case 0: /* D4 vblank */
7915 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7916 if (rdev->irq.crtc_vblank_int[3]) {
7917 drm_handle_vblank(rdev->ddev, 3);
7918 rdev->pm.vblank_sync = true;
7919 wake_up(&rdev->irq.vblank_queue);
7920 }
7921 if (atomic_read(&rdev->irq.pflip[3]))
7922 radeon_crtc_handle_vblank(rdev, 3);
7923 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7924 DRM_DEBUG("IH: D4 vblank\n");
7925 }
7926 break;
7927 case 1: /* D4 vline */
7928 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7929 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7930 DRM_DEBUG("IH: D4 vline\n");
7931 }
7932 break;
7933 default:
7934 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7935 break;
7936 }
7937 break;
7938 case 5: /* D5 vblank/vline */
7939 switch (src_data) {
7940 case 0: /* D5 vblank */
7941 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7942 if (rdev->irq.crtc_vblank_int[4]) {
7943 drm_handle_vblank(rdev->ddev, 4);
7944 rdev->pm.vblank_sync = true;
7945 wake_up(&rdev->irq.vblank_queue);
7946 }
7947 if (atomic_read(&rdev->irq.pflip[4]))
7948 radeon_crtc_handle_vblank(rdev, 4);
7949 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7950 DRM_DEBUG("IH: D5 vblank\n");
7951 }
7952 break;
7953 case 1: /* D5 vline */
7954 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7955 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7956 DRM_DEBUG("IH: D5 vline\n");
7957 }
7958 break;
7959 default:
7960 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7961 break;
7962 }
7963 break;
7964 case 6: /* D6 vblank/vline */
7965 switch (src_data) {
7966 case 0: /* D6 vblank */
7967 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7968 if (rdev->irq.crtc_vblank_int[5]) {
7969 drm_handle_vblank(rdev->ddev, 5);
7970 rdev->pm.vblank_sync = true;
7971 wake_up(&rdev->irq.vblank_queue);
7972 }
7973 if (atomic_read(&rdev->irq.pflip[5]))
7974 radeon_crtc_handle_vblank(rdev, 5);
7975 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7976 DRM_DEBUG("IH: D6 vblank\n");
7977 }
7978 break;
7979 case 1: /* D6 vline */
7980 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7981 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7982 DRM_DEBUG("IH: D6 vline\n");
7983 }
7984 break;
7985 default:
7986 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7987 break;
7988 }
7989 break;
7990 case 8: /* D1 page flip */
7991 case 10: /* D2 page flip */
7992 case 12: /* D3 page flip */
7993 case 14: /* D4 page flip */
7994 case 16: /* D5 page flip */
7995 case 18: /* D6 page flip */
7996 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7997 if (radeon_use_pflipirq > 0)
7998 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7999 break;
8000 case 42: /* HPD hotplug */
8001 switch (src_data) {
8002 case 0:
8003 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8004 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8005 queue_hotplug = true;
8006 DRM_DEBUG("IH: HPD1\n");
8007 }
8008 break;
8009 case 1:
8010 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8011 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8012 queue_hotplug = true;
8013 DRM_DEBUG("IH: HPD2\n");
8014 }
8015 break;
8016 case 2:
8017 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8018 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8019 queue_hotplug = true;
8020 DRM_DEBUG("IH: HPD3\n");
8021 }
8022 break;
8023 case 3:
8024 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8025 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8026 queue_hotplug = true;
8027 DRM_DEBUG("IH: HPD4\n");
8028 }
8029 break;
8030 case 4:
8031 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8032 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8033 queue_hotplug = true;
8034 DRM_DEBUG("IH: HPD5\n");
8035 }
8036 break;
8037 case 5:
8038 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8039 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8040 queue_hotplug = true;
8041 DRM_DEBUG("IH: HPD6\n");
8042 }
8043 break;
8044 default:
8045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8046 break;
8047 }
8048 break;
8049 case 124: /* UVD */
8050 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8051 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8052 break;
8053 case 146:
8054 case 147:
8055 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8056 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8057 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8058 /* reset addr and status */
8059 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8060 if (addr == 0x0 && status == 0x0)
8061 break;
8062 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8063 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8064 addr);
8065 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8066 status);
8067 cik_vm_decode_fault(rdev, status, addr, mc_client);
8068 break;
8069 case 167: /* VCE */
8070 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8071 switch (src_data) {
8072 case 0:
8073 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8074 break;
8075 case 1:
8076 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8077 break;
8078 default:
8079 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8080 break;
8081 }
8082 break;
8083 case 176: /* GFX RB CP_INT */
8084 case 177: /* GFX IB CP_INT */
8085 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8086 break;
8087 case 181: /* CP EOP event */
8088 DRM_DEBUG("IH: CP EOP\n");
8089 /* XXX check the bitfield order! */
8090 me_id = (ring_id & 0x60) >> 5;
8091 pipe_id = (ring_id & 0x18) >> 3;
8092 queue_id = (ring_id & 0x7) >> 0;
8093 switch (me_id) {
8094 case 0:
8095 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8096 break;
8097 case 1:
8098 case 2:
8099 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8100 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8101 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8102 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8103 break;
8104 }
8105 break;
8106 case 184: /* CP Privileged reg access */
8107 DRM_ERROR("Illegal register access in command stream\n");
8108 /* XXX check the bitfield order! */
8109 me_id = (ring_id & 0x60) >> 5;
8110 pipe_id = (ring_id & 0x18) >> 3;
8111 queue_id = (ring_id & 0x7) >> 0;
8112 switch (me_id) {
8113 case 0:
8114 /* This results in a full GPU reset, but all we need to do is soft
8115 * reset the CP for gfx
8116 */
8117 queue_reset = true;
8118 break;
8119 case 1:
8120 /* XXX compute */
8121 queue_reset = true;
8122 break;
8123 case 2:
8124 /* XXX compute */
8125 queue_reset = true;
8126 break;
8127 }
8128 break;
8129 case 185: /* CP Privileged inst */
8130 DRM_ERROR("Illegal instruction in command stream\n");
8131 /* XXX check the bitfield order! */
8132 me_id = (ring_id & 0x60) >> 5;
8133 pipe_id = (ring_id & 0x18) >> 3;
8134 queue_id = (ring_id & 0x7) >> 0;
8135 switch (me_id) {
8136 case 0:
8137 /* This results in a full GPU reset, but all we need to do is soft
8138 * reset the CP for gfx
8139 */
8140 queue_reset = true;
8141 break;
8142 case 1:
8143 /* XXX compute */
8144 queue_reset = true;
8145 break;
8146 case 2:
8147 /* XXX compute */
8148 queue_reset = true;
8149 break;
8150 }
8151 break;
8152 case 224: /* SDMA trap event */
8153 /* XXX check the bitfield order! */
8154 me_id = (ring_id & 0x3) >> 0;
8155 queue_id = (ring_id & 0xc) >> 2;
8156 DRM_DEBUG("IH: SDMA trap\n");
8157 switch (me_id) {
8158 case 0:
8159 switch (queue_id) {
8160 case 0:
8161 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8162 break;
8163 case 1:
8164 /* XXX compute */
8165 break;
8166 case 2:
8167 /* XXX compute */
8168 break;
8169 }
8170 break;
8171 case 1:
8172 switch (queue_id) {
8173 case 0:
8174 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8175 break;
8176 case 1:
8177 /* XXX compute */
8178 break;
8179 case 2:
8180 /* XXX compute */
8181 break;
8182 }
8183 break;
8184 }
8185 break;
8186 case 230: /* thermal low to high */
8187 DRM_DEBUG("IH: thermal low to high\n");
8188 rdev->pm.dpm.thermal.high_to_low = false;
8189 queue_thermal = true;
8190 break;
8191 case 231: /* thermal high to low */
8192 DRM_DEBUG("IH: thermal high to low\n");
8193 rdev->pm.dpm.thermal.high_to_low = true;
8194 queue_thermal = true;
8195 break;
8196 case 233: /* GUI IDLE */
8197 DRM_DEBUG("IH: GUI idle\n");
8198 break;
8199 case 241: /* SDMA Privileged inst */
8200 case 247: /* SDMA Privileged inst */
8201 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8202 /* XXX check the bitfield order! */
8203 me_id = (ring_id & 0x3) >> 0;
8204 queue_id = (ring_id & 0xc) >> 2;
8205 switch (me_id) {
8206 case 0:
8207 switch (queue_id) {
8208 case 0:
8209 queue_reset = true;
8210 break;
8211 case 1:
8212 /* XXX compute */
8213 queue_reset = true;
8214 break;
8215 case 2:
8216 /* XXX compute */
8217 queue_reset = true;
8218 break;
8219 }
8220 break;
8221 case 1:
8222 switch (queue_id) {
8223 case 0:
8224 queue_reset = true;
8225 break;
8226 case 1:
8227 /* XXX compute */
8228 queue_reset = true;
8229 break;
8230 case 2:
8231 /* XXX compute */
8232 queue_reset = true;
8233 break;
8234 }
8235 break;
8236 }
8237 break;
8238 default:
8239 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8240 break;
8241 }
8242
8243 /* wptr/rptr are in bytes! */
8244 rptr += 16;
8245 rptr &= rdev->ih.ptr_mask;
8246 }
8247 if (queue_hotplug)
8248 schedule_work(&rdev->hotplug_work);
8249 if (queue_reset)
8250 schedule_work(&rdev->reset_work);
8251 if (queue_thermal)
8252 schedule_work(&rdev->pm.dpm.thermal.work);
8253 rdev->ih.rptr = rptr;
8254 WREG32(IH_RB_RPTR, rdev->ih.rptr);
8255 atomic_set(&rdev->ih.lock, 0);
8256
8257 /* make sure wptr hasn't changed while processing */
8258 wptr = cik_get_ih_wptr(rdev);
8259 if (wptr != rptr)
8260 goto restart_ih;
8261
8262 return IRQ_HANDLED;
8263 }
8264
8265 /*
8266 * startup/shutdown callbacks
8267 */
8268 /**
8269 * cik_startup - program the asic to a functional state
8270 *
8271 * @rdev: radeon_device pointer
8272 *
8273 * Programs the asic to a functional state (CIK).
8274 * Called by cik_init() and cik_resume().
8275 * Returns 0 for success, error for failure.
8276 */
8277 static int cik_startup(struct radeon_device *rdev)
8278 {
8279 struct radeon_ring *ring;
8280 u32 nop;
8281 int r;
8282
8283 /* enable pcie gen2/3 link */
8284 cik_pcie_gen3_enable(rdev);
8285 /* enable aspm */
8286 cik_program_aspm(rdev);
8287
8288 /* scratch needs to be initialized before MC */
8289 r = r600_vram_scratch_init(rdev);
8290 if (r)
8291 return r;
8292
8293 cik_mc_program(rdev);
8294
8295 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8296 r = ci_mc_load_microcode(rdev);
8297 if (r) {
8298 DRM_ERROR("Failed to load MC firmware!\n");
8299 return r;
8300 }
8301 }
8302
8303 r = cik_pcie_gart_enable(rdev);
8304 if (r)
8305 return r;
8306 cik_gpu_init(rdev);
8307
8308 /* allocate rlc buffers */
8309 if (rdev->flags & RADEON_IS_IGP) {
8310 if (rdev->family == CHIP_KAVERI) {
8311 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8312 rdev->rlc.reg_list_size =
8313 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8314 } else {
8315 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8316 rdev->rlc.reg_list_size =
8317 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8318 }
8319 }
8320 rdev->rlc.cs_data = ci_cs_data;
8321 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8322 r = sumo_rlc_init(rdev);
8323 if (r) {
8324 DRM_ERROR("Failed to init rlc BOs!\n");
8325 return r;
8326 }
8327
8328 /* allocate wb buffer */
8329 r = radeon_wb_init(rdev);
8330 if (r)
8331 return r;
8332
8333 /* allocate mec buffers */
8334 r = cik_mec_init(rdev);
8335 if (r) {
8336 DRM_ERROR("Failed to init MEC BOs!\n");
8337 return r;
8338 }
8339
8340 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8341 if (r) {
8342 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8343 return r;
8344 }
8345
8346 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8347 if (r) {
8348 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8349 return r;
8350 }
8351
8352 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8353 if (r) {
8354 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8355 return r;
8356 }
8357
8358 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8359 if (r) {
8360 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8361 return r;
8362 }
8363
8364 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8365 if (r) {
8366 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8367 return r;
8368 }
8369
8370 r = radeon_uvd_resume(rdev);
8371 if (!r) {
8372 r = uvd_v4_2_resume(rdev);
8373 if (!r) {
8374 r = radeon_fence_driver_start_ring(rdev,
8375 R600_RING_TYPE_UVD_INDEX);
8376 if (r)
8377 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8378 }
8379 }
8380 if (r)
8381 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8382
8383 r = radeon_vce_resume(rdev);
8384 if (!r) {
8385 r = vce_v2_0_resume(rdev);
8386 if (!r)
8387 r = radeon_fence_driver_start_ring(rdev,
8388 TN_RING_TYPE_VCE1_INDEX);
8389 if (!r)
8390 r = radeon_fence_driver_start_ring(rdev,
8391 TN_RING_TYPE_VCE2_INDEX);
8392 }
8393 if (r) {
8394 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8395 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8396 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8397 }
8398
8399 /* Enable IRQ */
8400 if (!rdev->irq.installed) {
8401 r = radeon_irq_kms_init(rdev);
8402 if (r)
8403 return r;
8404 }
8405
8406 r = cik_irq_init(rdev);
8407 if (r) {
8408 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8409 radeon_irq_kms_fini(rdev);
8410 return r;
8411 }
8412 cik_irq_set(rdev);
8413
8414 if (rdev->family == CHIP_HAWAII) {
8415 if (rdev->new_fw)
8416 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8417 else
8418 nop = RADEON_CP_PACKET2;
8419 } else {
8420 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8421 }
8422
8423 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8424 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8425 nop);
8426 if (r)
8427 return r;
8428
8429 /* set up the compute queues */
8430 /* type-2 packets are deprecated on MEC, use type-3 instead */
8431 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8432 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8433 nop);
8434 if (r)
8435 return r;
8436 ring->me = 1; /* first MEC */
8437 ring->pipe = 0; /* first pipe */
8438 ring->queue = 0; /* first queue */
8439 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8440
8441 /* type-2 packets are deprecated on MEC, use type-3 instead */
8442 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8443 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8444 nop);
8445 if (r)
8446 return r;
8447 /* dGPU only have 1 MEC */
8448 ring->me = 1; /* first MEC */
8449 ring->pipe = 0; /* first pipe */
8450 ring->queue = 1; /* second queue */
8451 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8452
8453 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8454 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8455 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8456 if (r)
8457 return r;
8458
8459 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8460 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8461 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8462 if (r)
8463 return r;
8464
8465 r = cik_cp_resume(rdev);
8466 if (r)
8467 return r;
8468
8469 r = cik_sdma_resume(rdev);
8470 if (r)
8471 return r;
8472
8473 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8474 if (ring->ring_size) {
8475 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8476 RADEON_CP_PACKET2);
8477 if (!r)
8478 r = uvd_v1_0_init(rdev);
8479 if (r)
8480 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8481 }
8482
8483 r = -ENOENT;
8484
8485 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8486 if (ring->ring_size)
8487 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8488 VCE_CMD_NO_OP);
8489
8490 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8491 if (ring->ring_size)
8492 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8493 VCE_CMD_NO_OP);
8494
8495 if (!r)
8496 r = vce_v1_0_init(rdev);
8497 else if (r != -ENOENT)
8498 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8499
8500 r = radeon_ib_pool_init(rdev);
8501 if (r) {
8502 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8503 return r;
8504 }
8505
8506 r = radeon_vm_manager_init(rdev);
8507 if (r) {
8508 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8509 return r;
8510 }
8511
8512 r = dce6_audio_init(rdev);
8513 if (r)
8514 return r;
8515
8516 return 0;
8517 }
8518
8519 /**
8520 * cik_resume - resume the asic to a functional state
8521 *
8522 * @rdev: radeon_device pointer
8523 *
8524 * Programs the asic to a functional state (CIK).
8525 * Called at resume.
8526 * Returns 0 for success, error for failure.
8527 */
8528 int cik_resume(struct radeon_device *rdev)
8529 {
8530 int r;
8531
8532 /* post card */
8533 atom_asic_init(rdev->mode_info.atom_context);
8534
8535 /* init golden registers */
8536 cik_init_golden_registers(rdev);
8537
8538 if (rdev->pm.pm_method == PM_METHOD_DPM)
8539 radeon_pm_resume(rdev);
8540
8541 rdev->accel_working = true;
8542 r = cik_startup(rdev);
8543 if (r) {
8544 DRM_ERROR("cik startup failed on resume\n");
8545 rdev->accel_working = false;
8546 return r;
8547 }
8548
8549 return r;
8550
8551 }
8552
8553 /**
8554 * cik_suspend - suspend the asic
8555 *
8556 * @rdev: radeon_device pointer
8557 *
8558 * Bring the chip into a state suitable for suspend (CIK).
8559 * Called at suspend.
8560 * Returns 0 for success.
8561 */
8562 int cik_suspend(struct radeon_device *rdev)
8563 {
8564 radeon_pm_suspend(rdev);
8565 dce6_audio_fini(rdev);
8566 radeon_vm_manager_fini(rdev);
8567 cik_cp_enable(rdev, false);
8568 cik_sdma_enable(rdev, false);
8569 uvd_v1_0_fini(rdev);
8570 radeon_uvd_suspend(rdev);
8571 radeon_vce_suspend(rdev);
8572 cik_fini_pg(rdev);
8573 cik_fini_cg(rdev);
8574 cik_irq_suspend(rdev);
8575 radeon_wb_disable(rdev);
8576 cik_pcie_gart_disable(rdev);
8577 return 0;
8578 }
8579
8580 /* Plan is to move initialization in that function and use
8581 * helper function so that radeon_device_init pretty much
8582 * do nothing more than calling asic specific function. This
8583 * should also allow to remove a bunch of callback function
8584 * like vram_info.
8585 */
8586 /**
8587 * cik_init - asic specific driver and hw init
8588 *
8589 * @rdev: radeon_device pointer
8590 *
8591 * Setup asic specific driver variables and program the hw
8592 * to a functional state (CIK).
8593 * Called at driver startup.
8594 * Returns 0 for success, errors for failure.
8595 */
8596 int cik_init(struct radeon_device *rdev)
8597 {
8598 struct radeon_ring *ring;
8599 int r;
8600
8601 /* Read BIOS */
8602 if (!radeon_get_bios(rdev)) {
8603 if (ASIC_IS_AVIVO(rdev))
8604 return -EINVAL;
8605 }
8606 /* Must be an ATOMBIOS */
8607 if (!rdev->is_atom_bios) {
8608 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8609 return -EINVAL;
8610 }
8611 r = radeon_atombios_init(rdev);
8612 if (r)
8613 return r;
8614
8615 /* Post card if necessary */
8616 if (!radeon_card_posted(rdev)) {
8617 if (!rdev->bios) {
8618 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8619 return -EINVAL;
8620 }
8621 DRM_INFO("GPU not posted. posting now...\n");
8622 atom_asic_init(rdev->mode_info.atom_context);
8623 }
8624 /* init golden registers */
8625 cik_init_golden_registers(rdev);
8626 /* Initialize scratch registers */
8627 cik_scratch_init(rdev);
8628 /* Initialize surface registers */
8629 radeon_surface_init(rdev);
8630 /* Initialize clocks */
8631 radeon_get_clock_info(rdev->ddev);
8632
8633 /* Fence driver */
8634 r = radeon_fence_driver_init(rdev);
8635 if (r)
8636 return r;
8637
8638 /* initialize memory controller */
8639 r = cik_mc_init(rdev);
8640 if (r)
8641 return r;
8642 /* Memory manager */
8643 r = radeon_bo_init(rdev);
8644 if (r)
8645 return r;
8646
8647 if (rdev->flags & RADEON_IS_IGP) {
8648 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8649 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8650 r = cik_init_microcode(rdev);
8651 if (r) {
8652 DRM_ERROR("Failed to load firmware!\n");
8653 return r;
8654 }
8655 }
8656 } else {
8657 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8658 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8659 !rdev->mc_fw) {
8660 r = cik_init_microcode(rdev);
8661 if (r) {
8662 DRM_ERROR("Failed to load firmware!\n");
8663 return r;
8664 }
8665 }
8666 }
8667
8668 /* Initialize power management */
8669 radeon_pm_init(rdev);
8670
8671 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8672 ring->ring_obj = NULL;
8673 r600_ring_init(rdev, ring, 1024 * 1024);
8674
8675 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8676 ring->ring_obj = NULL;
8677 r600_ring_init(rdev, ring, 1024 * 1024);
8678 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8679 if (r)
8680 return r;
8681
8682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8683 ring->ring_obj = NULL;
8684 r600_ring_init(rdev, ring, 1024 * 1024);
8685 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8686 if (r)
8687 return r;
8688
8689 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 256 * 1024);
8692
8693 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8694 ring->ring_obj = NULL;
8695 r600_ring_init(rdev, ring, 256 * 1024);
8696
8697 r = radeon_uvd_init(rdev);
8698 if (!r) {
8699 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8700 ring->ring_obj = NULL;
8701 r600_ring_init(rdev, ring, 4096);
8702 }
8703
8704 r = radeon_vce_init(rdev);
8705 if (!r) {
8706 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8707 ring->ring_obj = NULL;
8708 r600_ring_init(rdev, ring, 4096);
8709
8710 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8711 ring->ring_obj = NULL;
8712 r600_ring_init(rdev, ring, 4096);
8713 }
8714
8715 rdev->ih.ring_obj = NULL;
8716 r600_ih_ring_init(rdev, 64 * 1024);
8717
8718 r = r600_pcie_gart_init(rdev);
8719 if (r)
8720 return r;
8721
8722 rdev->accel_working = true;
8723 r = cik_startup(rdev);
8724 if (r) {
8725 dev_err(rdev->dev, "disabling GPU acceleration\n");
8726 cik_cp_fini(rdev);
8727 cik_sdma_fini(rdev);
8728 cik_irq_fini(rdev);
8729 sumo_rlc_fini(rdev);
8730 cik_mec_fini(rdev);
8731 radeon_wb_fini(rdev);
8732 radeon_ib_pool_fini(rdev);
8733 radeon_vm_manager_fini(rdev);
8734 radeon_irq_kms_fini(rdev);
8735 cik_pcie_gart_fini(rdev);
8736 rdev->accel_working = false;
8737 }
8738
8739 /* Don't start up if the MC ucode is missing.
8740 * The default clocks and voltages before the MC ucode
8741 * is loaded are not suffient for advanced operations.
8742 */
8743 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8744 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8745 return -EINVAL;
8746 }
8747
8748 return 0;
8749 }
8750
8751 /**
8752 * cik_fini - asic specific driver and hw fini
8753 *
8754 * @rdev: radeon_device pointer
8755 *
8756 * Tear down the asic specific driver variables and program the hw
8757 * to an idle state (CIK).
8758 * Called at driver unload.
8759 */
8760 void cik_fini(struct radeon_device *rdev)
8761 {
8762 radeon_pm_fini(rdev);
8763 cik_cp_fini(rdev);
8764 cik_sdma_fini(rdev);
8765 cik_fini_pg(rdev);
8766 cik_fini_cg(rdev);
8767 cik_irq_fini(rdev);
8768 sumo_rlc_fini(rdev);
8769 cik_mec_fini(rdev);
8770 radeon_wb_fini(rdev);
8771 radeon_vm_manager_fini(rdev);
8772 radeon_ib_pool_fini(rdev);
8773 radeon_irq_kms_fini(rdev);
8774 uvd_v1_0_fini(rdev);
8775 radeon_uvd_fini(rdev);
8776 radeon_vce_fini(rdev);
8777 cik_pcie_gart_fini(rdev);
8778 r600_vram_scratch_fini(rdev);
8779 radeon_gem_fini(rdev);
8780 radeon_fence_driver_fini(rdev);
8781 radeon_bo_fini(rdev);
8782 radeon_atombios_fini(rdev);
8783 kfree(rdev->bios);
8784 rdev->bios = NULL;
8785 }
8786
8787 void dce8_program_fmt(struct drm_encoder *encoder)
8788 {
8789 struct drm_device *dev = encoder->dev;
8790 struct radeon_device *rdev = dev->dev_private;
8791 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8792 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8793 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8794 int bpc = 0;
8795 u32 tmp = 0;
8796 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8797
8798 if (connector) {
8799 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8800 bpc = radeon_get_monitor_bpc(connector);
8801 dither = radeon_connector->dither;
8802 }
8803
8804 /* LVDS/eDP FMT is set up by atom */
8805 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8806 return;
8807
8808 /* not needed for analog */
8809 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8810 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8811 return;
8812
8813 if (bpc == 0)
8814 return;
8815
8816 switch (bpc) {
8817 case 6:
8818 if (dither == RADEON_FMT_DITHER_ENABLE)
8819 /* XXX sort out optimal dither settings */
8820 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8822 else
8823 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8824 break;
8825 case 8:
8826 if (dither == RADEON_FMT_DITHER_ENABLE)
8827 /* XXX sort out optimal dither settings */
8828 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8829 FMT_RGB_RANDOM_ENABLE |
8830 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8831 else
8832 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8833 break;
8834 case 10:
8835 if (dither == RADEON_FMT_DITHER_ENABLE)
8836 /* XXX sort out optimal dither settings */
8837 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8838 FMT_RGB_RANDOM_ENABLE |
8839 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8840 else
8841 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8842 break;
8843 default:
8844 /* not needed */
8845 break;
8846 }
8847
8848 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8849 }
8850
8851 /* display watermark setup */
8852 /**
8853 * dce8_line_buffer_adjust - Set up the line buffer
8854 *
8855 * @rdev: radeon_device pointer
8856 * @radeon_crtc: the selected display controller
8857 * @mode: the current display mode on the selected display
8858 * controller
8859 *
8860 * Setup up the line buffer allocation for
8861 * the selected display controller (CIK).
8862 * Returns the line buffer size in pixels.
8863 */
8864 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8865 struct radeon_crtc *radeon_crtc,
8866 struct drm_display_mode *mode)
8867 {
8868 u32 tmp, buffer_alloc, i;
8869 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8870 /*
8871 * Line Buffer Setup
8872 * There are 6 line buffers, one for each display controllers.
8873 * There are 3 partitions per LB. Select the number of partitions
8874 * to enable based on the display width. For display widths larger
8875 * than 4096, you need use to use 2 display controllers and combine
8876 * them using the stereo blender.
8877 */
8878 if (radeon_crtc->base.enabled && mode) {
8879 if (mode->crtc_hdisplay < 1920) {
8880 tmp = 1;
8881 buffer_alloc = 2;
8882 } else if (mode->crtc_hdisplay < 2560) {
8883 tmp = 2;
8884 buffer_alloc = 2;
8885 } else if (mode->crtc_hdisplay < 4096) {
8886 tmp = 0;
8887 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8888 } else {
8889 DRM_DEBUG_KMS("Mode too big for LB!\n");
8890 tmp = 0;
8891 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8892 }
8893 } else {
8894 tmp = 1;
8895 buffer_alloc = 0;
8896 }
8897
8898 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8899 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8900
8901 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8902 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8903 for (i = 0; i < rdev->usec_timeout; i++) {
8904 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8905 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8906 break;
8907 udelay(1);
8908 }
8909
8910 if (radeon_crtc->base.enabled && mode) {
8911 switch (tmp) {
8912 case 0:
8913 default:
8914 return 4096 * 2;
8915 case 1:
8916 return 1920 * 2;
8917 case 2:
8918 return 2560 * 2;
8919 }
8920 }
8921
8922 /* controller not enabled, so no lb used */
8923 return 0;
8924 }
8925
8926 /**
8927 * cik_get_number_of_dram_channels - get the number of dram channels
8928 *
8929 * @rdev: radeon_device pointer
8930 *
8931 * Look up the number of video ram channels (CIK).
8932 * Used for display watermark bandwidth calculations
8933 * Returns the number of dram channels
8934 */
8935 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8936 {
8937 u32 tmp = RREG32(MC_SHARED_CHMAP);
8938
8939 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8940 case 0:
8941 default:
8942 return 1;
8943 case 1:
8944 return 2;
8945 case 2:
8946 return 4;
8947 case 3:
8948 return 8;
8949 case 4:
8950 return 3;
8951 case 5:
8952 return 6;
8953 case 6:
8954 return 10;
8955 case 7:
8956 return 12;
8957 case 8:
8958 return 16;
8959 }
8960 }
8961
8962 struct dce8_wm_params {
8963 u32 dram_channels; /* number of dram channels */
8964 u32 yclk; /* bandwidth per dram data pin in kHz */
8965 u32 sclk; /* engine clock in kHz */
8966 u32 disp_clk; /* display clock in kHz */
8967 u32 src_width; /* viewport width */
8968 u32 active_time; /* active display time in ns */
8969 u32 blank_time; /* blank time in ns */
8970 bool interlaced; /* mode is interlaced */
8971 fixed20_12 vsc; /* vertical scale ratio */
8972 u32 num_heads; /* number of active crtcs */
8973 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8974 u32 lb_size; /* line buffer allocated to pipe */
8975 u32 vtaps; /* vertical scaler taps */
8976 };
8977
8978 /**
8979 * dce8_dram_bandwidth - get the dram bandwidth
8980 *
8981 * @wm: watermark calculation data
8982 *
8983 * Calculate the raw dram bandwidth (CIK).
8984 * Used for display watermark bandwidth calculations
8985 * Returns the dram bandwidth in MBytes/s
8986 */
8987 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8988 {
8989 /* Calculate raw DRAM Bandwidth */
8990 fixed20_12 dram_efficiency; /* 0.7 */
8991 fixed20_12 yclk, dram_channels, bandwidth;
8992 fixed20_12 a;
8993
8994 a.full = dfixed_const(1000);
8995 yclk.full = dfixed_const(wm->yclk);
8996 yclk.full = dfixed_div(yclk, a);
8997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8998 a.full = dfixed_const(10);
8999 dram_efficiency.full = dfixed_const(7);
9000 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9001 bandwidth.full = dfixed_mul(dram_channels, yclk);
9002 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9003
9004 return dfixed_trunc(bandwidth);
9005 }
9006
9007 /**
9008 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9009 *
9010 * @wm: watermark calculation data
9011 *
9012 * Calculate the dram bandwidth used for display (CIK).
9013 * Used for display watermark bandwidth calculations
9014 * Returns the dram bandwidth for display in MBytes/s
9015 */
9016 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9017 {
9018 /* Calculate DRAM Bandwidth and the part allocated to display. */
9019 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9020 fixed20_12 yclk, dram_channels, bandwidth;
9021 fixed20_12 a;
9022
9023 a.full = dfixed_const(1000);
9024 yclk.full = dfixed_const(wm->yclk);
9025 yclk.full = dfixed_div(yclk, a);
9026 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9027 a.full = dfixed_const(10);
9028 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9029 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9030 bandwidth.full = dfixed_mul(dram_channels, yclk);
9031 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9032
9033 return dfixed_trunc(bandwidth);
9034 }
9035
9036 /**
9037 * dce8_data_return_bandwidth - get the data return bandwidth
9038 *
9039 * @wm: watermark calculation data
9040 *
9041 * Calculate the data return bandwidth used for display (CIK).
9042 * Used for display watermark bandwidth calculations
9043 * Returns the data return bandwidth in MBytes/s
9044 */
9045 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9046 {
9047 /* Calculate the display Data return Bandwidth */
9048 fixed20_12 return_efficiency; /* 0.8 */
9049 fixed20_12 sclk, bandwidth;
9050 fixed20_12 a;
9051
9052 a.full = dfixed_const(1000);
9053 sclk.full = dfixed_const(wm->sclk);
9054 sclk.full = dfixed_div(sclk, a);
9055 a.full = dfixed_const(10);
9056 return_efficiency.full = dfixed_const(8);
9057 return_efficiency.full = dfixed_div(return_efficiency, a);
9058 a.full = dfixed_const(32);
9059 bandwidth.full = dfixed_mul(a, sclk);
9060 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9061
9062 return dfixed_trunc(bandwidth);
9063 }
9064
9065 /**
9066 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9067 *
9068 * @wm: watermark calculation data
9069 *
9070 * Calculate the dmif bandwidth used for display (CIK).
9071 * Used for display watermark bandwidth calculations
9072 * Returns the dmif bandwidth in MBytes/s
9073 */
9074 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9075 {
9076 /* Calculate the DMIF Request Bandwidth */
9077 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9078 fixed20_12 disp_clk, bandwidth;
9079 fixed20_12 a, b;
9080
9081 a.full = dfixed_const(1000);
9082 disp_clk.full = dfixed_const(wm->disp_clk);
9083 disp_clk.full = dfixed_div(disp_clk, a);
9084 a.full = dfixed_const(32);
9085 b.full = dfixed_mul(a, disp_clk);
9086
9087 a.full = dfixed_const(10);
9088 disp_clk_request_efficiency.full = dfixed_const(8);
9089 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9090
9091 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9092
9093 return dfixed_trunc(bandwidth);
9094 }
9095
9096 /**
9097 * dce8_available_bandwidth - get the min available bandwidth
9098 *
9099 * @wm: watermark calculation data
9100 *
9101 * Calculate the min available bandwidth used for display (CIK).
9102 * Used for display watermark bandwidth calculations
9103 * Returns the min available bandwidth in MBytes/s
9104 */
9105 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9106 {
9107 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9108 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9109 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9110 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9111
9112 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9113 }
9114
9115 /**
9116 * dce8_average_bandwidth - get the average available bandwidth
9117 *
9118 * @wm: watermark calculation data
9119 *
9120 * Calculate the average available bandwidth used for display (CIK).
9121 * Used for display watermark bandwidth calculations
9122 * Returns the average available bandwidth in MBytes/s
9123 */
9124 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9125 {
9126 /* Calculate the display mode Average Bandwidth
9127 * DisplayMode should contain the source and destination dimensions,
9128 * timing, etc.
9129 */
9130 fixed20_12 bpp;
9131 fixed20_12 line_time;
9132 fixed20_12 src_width;
9133 fixed20_12 bandwidth;
9134 fixed20_12 a;
9135
9136 a.full = dfixed_const(1000);
9137 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9138 line_time.full = dfixed_div(line_time, a);
9139 bpp.full = dfixed_const(wm->bytes_per_pixel);
9140 src_width.full = dfixed_const(wm->src_width);
9141 bandwidth.full = dfixed_mul(src_width, bpp);
9142 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9143 bandwidth.full = dfixed_div(bandwidth, line_time);
9144
9145 return dfixed_trunc(bandwidth);
9146 }
9147
9148 /**
9149 * dce8_latency_watermark - get the latency watermark
9150 *
9151 * @wm: watermark calculation data
9152 *
9153 * Calculate the latency watermark (CIK).
9154 * Used for display watermark bandwidth calculations
9155 * Returns the latency watermark in ns
9156 */
9157 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9158 {
9159 /* First calculate the latency in ns */
9160 u32 mc_latency = 2000; /* 2000 ns. */
9161 u32 available_bandwidth = dce8_available_bandwidth(wm);
9162 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9163 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9164 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9165 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9166 (wm->num_heads * cursor_line_pair_return_time);
9167 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9168 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9169 u32 tmp, dmif_size = 12288;
9170 fixed20_12 a, b, c;
9171
9172 if (wm->num_heads == 0)
9173 return 0;
9174
9175 a.full = dfixed_const(2);
9176 b.full = dfixed_const(1);
9177 if ((wm->vsc.full > a.full) ||
9178 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9179 (wm->vtaps >= 5) ||
9180 ((wm->vsc.full >= a.full) && wm->interlaced))
9181 max_src_lines_per_dst_line = 4;
9182 else
9183 max_src_lines_per_dst_line = 2;
9184
9185 a.full = dfixed_const(available_bandwidth);
9186 b.full = dfixed_const(wm->num_heads);
9187 a.full = dfixed_div(a, b);
9188
9189 b.full = dfixed_const(mc_latency + 512);
9190 c.full = dfixed_const(wm->disp_clk);
9191 b.full = dfixed_div(b, c);
9192
9193 c.full = dfixed_const(dmif_size);
9194 b.full = dfixed_div(c, b);
9195
9196 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9197
9198 b.full = dfixed_const(1000);
9199 c.full = dfixed_const(wm->disp_clk);
9200 b.full = dfixed_div(c, b);
9201 c.full = dfixed_const(wm->bytes_per_pixel);
9202 b.full = dfixed_mul(b, c);
9203
9204 lb_fill_bw = min(tmp, dfixed_trunc(b));
9205
9206 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9207 b.full = dfixed_const(1000);
9208 c.full = dfixed_const(lb_fill_bw);
9209 b.full = dfixed_div(c, b);
9210 a.full = dfixed_div(a, b);
9211 line_fill_time = dfixed_trunc(a);
9212
9213 if (line_fill_time < wm->active_time)
9214 return latency;
9215 else
9216 return latency + (line_fill_time - wm->active_time);
9217
9218 }
9219
9220 /**
9221 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9222 * average and available dram bandwidth
9223 *
9224 * @wm: watermark calculation data
9225 *
9226 * Check if the display average bandwidth fits in the display
9227 * dram bandwidth (CIK).
9228 * Used for display watermark bandwidth calculations
9229 * Returns true if the display fits, false if not.
9230 */
9231 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9232 {
9233 if (dce8_average_bandwidth(wm) <=
9234 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9235 return true;
9236 else
9237 return false;
9238 }
9239
9240 /**
9241 * dce8_average_bandwidth_vs_available_bandwidth - check
9242 * average and available bandwidth
9243 *
9244 * @wm: watermark calculation data
9245 *
9246 * Check if the display average bandwidth fits in the display
9247 * available bandwidth (CIK).
9248 * Used for display watermark bandwidth calculations
9249 * Returns true if the display fits, false if not.
9250 */
9251 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9252 {
9253 if (dce8_average_bandwidth(wm) <=
9254 (dce8_available_bandwidth(wm) / wm->num_heads))
9255 return true;
9256 else
9257 return false;
9258 }
9259
9260 /**
9261 * dce8_check_latency_hiding - check latency hiding
9262 *
9263 * @wm: watermark calculation data
9264 *
9265 * Check latency hiding (CIK).
9266 * Used for display watermark bandwidth calculations
9267 * Returns true if the display fits, false if not.
9268 */
9269 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9270 {
9271 u32 lb_partitions = wm->lb_size / wm->src_width;
9272 u32 line_time = wm->active_time + wm->blank_time;
9273 u32 latency_tolerant_lines;
9274 u32 latency_hiding;
9275 fixed20_12 a;
9276
9277 a.full = dfixed_const(1);
9278 if (wm->vsc.full > a.full)
9279 latency_tolerant_lines = 1;
9280 else {
9281 if (lb_partitions <= (wm->vtaps + 1))
9282 latency_tolerant_lines = 1;
9283 else
9284 latency_tolerant_lines = 2;
9285 }
9286
9287 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9288
9289 if (dce8_latency_watermark(wm) <= latency_hiding)
9290 return true;
9291 else
9292 return false;
9293 }
9294
9295 /**
9296 * dce8_program_watermarks - program display watermarks
9297 *
9298 * @rdev: radeon_device pointer
9299 * @radeon_crtc: the selected display controller
9300 * @lb_size: line buffer size
9301 * @num_heads: number of display controllers in use
9302 *
9303 * Calculate and program the display watermarks for the
9304 * selected display controller (CIK).
9305 */
9306 static void dce8_program_watermarks(struct radeon_device *rdev,
9307 struct radeon_crtc *radeon_crtc,
9308 u32 lb_size, u32 num_heads)
9309 {
9310 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9311 struct dce8_wm_params wm_low, wm_high;
9312 u32 pixel_period;
9313 u32 line_time = 0;
9314 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9315 u32 tmp, wm_mask;
9316
9317 if (radeon_crtc->base.enabled && num_heads && mode) {
9318 pixel_period = 1000000 / (u32)mode->clock;
9319 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9320
9321 /* watermark for high clocks */
9322 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9323 rdev->pm.dpm_enabled) {
9324 wm_high.yclk =
9325 radeon_dpm_get_mclk(rdev, false) * 10;
9326 wm_high.sclk =
9327 radeon_dpm_get_sclk(rdev, false) * 10;
9328 } else {
9329 wm_high.yclk = rdev->pm.current_mclk * 10;
9330 wm_high.sclk = rdev->pm.current_sclk * 10;
9331 }
9332
9333 wm_high.disp_clk = mode->clock;
9334 wm_high.src_width = mode->crtc_hdisplay;
9335 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9336 wm_high.blank_time = line_time - wm_high.active_time;
9337 wm_high.interlaced = false;
9338 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9339 wm_high.interlaced = true;
9340 wm_high.vsc = radeon_crtc->vsc;
9341 wm_high.vtaps = 1;
9342 if (radeon_crtc->rmx_type != RMX_OFF)
9343 wm_high.vtaps = 2;
9344 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9345 wm_high.lb_size = lb_size;
9346 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9347 wm_high.num_heads = num_heads;
9348
9349 /* set for high clocks */
9350 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9351
9352 /* possibly force display priority to high */
9353 /* should really do this at mode validation time... */
9354 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9355 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9356 !dce8_check_latency_hiding(&wm_high) ||
9357 (rdev->disp_priority == 2)) {
9358 DRM_DEBUG_KMS("force priority to high\n");
9359 }
9360
9361 /* watermark for low clocks */
9362 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9363 rdev->pm.dpm_enabled) {
9364 wm_low.yclk =
9365 radeon_dpm_get_mclk(rdev, true) * 10;
9366 wm_low.sclk =
9367 radeon_dpm_get_sclk(rdev, true) * 10;
9368 } else {
9369 wm_low.yclk = rdev->pm.current_mclk * 10;
9370 wm_low.sclk = rdev->pm.current_sclk * 10;
9371 }
9372
9373 wm_low.disp_clk = mode->clock;
9374 wm_low.src_width = mode->crtc_hdisplay;
9375 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9376 wm_low.blank_time = line_time - wm_low.active_time;
9377 wm_low.interlaced = false;
9378 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9379 wm_low.interlaced = true;
9380 wm_low.vsc = radeon_crtc->vsc;
9381 wm_low.vtaps = 1;
9382 if (radeon_crtc->rmx_type != RMX_OFF)
9383 wm_low.vtaps = 2;
9384 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9385 wm_low.lb_size = lb_size;
9386 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9387 wm_low.num_heads = num_heads;
9388
9389 /* set for low clocks */
9390 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9391
9392 /* possibly force display priority to high */
9393 /* should really do this at mode validation time... */
9394 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9395 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9396 !dce8_check_latency_hiding(&wm_low) ||
9397 (rdev->disp_priority == 2)) {
9398 DRM_DEBUG_KMS("force priority to high\n");
9399 }
9400 }
9401
9402 /* select wm A */
9403 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9404 tmp = wm_mask;
9405 tmp &= ~LATENCY_WATERMARK_MASK(3);
9406 tmp |= LATENCY_WATERMARK_MASK(1);
9407 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9408 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9409 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9410 LATENCY_HIGH_WATERMARK(line_time)));
9411 /* select wm B */
9412 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9413 tmp &= ~LATENCY_WATERMARK_MASK(3);
9414 tmp |= LATENCY_WATERMARK_MASK(2);
9415 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9416 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9417 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9418 LATENCY_HIGH_WATERMARK(line_time)));
9419 /* restore original selection */
9420 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9421
9422 /* save values for DPM */
9423 radeon_crtc->line_time = line_time;
9424 radeon_crtc->wm_high = latency_watermark_a;
9425 radeon_crtc->wm_low = latency_watermark_b;
9426 }
9427
9428 /**
9429 * dce8_bandwidth_update - program display watermarks
9430 *
9431 * @rdev: radeon_device pointer
9432 *
9433 * Calculate and program the display watermarks and line
9434 * buffer allocation (CIK).
9435 */
9436 void dce8_bandwidth_update(struct radeon_device *rdev)
9437 {
9438 struct drm_display_mode *mode = NULL;
9439 u32 num_heads = 0, lb_size;
9440 int i;
9441
9442 radeon_update_display_priority(rdev);
9443
9444 for (i = 0; i < rdev->num_crtc; i++) {
9445 if (rdev->mode_info.crtcs[i]->base.enabled)
9446 num_heads++;
9447 }
9448 for (i = 0; i < rdev->num_crtc; i++) {
9449 mode = &rdev->mode_info.crtcs[i]->base.mode;
9450 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9451 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9452 }
9453 }
9454
9455 /**
9456 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9457 *
9458 * @rdev: radeon_device pointer
9459 *
9460 * Fetches a GPU clock counter snapshot (SI).
9461 * Returns the 64 bit clock counter snapshot.
9462 */
9463 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9464 {
9465 uint64_t clock;
9466
9467 mutex_lock(&rdev->gpu_clock_mutex);
9468 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9469 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9470 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9471 mutex_unlock(&rdev->gpu_clock_mutex);
9472 return clock;
9473 }
9474
9475 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9476 u32 cntl_reg, u32 status_reg)
9477 {
9478 int r, i;
9479 struct atom_clock_dividers dividers;
9480 uint32_t tmp;
9481
9482 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9483 clock, false, &dividers);
9484 if (r)
9485 return r;
9486
9487 tmp = RREG32_SMC(cntl_reg);
9488 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9489 tmp |= dividers.post_divider;
9490 WREG32_SMC(cntl_reg, tmp);
9491
9492 for (i = 0; i < 100; i++) {
9493 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9494 break;
9495 mdelay(10);
9496 }
9497 if (i == 100)
9498 return -ETIMEDOUT;
9499
9500 return 0;
9501 }
9502
9503 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9504 {
9505 int r = 0;
9506
9507 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9508 if (r)
9509 return r;
9510
9511 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9512 return r;
9513 }
9514
9515 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9516 {
9517 int r, i;
9518 struct atom_clock_dividers dividers;
9519 u32 tmp;
9520
9521 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9522 ecclk, false, &dividers);
9523 if (r)
9524 return r;
9525
9526 for (i = 0; i < 100; i++) {
9527 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9528 break;
9529 mdelay(10);
9530 }
9531 if (i == 100)
9532 return -ETIMEDOUT;
9533
9534 tmp = RREG32_SMC(CG_ECLK_CNTL);
9535 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9536 tmp |= dividers.post_divider;
9537 WREG32_SMC(CG_ECLK_CNTL, tmp);
9538
9539 for (i = 0; i < 100; i++) {
9540 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9541 break;
9542 mdelay(10);
9543 }
9544 if (i == 100)
9545 return -ETIMEDOUT;
9546
9547 return 0;
9548 }
9549
9550 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9551 {
9552 struct pci_dev *root = rdev->pdev->bus->self;
9553 int bridge_pos, gpu_pos;
9554 u32 speed_cntl, mask, current_data_rate;
9555 int ret, i;
9556 u16 tmp16;
9557
9558 if (radeon_pcie_gen2 == 0)
9559 return;
9560
9561 if (rdev->flags & RADEON_IS_IGP)
9562 return;
9563
9564 if (!(rdev->flags & RADEON_IS_PCIE))
9565 return;
9566
9567 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9568 if (ret != 0)
9569 return;
9570
9571 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9572 return;
9573
9574 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9575 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9576 LC_CURRENT_DATA_RATE_SHIFT;
9577 if (mask & DRM_PCIE_SPEED_80) {
9578 if (current_data_rate == 2) {
9579 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9580 return;
9581 }
9582 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9583 } else if (mask & DRM_PCIE_SPEED_50) {
9584 if (current_data_rate == 1) {
9585 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9586 return;
9587 }
9588 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9589 }
9590
9591 bridge_pos = pci_pcie_cap(root);
9592 if (!bridge_pos)
9593 return;
9594
9595 gpu_pos = pci_pcie_cap(rdev->pdev);
9596 if (!gpu_pos)
9597 return;
9598
9599 if (mask & DRM_PCIE_SPEED_80) {
9600 /* re-try equalization if gen3 is not already enabled */
9601 if (current_data_rate != 2) {
9602 u16 bridge_cfg, gpu_cfg;
9603 u16 bridge_cfg2, gpu_cfg2;
9604 u32 max_lw, current_lw, tmp;
9605
9606 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9607 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9608
9609 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9610 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9611
9612 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9613 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9614
9615 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9616 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9617 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9618
9619 if (current_lw < max_lw) {
9620 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9621 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9622 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9623 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9624 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9625 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9626 }
9627 }
9628
9629 for (i = 0; i < 10; i++) {
9630 /* check status */
9631 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9632 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9633 break;
9634
9635 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9636 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9637
9638 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9639 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9640
9641 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9642 tmp |= LC_SET_QUIESCE;
9643 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9644
9645 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9646 tmp |= LC_REDO_EQ;
9647 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9648
9649 mdelay(100);
9650
9651 /* linkctl */
9652 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9653 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9654 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9655 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9656
9657 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9658 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9659 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9660 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9661
9662 /* linkctl2 */
9663 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9664 tmp16 &= ~((1 << 4) | (7 << 9));
9665 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9666 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9667
9668 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9669 tmp16 &= ~((1 << 4) | (7 << 9));
9670 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9671 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9672
9673 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9674 tmp &= ~LC_SET_QUIESCE;
9675 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9676 }
9677 }
9678 }
9679
9680 /* set the link speed */
9681 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9682 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9683 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9684
9685 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9686 tmp16 &= ~0xf;
9687 if (mask & DRM_PCIE_SPEED_80)
9688 tmp16 |= 3; /* gen3 */
9689 else if (mask & DRM_PCIE_SPEED_50)
9690 tmp16 |= 2; /* gen2 */
9691 else
9692 tmp16 |= 1; /* gen1 */
9693 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9694
9695 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9696 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9697 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9698
9699 for (i = 0; i < rdev->usec_timeout; i++) {
9700 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9701 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9702 break;
9703 udelay(1);
9704 }
9705 }
9706
9707 static void cik_program_aspm(struct radeon_device *rdev)
9708 {
9709 u32 data, orig;
9710 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9711 bool disable_clkreq = false;
9712
9713 if (radeon_aspm == 0)
9714 return;
9715
9716 /* XXX double check IGPs */
9717 if (rdev->flags & RADEON_IS_IGP)
9718 return;
9719
9720 if (!(rdev->flags & RADEON_IS_PCIE))
9721 return;
9722
9723 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9724 data &= ~LC_XMIT_N_FTS_MASK;
9725 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9726 if (orig != data)
9727 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9728
9729 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9730 data |= LC_GO_TO_RECOVERY;
9731 if (orig != data)
9732 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9733
9734 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9735 data |= P_IGNORE_EDB_ERR;
9736 if (orig != data)
9737 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9738
9739 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9740 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9741 data |= LC_PMI_TO_L1_DIS;
9742 if (!disable_l0s)
9743 data |= LC_L0S_INACTIVITY(7);
9744
9745 if (!disable_l1) {
9746 data |= LC_L1_INACTIVITY(7);
9747 data &= ~LC_PMI_TO_L1_DIS;
9748 if (orig != data)
9749 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9750
9751 if (!disable_plloff_in_l1) {
9752 bool clk_req_support;
9753
9754 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9755 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9756 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9757 if (orig != data)
9758 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9759
9760 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9761 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9762 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9763 if (orig != data)
9764 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9765
9766 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9767 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9768 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9769 if (orig != data)
9770 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9771
9772 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9773 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9774 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9775 if (orig != data)
9776 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9777
9778 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9779 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9780 data |= LC_DYN_LANES_PWR_STATE(3);
9781 if (orig != data)
9782 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9783
9784 if (!disable_clkreq) {
9785 struct pci_dev *root = rdev->pdev->bus->self;
9786 u32 lnkcap;
9787
9788 clk_req_support = false;
9789 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9790 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9791 clk_req_support = true;
9792 } else {
9793 clk_req_support = false;
9794 }
9795
9796 if (clk_req_support) {
9797 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9798 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9799 if (orig != data)
9800 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9801
9802 orig = data = RREG32_SMC(THM_CLK_CNTL);
9803 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9804 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9805 if (orig != data)
9806 WREG32_SMC(THM_CLK_CNTL, data);
9807
9808 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9809 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9810 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9811 if (orig != data)
9812 WREG32_SMC(MISC_CLK_CTRL, data);
9813
9814 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9815 data &= ~BCLK_AS_XCLK;
9816 if (orig != data)
9817 WREG32_SMC(CG_CLKPIN_CNTL, data);
9818
9819 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9820 data &= ~FORCE_BIF_REFCLK_EN;
9821 if (orig != data)
9822 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9823
9824 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9825 data &= ~MPLL_CLKOUT_SEL_MASK;
9826 data |= MPLL_CLKOUT_SEL(4);
9827 if (orig != data)
9828 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9829 }
9830 }
9831 } else {
9832 if (orig != data)
9833 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9834 }
9835
9836 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9837 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9838 if (orig != data)
9839 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9840
9841 if (!disable_l0s) {
9842 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9843 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9844 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9845 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9846 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9847 data &= ~LC_L0S_INACTIVITY_MASK;
9848 if (orig != data)
9849 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9850 }
9851 }
9852 }
9853 }
Linux kernel - Deliverables
This page took 0.334217 seconds and 5 git commands to generate.