projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'fixes' of git://ftp.arm.linux.org.uk/~rmk/linux-arm
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
35
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
66
67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
72 extern void sumo_rlc_fini(struct radeon_device *rdev);
73 extern int sumo_rlc_init(struct radeon_device *rdev);
74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
75 extern void si_rlc_reset(struct radeon_device *rdev);
76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
77 extern int cik_sdma_resume(struct radeon_device *rdev);
78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
79 extern void cik_sdma_fini(struct radeon_device *rdev);
80 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
81 static void cik_rlc_stop(struct radeon_device *rdev);
82 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
83 static void cik_program_aspm(struct radeon_device *rdev);
84 static void cik_init_pg(struct radeon_device *rdev);
85 static void cik_init_cg(struct radeon_device *rdev);
86 static void cik_fini_pg(struct radeon_device *rdev);
87 static void cik_fini_cg(struct radeon_device *rdev);
88 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
89 bool enable);
90
91 /* get temperature in millidegrees */
92 int ci_get_temp(struct radeon_device *rdev)
93 {
94 u32 temp;
95 int actual_temp = 0;
96
97 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
98 CTF_TEMP_SHIFT;
99
100 if (temp & 0x200)
101 actual_temp = 255;
102 else
103 actual_temp = temp & 0x1ff;
104
105 actual_temp = actual_temp * 1000;
106
107 return actual_temp;
108 }
109
110 /* get temperature in millidegrees */
111 int kv_get_temp(struct radeon_device *rdev)
112 {
113 u32 temp;
114 int actual_temp = 0;
115
116 temp = RREG32_SMC(0xC0300E0C);
117
118 if (temp)
119 actual_temp = (temp / 8) - 49;
120 else
121 actual_temp = 0;
122
123 actual_temp = actual_temp * 1000;
124
125 return actual_temp;
126 }
127
128 /*
129 * Indirect registers accessor
130 */
131 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
132 {
133 unsigned long flags;
134 u32 r;
135
136 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
137 WREG32(PCIE_INDEX, reg);
138 (void)RREG32(PCIE_INDEX);
139 r = RREG32(PCIE_DATA);
140 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
141 return r;
142 }
143
144 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
145 {
146 unsigned long flags;
147
148 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
149 WREG32(PCIE_INDEX, reg);
150 (void)RREG32(PCIE_INDEX);
151 WREG32(PCIE_DATA, v);
152 (void)RREG32(PCIE_DATA);
153 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
154 }
155
156 static const u32 spectre_rlc_save_restore_register_list[] =
157 {
158 (0x0e00 << 16) | (0xc12c >> 2),
159 0x00000000,
160 (0x0e00 << 16) | (0xc140 >> 2),
161 0x00000000,
162 (0x0e00 << 16) | (0xc150 >> 2),
163 0x00000000,
164 (0x0e00 << 16) | (0xc15c >> 2),
165 0x00000000,
166 (0x0e00 << 16) | (0xc168 >> 2),
167 0x00000000,
168 (0x0e00 << 16) | (0xc170 >> 2),
169 0x00000000,
170 (0x0e00 << 16) | (0xc178 >> 2),
171 0x00000000,
172 (0x0e00 << 16) | (0xc204 >> 2),
173 0x00000000,
174 (0x0e00 << 16) | (0xc2b4 >> 2),
175 0x00000000,
176 (0x0e00 << 16) | (0xc2b8 >> 2),
177 0x00000000,
178 (0x0e00 << 16) | (0xc2bc >> 2),
179 0x00000000,
180 (0x0e00 << 16) | (0xc2c0 >> 2),
181 0x00000000,
182 (0x0e00 << 16) | (0x8228 >> 2),
183 0x00000000,
184 (0x0e00 << 16) | (0x829c >> 2),
185 0x00000000,
186 (0x0e00 << 16) | (0x869c >> 2),
187 0x00000000,
188 (0x0600 << 16) | (0x98f4 >> 2),
189 0x00000000,
190 (0x0e00 << 16) | (0x98f8 >> 2),
191 0x00000000,
192 (0x0e00 << 16) | (0x9900 >> 2),
193 0x00000000,
194 (0x0e00 << 16) | (0xc260 >> 2),
195 0x00000000,
196 (0x0e00 << 16) | (0x90e8 >> 2),
197 0x00000000,
198 (0x0e00 << 16) | (0x3c000 >> 2),
199 0x00000000,
200 (0x0e00 << 16) | (0x3c00c >> 2),
201 0x00000000,
202 (0x0e00 << 16) | (0x8c1c >> 2),
203 0x00000000,
204 (0x0e00 << 16) | (0x9700 >> 2),
205 0x00000000,
206 (0x0e00 << 16) | (0xcd20 >> 2),
207 0x00000000,
208 (0x4e00 << 16) | (0xcd20 >> 2),
209 0x00000000,
210 (0x5e00 << 16) | (0xcd20 >> 2),
211 0x00000000,
212 (0x6e00 << 16) | (0xcd20 >> 2),
213 0x00000000,
214 (0x7e00 << 16) | (0xcd20 >> 2),
215 0x00000000,
216 (0x8e00 << 16) | (0xcd20 >> 2),
217 0x00000000,
218 (0x9e00 << 16) | (0xcd20 >> 2),
219 0x00000000,
220 (0xae00 << 16) | (0xcd20 >> 2),
221 0x00000000,
222 (0xbe00 << 16) | (0xcd20 >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0x89bc >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0x8900 >> 2),
227 0x00000000,
228 0x3,
229 (0x0e00 << 16) | (0xc130 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc134 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc1fc >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0xc208 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0xc264 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xc268 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0xc26c >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0xc270 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc274 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc278 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0xc27c >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0xc280 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0xc284 >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0xc288 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xc28c >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xc290 >> 2),
260 0x00000000,
261 (0x0e00 << 16) | (0xc294 >> 2),
262 0x00000000,
263 (0x0e00 << 16) | (0xc298 >> 2),
264 0x00000000,
265 (0x0e00 << 16) | (0xc29c >> 2),
266 0x00000000,
267 (0x0e00 << 16) | (0xc2a0 >> 2),
268 0x00000000,
269 (0x0e00 << 16) | (0xc2a4 >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0xc2a8 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc2ac >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc2b0 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x301d0 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x30238 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0x30250 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0x30254 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0x30258 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0x3025c >> 2),
288 0x00000000,
289 (0x4e00 << 16) | (0xc900 >> 2),
290 0x00000000,
291 (0x5e00 << 16) | (0xc900 >> 2),
292 0x00000000,
293 (0x6e00 << 16) | (0xc900 >> 2),
294 0x00000000,
295 (0x7e00 << 16) | (0xc900 >> 2),
296 0x00000000,
297 (0x8e00 << 16) | (0xc900 >> 2),
298 0x00000000,
299 (0x9e00 << 16) | (0xc900 >> 2),
300 0x00000000,
301 (0xae00 << 16) | (0xc900 >> 2),
302 0x00000000,
303 (0xbe00 << 16) | (0xc900 >> 2),
304 0x00000000,
305 (0x4e00 << 16) | (0xc904 >> 2),
306 0x00000000,
307 (0x5e00 << 16) | (0xc904 >> 2),
308 0x00000000,
309 (0x6e00 << 16) | (0xc904 >> 2),
310 0x00000000,
311 (0x7e00 << 16) | (0xc904 >> 2),
312 0x00000000,
313 (0x8e00 << 16) | (0xc904 >> 2),
314 0x00000000,
315 (0x9e00 << 16) | (0xc904 >> 2),
316 0x00000000,
317 (0xae00 << 16) | (0xc904 >> 2),
318 0x00000000,
319 (0xbe00 << 16) | (0xc904 >> 2),
320 0x00000000,
321 (0x4e00 << 16) | (0xc908 >> 2),
322 0x00000000,
323 (0x5e00 << 16) | (0xc908 >> 2),
324 0x00000000,
325 (0x6e00 << 16) | (0xc908 >> 2),
326 0x00000000,
327 (0x7e00 << 16) | (0xc908 >> 2),
328 0x00000000,
329 (0x8e00 << 16) | (0xc908 >> 2),
330 0x00000000,
331 (0x9e00 << 16) | (0xc908 >> 2),
332 0x00000000,
333 (0xae00 << 16) | (0xc908 >> 2),
334 0x00000000,
335 (0xbe00 << 16) | (0xc908 >> 2),
336 0x00000000,
337 (0x4e00 << 16) | (0xc90c >> 2),
338 0x00000000,
339 (0x5e00 << 16) | (0xc90c >> 2),
340 0x00000000,
341 (0x6e00 << 16) | (0xc90c >> 2),
342 0x00000000,
343 (0x7e00 << 16) | (0xc90c >> 2),
344 0x00000000,
345 (0x8e00 << 16) | (0xc90c >> 2),
346 0x00000000,
347 (0x9e00 << 16) | (0xc90c >> 2),
348 0x00000000,
349 (0xae00 << 16) | (0xc90c >> 2),
350 0x00000000,
351 (0xbe00 << 16) | (0xc90c >> 2),
352 0x00000000,
353 (0x4e00 << 16) | (0xc910 >> 2),
354 0x00000000,
355 (0x5e00 << 16) | (0xc910 >> 2),
356 0x00000000,
357 (0x6e00 << 16) | (0xc910 >> 2),
358 0x00000000,
359 (0x7e00 << 16) | (0xc910 >> 2),
360 0x00000000,
361 (0x8e00 << 16) | (0xc910 >> 2),
362 0x00000000,
363 (0x9e00 << 16) | (0xc910 >> 2),
364 0x00000000,
365 (0xae00 << 16) | (0xc910 >> 2),
366 0x00000000,
367 (0xbe00 << 16) | (0xc910 >> 2),
368 0x00000000,
369 (0x0e00 << 16) | (0xc99c >> 2),
370 0x00000000,
371 (0x0e00 << 16) | (0x9834 >> 2),
372 0x00000000,
373 (0x0000 << 16) | (0x30f00 >> 2),
374 0x00000000,
375 (0x0001 << 16) | (0x30f00 >> 2),
376 0x00000000,
377 (0x0000 << 16) | (0x30f04 >> 2),
378 0x00000000,
379 (0x0001 << 16) | (0x30f04 >> 2),
380 0x00000000,
381 (0x0000 << 16) | (0x30f08 >> 2),
382 0x00000000,
383 (0x0001 << 16) | (0x30f08 >> 2),
384 0x00000000,
385 (0x0000 << 16) | (0x30f0c >> 2),
386 0x00000000,
387 (0x0001 << 16) | (0x30f0c >> 2),
388 0x00000000,
389 (0x0600 << 16) | (0x9b7c >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0x8a14 >> 2),
392 0x00000000,
393 (0x0e00 << 16) | (0x8a18 >> 2),
394 0x00000000,
395 (0x0600 << 16) | (0x30a00 >> 2),
396 0x00000000,
397 (0x0e00 << 16) | (0x8bf0 >> 2),
398 0x00000000,
399 (0x0e00 << 16) | (0x8bcc >> 2),
400 0x00000000,
401 (0x0e00 << 16) | (0x8b24 >> 2),
402 0x00000000,
403 (0x0e00 << 16) | (0x30a04 >> 2),
404 0x00000000,
405 (0x0600 << 16) | (0x30a10 >> 2),
406 0x00000000,
407 (0x0600 << 16) | (0x30a14 >> 2),
408 0x00000000,
409 (0x0600 << 16) | (0x30a18 >> 2),
410 0x00000000,
411 (0x0600 << 16) | (0x30a2c >> 2),
412 0x00000000,
413 (0x0e00 << 16) | (0xc700 >> 2),
414 0x00000000,
415 (0x0e00 << 16) | (0xc704 >> 2),
416 0x00000000,
417 (0x0e00 << 16) | (0xc708 >> 2),
418 0x00000000,
419 (0x0e00 << 16) | (0xc768 >> 2),
420 0x00000000,
421 (0x0400 << 16) | (0xc770 >> 2),
422 0x00000000,
423 (0x0400 << 16) | (0xc774 >> 2),
424 0x00000000,
425 (0x0400 << 16) | (0xc778 >> 2),
426 0x00000000,
427 (0x0400 << 16) | (0xc77c >> 2),
428 0x00000000,
429 (0x0400 << 16) | (0xc780 >> 2),
430 0x00000000,
431 (0x0400 << 16) | (0xc784 >> 2),
432 0x00000000,
433 (0x0400 << 16) | (0xc788 >> 2),
434 0x00000000,
435 (0x0400 << 16) | (0xc78c >> 2),
436 0x00000000,
437 (0x0400 << 16) | (0xc798 >> 2),
438 0x00000000,
439 (0x0400 << 16) | (0xc79c >> 2),
440 0x00000000,
441 (0x0400 << 16) | (0xc7a0 >> 2),
442 0x00000000,
443 (0x0400 << 16) | (0xc7a4 >> 2),
444 0x00000000,
445 (0x0400 << 16) | (0xc7a8 >> 2),
446 0x00000000,
447 (0x0400 << 16) | (0xc7ac >> 2),
448 0x00000000,
449 (0x0400 << 16) | (0xc7b0 >> 2),
450 0x00000000,
451 (0x0400 << 16) | (0xc7b4 >> 2),
452 0x00000000,
453 (0x0e00 << 16) | (0x9100 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0x3c010 >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x92a8 >> 2),
458 0x00000000,
459 (0x0e00 << 16) | (0x92ac >> 2),
460 0x00000000,
461 (0x0e00 << 16) | (0x92b4 >> 2),
462 0x00000000,
463 (0x0e00 << 16) | (0x92b8 >> 2),
464 0x00000000,
465 (0x0e00 << 16) | (0x92bc >> 2),
466 0x00000000,
467 (0x0e00 << 16) | (0x92c0 >> 2),
468 0x00000000,
469 (0x0e00 << 16) | (0x92c4 >> 2),
470 0x00000000,
471 (0x0e00 << 16) | (0x92c8 >> 2),
472 0x00000000,
473 (0x0e00 << 16) | (0x92cc >> 2),
474 0x00000000,
475 (0x0e00 << 16) | (0x92d0 >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8c00 >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0x8c04 >> 2),
480 0x00000000,
481 (0x0e00 << 16) | (0x8c20 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0x8c38 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0x8c3c >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0xae00 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0x9604 >> 2),
490 0x00000000,
491 (0x0e00 << 16) | (0xac08 >> 2),
492 0x00000000,
493 (0x0e00 << 16) | (0xac0c >> 2),
494 0x00000000,
495 (0x0e00 << 16) | (0xac10 >> 2),
496 0x00000000,
497 (0x0e00 << 16) | (0xac14 >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xac58 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xac68 >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xac6c >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xac70 >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0xac74 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0xac78 >> 2),
510 0x00000000,
511 (0x0e00 << 16) | (0xac7c >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0xac80 >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0xac84 >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0xac88 >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0xac8c >> 2),
520 0x00000000,
521 (0x0e00 << 16) | (0x970c >> 2),
522 0x00000000,
523 (0x0e00 << 16) | (0x9714 >> 2),
524 0x00000000,
525 (0x0e00 << 16) | (0x9718 >> 2),
526 0x00000000,
527 (0x0e00 << 16) | (0x971c >> 2),
528 0x00000000,
529 (0x0e00 << 16) | (0x31068 >> 2),
530 0x00000000,
531 (0x4e00 << 16) | (0x31068 >> 2),
532 0x00000000,
533 (0x5e00 << 16) | (0x31068 >> 2),
534 0x00000000,
535 (0x6e00 << 16) | (0x31068 >> 2),
536 0x00000000,
537 (0x7e00 << 16) | (0x31068 >> 2),
538 0x00000000,
539 (0x8e00 << 16) | (0x31068 >> 2),
540 0x00000000,
541 (0x9e00 << 16) | (0x31068 >> 2),
542 0x00000000,
543 (0xae00 << 16) | (0x31068 >> 2),
544 0x00000000,
545 (0xbe00 << 16) | (0x31068 >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0xcd10 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0xcd14 >> 2),
550 0x00000000,
551 (0x0e00 << 16) | (0x88b0 >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x88b4 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x88b8 >> 2),
556 0x00000000,
557 (0x0e00 << 16) | (0x88bc >> 2),
558 0x00000000,
559 (0x0400 << 16) | (0x89c0 >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x88c4 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x88c8 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x88d0 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x88d4 >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x88d8 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x8980 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x30938 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x3093c >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x30940 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x89a0 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x30900 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x30904 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x89b4 >> 2),
586 0x00000000,
587 (0x0e00 << 16) | (0x3c210 >> 2),
588 0x00000000,
589 (0x0e00 << 16) | (0x3c214 >> 2),
590 0x00000000,
591 (0x0e00 << 16) | (0x3c218 >> 2),
592 0x00000000,
593 (0x0e00 << 16) | (0x8904 >> 2),
594 0x00000000,
595 0x5,
596 (0x0e00 << 16) | (0x8c28 >> 2),
597 (0x0e00 << 16) | (0x8c2c >> 2),
598 (0x0e00 << 16) | (0x8c30 >> 2),
599 (0x0e00 << 16) | (0x8c34 >> 2),
600 (0x0e00 << 16) | (0x9600 >> 2),
601 };
602
603 static const u32 kalindi_rlc_save_restore_register_list[] =
604 {
605 (0x0e00 << 16) | (0xc12c >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xc140 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xc150 >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xc15c >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xc168 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xc170 >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xc204 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0xc2b4 >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0xc2b8 >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0xc2bc >> 2),
624 0x00000000,
625 (0x0e00 << 16) | (0xc2c0 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0x8228 >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0x829c >> 2),
630 0x00000000,
631 (0x0e00 << 16) | (0x869c >> 2),
632 0x00000000,
633 (0x0600 << 16) | (0x98f4 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0x98f8 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x9900 >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0xc260 >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x90e8 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0x3c000 >> 2),
644 0x00000000,
645 (0x0e00 << 16) | (0x3c00c >> 2),
646 0x00000000,
647 (0x0e00 << 16) | (0x8c1c >> 2),
648 0x00000000,
649 (0x0e00 << 16) | (0x9700 >> 2),
650 0x00000000,
651 (0x0e00 << 16) | (0xcd20 >> 2),
652 0x00000000,
653 (0x4e00 << 16) | (0xcd20 >> 2),
654 0x00000000,
655 (0x5e00 << 16) | (0xcd20 >> 2),
656 0x00000000,
657 (0x6e00 << 16) | (0xcd20 >> 2),
658 0x00000000,
659 (0x7e00 << 16) | (0xcd20 >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0x89bc >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0x8900 >> 2),
664 0x00000000,
665 0x3,
666 (0x0e00 << 16) | (0xc130 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc134 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc1fc >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc208 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc264 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc268 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc26c >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0xc270 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0xc274 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xc28c >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0xc290 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0xc294 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc298 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0xc2a0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0xc2a4 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0xc2a8 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0xc2ac >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x301d0 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x30238 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x30250 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x30254 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x30258 >> 2),
709 0x00000000,
710 (0x0e00 << 16) | (0x3025c >> 2),
711 0x00000000,
712 (0x4e00 << 16) | (0xc900 >> 2),
713 0x00000000,
714 (0x5e00 << 16) | (0xc900 >> 2),
715 0x00000000,
716 (0x6e00 << 16) | (0xc900 >> 2),
717 0x00000000,
718 (0x7e00 << 16) | (0xc900 >> 2),
719 0x00000000,
720 (0x4e00 << 16) | (0xc904 >> 2),
721 0x00000000,
722 (0x5e00 << 16) | (0xc904 >> 2),
723 0x00000000,
724 (0x6e00 << 16) | (0xc904 >> 2),
725 0x00000000,
726 (0x7e00 << 16) | (0xc904 >> 2),
727 0x00000000,
728 (0x4e00 << 16) | (0xc908 >> 2),
729 0x00000000,
730 (0x5e00 << 16) | (0xc908 >> 2),
731 0x00000000,
732 (0x6e00 << 16) | (0xc908 >> 2),
733 0x00000000,
734 (0x7e00 << 16) | (0xc908 >> 2),
735 0x00000000,
736 (0x4e00 << 16) | (0xc90c >> 2),
737 0x00000000,
738 (0x5e00 << 16) | (0xc90c >> 2),
739 0x00000000,
740 (0x6e00 << 16) | (0xc90c >> 2),
741 0x00000000,
742 (0x7e00 << 16) | (0xc90c >> 2),
743 0x00000000,
744 (0x4e00 << 16) | (0xc910 >> 2),
745 0x00000000,
746 (0x5e00 << 16) | (0xc910 >> 2),
747 0x00000000,
748 (0x6e00 << 16) | (0xc910 >> 2),
749 0x00000000,
750 (0x7e00 << 16) | (0xc910 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0xc99c >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x9834 >> 2),
755 0x00000000,
756 (0x0000 << 16) | (0x30f00 >> 2),
757 0x00000000,
758 (0x0000 << 16) | (0x30f04 >> 2),
759 0x00000000,
760 (0x0000 << 16) | (0x30f08 >> 2),
761 0x00000000,
762 (0x0000 << 16) | (0x30f0c >> 2),
763 0x00000000,
764 (0x0600 << 16) | (0x9b7c >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0x8a14 >> 2),
767 0x00000000,
768 (0x0e00 << 16) | (0x8a18 >> 2),
769 0x00000000,
770 (0x0600 << 16) | (0x30a00 >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0x8bf0 >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x8bcc >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x8b24 >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0x30a04 >> 2),
779 0x00000000,
780 (0x0600 << 16) | (0x30a10 >> 2),
781 0x00000000,
782 (0x0600 << 16) | (0x30a14 >> 2),
783 0x00000000,
784 (0x0600 << 16) | (0x30a18 >> 2),
785 0x00000000,
786 (0x0600 << 16) | (0x30a2c >> 2),
787 0x00000000,
788 (0x0e00 << 16) | (0xc700 >> 2),
789 0x00000000,
790 (0x0e00 << 16) | (0xc704 >> 2),
791 0x00000000,
792 (0x0e00 << 16) | (0xc708 >> 2),
793 0x00000000,
794 (0x0e00 << 16) | (0xc768 >> 2),
795 0x00000000,
796 (0x0400 << 16) | (0xc770 >> 2),
797 0x00000000,
798 (0x0400 << 16) | (0xc774 >> 2),
799 0x00000000,
800 (0x0400 << 16) | (0xc798 >> 2),
801 0x00000000,
802 (0x0400 << 16) | (0xc79c >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0x9100 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0x3c010 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0x8c00 >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0x8c04 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0x8c20 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0x8c38 >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0x8c3c >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0xae00 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0x9604 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0xac08 >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0xac0c >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0xac10 >> 2),
827 0x00000000,
828 (0x0e00 << 16) | (0xac14 >> 2),
829 0x00000000,
830 (0x0e00 << 16) | (0xac58 >> 2),
831 0x00000000,
832 (0x0e00 << 16) | (0xac68 >> 2),
833 0x00000000,
834 (0x0e00 << 16) | (0xac6c >> 2),
835 0x00000000,
836 (0x0e00 << 16) | (0xac70 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xac74 >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0xac78 >> 2),
841 0x00000000,
842 (0x0e00 << 16) | (0xac7c >> 2),
843 0x00000000,
844 (0x0e00 << 16) | (0xac80 >> 2),
845 0x00000000,
846 (0x0e00 << 16) | (0xac84 >> 2),
847 0x00000000,
848 (0x0e00 << 16) | (0xac88 >> 2),
849 0x00000000,
850 (0x0e00 << 16) | (0xac8c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x970c >> 2),
853 0x00000000,
854 (0x0e00 << 16) | (0x9714 >> 2),
855 0x00000000,
856 (0x0e00 << 16) | (0x9718 >> 2),
857 0x00000000,
858 (0x0e00 << 16) | (0x971c >> 2),
859 0x00000000,
860 (0x0e00 << 16) | (0x31068 >> 2),
861 0x00000000,
862 (0x4e00 << 16) | (0x31068 >> 2),
863 0x00000000,
864 (0x5e00 << 16) | (0x31068 >> 2),
865 0x00000000,
866 (0x6e00 << 16) | (0x31068 >> 2),
867 0x00000000,
868 (0x7e00 << 16) | (0x31068 >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0xcd10 >> 2),
871 0x00000000,
872 (0x0e00 << 16) | (0xcd14 >> 2),
873 0x00000000,
874 (0x0e00 << 16) | (0x88b0 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0x88b4 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0x88b8 >> 2),
879 0x00000000,
880 (0x0e00 << 16) | (0x88bc >> 2),
881 0x00000000,
882 (0x0400 << 16) | (0x89c0 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x88c4 >> 2),
885 0x00000000,
886 (0x0e00 << 16) | (0x88c8 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x88d0 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x88d4 >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x88d8 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x8980 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x30938 >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x3093c >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x30940 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x89a0 >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0x30900 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x30904 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0x89b4 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0x3e1fc >> 2),
911 0x00000000,
912 (0x0e00 << 16) | (0x3c210 >> 2),
913 0x00000000,
914 (0x0e00 << 16) | (0x3c214 >> 2),
915 0x00000000,
916 (0x0e00 << 16) | (0x3c218 >> 2),
917 0x00000000,
918 (0x0e00 << 16) | (0x8904 >> 2),
919 0x00000000,
920 0x5,
921 (0x0e00 << 16) | (0x8c28 >> 2),
922 (0x0e00 << 16) | (0x8c2c >> 2),
923 (0x0e00 << 16) | (0x8c30 >> 2),
924 (0x0e00 << 16) | (0x8c34 >> 2),
925 (0x0e00 << 16) | (0x9600 >> 2),
926 };
927
928 static const u32 bonaire_golden_spm_registers[] =
929 {
930 0x30800, 0xe0ffffff, 0xe0000000
931 };
932
933 static const u32 bonaire_golden_common_registers[] =
934 {
935 0xc770, 0xffffffff, 0x00000800,
936 0xc774, 0xffffffff, 0x00000800,
937 0xc798, 0xffffffff, 0x00007fbf,
938 0xc79c, 0xffffffff, 0x00007faf
939 };
940
941 static const u32 bonaire_golden_registers[] =
942 {
943 0x3354, 0x00000333, 0x00000333,
944 0x3350, 0x000c0fc0, 0x00040200,
945 0x9a10, 0x00010000, 0x00058208,
946 0x3c000, 0xffff1fff, 0x00140000,
947 0x3c200, 0xfdfc0fff, 0x00000100,
948 0x3c234, 0x40000000, 0x40000200,
949 0x9830, 0xffffffff, 0x00000000,
950 0x9834, 0xf00fffff, 0x00000400,
951 0x9838, 0x0002021c, 0x00020200,
952 0xc78, 0x00000080, 0x00000000,
953 0x5bb0, 0x000000f0, 0x00000070,
954 0x5bc0, 0xf0311fff, 0x80300000,
955 0x98f8, 0x73773777, 0x12010001,
956 0x350c, 0x00810000, 0x408af000,
957 0x7030, 0x31000111, 0x00000011,
958 0x2f48, 0x73773777, 0x12010001,
959 0x220c, 0x00007fb6, 0x0021a1b1,
960 0x2210, 0x00007fb6, 0x002021b1,
961 0x2180, 0x00007fb6, 0x00002191,
962 0x2218, 0x00007fb6, 0x002121b1,
963 0x221c, 0x00007fb6, 0x002021b1,
964 0x21dc, 0x00007fb6, 0x00002191,
965 0x21e0, 0x00007fb6, 0x00002191,
966 0x3628, 0x0000003f, 0x0000000a,
967 0x362c, 0x0000003f, 0x0000000a,
968 0x2ae4, 0x00073ffe, 0x000022a2,
969 0x240c, 0x000007ff, 0x00000000,
970 0x8a14, 0xf000003f, 0x00000007,
971 0x8bf0, 0x00002001, 0x00000001,
972 0x8b24, 0xffffffff, 0x00ffffff,
973 0x30a04, 0x0000ff0f, 0x00000000,
974 0x28a4c, 0x07ffffff, 0x06000000,
975 0x4d8, 0x00000fff, 0x00000100,
976 0x3e78, 0x00000001, 0x00000002,
977 0x9100, 0x03000000, 0x0362c688,
978 0x8c00, 0x000000ff, 0x00000001,
979 0xe40, 0x00001fff, 0x00001fff,
980 0x9060, 0x0000007f, 0x00000020,
981 0x9508, 0x00010000, 0x00010000,
982 0xac14, 0x000003ff, 0x000000f3,
983 0xac0c, 0xffffffff, 0x00001032
984 };
985
986 static const u32 bonaire_mgcg_cgcg_init[] =
987 {
988 0xc420, 0xffffffff, 0xfffffffc,
989 0x30800, 0xffffffff, 0xe0000000,
990 0x3c2a0, 0xffffffff, 0x00000100,
991 0x3c208, 0xffffffff, 0x00000100,
992 0x3c2c0, 0xffffffff, 0xc0000100,
993 0x3c2c8, 0xffffffff, 0xc0000100,
994 0x3c2c4, 0xffffffff, 0xc0000100,
995 0x55e4, 0xffffffff, 0x00600100,
996 0x3c280, 0xffffffff, 0x00000100,
997 0x3c214, 0xffffffff, 0x06000100,
998 0x3c220, 0xffffffff, 0x00000100,
999 0x3c218, 0xffffffff, 0x06000100,
1000 0x3c204, 0xffffffff, 0x00000100,
1001 0x3c2e0, 0xffffffff, 0x00000100,
1002 0x3c224, 0xffffffff, 0x00000100,
1003 0x3c200, 0xffffffff, 0x00000100,
1004 0x3c230, 0xffffffff, 0x00000100,
1005 0x3c234, 0xffffffff, 0x00000100,
1006 0x3c250, 0xffffffff, 0x00000100,
1007 0x3c254, 0xffffffff, 0x00000100,
1008 0x3c258, 0xffffffff, 0x00000100,
1009 0x3c25c, 0xffffffff, 0x00000100,
1010 0x3c260, 0xffffffff, 0x00000100,
1011 0x3c27c, 0xffffffff, 0x00000100,
1012 0x3c278, 0xffffffff, 0x00000100,
1013 0x3c210, 0xffffffff, 0x06000100,
1014 0x3c290, 0xffffffff, 0x00000100,
1015 0x3c274, 0xffffffff, 0x00000100,
1016 0x3c2b4, 0xffffffff, 0x00000100,
1017 0x3c2b0, 0xffffffff, 0x00000100,
1018 0x3c270, 0xffffffff, 0x00000100,
1019 0x30800, 0xffffffff, 0xe0000000,
1020 0x3c020, 0xffffffff, 0x00010000,
1021 0x3c024, 0xffffffff, 0x00030002,
1022 0x3c028, 0xffffffff, 0x00040007,
1023 0x3c02c, 0xffffffff, 0x00060005,
1024 0x3c030, 0xffffffff, 0x00090008,
1025 0x3c034, 0xffffffff, 0x00010000,
1026 0x3c038, 0xffffffff, 0x00030002,
1027 0x3c03c, 0xffffffff, 0x00040007,
1028 0x3c040, 0xffffffff, 0x00060005,
1029 0x3c044, 0xffffffff, 0x00090008,
1030 0x3c048, 0xffffffff, 0x00010000,
1031 0x3c04c, 0xffffffff, 0x00030002,
1032 0x3c050, 0xffffffff, 0x00040007,
1033 0x3c054, 0xffffffff, 0x00060005,
1034 0x3c058, 0xffffffff, 0x00090008,
1035 0x3c05c, 0xffffffff, 0x00010000,
1036 0x3c060, 0xffffffff, 0x00030002,
1037 0x3c064, 0xffffffff, 0x00040007,
1038 0x3c068, 0xffffffff, 0x00060005,
1039 0x3c06c, 0xffffffff, 0x00090008,
1040 0x3c070, 0xffffffff, 0x00010000,
1041 0x3c074, 0xffffffff, 0x00030002,
1042 0x3c078, 0xffffffff, 0x00040007,
1043 0x3c07c, 0xffffffff, 0x00060005,
1044 0x3c080, 0xffffffff, 0x00090008,
1045 0x3c084, 0xffffffff, 0x00010000,
1046 0x3c088, 0xffffffff, 0x00030002,
1047 0x3c08c, 0xffffffff, 0x00040007,
1048 0x3c090, 0xffffffff, 0x00060005,
1049 0x3c094, 0xffffffff, 0x00090008,
1050 0x3c098, 0xffffffff, 0x00010000,
1051 0x3c09c, 0xffffffff, 0x00030002,
1052 0x3c0a0, 0xffffffff, 0x00040007,
1053 0x3c0a4, 0xffffffff, 0x00060005,
1054 0x3c0a8, 0xffffffff, 0x00090008,
1055 0x3c000, 0xffffffff, 0x96e00200,
1056 0x8708, 0xffffffff, 0x00900100,
1057 0xc424, 0xffffffff, 0x0020003f,
1058 0x38, 0xffffffff, 0x0140001c,
1059 0x3c, 0x000f0000, 0x000f0000,
1060 0x220, 0xffffffff, 0xC060000C,
1061 0x224, 0xc0000fff, 0x00000100,
1062 0xf90, 0xffffffff, 0x00000100,
1063 0xf98, 0x00000101, 0x00000000,
1064 0x20a8, 0xffffffff, 0x00000104,
1065 0x55e4, 0xff000fff, 0x00000100,
1066 0x30cc, 0xc0000fff, 0x00000104,
1067 0xc1e4, 0x00000001, 0x00000001,
1068 0xd00c, 0xff000ff0, 0x00000100,
1069 0xd80c, 0xff000ff0, 0x00000100
1070 };
1071
1072 static const u32 spectre_golden_spm_registers[] =
1073 {
1074 0x30800, 0xe0ffffff, 0xe0000000
1075 };
1076
1077 static const u32 spectre_golden_common_registers[] =
1078 {
1079 0xc770, 0xffffffff, 0x00000800,
1080 0xc774, 0xffffffff, 0x00000800,
1081 0xc798, 0xffffffff, 0x00007fbf,
1082 0xc79c, 0xffffffff, 0x00007faf
1083 };
1084
1085 static const u32 spectre_golden_registers[] =
1086 {
1087 0x3c000, 0xffff1fff, 0x96940200,
1088 0x3c00c, 0xffff0001, 0xff000000,
1089 0x3c200, 0xfffc0fff, 0x00000100,
1090 0x6ed8, 0x00010101, 0x00010000,
1091 0x9834, 0xf00fffff, 0x00000400,
1092 0x9838, 0xfffffffc, 0x00020200,
1093 0x5bb0, 0x000000f0, 0x00000070,
1094 0x5bc0, 0xf0311fff, 0x80300000,
1095 0x98f8, 0x73773777, 0x12010001,
1096 0x9b7c, 0x00ff0000, 0x00fc0000,
1097 0x2f48, 0x73773777, 0x12010001,
1098 0x8a14, 0xf000003f, 0x00000007,
1099 0x8b24, 0xffffffff, 0x00ffffff,
1100 0x28350, 0x3f3f3fff, 0x00000082,
1101 0x28354, 0x0000003f, 0x00000000,
1102 0x3e78, 0x00000001, 0x00000002,
1103 0x913c, 0xffff03df, 0x00000004,
1104 0xc768, 0x00000008, 0x00000008,
1105 0x8c00, 0x000008ff, 0x00000800,
1106 0x9508, 0x00010000, 0x00010000,
1107 0xac0c, 0xffffffff, 0x54763210,
1108 0x214f8, 0x01ff01ff, 0x00000002,
1109 0x21498, 0x007ff800, 0x00200000,
1110 0x2015c, 0xffffffff, 0x00000f40,
1111 0x30934, 0xffffffff, 0x00000001
1112 };
1113
1114 static const u32 spectre_mgcg_cgcg_init[] =
1115 {
1116 0xc420, 0xffffffff, 0xfffffffc,
1117 0x30800, 0xffffffff, 0xe0000000,
1118 0x3c2a0, 0xffffffff, 0x00000100,
1119 0x3c208, 0xffffffff, 0x00000100,
1120 0x3c2c0, 0xffffffff, 0x00000100,
1121 0x3c2c8, 0xffffffff, 0x00000100,
1122 0x3c2c4, 0xffffffff, 0x00000100,
1123 0x55e4, 0xffffffff, 0x00600100,
1124 0x3c280, 0xffffffff, 0x00000100,
1125 0x3c214, 0xffffffff, 0x06000100,
1126 0x3c220, 0xffffffff, 0x00000100,
1127 0x3c218, 0xffffffff, 0x06000100,
1128 0x3c204, 0xffffffff, 0x00000100,
1129 0x3c2e0, 0xffffffff, 0x00000100,
1130 0x3c224, 0xffffffff, 0x00000100,
1131 0x3c200, 0xffffffff, 0x00000100,
1132 0x3c230, 0xffffffff, 0x00000100,
1133 0x3c234, 0xffffffff, 0x00000100,
1134 0x3c250, 0xffffffff, 0x00000100,
1135 0x3c254, 0xffffffff, 0x00000100,
1136 0x3c258, 0xffffffff, 0x00000100,
1137 0x3c25c, 0xffffffff, 0x00000100,
1138 0x3c260, 0xffffffff, 0x00000100,
1139 0x3c27c, 0xffffffff, 0x00000100,
1140 0x3c278, 0xffffffff, 0x00000100,
1141 0x3c210, 0xffffffff, 0x06000100,
1142 0x3c290, 0xffffffff, 0x00000100,
1143 0x3c274, 0xffffffff, 0x00000100,
1144 0x3c2b4, 0xffffffff, 0x00000100,
1145 0x3c2b0, 0xffffffff, 0x00000100,
1146 0x3c270, 0xffffffff, 0x00000100,
1147 0x30800, 0xffffffff, 0xe0000000,
1148 0x3c020, 0xffffffff, 0x00010000,
1149 0x3c024, 0xffffffff, 0x00030002,
1150 0x3c028, 0xffffffff, 0x00040007,
1151 0x3c02c, 0xffffffff, 0x00060005,
1152 0x3c030, 0xffffffff, 0x00090008,
1153 0x3c034, 0xffffffff, 0x00010000,
1154 0x3c038, 0xffffffff, 0x00030002,
1155 0x3c03c, 0xffffffff, 0x00040007,
1156 0x3c040, 0xffffffff, 0x00060005,
1157 0x3c044, 0xffffffff, 0x00090008,
1158 0x3c048, 0xffffffff, 0x00010000,
1159 0x3c04c, 0xffffffff, 0x00030002,
1160 0x3c050, 0xffffffff, 0x00040007,
1161 0x3c054, 0xffffffff, 0x00060005,
1162 0x3c058, 0xffffffff, 0x00090008,
1163 0x3c05c, 0xffffffff, 0x00010000,
1164 0x3c060, 0xffffffff, 0x00030002,
1165 0x3c064, 0xffffffff, 0x00040007,
1166 0x3c068, 0xffffffff, 0x00060005,
1167 0x3c06c, 0xffffffff, 0x00090008,
1168 0x3c070, 0xffffffff, 0x00010000,
1169 0x3c074, 0xffffffff, 0x00030002,
1170 0x3c078, 0xffffffff, 0x00040007,
1171 0x3c07c, 0xffffffff, 0x00060005,
1172 0x3c080, 0xffffffff, 0x00090008,
1173 0x3c084, 0xffffffff, 0x00010000,
1174 0x3c088, 0xffffffff, 0x00030002,
1175 0x3c08c, 0xffffffff, 0x00040007,
1176 0x3c090, 0xffffffff, 0x00060005,
1177 0x3c094, 0xffffffff, 0x00090008,
1178 0x3c098, 0xffffffff, 0x00010000,
1179 0x3c09c, 0xffffffff, 0x00030002,
1180 0x3c0a0, 0xffffffff, 0x00040007,
1181 0x3c0a4, 0xffffffff, 0x00060005,
1182 0x3c0a8, 0xffffffff, 0x00090008,
1183 0x3c0ac, 0xffffffff, 0x00010000,
1184 0x3c0b0, 0xffffffff, 0x00030002,
1185 0x3c0b4, 0xffffffff, 0x00040007,
1186 0x3c0b8, 0xffffffff, 0x00060005,
1187 0x3c0bc, 0xffffffff, 0x00090008,
1188 0x3c000, 0xffffffff, 0x96e00200,
1189 0x8708, 0xffffffff, 0x00900100,
1190 0xc424, 0xffffffff, 0x0020003f,
1191 0x38, 0xffffffff, 0x0140001c,
1192 0x3c, 0x000f0000, 0x000f0000,
1193 0x220, 0xffffffff, 0xC060000C,
1194 0x224, 0xc0000fff, 0x00000100,
1195 0xf90, 0xffffffff, 0x00000100,
1196 0xf98, 0x00000101, 0x00000000,
1197 0x20a8, 0xffffffff, 0x00000104,
1198 0x55e4, 0xff000fff, 0x00000100,
1199 0x30cc, 0xc0000fff, 0x00000104,
1200 0xc1e4, 0x00000001, 0x00000001,
1201 0xd00c, 0xff000ff0, 0x00000100,
1202 0xd80c, 0xff000ff0, 0x00000100
1203 };
1204
1205 static const u32 kalindi_golden_spm_registers[] =
1206 {
1207 0x30800, 0xe0ffffff, 0xe0000000
1208 };
1209
1210 static const u32 kalindi_golden_common_registers[] =
1211 {
1212 0xc770, 0xffffffff, 0x00000800,
1213 0xc774, 0xffffffff, 0x00000800,
1214 0xc798, 0xffffffff, 0x00007fbf,
1215 0xc79c, 0xffffffff, 0x00007faf
1216 };
1217
1218 static const u32 kalindi_golden_registers[] =
1219 {
1220 0x3c000, 0xffffdfff, 0x6e944040,
1221 0x55e4, 0xff607fff, 0xfc000100,
1222 0x3c220, 0xff000fff, 0x00000100,
1223 0x3c224, 0xff000fff, 0x00000100,
1224 0x3c200, 0xfffc0fff, 0x00000100,
1225 0x6ed8, 0x00010101, 0x00010000,
1226 0x9830, 0xffffffff, 0x00000000,
1227 0x9834, 0xf00fffff, 0x00000400,
1228 0x5bb0, 0x000000f0, 0x00000070,
1229 0x5bc0, 0xf0311fff, 0x80300000,
1230 0x98f8, 0x73773777, 0x12010001,
1231 0x98fc, 0xffffffff, 0x00000010,
1232 0x9b7c, 0x00ff0000, 0x00fc0000,
1233 0x8030, 0x00001f0f, 0x0000100a,
1234 0x2f48, 0x73773777, 0x12010001,
1235 0x2408, 0x000fffff, 0x000c007f,
1236 0x8a14, 0xf000003f, 0x00000007,
1237 0x8b24, 0x3fff3fff, 0x00ffcfff,
1238 0x30a04, 0x0000ff0f, 0x00000000,
1239 0x28a4c, 0x07ffffff, 0x06000000,
1240 0x4d8, 0x00000fff, 0x00000100,
1241 0x3e78, 0x00000001, 0x00000002,
1242 0xc768, 0x00000008, 0x00000008,
1243 0x8c00, 0x000000ff, 0x00000003,
1244 0x214f8, 0x01ff01ff, 0x00000002,
1245 0x21498, 0x007ff800, 0x00200000,
1246 0x2015c, 0xffffffff, 0x00000f40,
1247 0x88c4, 0x001f3ae3, 0x00000082,
1248 0x88d4, 0x0000001f, 0x00000010,
1249 0x30934, 0xffffffff, 0x00000000
1250 };
1251
1252 static const u32 kalindi_mgcg_cgcg_init[] =
1253 {
1254 0xc420, 0xffffffff, 0xfffffffc,
1255 0x30800, 0xffffffff, 0xe0000000,
1256 0x3c2a0, 0xffffffff, 0x00000100,
1257 0x3c208, 0xffffffff, 0x00000100,
1258 0x3c2c0, 0xffffffff, 0x00000100,
1259 0x3c2c8, 0xffffffff, 0x00000100,
1260 0x3c2c4, 0xffffffff, 0x00000100,
1261 0x55e4, 0xffffffff, 0x00600100,
1262 0x3c280, 0xffffffff, 0x00000100,
1263 0x3c214, 0xffffffff, 0x06000100,
1264 0x3c220, 0xffffffff, 0x00000100,
1265 0x3c218, 0xffffffff, 0x06000100,
1266 0x3c204, 0xffffffff, 0x00000100,
1267 0x3c2e0, 0xffffffff, 0x00000100,
1268 0x3c224, 0xffffffff, 0x00000100,
1269 0x3c200, 0xffffffff, 0x00000100,
1270 0x3c230, 0xffffffff, 0x00000100,
1271 0x3c234, 0xffffffff, 0x00000100,
1272 0x3c250, 0xffffffff, 0x00000100,
1273 0x3c254, 0xffffffff, 0x00000100,
1274 0x3c258, 0xffffffff, 0x00000100,
1275 0x3c25c, 0xffffffff, 0x00000100,
1276 0x3c260, 0xffffffff, 0x00000100,
1277 0x3c27c, 0xffffffff, 0x00000100,
1278 0x3c278, 0xffffffff, 0x00000100,
1279 0x3c210, 0xffffffff, 0x06000100,
1280 0x3c290, 0xffffffff, 0x00000100,
1281 0x3c274, 0xffffffff, 0x00000100,
1282 0x3c2b4, 0xffffffff, 0x00000100,
1283 0x3c2b0, 0xffffffff, 0x00000100,
1284 0x3c270, 0xffffffff, 0x00000100,
1285 0x30800, 0xffffffff, 0xe0000000,
1286 0x3c020, 0xffffffff, 0x00010000,
1287 0x3c024, 0xffffffff, 0x00030002,
1288 0x3c028, 0xffffffff, 0x00040007,
1289 0x3c02c, 0xffffffff, 0x00060005,
1290 0x3c030, 0xffffffff, 0x00090008,
1291 0x3c034, 0xffffffff, 0x00010000,
1292 0x3c038, 0xffffffff, 0x00030002,
1293 0x3c03c, 0xffffffff, 0x00040007,
1294 0x3c040, 0xffffffff, 0x00060005,
1295 0x3c044, 0xffffffff, 0x00090008,
1296 0x3c000, 0xffffffff, 0x96e00200,
1297 0x8708, 0xffffffff, 0x00900100,
1298 0xc424, 0xffffffff, 0x0020003f,
1299 0x38, 0xffffffff, 0x0140001c,
1300 0x3c, 0x000f0000, 0x000f0000,
1301 0x220, 0xffffffff, 0xC060000C,
1302 0x224, 0xc0000fff, 0x00000100,
1303 0x20a8, 0xffffffff, 0x00000104,
1304 0x55e4, 0xff000fff, 0x00000100,
1305 0x30cc, 0xc0000fff, 0x00000104,
1306 0xc1e4, 0x00000001, 0x00000001,
1307 0xd00c, 0xff000ff0, 0x00000100,
1308 0xd80c, 0xff000ff0, 0x00000100
1309 };
1310
1311 static const u32 hawaii_golden_spm_registers[] =
1312 {
1313 0x30800, 0xe0ffffff, 0xe0000000
1314 };
1315
1316 static const u32 hawaii_golden_common_registers[] =
1317 {
1318 0x30800, 0xffffffff, 0xe0000000,
1319 0x28350, 0xffffffff, 0x3a00161a,
1320 0x28354, 0xffffffff, 0x0000002e,
1321 0x9a10, 0xffffffff, 0x00018208,
1322 0x98f8, 0xffffffff, 0x12011003
1323 };
1324
1325 static const u32 hawaii_golden_registers[] =
1326 {
1327 0x3354, 0x00000333, 0x00000333,
1328 0x9a10, 0x00010000, 0x00058208,
1329 0x9830, 0xffffffff, 0x00000000,
1330 0x9834, 0xf00fffff, 0x00000400,
1331 0x9838, 0x0002021c, 0x00020200,
1332 0xc78, 0x00000080, 0x00000000,
1333 0x5bb0, 0x000000f0, 0x00000070,
1334 0x5bc0, 0xf0311fff, 0x80300000,
1335 0x350c, 0x00810000, 0x408af000,
1336 0x7030, 0x31000111, 0x00000011,
1337 0x2f48, 0x73773777, 0x12010001,
1338 0x2120, 0x0000007f, 0x0000001b,
1339 0x21dc, 0x00007fb6, 0x00002191,
1340 0x3628, 0x0000003f, 0x0000000a,
1341 0x362c, 0x0000003f, 0x0000000a,
1342 0x2ae4, 0x00073ffe, 0x000022a2,
1343 0x240c, 0x000007ff, 0x00000000,
1344 0x8bf0, 0x00002001, 0x00000001,
1345 0x8b24, 0xffffffff, 0x00ffffff,
1346 0x30a04, 0x0000ff0f, 0x00000000,
1347 0x28a4c, 0x07ffffff, 0x06000000,
1348 0x3e78, 0x00000001, 0x00000002,
1349 0xc768, 0x00000008, 0x00000008,
1350 0xc770, 0x00000f00, 0x00000800,
1351 0xc774, 0x00000f00, 0x00000800,
1352 0xc798, 0x00ffffff, 0x00ff7fbf,
1353 0xc79c, 0x00ffffff, 0x00ff7faf,
1354 0x8c00, 0x000000ff, 0x00000800,
1355 0xe40, 0x00001fff, 0x00001fff,
1356 0x9060, 0x0000007f, 0x00000020,
1357 0x9508, 0x00010000, 0x00010000,
1358 0xae00, 0x00100000, 0x000ff07c,
1359 0xac14, 0x000003ff, 0x0000000f,
1360 0xac10, 0xffffffff, 0x7564fdec,
1361 0xac0c, 0xffffffff, 0x3120b9a8,
1362 0xac08, 0x20000000, 0x0f9c0000
1363 };
1364
1365 static const u32 hawaii_mgcg_cgcg_init[] =
1366 {
1367 0xc420, 0xffffffff, 0xfffffffd,
1368 0x30800, 0xffffffff, 0xe0000000,
1369 0x3c2a0, 0xffffffff, 0x00000100,
1370 0x3c208, 0xffffffff, 0x00000100,
1371 0x3c2c0, 0xffffffff, 0x00000100,
1372 0x3c2c8, 0xffffffff, 0x00000100,
1373 0x3c2c4, 0xffffffff, 0x00000100,
1374 0x55e4, 0xffffffff, 0x00200100,
1375 0x3c280, 0xffffffff, 0x00000100,
1376 0x3c214, 0xffffffff, 0x06000100,
1377 0x3c220, 0xffffffff, 0x00000100,
1378 0x3c218, 0xffffffff, 0x06000100,
1379 0x3c204, 0xffffffff, 0x00000100,
1380 0x3c2e0, 0xffffffff, 0x00000100,
1381 0x3c224, 0xffffffff, 0x00000100,
1382 0x3c200, 0xffffffff, 0x00000100,
1383 0x3c230, 0xffffffff, 0x00000100,
1384 0x3c234, 0xffffffff, 0x00000100,
1385 0x3c250, 0xffffffff, 0x00000100,
1386 0x3c254, 0xffffffff, 0x00000100,
1387 0x3c258, 0xffffffff, 0x00000100,
1388 0x3c25c, 0xffffffff, 0x00000100,
1389 0x3c260, 0xffffffff, 0x00000100,
1390 0x3c27c, 0xffffffff, 0x00000100,
1391 0x3c278, 0xffffffff, 0x00000100,
1392 0x3c210, 0xffffffff, 0x06000100,
1393 0x3c290, 0xffffffff, 0x00000100,
1394 0x3c274, 0xffffffff, 0x00000100,
1395 0x3c2b4, 0xffffffff, 0x00000100,
1396 0x3c2b0, 0xffffffff, 0x00000100,
1397 0x3c270, 0xffffffff, 0x00000100,
1398 0x30800, 0xffffffff, 0xe0000000,
1399 0x3c020, 0xffffffff, 0x00010000,
1400 0x3c024, 0xffffffff, 0x00030002,
1401 0x3c028, 0xffffffff, 0x00040007,
1402 0x3c02c, 0xffffffff, 0x00060005,
1403 0x3c030, 0xffffffff, 0x00090008,
1404 0x3c034, 0xffffffff, 0x00010000,
1405 0x3c038, 0xffffffff, 0x00030002,
1406 0x3c03c, 0xffffffff, 0x00040007,
1407 0x3c040, 0xffffffff, 0x00060005,
1408 0x3c044, 0xffffffff, 0x00090008,
1409 0x3c048, 0xffffffff, 0x00010000,
1410 0x3c04c, 0xffffffff, 0x00030002,
1411 0x3c050, 0xffffffff, 0x00040007,
1412 0x3c054, 0xffffffff, 0x00060005,
1413 0x3c058, 0xffffffff, 0x00090008,
1414 0x3c05c, 0xffffffff, 0x00010000,
1415 0x3c060, 0xffffffff, 0x00030002,
1416 0x3c064, 0xffffffff, 0x00040007,
1417 0x3c068, 0xffffffff, 0x00060005,
1418 0x3c06c, 0xffffffff, 0x00090008,
1419 0x3c070, 0xffffffff, 0x00010000,
1420 0x3c074, 0xffffffff, 0x00030002,
1421 0x3c078, 0xffffffff, 0x00040007,
1422 0x3c07c, 0xffffffff, 0x00060005,
1423 0x3c080, 0xffffffff, 0x00090008,
1424 0x3c084, 0xffffffff, 0x00010000,
1425 0x3c088, 0xffffffff, 0x00030002,
1426 0x3c08c, 0xffffffff, 0x00040007,
1427 0x3c090, 0xffffffff, 0x00060005,
1428 0x3c094, 0xffffffff, 0x00090008,
1429 0x3c098, 0xffffffff, 0x00010000,
1430 0x3c09c, 0xffffffff, 0x00030002,
1431 0x3c0a0, 0xffffffff, 0x00040007,
1432 0x3c0a4, 0xffffffff, 0x00060005,
1433 0x3c0a8, 0xffffffff, 0x00090008,
1434 0x3c0ac, 0xffffffff, 0x00010000,
1435 0x3c0b0, 0xffffffff, 0x00030002,
1436 0x3c0b4, 0xffffffff, 0x00040007,
1437 0x3c0b8, 0xffffffff, 0x00060005,
1438 0x3c0bc, 0xffffffff, 0x00090008,
1439 0x3c0c0, 0xffffffff, 0x00010000,
1440 0x3c0c4, 0xffffffff, 0x00030002,
1441 0x3c0c8, 0xffffffff, 0x00040007,
1442 0x3c0cc, 0xffffffff, 0x00060005,
1443 0x3c0d0, 0xffffffff, 0x00090008,
1444 0x3c0d4, 0xffffffff, 0x00010000,
1445 0x3c0d8, 0xffffffff, 0x00030002,
1446 0x3c0dc, 0xffffffff, 0x00040007,
1447 0x3c0e0, 0xffffffff, 0x00060005,
1448 0x3c0e4, 0xffffffff, 0x00090008,
1449 0x3c0e8, 0xffffffff, 0x00010000,
1450 0x3c0ec, 0xffffffff, 0x00030002,
1451 0x3c0f0, 0xffffffff, 0x00040007,
1452 0x3c0f4, 0xffffffff, 0x00060005,
1453 0x3c0f8, 0xffffffff, 0x00090008,
1454 0xc318, 0xffffffff, 0x00020200,
1455 0x3350, 0xffffffff, 0x00000200,
1456 0x15c0, 0xffffffff, 0x00000400,
1457 0x55e8, 0xffffffff, 0x00000000,
1458 0x2f50, 0xffffffff, 0x00000902,
1459 0x3c000, 0xffffffff, 0x96940200,
1460 0x8708, 0xffffffff, 0x00900100,
1461 0xc424, 0xffffffff, 0x0020003f,
1462 0x38, 0xffffffff, 0x0140001c,
1463 0x3c, 0x000f0000, 0x000f0000,
1464 0x220, 0xffffffff, 0xc060000c,
1465 0x224, 0xc0000fff, 0x00000100,
1466 0xf90, 0xffffffff, 0x00000100,
1467 0xf98, 0x00000101, 0x00000000,
1468 0x20a8, 0xffffffff, 0x00000104,
1469 0x55e4, 0xff000fff, 0x00000100,
1470 0x30cc, 0xc0000fff, 0x00000104,
1471 0xc1e4, 0x00000001, 0x00000001,
1472 0xd00c, 0xff000ff0, 0x00000100,
1473 0xd80c, 0xff000ff0, 0x00000100
1474 };
1475
1476 static void cik_init_golden_registers(struct radeon_device *rdev)
1477 {
1478 switch (rdev->family) {
1479 case CHIP_BONAIRE:
1480 radeon_program_register_sequence(rdev,
1481 bonaire_mgcg_cgcg_init,
1482 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_common_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1489 radeon_program_register_sequence(rdev,
1490 bonaire_golden_spm_registers,
1491 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1492 break;
1493 case CHIP_KABINI:
1494 radeon_program_register_sequence(rdev,
1495 kalindi_mgcg_cgcg_init,
1496 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_common_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1503 radeon_program_register_sequence(rdev,
1504 kalindi_golden_spm_registers,
1505 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1506 break;
1507 case CHIP_KAVERI:
1508 radeon_program_register_sequence(rdev,
1509 spectre_mgcg_cgcg_init,
1510 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_common_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1517 radeon_program_register_sequence(rdev,
1518 spectre_golden_spm_registers,
1519 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1520 break;
1521 case CHIP_HAWAII:
1522 radeon_program_register_sequence(rdev,
1523 hawaii_mgcg_cgcg_init,
1524 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_common_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1531 radeon_program_register_sequence(rdev,
1532 hawaii_golden_spm_registers,
1533 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1534 break;
1535 default:
1536 break;
1537 }
1538 }
1539
1540 /**
1541 * cik_get_xclk - get the xclk
1542 *
1543 * @rdev: radeon_device pointer
1544 *
1545 * Returns the reference clock used by the gfx engine
1546 * (CIK).
1547 */
1548 u32 cik_get_xclk(struct radeon_device *rdev)
1549 {
1550 u32 reference_clock = rdev->clock.spll.reference_freq;
1551
1552 if (rdev->flags & RADEON_IS_IGP) {
1553 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1554 return reference_clock / 2;
1555 } else {
1556 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1557 return reference_clock / 4;
1558 }
1559 return reference_clock;
1560 }
1561
1562 /**
1563 * cik_mm_rdoorbell - read a doorbell dword
1564 *
1565 * @rdev: radeon_device pointer
1566 * @index: doorbell index
1567 *
1568 * Returns the value in the doorbell aperture at the
1569 * requested doorbell index (CIK).
1570 */
1571 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1572 {
1573 if (index < rdev->doorbell.num_doorbells) {
1574 return readl(rdev->doorbell.ptr + index);
1575 } else {
1576 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1577 return 0;
1578 }
1579 }
1580
1581 /**
1582 * cik_mm_wdoorbell - write a doorbell dword
1583 *
1584 * @rdev: radeon_device pointer
1585 * @index: doorbell index
1586 * @v: value to write
1587 *
1588 * Writes @v to the doorbell aperture at the
1589 * requested doorbell index (CIK).
1590 */
1591 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1592 {
1593 if (index < rdev->doorbell.num_doorbells) {
1594 writel(v, rdev->doorbell.ptr + index);
1595 } else {
1596 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1597 }
1598 }
1599
1600 #define BONAIRE_IO_MC_REGS_SIZE 36
1601
1602 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1603 {
1604 {0x00000070, 0x04400000},
1605 {0x00000071, 0x80c01803},
1606 {0x00000072, 0x00004004},
1607 {0x00000073, 0x00000100},
1608 {0x00000074, 0x00ff0000},
1609 {0x00000075, 0x34000000},
1610 {0x00000076, 0x08000014},
1611 {0x00000077, 0x00cc08ec},
1612 {0x00000078, 0x00000400},
1613 {0x00000079, 0x00000000},
1614 {0x0000007a, 0x04090000},
1615 {0x0000007c, 0x00000000},
1616 {0x0000007e, 0x4408a8e8},
1617 {0x0000007f, 0x00000304},
1618 {0x00000080, 0x00000000},
1619 {0x00000082, 0x00000001},
1620 {0x00000083, 0x00000002},
1621 {0x00000084, 0xf3e4f400},
1622 {0x00000085, 0x052024e3},
1623 {0x00000087, 0x00000000},
1624 {0x00000088, 0x01000000},
1625 {0x0000008a, 0x1c0a0000},
1626 {0x0000008b, 0xff010000},
1627 {0x0000008d, 0xffffefff},
1628 {0x0000008e, 0xfff3efff},
1629 {0x0000008f, 0xfff3efbf},
1630 {0x00000092, 0xf7ffffff},
1631 {0x00000093, 0xffffff7f},
1632 {0x00000095, 0x00101101},
1633 {0x00000096, 0x00000fff},
1634 {0x00000097, 0x00116fff},
1635 {0x00000098, 0x60010000},
1636 {0x00000099, 0x10010000},
1637 {0x0000009a, 0x00006000},
1638 {0x0000009b, 0x00001000},
1639 {0x0000009f, 0x00b48000}
1640 };
1641
1642 #define HAWAII_IO_MC_REGS_SIZE 22
1643
1644 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1645 {
1646 {0x0000007d, 0x40000000},
1647 {0x0000007e, 0x40180304},
1648 {0x0000007f, 0x0000ff00},
1649 {0x00000081, 0x00000000},
1650 {0x00000083, 0x00000800},
1651 {0x00000086, 0x00000000},
1652 {0x00000087, 0x00000100},
1653 {0x00000088, 0x00020100},
1654 {0x00000089, 0x00000000},
1655 {0x0000008b, 0x00040000},
1656 {0x0000008c, 0x00000100},
1657 {0x0000008e, 0xff010000},
1658 {0x00000090, 0xffffefff},
1659 {0x00000091, 0xfff3efff},
1660 {0x00000092, 0xfff3efbf},
1661 {0x00000093, 0xf7ffffff},
1662 {0x00000094, 0xffffff7f},
1663 {0x00000095, 0x00000fff},
1664 {0x00000096, 0x00116fff},
1665 {0x00000097, 0x60010000},
1666 {0x00000098, 0x10010000},
1667 {0x0000009f, 0x00c79000}
1668 };
1669
1670
1671 /**
1672 * cik_srbm_select - select specific register instances
1673 *
1674 * @rdev: radeon_device pointer
1675 * @me: selected ME (micro engine)
1676 * @pipe: pipe
1677 * @queue: queue
1678 * @vmid: VMID
1679 *
1680 * Switches the currently active registers instances. Some
1681 * registers are instanced per VMID, others are instanced per
1682 * me/pipe/queue combination.
1683 */
1684 static void cik_srbm_select(struct radeon_device *rdev,
1685 u32 me, u32 pipe, u32 queue, u32 vmid)
1686 {
1687 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1688 MEID(me & 0x3) |
1689 VMID(vmid & 0xf) |
1690 QUEUEID(queue & 0x7));
1691 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1692 }
1693
1694 /* ucode loading */
1695 /**
1696 * ci_mc_load_microcode - load MC ucode into the hw
1697 *
1698 * @rdev: radeon_device pointer
1699 *
1700 * Load the GDDR MC ucode into the hw (CIK).
1701 * Returns 0 on success, error on failure.
1702 */
1703 int ci_mc_load_microcode(struct radeon_device *rdev)
1704 {
1705 const __be32 *fw_data;
1706 u32 running, blackout = 0;
1707 u32 *io_mc_regs;
1708 int i, regs_size, ucode_size;
1709
1710 if (!rdev->mc_fw)
1711 return -EINVAL;
1712
1713 ucode_size = rdev->mc_fw->size / 4;
1714
1715 switch (rdev->family) {
1716 case CHIP_BONAIRE:
1717 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1718 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1719 break;
1720 case CHIP_HAWAII:
1721 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1722 regs_size = HAWAII_IO_MC_REGS_SIZE;
1723 break;
1724 default:
1725 return -EINVAL;
1726 }
1727
1728 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1729
1730 if (running == 0) {
1731 if (running) {
1732 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1733 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1734 }
1735
1736 /* reset the engine and set to writable */
1737 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1738 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1739
1740 /* load mc io regs */
1741 for (i = 0; i < regs_size; i++) {
1742 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1743 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1744 }
1745 /* load the MC ucode */
1746 fw_data = (const __be32 *)rdev->mc_fw->data;
1747 for (i = 0; i < ucode_size; i++)
1748 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1749
1750 /* put the engine back into the active state */
1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1752 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1753 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1754
1755 /* wait for training to complete */
1756 for (i = 0; i < rdev->usec_timeout; i++) {
1757 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1758 break;
1759 udelay(1);
1760 }
1761 for (i = 0; i < rdev->usec_timeout; i++) {
1762 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1763 break;
1764 udelay(1);
1765 }
1766
1767 if (running)
1768 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1769 }
1770
1771 return 0;
1772 }
1773
1774 /**
1775 * cik_init_microcode - load ucode images from disk
1776 *
1777 * @rdev: radeon_device pointer
1778 *
1779 * Use the firmware interface to load the ucode images into
1780 * the driver (not loaded into hw).
1781 * Returns 0 on success, error on failure.
1782 */
1783 static int cik_init_microcode(struct radeon_device *rdev)
1784 {
1785 const char *chip_name;
1786 size_t pfp_req_size, me_req_size, ce_req_size,
1787 mec_req_size, rlc_req_size, mc_req_size = 0,
1788 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1789 char fw_name[30];
1790 int err;
1791
1792 DRM_DEBUG("\n");
1793
1794 switch (rdev->family) {
1795 case CHIP_BONAIRE:
1796 chip_name = "BONAIRE";
1797 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1798 me_req_size = CIK_ME_UCODE_SIZE * 4;
1799 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1800 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1801 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1802 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1803 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1804 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1805 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1806 break;
1807 case CHIP_HAWAII:
1808 chip_name = "HAWAII";
1809 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1810 me_req_size = CIK_ME_UCODE_SIZE * 4;
1811 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1812 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1813 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1814 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1815 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1816 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1817 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1818 break;
1819 case CHIP_KAVERI:
1820 chip_name = "KAVERI";
1821 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1822 me_req_size = CIK_ME_UCODE_SIZE * 4;
1823 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1824 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1825 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1826 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1827 break;
1828 case CHIP_KABINI:
1829 chip_name = "KABINI";
1830 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1831 me_req_size = CIK_ME_UCODE_SIZE * 4;
1832 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1833 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1834 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1835 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1836 break;
1837 default: BUG();
1838 }
1839
1840 DRM_INFO("Loading %s Microcode\n", chip_name);
1841
1842 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1843 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1844 if (err)
1845 goto out;
1846 if (rdev->pfp_fw->size != pfp_req_size) {
1847 printk(KERN_ERR
1848 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1849 rdev->pfp_fw->size, fw_name);
1850 err = -EINVAL;
1851 goto out;
1852 }
1853
1854 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1855 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1856 if (err)
1857 goto out;
1858 if (rdev->me_fw->size != me_req_size) {
1859 printk(KERN_ERR
1860 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1861 rdev->me_fw->size, fw_name);
1862 err = -EINVAL;
1863 }
1864
1865 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1866 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1867 if (err)
1868 goto out;
1869 if (rdev->ce_fw->size != ce_req_size) {
1870 printk(KERN_ERR
1871 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1872 rdev->ce_fw->size, fw_name);
1873 err = -EINVAL;
1874 }
1875
1876 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1877 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1878 if (err)
1879 goto out;
1880 if (rdev->mec_fw->size != mec_req_size) {
1881 printk(KERN_ERR
1882 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1883 rdev->mec_fw->size, fw_name);
1884 err = -EINVAL;
1885 }
1886
1887 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1888 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1889 if (err)
1890 goto out;
1891 if (rdev->rlc_fw->size != rlc_req_size) {
1892 printk(KERN_ERR
1893 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1894 rdev->rlc_fw->size, fw_name);
1895 err = -EINVAL;
1896 }
1897
1898 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1899 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1900 if (err)
1901 goto out;
1902 if (rdev->sdma_fw->size != sdma_req_size) {
1903 printk(KERN_ERR
1904 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1905 rdev->sdma_fw->size, fw_name);
1906 err = -EINVAL;
1907 }
1908
1909 /* No SMC, MC ucode on APUs */
1910 if (!(rdev->flags & RADEON_IS_IGP)) {
1911 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1912 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1913 if (err) {
1914 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1915 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1916 if (err)
1917 goto out;
1918 }
1919 if ((rdev->mc_fw->size != mc_req_size) &&
1920 (rdev->mc_fw->size != mc2_req_size)){
1921 printk(KERN_ERR
1922 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1923 rdev->mc_fw->size, fw_name);
1924 err = -EINVAL;
1925 }
1926 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1927
1928 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1929 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1930 if (err) {
1931 printk(KERN_ERR
1932 "smc: error loading firmware \"%s\"\n",
1933 fw_name);
1934 release_firmware(rdev->smc_fw);
1935 rdev->smc_fw = NULL;
1936 err = 0;
1937 } else if (rdev->smc_fw->size != smc_req_size) {
1938 printk(KERN_ERR
1939 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1940 rdev->smc_fw->size, fw_name);
1941 err = -EINVAL;
1942 }
1943 }
1944
1945 out:
1946 if (err) {
1947 if (err != -EINVAL)
1948 printk(KERN_ERR
1949 "cik_cp: Failed to load firmware \"%s\"\n",
1950 fw_name);
1951 release_firmware(rdev->pfp_fw);
1952 rdev->pfp_fw = NULL;
1953 release_firmware(rdev->me_fw);
1954 rdev->me_fw = NULL;
1955 release_firmware(rdev->ce_fw);
1956 rdev->ce_fw = NULL;
1957 release_firmware(rdev->rlc_fw);
1958 rdev->rlc_fw = NULL;
1959 release_firmware(rdev->mc_fw);
1960 rdev->mc_fw = NULL;
1961 release_firmware(rdev->smc_fw);
1962 rdev->smc_fw = NULL;
1963 }
1964 return err;
1965 }
1966
1967 /*
1968 * Core functions
1969 */
1970 /**
1971 * cik_tiling_mode_table_init - init the hw tiling table
1972 *
1973 * @rdev: radeon_device pointer
1974 *
1975 * Starting with SI, the tiling setup is done globally in a
1976 * set of 32 tiling modes. Rather than selecting each set of
1977 * parameters per surface as on older asics, we just select
1978 * which index in the tiling table we want to use, and the
1979 * surface uses those parameters (CIK).
1980 */
1981 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1982 {
1983 const u32 num_tile_mode_states = 32;
1984 const u32 num_secondary_tile_mode_states = 16;
1985 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1986 u32 num_pipe_configs;
1987 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1988 rdev->config.cik.max_shader_engines;
1989
1990 switch (rdev->config.cik.mem_row_size_in_kb) {
1991 case 1:
1992 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1993 break;
1994 case 2:
1995 default:
1996 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1997 break;
1998 case 4:
1999 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2000 break;
2001 }
2002
2003 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2004 if (num_pipe_configs > 8)
2005 num_pipe_configs = 16;
2006
2007 if (num_pipe_configs == 16) {
2008 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2009 switch (reg_offset) {
2010 case 0:
2011 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2012 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2013 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2014 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2015 break;
2016 case 1:
2017 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2018 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2019 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2020 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2021 break;
2022 case 2:
2023 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2024 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2025 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2026 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2027 break;
2028 case 3:
2029 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2030 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2031 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2032 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2033 break;
2034 case 4:
2035 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2036 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2037 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2038 TILE_SPLIT(split_equal_to_row_size));
2039 break;
2040 case 5:
2041 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2044 break;
2045 case 6:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2047 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2048 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2049 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2050 break;
2051 case 7:
2052 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2053 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2054 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2055 TILE_SPLIT(split_equal_to_row_size));
2056 break;
2057 case 8:
2058 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2059 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2060 break;
2061 case 9:
2062 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2063 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2064 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2065 break;
2066 case 10:
2067 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2068 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2069 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071 break;
2072 case 11:
2073 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2074 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2075 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2076 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2077 break;
2078 case 12:
2079 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2080 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2081 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2082 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2083 break;
2084 case 13:
2085 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2086 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2088 break;
2089 case 14:
2090 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2092 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2094 break;
2095 case 16:
2096 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2097 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2098 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2099 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2100 break;
2101 case 17:
2102 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2103 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2104 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2106 break;
2107 case 27:
2108 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2111 break;
2112 case 28:
2113 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2114 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2115 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2117 break;
2118 case 29:
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2120 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2122 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2123 break;
2124 case 30:
2125 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2126 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2127 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2128 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2129 break;
2130 default:
2131 gb_tile_moden = 0;
2132 break;
2133 }
2134 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2135 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2136 }
2137 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2138 switch (reg_offset) {
2139 case 0:
2140 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2141 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2142 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2143 NUM_BANKS(ADDR_SURF_16_BANK));
2144 break;
2145 case 1:
2146 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2147 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2148 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2149 NUM_BANKS(ADDR_SURF_16_BANK));
2150 break;
2151 case 2:
2152 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2153 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2154 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2155 NUM_BANKS(ADDR_SURF_16_BANK));
2156 break;
2157 case 3:
2158 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2159 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2160 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2161 NUM_BANKS(ADDR_SURF_16_BANK));
2162 break;
2163 case 4:
2164 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2167 NUM_BANKS(ADDR_SURF_8_BANK));
2168 break;
2169 case 5:
2170 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2171 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2172 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2173 NUM_BANKS(ADDR_SURF_4_BANK));
2174 break;
2175 case 6:
2176 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2177 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2178 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2179 NUM_BANKS(ADDR_SURF_2_BANK));
2180 break;
2181 case 8:
2182 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2183 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2184 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2185 NUM_BANKS(ADDR_SURF_16_BANK));
2186 break;
2187 case 9:
2188 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2189 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2190 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2191 NUM_BANKS(ADDR_SURF_16_BANK));
2192 break;
2193 case 10:
2194 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2195 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2196 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2197 NUM_BANKS(ADDR_SURF_16_BANK));
2198 break;
2199 case 11:
2200 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2201 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2202 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2203 NUM_BANKS(ADDR_SURF_8_BANK));
2204 break;
2205 case 12:
2206 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2209 NUM_BANKS(ADDR_SURF_4_BANK));
2210 break;
2211 case 13:
2212 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2213 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2214 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2215 NUM_BANKS(ADDR_SURF_2_BANK));
2216 break;
2217 case 14:
2218 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2221 NUM_BANKS(ADDR_SURF_2_BANK));
2222 break;
2223 default:
2224 gb_tile_moden = 0;
2225 break;
2226 }
2227 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2228 }
2229 } else if (num_pipe_configs == 8) {
2230 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2231 switch (reg_offset) {
2232 case 0:
2233 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2234 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2235 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2236 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2237 break;
2238 case 1:
2239 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2241 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2242 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2243 break;
2244 case 2:
2245 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2246 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2247 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2248 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2249 break;
2250 case 3:
2251 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2253 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2254 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2255 break;
2256 case 4:
2257 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2259 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2260 TILE_SPLIT(split_equal_to_row_size));
2261 break;
2262 case 5:
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2264 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2266 break;
2267 case 6:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2272 break;
2273 case 7:
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277 TILE_SPLIT(split_equal_to_row_size));
2278 break;
2279 case 8:
2280 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2282 break;
2283 case 9:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2287 break;
2288 case 10:
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2293 break;
2294 case 11:
2295 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2296 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2299 break;
2300 case 12:
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2305 break;
2306 case 13:
2307 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2310 break;
2311 case 14:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2314 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2316 break;
2317 case 16:
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2319 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2322 break;
2323 case 17:
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2328 break;
2329 case 27:
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2333 break;
2334 case 28:
2335 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2337 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2339 break;
2340 case 29:
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2342 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2345 break;
2346 case 30:
2347 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2349 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2350 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2351 break;
2352 default:
2353 gb_tile_moden = 0;
2354 break;
2355 }
2356 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2357 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2358 }
2359 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2360 switch (reg_offset) {
2361 case 0:
2362 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2365 NUM_BANKS(ADDR_SURF_16_BANK));
2366 break;
2367 case 1:
2368 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2371 NUM_BANKS(ADDR_SURF_16_BANK));
2372 break;
2373 case 2:
2374 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2377 NUM_BANKS(ADDR_SURF_16_BANK));
2378 break;
2379 case 3:
2380 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2383 NUM_BANKS(ADDR_SURF_16_BANK));
2384 break;
2385 case 4:
2386 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2389 NUM_BANKS(ADDR_SURF_8_BANK));
2390 break;
2391 case 5:
2392 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2395 NUM_BANKS(ADDR_SURF_4_BANK));
2396 break;
2397 case 6:
2398 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2401 NUM_BANKS(ADDR_SURF_2_BANK));
2402 break;
2403 case 8:
2404 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2407 NUM_BANKS(ADDR_SURF_16_BANK));
2408 break;
2409 case 9:
2410 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2413 NUM_BANKS(ADDR_SURF_16_BANK));
2414 break;
2415 case 10:
2416 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2419 NUM_BANKS(ADDR_SURF_16_BANK));
2420 break;
2421 case 11:
2422 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425 NUM_BANKS(ADDR_SURF_16_BANK));
2426 break;
2427 case 12:
2428 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2431 NUM_BANKS(ADDR_SURF_8_BANK));
2432 break;
2433 case 13:
2434 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2437 NUM_BANKS(ADDR_SURF_4_BANK));
2438 break;
2439 case 14:
2440 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2443 NUM_BANKS(ADDR_SURF_2_BANK));
2444 break;
2445 default:
2446 gb_tile_moden = 0;
2447 break;
2448 }
2449 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2450 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2451 }
2452 } else if (num_pipe_configs == 4) {
2453 if (num_rbs == 4) {
2454 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2455 switch (reg_offset) {
2456 case 0:
2457 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2459 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2461 break;
2462 case 1:
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2467 break;
2468 case 2:
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2473 break;
2474 case 3:
2475 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2477 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2478 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2479 break;
2480 case 4:
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484 TILE_SPLIT(split_equal_to_row_size));
2485 break;
2486 case 5:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2490 break;
2491 case 6:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2496 break;
2497 case 7:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2501 TILE_SPLIT(split_equal_to_row_size));
2502 break;
2503 case 8:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2506 break;
2507 case 9:
2508 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2509 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2511 break;
2512 case 10:
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517 break;
2518 case 11:
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523 break;
2524 case 12:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529 break;
2530 case 13:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2534 break;
2535 case 14:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 16:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 17:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 case 27:
2554 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2557 break;
2558 case 28:
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2563 break;
2564 case 29:
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2569 break;
2570 case 30:
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575 break;
2576 default:
2577 gb_tile_moden = 0;
2578 break;
2579 }
2580 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2581 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2582 }
2583 } else if (num_rbs < 4) {
2584 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2585 switch (reg_offset) {
2586 case 0:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2591 break;
2592 case 1:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2597 break;
2598 case 2:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2603 break;
2604 case 3:
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2609 break;
2610 case 4:
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2613 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614 TILE_SPLIT(split_equal_to_row_size));
2615 break;
2616 case 5:
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2618 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2620 break;
2621 case 6:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2626 break;
2627 case 7:
2628 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2629 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2630 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2631 TILE_SPLIT(split_equal_to_row_size));
2632 break;
2633 case 8:
2634 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2636 break;
2637 case 9:
2638 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2639 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2641 break;
2642 case 10:
2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 break;
2648 case 11:
2649 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2650 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2652 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2653 break;
2654 case 12:
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2659 break;
2660 case 13:
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2664 break;
2665 case 14:
2666 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2670 break;
2671 case 16:
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2676 break;
2677 case 17:
2678 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2679 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2680 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2682 break;
2683 case 27:
2684 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2687 break;
2688 case 28:
2689 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2691 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2693 break;
2694 case 29:
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2696 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699 break;
2700 case 30:
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2702 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2705 break;
2706 default:
2707 gb_tile_moden = 0;
2708 break;
2709 }
2710 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2711 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2712 }
2713 }
2714 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2715 switch (reg_offset) {
2716 case 0:
2717 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2720 NUM_BANKS(ADDR_SURF_16_BANK));
2721 break;
2722 case 1:
2723 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2725 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2726 NUM_BANKS(ADDR_SURF_16_BANK));
2727 break;
2728 case 2:
2729 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2731 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2732 NUM_BANKS(ADDR_SURF_16_BANK));
2733 break;
2734 case 3:
2735 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2737 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2738 NUM_BANKS(ADDR_SURF_16_BANK));
2739 break;
2740 case 4:
2741 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2744 NUM_BANKS(ADDR_SURF_16_BANK));
2745 break;
2746 case 5:
2747 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2748 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2749 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2750 NUM_BANKS(ADDR_SURF_8_BANK));
2751 break;
2752 case 6:
2753 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2756 NUM_BANKS(ADDR_SURF_4_BANK));
2757 break;
2758 case 8:
2759 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2760 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2761 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2762 NUM_BANKS(ADDR_SURF_16_BANK));
2763 break;
2764 case 9:
2765 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2766 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2767 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2768 NUM_BANKS(ADDR_SURF_16_BANK));
2769 break;
2770 case 10:
2771 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2773 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774 NUM_BANKS(ADDR_SURF_16_BANK));
2775 break;
2776 case 11:
2777 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2778 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2779 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2780 NUM_BANKS(ADDR_SURF_16_BANK));
2781 break;
2782 case 12:
2783 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2785 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2786 NUM_BANKS(ADDR_SURF_16_BANK));
2787 break;
2788 case 13:
2789 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2792 NUM_BANKS(ADDR_SURF_8_BANK));
2793 break;
2794 case 14:
2795 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2798 NUM_BANKS(ADDR_SURF_4_BANK));
2799 break;
2800 default:
2801 gb_tile_moden = 0;
2802 break;
2803 }
2804 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2805 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2806 }
2807 } else if (num_pipe_configs == 2) {
2808 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2809 switch (reg_offset) {
2810 case 0:
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P2) |
2814 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2815 break;
2816 case 1:
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P2) |
2820 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2821 break;
2822 case 2:
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2824 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2825 PIPE_CONFIG(ADDR_SURF_P2) |
2826 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2827 break;
2828 case 3:
2829 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2831 PIPE_CONFIG(ADDR_SURF_P2) |
2832 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2833 break;
2834 case 4:
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 TILE_SPLIT(split_equal_to_row_size));
2839 break;
2840 case 5:
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2842 PIPE_CONFIG(ADDR_SURF_P2) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2844 break;
2845 case 6:
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2847 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P2) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2850 break;
2851 case 7:
2852 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2854 PIPE_CONFIG(ADDR_SURF_P2) |
2855 TILE_SPLIT(split_equal_to_row_size));
2856 break;
2857 case 8:
2858 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2859 PIPE_CONFIG(ADDR_SURF_P2);
2860 break;
2861 case 9:
2862 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2863 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2864 PIPE_CONFIG(ADDR_SURF_P2));
2865 break;
2866 case 10:
2867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2869 PIPE_CONFIG(ADDR_SURF_P2) |
2870 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871 break;
2872 case 11:
2873 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P2) |
2876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877 break;
2878 case 12:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883 break;
2884 case 13:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2886 PIPE_CONFIG(ADDR_SURF_P2) |
2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2888 break;
2889 case 14:
2890 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892 PIPE_CONFIG(ADDR_SURF_P2) |
2893 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2894 break;
2895 case 16:
2896 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2900 break;
2901 case 17:
2902 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2903 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2904 PIPE_CONFIG(ADDR_SURF_P2) |
2905 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2906 break;
2907 case 27:
2908 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2910 PIPE_CONFIG(ADDR_SURF_P2));
2911 break;
2912 case 28:
2913 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917 break;
2918 case 29:
2919 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2920 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2921 PIPE_CONFIG(ADDR_SURF_P2) |
2922 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2923 break;
2924 case 30:
2925 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2926 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2927 PIPE_CONFIG(ADDR_SURF_P2) |
2928 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2929 break;
2930 default:
2931 gb_tile_moden = 0;
2932 break;
2933 }
2934 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2935 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2936 }
2937 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2938 switch (reg_offset) {
2939 case 0:
2940 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2941 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2942 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2943 NUM_BANKS(ADDR_SURF_16_BANK));
2944 break;
2945 case 1:
2946 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2947 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2948 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2949 NUM_BANKS(ADDR_SURF_16_BANK));
2950 break;
2951 case 2:
2952 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2953 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2954 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2955 NUM_BANKS(ADDR_SURF_16_BANK));
2956 break;
2957 case 3:
2958 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2959 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2960 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961 NUM_BANKS(ADDR_SURF_16_BANK));
2962 break;
2963 case 4:
2964 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967 NUM_BANKS(ADDR_SURF_16_BANK));
2968 break;
2969 case 5:
2970 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2971 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2972 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2973 NUM_BANKS(ADDR_SURF_16_BANK));
2974 break;
2975 case 6:
2976 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2979 NUM_BANKS(ADDR_SURF_8_BANK));
2980 break;
2981 case 8:
2982 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2983 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2984 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985 NUM_BANKS(ADDR_SURF_16_BANK));
2986 break;
2987 case 9:
2988 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991 NUM_BANKS(ADDR_SURF_16_BANK));
2992 break;
2993 case 10:
2994 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997 NUM_BANKS(ADDR_SURF_16_BANK));
2998 break;
2999 case 11:
3000 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003 NUM_BANKS(ADDR_SURF_16_BANK));
3004 break;
3005 case 12:
3006 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009 NUM_BANKS(ADDR_SURF_16_BANK));
3010 break;
3011 case 13:
3012 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015 NUM_BANKS(ADDR_SURF_16_BANK));
3016 break;
3017 case 14:
3018 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3021 NUM_BANKS(ADDR_SURF_8_BANK));
3022 break;
3023 default:
3024 gb_tile_moden = 0;
3025 break;
3026 }
3027 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3028 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3029 }
3030 } else
3031 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3032 }
3033
3034 /**
3035 * cik_select_se_sh - select which SE, SH to address
3036 *
3037 * @rdev: radeon_device pointer
3038 * @se_num: shader engine to address
3039 * @sh_num: sh block to address
3040 *
3041 * Select which SE, SH combinations to address. Certain
3042 * registers are instanced per SE or SH. 0xffffffff means
3043 * broadcast to all SEs or SHs (CIK).
3044 */
3045 static void cik_select_se_sh(struct radeon_device *rdev,
3046 u32 se_num, u32 sh_num)
3047 {
3048 u32 data = INSTANCE_BROADCAST_WRITES;
3049
3050 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3051 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3052 else if (se_num == 0xffffffff)
3053 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3054 else if (sh_num == 0xffffffff)
3055 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3056 else
3057 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3058 WREG32(GRBM_GFX_INDEX, data);
3059 }
3060
3061 /**
3062 * cik_create_bitmask - create a bitmask
3063 *
3064 * @bit_width: length of the mask
3065 *
3066 * create a variable length bit mask (CIK).
3067 * Returns the bitmask.
3068 */
3069 static u32 cik_create_bitmask(u32 bit_width)
3070 {
3071 u32 i, mask = 0;
3072
3073 for (i = 0; i < bit_width; i++) {
3074 mask <<= 1;
3075 mask |= 1;
3076 }
3077 return mask;
3078 }
3079
3080 /**
3081 * cik_get_rb_disabled - computes the mask of disabled RBs
3082 *
3083 * @rdev: radeon_device pointer
3084 * @max_rb_num: max RBs (render backends) for the asic
3085 * @se_num: number of SEs (shader engines) for the asic
3086 * @sh_per_se: number of SH blocks per SE for the asic
3087 *
3088 * Calculates the bitmask of disabled RBs (CIK).
3089 * Returns the disabled RB bitmask.
3090 */
3091 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3092 u32 max_rb_num_per_se,
3093 u32 sh_per_se)
3094 {
3095 u32 data, mask;
3096
3097 data = RREG32(CC_RB_BACKEND_DISABLE);
3098 if (data & 1)
3099 data &= BACKEND_DISABLE_MASK;
3100 else
3101 data = 0;
3102 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3103
3104 data >>= BACKEND_DISABLE_SHIFT;
3105
3106 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3107
3108 return data & mask;
3109 }
3110
3111 /**
3112 * cik_setup_rb - setup the RBs on the asic
3113 *
3114 * @rdev: radeon_device pointer
3115 * @se_num: number of SEs (shader engines) for the asic
3116 * @sh_per_se: number of SH blocks per SE for the asic
3117 * @max_rb_num: max RBs (render backends) for the asic
3118 *
3119 * Configures per-SE/SH RB registers (CIK).
3120 */
3121 static void cik_setup_rb(struct radeon_device *rdev,
3122 u32 se_num, u32 sh_per_se,
3123 u32 max_rb_num_per_se)
3124 {
3125 int i, j;
3126 u32 data, mask;
3127 u32 disabled_rbs = 0;
3128 u32 enabled_rbs = 0;
3129
3130 for (i = 0; i < se_num; i++) {
3131 for (j = 0; j < sh_per_se; j++) {
3132 cik_select_se_sh(rdev, i, j);
3133 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3134 if (rdev->family == CHIP_HAWAII)
3135 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3136 else
3137 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3138 }
3139 }
3140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141
3142 mask = 1;
3143 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3144 if (!(disabled_rbs & mask))
3145 enabled_rbs |= mask;
3146 mask <<= 1;
3147 }
3148
3149 rdev->config.cik.backend_enable_mask = enabled_rbs;
3150
3151 for (i = 0; i < se_num; i++) {
3152 cik_select_se_sh(rdev, i, 0xffffffff);
3153 data = 0;
3154 for (j = 0; j < sh_per_se; j++) {
3155 switch (enabled_rbs & 3) {
3156 case 0:
3157 if (j == 0)
3158 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3159 else
3160 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3161 break;
3162 case 1:
3163 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3164 break;
3165 case 2:
3166 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3167 break;
3168 case 3:
3169 default:
3170 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3171 break;
3172 }
3173 enabled_rbs >>= 2;
3174 }
3175 WREG32(PA_SC_RASTER_CONFIG, data);
3176 }
3177 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3178 }
3179
3180 /**
3181 * cik_gpu_init - setup the 3D engine
3182 *
3183 * @rdev: radeon_device pointer
3184 *
3185 * Configures the 3D engine and tiling configuration
3186 * registers so that the 3D engine is usable.
3187 */
3188 static void cik_gpu_init(struct radeon_device *rdev)
3189 {
3190 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3191 u32 mc_shared_chmap, mc_arb_ramcfg;
3192 u32 hdp_host_path_cntl;
3193 u32 tmp;
3194 int i, j;
3195
3196 switch (rdev->family) {
3197 case CHIP_BONAIRE:
3198 rdev->config.cik.max_shader_engines = 2;
3199 rdev->config.cik.max_tile_pipes = 4;
3200 rdev->config.cik.max_cu_per_sh = 7;
3201 rdev->config.cik.max_sh_per_se = 1;
3202 rdev->config.cik.max_backends_per_se = 2;
3203 rdev->config.cik.max_texture_channel_caches = 4;
3204 rdev->config.cik.max_gprs = 256;
3205 rdev->config.cik.max_gs_threads = 32;
3206 rdev->config.cik.max_hw_contexts = 8;
3207
3208 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3209 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3210 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3211 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3212 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3213 break;
3214 case CHIP_HAWAII:
3215 rdev->config.cik.max_shader_engines = 4;
3216 rdev->config.cik.max_tile_pipes = 16;
3217 rdev->config.cik.max_cu_per_sh = 11;
3218 rdev->config.cik.max_sh_per_se = 1;
3219 rdev->config.cik.max_backends_per_se = 4;
3220 rdev->config.cik.max_texture_channel_caches = 16;
3221 rdev->config.cik.max_gprs = 256;
3222 rdev->config.cik.max_gs_threads = 32;
3223 rdev->config.cik.max_hw_contexts = 8;
3224
3225 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3226 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3227 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3228 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3229 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3230 break;
3231 case CHIP_KAVERI:
3232 rdev->config.cik.max_shader_engines = 1;
3233 rdev->config.cik.max_tile_pipes = 4;
3234 if ((rdev->pdev->device == 0x1304) ||
3235 (rdev->pdev->device == 0x1305) ||
3236 (rdev->pdev->device == 0x130C) ||
3237 (rdev->pdev->device == 0x130F) ||
3238 (rdev->pdev->device == 0x1310) ||
3239 (rdev->pdev->device == 0x1311) ||
3240 (rdev->pdev->device == 0x131C)) {
3241 rdev->config.cik.max_cu_per_sh = 8;
3242 rdev->config.cik.max_backends_per_se = 2;
3243 } else if ((rdev->pdev->device == 0x1309) ||
3244 (rdev->pdev->device == 0x130A) ||
3245 (rdev->pdev->device == 0x130D) ||
3246 (rdev->pdev->device == 0x1313) ||
3247 (rdev->pdev->device == 0x131D)) {
3248 rdev->config.cik.max_cu_per_sh = 6;
3249 rdev->config.cik.max_backends_per_se = 2;
3250 } else if ((rdev->pdev->device == 0x1306) ||
3251 (rdev->pdev->device == 0x1307) ||
3252 (rdev->pdev->device == 0x130B) ||
3253 (rdev->pdev->device == 0x130E) ||
3254 (rdev->pdev->device == 0x1315) ||
3255 (rdev->pdev->device == 0x131B)) {
3256 rdev->config.cik.max_cu_per_sh = 4;
3257 rdev->config.cik.max_backends_per_se = 1;
3258 } else {
3259 rdev->config.cik.max_cu_per_sh = 3;
3260 rdev->config.cik.max_backends_per_se = 1;
3261 }
3262 rdev->config.cik.max_sh_per_se = 1;
3263 rdev->config.cik.max_texture_channel_caches = 4;
3264 rdev->config.cik.max_gprs = 256;
3265 rdev->config.cik.max_gs_threads = 16;
3266 rdev->config.cik.max_hw_contexts = 8;
3267
3268 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3269 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3270 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3271 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3272 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3273 break;
3274 case CHIP_KABINI:
3275 default:
3276 rdev->config.cik.max_shader_engines = 1;
3277 rdev->config.cik.max_tile_pipes = 2;
3278 rdev->config.cik.max_cu_per_sh = 2;
3279 rdev->config.cik.max_sh_per_se = 1;
3280 rdev->config.cik.max_backends_per_se = 1;
3281 rdev->config.cik.max_texture_channel_caches = 2;
3282 rdev->config.cik.max_gprs = 256;
3283 rdev->config.cik.max_gs_threads = 16;
3284 rdev->config.cik.max_hw_contexts = 8;
3285
3286 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3287 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3288 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3289 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3290 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3291 break;
3292 }
3293
3294 /* Initialize HDP */
3295 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3296 WREG32((0x2c14 + j), 0x00000000);
3297 WREG32((0x2c18 + j), 0x00000000);
3298 WREG32((0x2c1c + j), 0x00000000);
3299 WREG32((0x2c20 + j), 0x00000000);
3300 WREG32((0x2c24 + j), 0x00000000);
3301 }
3302
3303 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3304
3305 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3306
3307 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3308 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3309
3310 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3311 rdev->config.cik.mem_max_burst_length_bytes = 256;
3312 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3313 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3314 if (rdev->config.cik.mem_row_size_in_kb > 4)
3315 rdev->config.cik.mem_row_size_in_kb = 4;
3316 /* XXX use MC settings? */
3317 rdev->config.cik.shader_engine_tile_size = 32;
3318 rdev->config.cik.num_gpus = 1;
3319 rdev->config.cik.multi_gpu_tile_size = 64;
3320
3321 /* fix up row size */
3322 gb_addr_config &= ~ROW_SIZE_MASK;
3323 switch (rdev->config.cik.mem_row_size_in_kb) {
3324 case 1:
3325 default:
3326 gb_addr_config |= ROW_SIZE(0);
3327 break;
3328 case 2:
3329 gb_addr_config |= ROW_SIZE(1);
3330 break;
3331 case 4:
3332 gb_addr_config |= ROW_SIZE(2);
3333 break;
3334 }
3335
3336 /* setup tiling info dword. gb_addr_config is not adequate since it does
3337 * not have bank info, so create a custom tiling dword.
3338 * bits 3:0 num_pipes
3339 * bits 7:4 num_banks
3340 * bits 11:8 group_size
3341 * bits 15:12 row_size
3342 */
3343 rdev->config.cik.tile_config = 0;
3344 switch (rdev->config.cik.num_tile_pipes) {
3345 case 1:
3346 rdev->config.cik.tile_config |= (0 << 0);
3347 break;
3348 case 2:
3349 rdev->config.cik.tile_config |= (1 << 0);
3350 break;
3351 case 4:
3352 rdev->config.cik.tile_config |= (2 << 0);
3353 break;
3354 case 8:
3355 default:
3356 /* XXX what about 12? */
3357 rdev->config.cik.tile_config |= (3 << 0);
3358 break;
3359 }
3360 rdev->config.cik.tile_config |=
3361 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3362 rdev->config.cik.tile_config |=
3363 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3364 rdev->config.cik.tile_config |=
3365 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3366
3367 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3368 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3369 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3370 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3371 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3372 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3373 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3374 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3375
3376 cik_tiling_mode_table_init(rdev);
3377
3378 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3379 rdev->config.cik.max_sh_per_se,
3380 rdev->config.cik.max_backends_per_se);
3381
3382 /* set HW defaults for 3D engine */
3383 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3384
3385 WREG32(SX_DEBUG_1, 0x20);
3386
3387 WREG32(TA_CNTL_AUX, 0x00010000);
3388
3389 tmp = RREG32(SPI_CONFIG_CNTL);
3390 tmp |= 0x03000000;
3391 WREG32(SPI_CONFIG_CNTL, tmp);
3392
3393 WREG32(SQ_CONFIG, 1);
3394
3395 WREG32(DB_DEBUG, 0);
3396
3397 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3398 tmp |= 0x00000400;
3399 WREG32(DB_DEBUG2, tmp);
3400
3401 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3402 tmp |= 0x00020200;
3403 WREG32(DB_DEBUG3, tmp);
3404
3405 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3406 tmp |= 0x00018208;
3407 WREG32(CB_HW_CONTROL, tmp);
3408
3409 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3410
3411 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3412 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3413 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3414 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3415
3416 WREG32(VGT_NUM_INSTANCES, 1);
3417
3418 WREG32(CP_PERFMON_CNTL, 0);
3419
3420 WREG32(SQ_CONFIG, 0);
3421
3422 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3423 FORCE_EOV_MAX_REZ_CNT(255)));
3424
3425 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3426 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3427
3428 WREG32(VGT_GS_VERTEX_REUSE, 16);
3429 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3430
3431 tmp = RREG32(HDP_MISC_CNTL);
3432 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3433 WREG32(HDP_MISC_CNTL, tmp);
3434
3435 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3436 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3437
3438 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3439 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3440
3441 udelay(50);
3442 }
3443
3444 /*
3445 * GPU scratch registers helpers function.
3446 */
3447 /**
3448 * cik_scratch_init - setup driver info for CP scratch regs
3449 *
3450 * @rdev: radeon_device pointer
3451 *
3452 * Set up the number and offset of the CP scratch registers.
3453 * NOTE: use of CP scratch registers is a legacy inferface and
3454 * is not used by default on newer asics (r6xx+). On newer asics,
3455 * memory buffers are used for fences rather than scratch regs.
3456 */
3457 static void cik_scratch_init(struct radeon_device *rdev)
3458 {
3459 int i;
3460
3461 rdev->scratch.num_reg = 7;
3462 rdev->scratch.reg_base = SCRATCH_REG0;
3463 for (i = 0; i < rdev->scratch.num_reg; i++) {
3464 rdev->scratch.free[i] = true;
3465 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3466 }
3467 }
3468
3469 /**
3470 * cik_ring_test - basic gfx ring test
3471 *
3472 * @rdev: radeon_device pointer
3473 * @ring: radeon_ring structure holding ring information
3474 *
3475 * Allocate a scratch register and write to it using the gfx ring (CIK).
3476 * Provides a basic gfx ring test to verify that the ring is working.
3477 * Used by cik_cp_gfx_resume();
3478 * Returns 0 on success, error on failure.
3479 */
3480 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3481 {
3482 uint32_t scratch;
3483 uint32_t tmp = 0;
3484 unsigned i;
3485 int r;
3486
3487 r = radeon_scratch_get(rdev, &scratch);
3488 if (r) {
3489 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3490 return r;
3491 }
3492 WREG32(scratch, 0xCAFEDEAD);
3493 r = radeon_ring_lock(rdev, ring, 3);
3494 if (r) {
3495 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3496 radeon_scratch_free(rdev, scratch);
3497 return r;
3498 }
3499 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3500 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3501 radeon_ring_write(ring, 0xDEADBEEF);
3502 radeon_ring_unlock_commit(rdev, ring);
3503
3504 for (i = 0; i < rdev->usec_timeout; i++) {
3505 tmp = RREG32(scratch);
3506 if (tmp == 0xDEADBEEF)
3507 break;
3508 DRM_UDELAY(1);
3509 }
3510 if (i < rdev->usec_timeout) {
3511 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3512 } else {
3513 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3514 ring->idx, scratch, tmp);
3515 r = -EINVAL;
3516 }
3517 radeon_scratch_free(rdev, scratch);
3518 return r;
3519 }
3520
3521 /**
3522 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3523 *
3524 * @rdev: radeon_device pointer
3525 * @ridx: radeon ring index
3526 *
3527 * Emits an hdp flush on the cp.
3528 */
3529 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3530 int ridx)
3531 {
3532 struct radeon_ring *ring = &rdev->ring[ridx];
3533 u32 ref_and_mask;
3534
3535 switch (ring->idx) {
3536 case CAYMAN_RING_TYPE_CP1_INDEX:
3537 case CAYMAN_RING_TYPE_CP2_INDEX:
3538 default:
3539 switch (ring->me) {
3540 case 0:
3541 ref_and_mask = CP2 << ring->pipe;
3542 break;
3543 case 1:
3544 ref_and_mask = CP6 << ring->pipe;
3545 break;
3546 default:
3547 return;
3548 }
3549 break;
3550 case RADEON_RING_TYPE_GFX_INDEX:
3551 ref_and_mask = CP0;
3552 break;
3553 }
3554
3555 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3556 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3557 WAIT_REG_MEM_FUNCTION(3) | /* == */
3558 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3559 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3560 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3561 radeon_ring_write(ring, ref_and_mask);
3562 radeon_ring_write(ring, ref_and_mask);
3563 radeon_ring_write(ring, 0x20); /* poll interval */
3564 }
3565
3566 /**
3567 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3568 *
3569 * @rdev: radeon_device pointer
3570 * @fence: radeon fence object
3571 *
3572 * Emits a fence sequnce number on the gfx ring and flushes
3573 * GPU caches.
3574 */
3575 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3576 struct radeon_fence *fence)
3577 {
3578 struct radeon_ring *ring = &rdev->ring[fence->ring];
3579 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3580
3581 /* EVENT_WRITE_EOP - flush caches, send int */
3582 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3583 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3584 EOP_TC_ACTION_EN |
3585 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3586 EVENT_INDEX(5)));
3587 radeon_ring_write(ring, addr & 0xfffffffc);
3588 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3589 radeon_ring_write(ring, fence->seq);
3590 radeon_ring_write(ring, 0);
3591 /* HDP flush */
3592 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3593 }
3594
3595 /**
3596 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3597 *
3598 * @rdev: radeon_device pointer
3599 * @fence: radeon fence object
3600 *
3601 * Emits a fence sequnce number on the compute ring and flushes
3602 * GPU caches.
3603 */
3604 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3605 struct radeon_fence *fence)
3606 {
3607 struct radeon_ring *ring = &rdev->ring[fence->ring];
3608 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3609
3610 /* RELEASE_MEM - flush caches, send int */
3611 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3612 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3613 EOP_TC_ACTION_EN |
3614 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3615 EVENT_INDEX(5)));
3616 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3617 radeon_ring_write(ring, addr & 0xfffffffc);
3618 radeon_ring_write(ring, upper_32_bits(addr));
3619 radeon_ring_write(ring, fence->seq);
3620 radeon_ring_write(ring, 0);
3621 /* HDP flush */
3622 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3623 }
3624
3625 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3626 struct radeon_ring *ring,
3627 struct radeon_semaphore *semaphore,
3628 bool emit_wait)
3629 {
3630 uint64_t addr = semaphore->gpu_addr;
3631 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3632
3633 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3634 radeon_ring_write(ring, addr & 0xffffffff);
3635 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3636
3637 return true;
3638 }
3639
3640 /**
3641 * cik_copy_cpdma - copy pages using the CP DMA engine
3642 *
3643 * @rdev: radeon_device pointer
3644 * @src_offset: src GPU address
3645 * @dst_offset: dst GPU address
3646 * @num_gpu_pages: number of GPU pages to xfer
3647 * @fence: radeon fence object
3648 *
3649 * Copy GPU paging using the CP DMA engine (CIK+).
3650 * Used by the radeon ttm implementation to move pages if
3651 * registered as the asic copy callback.
3652 */
3653 int cik_copy_cpdma(struct radeon_device *rdev,
3654 uint64_t src_offset, uint64_t dst_offset,
3655 unsigned num_gpu_pages,
3656 struct radeon_fence **fence)
3657 {
3658 struct radeon_semaphore *sem = NULL;
3659 int ring_index = rdev->asic->copy.blit_ring_index;
3660 struct radeon_ring *ring = &rdev->ring[ring_index];
3661 u32 size_in_bytes, cur_size_in_bytes, control;
3662 int i, num_loops;
3663 int r = 0;
3664
3665 r = radeon_semaphore_create(rdev, &sem);
3666 if (r) {
3667 DRM_ERROR("radeon: moving bo (%d).\n", r);
3668 return r;
3669 }
3670
3671 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3672 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3673 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3674 if (r) {
3675 DRM_ERROR("radeon: moving bo (%d).\n", r);
3676 radeon_semaphore_free(rdev, &sem, NULL);
3677 return r;
3678 }
3679
3680 radeon_semaphore_sync_to(sem, *fence);
3681 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3682
3683 for (i = 0; i < num_loops; i++) {
3684 cur_size_in_bytes = size_in_bytes;
3685 if (cur_size_in_bytes > 0x1fffff)
3686 cur_size_in_bytes = 0x1fffff;
3687 size_in_bytes -= cur_size_in_bytes;
3688 control = 0;
3689 if (size_in_bytes == 0)
3690 control |= PACKET3_DMA_DATA_CP_SYNC;
3691 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3692 radeon_ring_write(ring, control);
3693 radeon_ring_write(ring, lower_32_bits(src_offset));
3694 radeon_ring_write(ring, upper_32_bits(src_offset));
3695 radeon_ring_write(ring, lower_32_bits(dst_offset));
3696 radeon_ring_write(ring, upper_32_bits(dst_offset));
3697 radeon_ring_write(ring, cur_size_in_bytes);
3698 src_offset += cur_size_in_bytes;
3699 dst_offset += cur_size_in_bytes;
3700 }
3701
3702 r = radeon_fence_emit(rdev, fence, ring->idx);
3703 if (r) {
3704 radeon_ring_unlock_undo(rdev, ring);
3705 return r;
3706 }
3707
3708 radeon_ring_unlock_commit(rdev, ring);
3709 radeon_semaphore_free(rdev, &sem, *fence);
3710
3711 return r;
3712 }
3713
3714 /*
3715 * IB stuff
3716 */
3717 /**
3718 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3719 *
3720 * @rdev: radeon_device pointer
3721 * @ib: radeon indirect buffer object
3722 *
3723 * Emits an DE (drawing engine) or CE (constant engine) IB
3724 * on the gfx ring. IBs are usually generated by userspace
3725 * acceleration drivers and submitted to the kernel for
3726 * sheduling on the ring. This function schedules the IB
3727 * on the gfx ring for execution by the GPU.
3728 */
3729 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3730 {
3731 struct radeon_ring *ring = &rdev->ring[ib->ring];
3732 u32 header, control = INDIRECT_BUFFER_VALID;
3733
3734 if (ib->is_const_ib) {
3735 /* set switch buffer packet before const IB */
3736 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3737 radeon_ring_write(ring, 0);
3738
3739 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3740 } else {
3741 u32 next_rptr;
3742 if (ring->rptr_save_reg) {
3743 next_rptr = ring->wptr + 3 + 4;
3744 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3745 radeon_ring_write(ring, ((ring->rptr_save_reg -
3746 PACKET3_SET_UCONFIG_REG_START) >> 2));
3747 radeon_ring_write(ring, next_rptr);
3748 } else if (rdev->wb.enabled) {
3749 next_rptr = ring->wptr + 5 + 4;
3750 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3751 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3752 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3753 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3754 radeon_ring_write(ring, next_rptr);
3755 }
3756
3757 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3758 }
3759
3760 control |= ib->length_dw |
3761 (ib->vm ? (ib->vm->id << 24) : 0);
3762
3763 radeon_ring_write(ring, header);
3764 radeon_ring_write(ring,
3765 #ifdef __BIG_ENDIAN
3766 (2 << 0) |
3767 #endif
3768 (ib->gpu_addr & 0xFFFFFFFC));
3769 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3770 radeon_ring_write(ring, control);
3771 }
3772
3773 /**
3774 * cik_ib_test - basic gfx ring IB test
3775 *
3776 * @rdev: radeon_device pointer
3777 * @ring: radeon_ring structure holding ring information
3778 *
3779 * Allocate an IB and execute it on the gfx ring (CIK).
3780 * Provides a basic gfx ring test to verify that IBs are working.
3781 * Returns 0 on success, error on failure.
3782 */
3783 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3784 {
3785 struct radeon_ib ib;
3786 uint32_t scratch;
3787 uint32_t tmp = 0;
3788 unsigned i;
3789 int r;
3790
3791 r = radeon_scratch_get(rdev, &scratch);
3792 if (r) {
3793 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3794 return r;
3795 }
3796 WREG32(scratch, 0xCAFEDEAD);
3797 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3798 if (r) {
3799 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3800 radeon_scratch_free(rdev, scratch);
3801 return r;
3802 }
3803 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3804 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3805 ib.ptr[2] = 0xDEADBEEF;
3806 ib.length_dw = 3;
3807 r = radeon_ib_schedule(rdev, &ib, NULL);
3808 if (r) {
3809 radeon_scratch_free(rdev, scratch);
3810 radeon_ib_free(rdev, &ib);
3811 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3812 return r;
3813 }
3814 r = radeon_fence_wait(ib.fence, false);
3815 if (r) {
3816 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3817 radeon_scratch_free(rdev, scratch);
3818 radeon_ib_free(rdev, &ib);
3819 return r;
3820 }
3821 for (i = 0; i < rdev->usec_timeout; i++) {
3822 tmp = RREG32(scratch);
3823 if (tmp == 0xDEADBEEF)
3824 break;
3825 DRM_UDELAY(1);
3826 }
3827 if (i < rdev->usec_timeout) {
3828 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3829 } else {
3830 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3831 scratch, tmp);
3832 r = -EINVAL;
3833 }
3834 radeon_scratch_free(rdev, scratch);
3835 radeon_ib_free(rdev, &ib);
3836 return r;
3837 }
3838
3839 /*
3840 * CP.
3841 * On CIK, gfx and compute now have independant command processors.
3842 *
3843 * GFX
3844 * Gfx consists of a single ring and can process both gfx jobs and
3845 * compute jobs. The gfx CP consists of three microengines (ME):
3846 * PFP - Pre-Fetch Parser
3847 * ME - Micro Engine
3848 * CE - Constant Engine
3849 * The PFP and ME make up what is considered the Drawing Engine (DE).
3850 * The CE is an asynchronous engine used for updating buffer desciptors
3851 * used by the DE so that they can be loaded into cache in parallel
3852 * while the DE is processing state update packets.
3853 *
3854 * Compute
3855 * The compute CP consists of two microengines (ME):
3856 * MEC1 - Compute MicroEngine 1
3857 * MEC2 - Compute MicroEngine 2
3858 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3859 * The queues are exposed to userspace and are programmed directly
3860 * by the compute runtime.
3861 */
3862 /**
3863 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3864 *
3865 * @rdev: radeon_device pointer
3866 * @enable: enable or disable the MEs
3867 *
3868 * Halts or unhalts the gfx MEs.
3869 */
3870 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3871 {
3872 if (enable)
3873 WREG32(CP_ME_CNTL, 0);
3874 else {
3875 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3876 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3877 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3878 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3879 }
3880 udelay(50);
3881 }
3882
3883 /**
3884 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3885 *
3886 * @rdev: radeon_device pointer
3887 *
3888 * Loads the gfx PFP, ME, and CE ucode.
3889 * Returns 0 for success, -EINVAL if the ucode is not available.
3890 */
3891 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3892 {
3893 const __be32 *fw_data;
3894 int i;
3895
3896 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3897 return -EINVAL;
3898
3899 cik_cp_gfx_enable(rdev, false);
3900
3901 /* PFP */
3902 fw_data = (const __be32 *)rdev->pfp_fw->data;
3903 WREG32(CP_PFP_UCODE_ADDR, 0);
3904 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3905 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3906 WREG32(CP_PFP_UCODE_ADDR, 0);
3907
3908 /* CE */
3909 fw_data = (const __be32 *)rdev->ce_fw->data;
3910 WREG32(CP_CE_UCODE_ADDR, 0);
3911 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3912 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3913 WREG32(CP_CE_UCODE_ADDR, 0);
3914
3915 /* ME */
3916 fw_data = (const __be32 *)rdev->me_fw->data;
3917 WREG32(CP_ME_RAM_WADDR, 0);
3918 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3919 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3920 WREG32(CP_ME_RAM_WADDR, 0);
3921
3922 WREG32(CP_PFP_UCODE_ADDR, 0);
3923 WREG32(CP_CE_UCODE_ADDR, 0);
3924 WREG32(CP_ME_RAM_WADDR, 0);
3925 WREG32(CP_ME_RAM_RADDR, 0);
3926 return 0;
3927 }
3928
3929 /**
3930 * cik_cp_gfx_start - start the gfx ring
3931 *
3932 * @rdev: radeon_device pointer
3933 *
3934 * Enables the ring and loads the clear state context and other
3935 * packets required to init the ring.
3936 * Returns 0 for success, error for failure.
3937 */
3938 static int cik_cp_gfx_start(struct radeon_device *rdev)
3939 {
3940 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3941 int r, i;
3942
3943 /* init the CP */
3944 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3945 WREG32(CP_ENDIAN_SWAP, 0);
3946 WREG32(CP_DEVICE_ID, 1);
3947
3948 cik_cp_gfx_enable(rdev, true);
3949
3950 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3951 if (r) {
3952 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3953 return r;
3954 }
3955
3956 /* init the CE partitions. CE only used for gfx on CIK */
3957 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3958 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3959 radeon_ring_write(ring, 0xc000);
3960 radeon_ring_write(ring, 0xc000);
3961
3962 /* setup clear context state */
3963 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3964 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3965
3966 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3967 radeon_ring_write(ring, 0x80000000);
3968 radeon_ring_write(ring, 0x80000000);
3969
3970 for (i = 0; i < cik_default_size; i++)
3971 radeon_ring_write(ring, cik_default_state[i]);
3972
3973 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3974 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3975
3976 /* set clear context state */
3977 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3978 radeon_ring_write(ring, 0);
3979
3980 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3981 radeon_ring_write(ring, 0x00000316);
3982 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3983 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3984
3985 radeon_ring_unlock_commit(rdev, ring);
3986
3987 return 0;
3988 }
3989
3990 /**
3991 * cik_cp_gfx_fini - stop the gfx ring
3992 *
3993 * @rdev: radeon_device pointer
3994 *
3995 * Stop the gfx ring and tear down the driver ring
3996 * info.
3997 */
3998 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3999 {
4000 cik_cp_gfx_enable(rdev, false);
4001 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4002 }
4003
4004 /**
4005 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4006 *
4007 * @rdev: radeon_device pointer
4008 *
4009 * Program the location and size of the gfx ring buffer
4010 * and test it to make sure it's working.
4011 * Returns 0 for success, error for failure.
4012 */
4013 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4014 {
4015 struct radeon_ring *ring;
4016 u32 tmp;
4017 u32 rb_bufsz;
4018 u64 rb_addr;
4019 int r;
4020
4021 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4022 if (rdev->family != CHIP_HAWAII)
4023 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4024
4025 /* Set the write pointer delay */
4026 WREG32(CP_RB_WPTR_DELAY, 0);
4027
4028 /* set the RB to use vmid 0 */
4029 WREG32(CP_RB_VMID, 0);
4030
4031 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4032
4033 /* ring 0 - compute and gfx */
4034 /* Set ring buffer size */
4035 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4036 rb_bufsz = order_base_2(ring->ring_size / 8);
4037 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4038 #ifdef __BIG_ENDIAN
4039 tmp |= BUF_SWAP_32BIT;
4040 #endif
4041 WREG32(CP_RB0_CNTL, tmp);
4042
4043 /* Initialize the ring buffer's read and write pointers */
4044 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4045 ring->wptr = 0;
4046 WREG32(CP_RB0_WPTR, ring->wptr);
4047
4048 /* set the wb address wether it's enabled or not */
4049 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4050 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4051
4052 /* scratch register shadowing is no longer supported */
4053 WREG32(SCRATCH_UMSK, 0);
4054
4055 if (!rdev->wb.enabled)
4056 tmp |= RB_NO_UPDATE;
4057
4058 mdelay(1);
4059 WREG32(CP_RB0_CNTL, tmp);
4060
4061 rb_addr = ring->gpu_addr >> 8;
4062 WREG32(CP_RB0_BASE, rb_addr);
4063 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4064
4065 /* start the ring */
4066 cik_cp_gfx_start(rdev);
4067 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4068 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4069 if (r) {
4070 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4071 return r;
4072 }
4073
4074 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4075 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4076
4077 return 0;
4078 }
4079
4080 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4081 struct radeon_ring *ring)
4082 {
4083 u32 rptr;
4084
4085 if (rdev->wb.enabled)
4086 rptr = rdev->wb.wb[ring->rptr_offs/4];
4087 else
4088 rptr = RREG32(CP_RB0_RPTR);
4089
4090 return rptr;
4091 }
4092
4093 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4094 struct radeon_ring *ring)
4095 {
4096 u32 wptr;
4097
4098 wptr = RREG32(CP_RB0_WPTR);
4099
4100 return wptr;
4101 }
4102
4103 void cik_gfx_set_wptr(struct radeon_device *rdev,
4104 struct radeon_ring *ring)
4105 {
4106 WREG32(CP_RB0_WPTR, ring->wptr);
4107 (void)RREG32(CP_RB0_WPTR);
4108 }
4109
4110 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4111 struct radeon_ring *ring)
4112 {
4113 u32 rptr;
4114
4115 if (rdev->wb.enabled) {
4116 rptr = rdev->wb.wb[ring->rptr_offs/4];
4117 } else {
4118 mutex_lock(&rdev->srbm_mutex);
4119 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4120 rptr = RREG32(CP_HQD_PQ_RPTR);
4121 cik_srbm_select(rdev, 0, 0, 0, 0);
4122 mutex_unlock(&rdev->srbm_mutex);
4123 }
4124
4125 return rptr;
4126 }
4127
4128 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4129 struct radeon_ring *ring)
4130 {
4131 u32 wptr;
4132
4133 if (rdev->wb.enabled) {
4134 /* XXX check if swapping is necessary on BE */
4135 wptr = rdev->wb.wb[ring->wptr_offs/4];
4136 } else {
4137 mutex_lock(&rdev->srbm_mutex);
4138 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4139 wptr = RREG32(CP_HQD_PQ_WPTR);
4140 cik_srbm_select(rdev, 0, 0, 0, 0);
4141 mutex_unlock(&rdev->srbm_mutex);
4142 }
4143
4144 return wptr;
4145 }
4146
4147 void cik_compute_set_wptr(struct radeon_device *rdev,
4148 struct radeon_ring *ring)
4149 {
4150 /* XXX check if swapping is necessary on BE */
4151 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4152 WDOORBELL32(ring->doorbell_index, ring->wptr);
4153 }
4154
4155 /**
4156 * cik_cp_compute_enable - enable/disable the compute CP MEs
4157 *
4158 * @rdev: radeon_device pointer
4159 * @enable: enable or disable the MEs
4160 *
4161 * Halts or unhalts the compute MEs.
4162 */
4163 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4164 {
4165 if (enable)
4166 WREG32(CP_MEC_CNTL, 0);
4167 else {
4168 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4169 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4170 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4171 }
4172 udelay(50);
4173 }
4174
4175 /**
4176 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4177 *
4178 * @rdev: radeon_device pointer
4179 *
4180 * Loads the compute MEC1&2 ucode.
4181 * Returns 0 for success, -EINVAL if the ucode is not available.
4182 */
4183 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4184 {
4185 const __be32 *fw_data;
4186 int i;
4187
4188 if (!rdev->mec_fw)
4189 return -EINVAL;
4190
4191 cik_cp_compute_enable(rdev, false);
4192
4193 /* MEC1 */
4194 fw_data = (const __be32 *)rdev->mec_fw->data;
4195 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4196 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4197 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4198 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4199
4200 if (rdev->family == CHIP_KAVERI) {
4201 /* MEC2 */
4202 fw_data = (const __be32 *)rdev->mec_fw->data;
4203 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4204 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4205 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4206 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4207 }
4208
4209 return 0;
4210 }
4211
4212 /**
4213 * cik_cp_compute_start - start the compute queues
4214 *
4215 * @rdev: radeon_device pointer
4216 *
4217 * Enable the compute queues.
4218 * Returns 0 for success, error for failure.
4219 */
4220 static int cik_cp_compute_start(struct radeon_device *rdev)
4221 {
4222 cik_cp_compute_enable(rdev, true);
4223
4224 return 0;
4225 }
4226
4227 /**
4228 * cik_cp_compute_fini - stop the compute queues
4229 *
4230 * @rdev: radeon_device pointer
4231 *
4232 * Stop the compute queues and tear down the driver queue
4233 * info.
4234 */
4235 static void cik_cp_compute_fini(struct radeon_device *rdev)
4236 {
4237 int i, idx, r;
4238
4239 cik_cp_compute_enable(rdev, false);
4240
4241 for (i = 0; i < 2; i++) {
4242 if (i == 0)
4243 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4244 else
4245 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4246
4247 if (rdev->ring[idx].mqd_obj) {
4248 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4249 if (unlikely(r != 0))
4250 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4251
4252 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4253 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4254
4255 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4256 rdev->ring[idx].mqd_obj = NULL;
4257 }
4258 }
4259 }
4260
4261 static void cik_mec_fini(struct radeon_device *rdev)
4262 {
4263 int r;
4264
4265 if (rdev->mec.hpd_eop_obj) {
4266 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4267 if (unlikely(r != 0))
4268 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4269 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4270 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4271
4272 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4273 rdev->mec.hpd_eop_obj = NULL;
4274 }
4275 }
4276
4277 #define MEC_HPD_SIZE 2048
4278
4279 static int cik_mec_init(struct radeon_device *rdev)
4280 {
4281 int r;
4282 u32 *hpd;
4283
4284 /*
4285 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4286 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4287 */
4288 if (rdev->family == CHIP_KAVERI)
4289 rdev->mec.num_mec = 2;
4290 else
4291 rdev->mec.num_mec = 1;
4292 rdev->mec.num_pipe = 4;
4293 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4294
4295 if (rdev->mec.hpd_eop_obj == NULL) {
4296 r = radeon_bo_create(rdev,
4297 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4298 PAGE_SIZE, true,
4299 RADEON_GEM_DOMAIN_GTT, NULL,
4300 &rdev->mec.hpd_eop_obj);
4301 if (r) {
4302 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4303 return r;
4304 }
4305 }
4306
4307 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4308 if (unlikely(r != 0)) {
4309 cik_mec_fini(rdev);
4310 return r;
4311 }
4312 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4313 &rdev->mec.hpd_eop_gpu_addr);
4314 if (r) {
4315 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4316 cik_mec_fini(rdev);
4317 return r;
4318 }
4319 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4320 if (r) {
4321 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4322 cik_mec_fini(rdev);
4323 return r;
4324 }
4325
4326 /* clear memory. Not sure if this is required or not */
4327 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4328
4329 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4330 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4331
4332 return 0;
4333 }
4334
4335 struct hqd_registers
4336 {
4337 u32 cp_mqd_base_addr;
4338 u32 cp_mqd_base_addr_hi;
4339 u32 cp_hqd_active;
4340 u32 cp_hqd_vmid;
4341 u32 cp_hqd_persistent_state;
4342 u32 cp_hqd_pipe_priority;
4343 u32 cp_hqd_queue_priority;
4344 u32 cp_hqd_quantum;
4345 u32 cp_hqd_pq_base;
4346 u32 cp_hqd_pq_base_hi;
4347 u32 cp_hqd_pq_rptr;
4348 u32 cp_hqd_pq_rptr_report_addr;
4349 u32 cp_hqd_pq_rptr_report_addr_hi;
4350 u32 cp_hqd_pq_wptr_poll_addr;
4351 u32 cp_hqd_pq_wptr_poll_addr_hi;
4352 u32 cp_hqd_pq_doorbell_control;
4353 u32 cp_hqd_pq_wptr;
4354 u32 cp_hqd_pq_control;
4355 u32 cp_hqd_ib_base_addr;
4356 u32 cp_hqd_ib_base_addr_hi;
4357 u32 cp_hqd_ib_rptr;
4358 u32 cp_hqd_ib_control;
4359 u32 cp_hqd_iq_timer;
4360 u32 cp_hqd_iq_rptr;
4361 u32 cp_hqd_dequeue_request;
4362 u32 cp_hqd_dma_offload;
4363 u32 cp_hqd_sema_cmd;
4364 u32 cp_hqd_msg_type;
4365 u32 cp_hqd_atomic0_preop_lo;
4366 u32 cp_hqd_atomic0_preop_hi;
4367 u32 cp_hqd_atomic1_preop_lo;
4368 u32 cp_hqd_atomic1_preop_hi;
4369 u32 cp_hqd_hq_scheduler0;
4370 u32 cp_hqd_hq_scheduler1;
4371 u32 cp_mqd_control;
4372 };
4373
4374 struct bonaire_mqd
4375 {
4376 u32 header;
4377 u32 dispatch_initiator;
4378 u32 dimensions[3];
4379 u32 start_idx[3];
4380 u32 num_threads[3];
4381 u32 pipeline_stat_enable;
4382 u32 perf_counter_enable;
4383 u32 pgm[2];
4384 u32 tba[2];
4385 u32 tma[2];
4386 u32 pgm_rsrc[2];
4387 u32 vmid;
4388 u32 resource_limits;
4389 u32 static_thread_mgmt01[2];
4390 u32 tmp_ring_size;
4391 u32 static_thread_mgmt23[2];
4392 u32 restart[3];
4393 u32 thread_trace_enable;
4394 u32 reserved1;
4395 u32 user_data[16];
4396 u32 vgtcs_invoke_count[2];
4397 struct hqd_registers queue_state;
4398 u32 dequeue_cntr;
4399 u32 interrupt_queue[64];
4400 };
4401
4402 /**
4403 * cik_cp_compute_resume - setup the compute queue registers
4404 *
4405 * @rdev: radeon_device pointer
4406 *
4407 * Program the compute queues and test them to make sure they
4408 * are working.
4409 * Returns 0 for success, error for failure.
4410 */
4411 static int cik_cp_compute_resume(struct radeon_device *rdev)
4412 {
4413 int r, i, idx;
4414 u32 tmp;
4415 bool use_doorbell = true;
4416 u64 hqd_gpu_addr;
4417 u64 mqd_gpu_addr;
4418 u64 eop_gpu_addr;
4419 u64 wb_gpu_addr;
4420 u32 *buf;
4421 struct bonaire_mqd *mqd;
4422
4423 r = cik_cp_compute_start(rdev);
4424 if (r)
4425 return r;
4426
4427 /* fix up chicken bits */
4428 tmp = RREG32(CP_CPF_DEBUG);
4429 tmp |= (1 << 23);
4430 WREG32(CP_CPF_DEBUG, tmp);
4431
4432 /* init the pipes */
4433 mutex_lock(&rdev->srbm_mutex);
4434 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4435 int me = (i < 4) ? 1 : 2;
4436 int pipe = (i < 4) ? i : (i - 4);
4437
4438 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4439
4440 cik_srbm_select(rdev, me, pipe, 0, 0);
4441
4442 /* write the EOP addr */
4443 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4444 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4445
4446 /* set the VMID assigned */
4447 WREG32(CP_HPD_EOP_VMID, 0);
4448
4449 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4450 tmp = RREG32(CP_HPD_EOP_CONTROL);
4451 tmp &= ~EOP_SIZE_MASK;
4452 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4453 WREG32(CP_HPD_EOP_CONTROL, tmp);
4454 }
4455 cik_srbm_select(rdev, 0, 0, 0, 0);
4456 mutex_unlock(&rdev->srbm_mutex);
4457
4458 /* init the queues. Just two for now. */
4459 for (i = 0; i < 2; i++) {
4460 if (i == 0)
4461 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4462 else
4463 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4464
4465 if (rdev->ring[idx].mqd_obj == NULL) {
4466 r = radeon_bo_create(rdev,
4467 sizeof(struct bonaire_mqd),
4468 PAGE_SIZE, true,
4469 RADEON_GEM_DOMAIN_GTT, NULL,
4470 &rdev->ring[idx].mqd_obj);
4471 if (r) {
4472 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4473 return r;
4474 }
4475 }
4476
4477 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4478 if (unlikely(r != 0)) {
4479 cik_cp_compute_fini(rdev);
4480 return r;
4481 }
4482 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4483 &mqd_gpu_addr);
4484 if (r) {
4485 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4486 cik_cp_compute_fini(rdev);
4487 return r;
4488 }
4489 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4490 if (r) {
4491 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4492 cik_cp_compute_fini(rdev);
4493 return r;
4494 }
4495
4496 /* init the mqd struct */
4497 memset(buf, 0, sizeof(struct bonaire_mqd));
4498
4499 mqd = (struct bonaire_mqd *)buf;
4500 mqd->header = 0xC0310800;
4501 mqd->static_thread_mgmt01[0] = 0xffffffff;
4502 mqd->static_thread_mgmt01[1] = 0xffffffff;
4503 mqd->static_thread_mgmt23[0] = 0xffffffff;
4504 mqd->static_thread_mgmt23[1] = 0xffffffff;
4505
4506 mutex_lock(&rdev->srbm_mutex);
4507 cik_srbm_select(rdev, rdev->ring[idx].me,
4508 rdev->ring[idx].pipe,
4509 rdev->ring[idx].queue, 0);
4510
4511 /* disable wptr polling */
4512 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4513 tmp &= ~WPTR_POLL_EN;
4514 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4515
4516 /* enable doorbell? */
4517 mqd->queue_state.cp_hqd_pq_doorbell_control =
4518 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4519 if (use_doorbell)
4520 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4521 else
4522 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4523 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4524 mqd->queue_state.cp_hqd_pq_doorbell_control);
4525
4526 /* disable the queue if it's active */
4527 mqd->queue_state.cp_hqd_dequeue_request = 0;
4528 mqd->queue_state.cp_hqd_pq_rptr = 0;
4529 mqd->queue_state.cp_hqd_pq_wptr= 0;
4530 if (RREG32(CP_HQD_ACTIVE) & 1) {
4531 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4532 for (i = 0; i < rdev->usec_timeout; i++) {
4533 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4534 break;
4535 udelay(1);
4536 }
4537 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4538 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4539 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4540 }
4541
4542 /* set the pointer to the MQD */
4543 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4544 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4545 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4546 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4547 /* set MQD vmid to 0 */
4548 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4549 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4550 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4551
4552 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4553 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4554 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4555 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4556 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4557 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4558
4559 /* set up the HQD, this is similar to CP_RB0_CNTL */
4560 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4561 mqd->queue_state.cp_hqd_pq_control &=
4562 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4563
4564 mqd->queue_state.cp_hqd_pq_control |=
4565 order_base_2(rdev->ring[idx].ring_size / 8);
4566 mqd->queue_state.cp_hqd_pq_control |=
4567 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4568 #ifdef __BIG_ENDIAN
4569 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4570 #endif
4571 mqd->queue_state.cp_hqd_pq_control &=
4572 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4573 mqd->queue_state.cp_hqd_pq_control |=
4574 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4575 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4576
4577 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4578 if (i == 0)
4579 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4580 else
4581 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4582 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4583 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4584 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4585 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4586 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4587
4588 /* set the wb address wether it's enabled or not */
4589 if (i == 0)
4590 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4591 else
4592 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4593 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4594 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4595 upper_32_bits(wb_gpu_addr) & 0xffff;
4596 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4597 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4598 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4599 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4600
4601 /* enable the doorbell if requested */
4602 if (use_doorbell) {
4603 mqd->queue_state.cp_hqd_pq_doorbell_control =
4604 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4605 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4606 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4607 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4608 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4609 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4610 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4611
4612 } else {
4613 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4614 }
4615 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4616 mqd->queue_state.cp_hqd_pq_doorbell_control);
4617
4618 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4619 rdev->ring[idx].wptr = 0;
4620 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4621 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4622 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4623
4624 /* set the vmid for the queue */
4625 mqd->queue_state.cp_hqd_vmid = 0;
4626 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4627
4628 /* activate the queue */
4629 mqd->queue_state.cp_hqd_active = 1;
4630 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4631
4632 cik_srbm_select(rdev, 0, 0, 0, 0);
4633 mutex_unlock(&rdev->srbm_mutex);
4634
4635 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4636 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4637
4638 rdev->ring[idx].ready = true;
4639 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4640 if (r)
4641 rdev->ring[idx].ready = false;
4642 }
4643
4644 return 0;
4645 }
4646
4647 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4648 {
4649 cik_cp_gfx_enable(rdev, enable);
4650 cik_cp_compute_enable(rdev, enable);
4651 }
4652
4653 static int cik_cp_load_microcode(struct radeon_device *rdev)
4654 {
4655 int r;
4656
4657 r = cik_cp_gfx_load_microcode(rdev);
4658 if (r)
4659 return r;
4660 r = cik_cp_compute_load_microcode(rdev);
4661 if (r)
4662 return r;
4663
4664 return 0;
4665 }
4666
4667 static void cik_cp_fini(struct radeon_device *rdev)
4668 {
4669 cik_cp_gfx_fini(rdev);
4670 cik_cp_compute_fini(rdev);
4671 }
4672
4673 static int cik_cp_resume(struct radeon_device *rdev)
4674 {
4675 int r;
4676
4677 cik_enable_gui_idle_interrupt(rdev, false);
4678
4679 r = cik_cp_load_microcode(rdev);
4680 if (r)
4681 return r;
4682
4683 r = cik_cp_gfx_resume(rdev);
4684 if (r)
4685 return r;
4686 r = cik_cp_compute_resume(rdev);
4687 if (r)
4688 return r;
4689
4690 cik_enable_gui_idle_interrupt(rdev, true);
4691
4692 return 0;
4693 }
4694
4695 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4696 {
4697 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4698 RREG32(GRBM_STATUS));
4699 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4700 RREG32(GRBM_STATUS2));
4701 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4702 RREG32(GRBM_STATUS_SE0));
4703 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4704 RREG32(GRBM_STATUS_SE1));
4705 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4706 RREG32(GRBM_STATUS_SE2));
4707 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4708 RREG32(GRBM_STATUS_SE3));
4709 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4710 RREG32(SRBM_STATUS));
4711 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4712 RREG32(SRBM_STATUS2));
4713 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4714 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4715 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4716 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4717 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4718 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4719 RREG32(CP_STALLED_STAT1));
4720 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4721 RREG32(CP_STALLED_STAT2));
4722 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4723 RREG32(CP_STALLED_STAT3));
4724 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4725 RREG32(CP_CPF_BUSY_STAT));
4726 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4727 RREG32(CP_CPF_STALLED_STAT1));
4728 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4729 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4730 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4731 RREG32(CP_CPC_STALLED_STAT1));
4732 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4733 }
4734
4735 /**
4736 * cik_gpu_check_soft_reset - check which blocks are busy
4737 *
4738 * @rdev: radeon_device pointer
4739 *
4740 * Check which blocks are busy and return the relevant reset
4741 * mask to be used by cik_gpu_soft_reset().
4742 * Returns a mask of the blocks to be reset.
4743 */
4744 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4745 {
4746 u32 reset_mask = 0;
4747 u32 tmp;
4748
4749 /* GRBM_STATUS */
4750 tmp = RREG32(GRBM_STATUS);
4751 if (tmp & (PA_BUSY | SC_BUSY |
4752 BCI_BUSY | SX_BUSY |
4753 TA_BUSY | VGT_BUSY |
4754 DB_BUSY | CB_BUSY |
4755 GDS_BUSY | SPI_BUSY |
4756 IA_BUSY | IA_BUSY_NO_DMA))
4757 reset_mask |= RADEON_RESET_GFX;
4758
4759 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4760 reset_mask |= RADEON_RESET_CP;
4761
4762 /* GRBM_STATUS2 */
4763 tmp = RREG32(GRBM_STATUS2);
4764 if (tmp & RLC_BUSY)
4765 reset_mask |= RADEON_RESET_RLC;
4766
4767 /* SDMA0_STATUS_REG */
4768 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4769 if (!(tmp & SDMA_IDLE))
4770 reset_mask |= RADEON_RESET_DMA;
4771
4772 /* SDMA1_STATUS_REG */
4773 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4774 if (!(tmp & SDMA_IDLE))
4775 reset_mask |= RADEON_RESET_DMA1;
4776
4777 /* SRBM_STATUS2 */
4778 tmp = RREG32(SRBM_STATUS2);
4779 if (tmp & SDMA_BUSY)
4780 reset_mask |= RADEON_RESET_DMA;
4781
4782 if (tmp & SDMA1_BUSY)
4783 reset_mask |= RADEON_RESET_DMA1;
4784
4785 /* SRBM_STATUS */
4786 tmp = RREG32(SRBM_STATUS);
4787
4788 if (tmp & IH_BUSY)
4789 reset_mask |= RADEON_RESET_IH;
4790
4791 if (tmp & SEM_BUSY)
4792 reset_mask |= RADEON_RESET_SEM;
4793
4794 if (tmp & GRBM_RQ_PENDING)
4795 reset_mask |= RADEON_RESET_GRBM;
4796
4797 if (tmp & VMC_BUSY)
4798 reset_mask |= RADEON_RESET_VMC;
4799
4800 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4801 MCC_BUSY | MCD_BUSY))
4802 reset_mask |= RADEON_RESET_MC;
4803
4804 if (evergreen_is_display_hung(rdev))
4805 reset_mask |= RADEON_RESET_DISPLAY;
4806
4807 /* Skip MC reset as it's mostly likely not hung, just busy */
4808 if (reset_mask & RADEON_RESET_MC) {
4809 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4810 reset_mask &= ~RADEON_RESET_MC;
4811 }
4812
4813 return reset_mask;
4814 }
4815
4816 /**
4817 * cik_gpu_soft_reset - soft reset GPU
4818 *
4819 * @rdev: radeon_device pointer
4820 * @reset_mask: mask of which blocks to reset
4821 *
4822 * Soft reset the blocks specified in @reset_mask.
4823 */
4824 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4825 {
4826 struct evergreen_mc_save save;
4827 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4828 u32 tmp;
4829
4830 if (reset_mask == 0)
4831 return;
4832
4833 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4834
4835 cik_print_gpu_status_regs(rdev);
4836 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4837 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4838 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4839 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4840
4841 /* disable CG/PG */
4842 cik_fini_pg(rdev);
4843 cik_fini_cg(rdev);
4844
4845 /* stop the rlc */
4846 cik_rlc_stop(rdev);
4847
4848 /* Disable GFX parsing/prefetching */
4849 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4850
4851 /* Disable MEC parsing/prefetching */
4852 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4853
4854 if (reset_mask & RADEON_RESET_DMA) {
4855 /* sdma0 */
4856 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4857 tmp |= SDMA_HALT;
4858 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4859 }
4860 if (reset_mask & RADEON_RESET_DMA1) {
4861 /* sdma1 */
4862 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4863 tmp |= SDMA_HALT;
4864 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4865 }
4866
4867 evergreen_mc_stop(rdev, &save);
4868 if (evergreen_mc_wait_for_idle(rdev)) {
4869 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4870 }
4871
4872 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4873 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4874
4875 if (reset_mask & RADEON_RESET_CP) {
4876 grbm_soft_reset |= SOFT_RESET_CP;
4877
4878 srbm_soft_reset |= SOFT_RESET_GRBM;
4879 }
4880
4881 if (reset_mask & RADEON_RESET_DMA)
4882 srbm_soft_reset |= SOFT_RESET_SDMA;
4883
4884 if (reset_mask & RADEON_RESET_DMA1)
4885 srbm_soft_reset |= SOFT_RESET_SDMA1;
4886
4887 if (reset_mask & RADEON_RESET_DISPLAY)
4888 srbm_soft_reset |= SOFT_RESET_DC;
4889
4890 if (reset_mask & RADEON_RESET_RLC)
4891 grbm_soft_reset |= SOFT_RESET_RLC;
4892
4893 if (reset_mask & RADEON_RESET_SEM)
4894 srbm_soft_reset |= SOFT_RESET_SEM;
4895
4896 if (reset_mask & RADEON_RESET_IH)
4897 srbm_soft_reset |= SOFT_RESET_IH;
4898
4899 if (reset_mask & RADEON_RESET_GRBM)
4900 srbm_soft_reset |= SOFT_RESET_GRBM;
4901
4902 if (reset_mask & RADEON_RESET_VMC)
4903 srbm_soft_reset |= SOFT_RESET_VMC;
4904
4905 if (!(rdev->flags & RADEON_IS_IGP)) {
4906 if (reset_mask & RADEON_RESET_MC)
4907 srbm_soft_reset |= SOFT_RESET_MC;
4908 }
4909
4910 if (grbm_soft_reset) {
4911 tmp = RREG32(GRBM_SOFT_RESET);
4912 tmp |= grbm_soft_reset;
4913 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4914 WREG32(GRBM_SOFT_RESET, tmp);
4915 tmp = RREG32(GRBM_SOFT_RESET);
4916
4917 udelay(50);
4918
4919 tmp &= ~grbm_soft_reset;
4920 WREG32(GRBM_SOFT_RESET, tmp);
4921 tmp = RREG32(GRBM_SOFT_RESET);
4922 }
4923
4924 if (srbm_soft_reset) {
4925 tmp = RREG32(SRBM_SOFT_RESET);
4926 tmp |= srbm_soft_reset;
4927 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4928 WREG32(SRBM_SOFT_RESET, tmp);
4929 tmp = RREG32(SRBM_SOFT_RESET);
4930
4931 udelay(50);
4932
4933 tmp &= ~srbm_soft_reset;
4934 WREG32(SRBM_SOFT_RESET, tmp);
4935 tmp = RREG32(SRBM_SOFT_RESET);
4936 }
4937
4938 /* Wait a little for things to settle down */
4939 udelay(50);
4940
4941 evergreen_mc_resume(rdev, &save);
4942 udelay(50);
4943
4944 cik_print_gpu_status_regs(rdev);
4945 }
4946
4947 struct kv_reset_save_regs {
4948 u32 gmcon_reng_execute;
4949 u32 gmcon_misc;
4950 u32 gmcon_misc3;
4951 };
4952
4953 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4954 struct kv_reset_save_regs *save)
4955 {
4956 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4957 save->gmcon_misc = RREG32(GMCON_MISC);
4958 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4959
4960 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4961 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4962 STCTRL_STUTTER_EN));
4963 }
4964
4965 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4966 struct kv_reset_save_regs *save)
4967 {
4968 int i;
4969
4970 WREG32(GMCON_PGFSM_WRITE, 0);
4971 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4972
4973 for (i = 0; i < 5; i++)
4974 WREG32(GMCON_PGFSM_WRITE, 0);
4975
4976 WREG32(GMCON_PGFSM_WRITE, 0);
4977 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4978
4979 for (i = 0; i < 5; i++)
4980 WREG32(GMCON_PGFSM_WRITE, 0);
4981
4982 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4983 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4984
4985 for (i = 0; i < 5; i++)
4986 WREG32(GMCON_PGFSM_WRITE, 0);
4987
4988 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4989 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4990
4991 for (i = 0; i < 5; i++)
4992 WREG32(GMCON_PGFSM_WRITE, 0);
4993
4994 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4995 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4996
4997 for (i = 0; i < 5; i++)
4998 WREG32(GMCON_PGFSM_WRITE, 0);
4999
5000 WREG32(GMCON_PGFSM_WRITE, 0);
5001 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5002
5003 for (i = 0; i < 5; i++)
5004 WREG32(GMCON_PGFSM_WRITE, 0);
5005
5006 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5007 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5008
5009 for (i = 0; i < 5; i++)
5010 WREG32(GMCON_PGFSM_WRITE, 0);
5011
5012 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5013 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5014
5015 for (i = 0; i < 5; i++)
5016 WREG32(GMCON_PGFSM_WRITE, 0);
5017
5018 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5019 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5020
5021 for (i = 0; i < 5; i++)
5022 WREG32(GMCON_PGFSM_WRITE, 0);
5023
5024 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5025 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5026
5027 for (i = 0; i < 5; i++)
5028 WREG32(GMCON_PGFSM_WRITE, 0);
5029
5030 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5031 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5032
5033 WREG32(GMCON_MISC3, save->gmcon_misc3);
5034 WREG32(GMCON_MISC, save->gmcon_misc);
5035 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5036 }
5037
5038 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5039 {
5040 struct evergreen_mc_save save;
5041 struct kv_reset_save_regs kv_save = { 0 };
5042 u32 tmp, i;
5043
5044 dev_info(rdev->dev, "GPU pci config reset\n");
5045
5046 /* disable dpm? */
5047
5048 /* disable cg/pg */
5049 cik_fini_pg(rdev);
5050 cik_fini_cg(rdev);
5051
5052 /* Disable GFX parsing/prefetching */
5053 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5054
5055 /* Disable MEC parsing/prefetching */
5056 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5057
5058 /* sdma0 */
5059 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5060 tmp |= SDMA_HALT;
5061 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5062 /* sdma1 */
5063 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5064 tmp |= SDMA_HALT;
5065 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5066 /* XXX other engines? */
5067
5068 /* halt the rlc, disable cp internal ints */
5069 cik_rlc_stop(rdev);
5070
5071 udelay(50);
5072
5073 /* disable mem access */
5074 evergreen_mc_stop(rdev, &save);
5075 if (evergreen_mc_wait_for_idle(rdev)) {
5076 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5077 }
5078
5079 if (rdev->flags & RADEON_IS_IGP)
5080 kv_save_regs_for_reset(rdev, &kv_save);
5081
5082 /* disable BM */
5083 pci_clear_master(rdev->pdev);
5084 /* reset */
5085 radeon_pci_config_reset(rdev);
5086
5087 udelay(100);
5088
5089 /* wait for asic to come out of reset */
5090 for (i = 0; i < rdev->usec_timeout; i++) {
5091 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5092 break;
5093 udelay(1);
5094 }
5095
5096 /* does asic init need to be run first??? */
5097 if (rdev->flags & RADEON_IS_IGP)
5098 kv_restore_regs_for_reset(rdev, &kv_save);
5099 }
5100
5101 /**
5102 * cik_asic_reset - soft reset GPU
5103 *
5104 * @rdev: radeon_device pointer
5105 *
5106 * Look up which blocks are hung and attempt
5107 * to reset them.
5108 * Returns 0 for success.
5109 */
5110 int cik_asic_reset(struct radeon_device *rdev)
5111 {
5112 u32 reset_mask;
5113
5114 reset_mask = cik_gpu_check_soft_reset(rdev);
5115
5116 if (reset_mask)
5117 r600_set_bios_scratch_engine_hung(rdev, true);
5118
5119 /* try soft reset */
5120 cik_gpu_soft_reset(rdev, reset_mask);
5121
5122 reset_mask = cik_gpu_check_soft_reset(rdev);
5123
5124 /* try pci config reset */
5125 if (reset_mask && radeon_hard_reset)
5126 cik_gpu_pci_config_reset(rdev);
5127
5128 reset_mask = cik_gpu_check_soft_reset(rdev);
5129
5130 if (!reset_mask)
5131 r600_set_bios_scratch_engine_hung(rdev, false);
5132
5133 return 0;
5134 }
5135
5136 /**
5137 * cik_gfx_is_lockup - check if the 3D engine is locked up
5138 *
5139 * @rdev: radeon_device pointer
5140 * @ring: radeon_ring structure holding ring information
5141 *
5142 * Check if the 3D engine is locked up (CIK).
5143 * Returns true if the engine is locked, false if not.
5144 */
5145 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5146 {
5147 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5148
5149 if (!(reset_mask & (RADEON_RESET_GFX |
5150 RADEON_RESET_COMPUTE |
5151 RADEON_RESET_CP))) {
5152 radeon_ring_lockup_update(rdev, ring);
5153 return false;
5154 }
5155 return radeon_ring_test_lockup(rdev, ring);
5156 }
5157
5158 /* MC */
5159 /**
5160 * cik_mc_program - program the GPU memory controller
5161 *
5162 * @rdev: radeon_device pointer
5163 *
5164 * Set the location of vram, gart, and AGP in the GPU's
5165 * physical address space (CIK).
5166 */
5167 static void cik_mc_program(struct radeon_device *rdev)
5168 {
5169 struct evergreen_mc_save save;
5170 u32 tmp;
5171 int i, j;
5172
5173 /* Initialize HDP */
5174 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5175 WREG32((0x2c14 + j), 0x00000000);
5176 WREG32((0x2c18 + j), 0x00000000);
5177 WREG32((0x2c1c + j), 0x00000000);
5178 WREG32((0x2c20 + j), 0x00000000);
5179 WREG32((0x2c24 + j), 0x00000000);
5180 }
5181 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5182
5183 evergreen_mc_stop(rdev, &save);
5184 if (radeon_mc_wait_for_idle(rdev)) {
5185 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5186 }
5187 /* Lockout access through VGA aperture*/
5188 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5189 /* Update configuration */
5190 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5191 rdev->mc.vram_start >> 12);
5192 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5193 rdev->mc.vram_end >> 12);
5194 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5195 rdev->vram_scratch.gpu_addr >> 12);
5196 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5197 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5198 WREG32(MC_VM_FB_LOCATION, tmp);
5199 /* XXX double check these! */
5200 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5201 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5202 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5203 WREG32(MC_VM_AGP_BASE, 0);
5204 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5205 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5206 if (radeon_mc_wait_for_idle(rdev)) {
5207 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5208 }
5209 evergreen_mc_resume(rdev, &save);
5210 /* we need to own VRAM, so turn off the VGA renderer here
5211 * to stop it overwriting our objects */
5212 rv515_vga_render_disable(rdev);
5213 }
5214
5215 /**
5216 * cik_mc_init - initialize the memory controller driver params
5217 *
5218 * @rdev: radeon_device pointer
5219 *
5220 * Look up the amount of vram, vram width, and decide how to place
5221 * vram and gart within the GPU's physical address space (CIK).
5222 * Returns 0 for success.
5223 */
5224 static int cik_mc_init(struct radeon_device *rdev)
5225 {
5226 u32 tmp;
5227 int chansize, numchan;
5228
5229 /* Get VRAM informations */
5230 rdev->mc.vram_is_ddr = true;
5231 tmp = RREG32(MC_ARB_RAMCFG);
5232 if (tmp & CHANSIZE_MASK) {
5233 chansize = 64;
5234 } else {
5235 chansize = 32;
5236 }
5237 tmp = RREG32(MC_SHARED_CHMAP);
5238 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5239 case 0:
5240 default:
5241 numchan = 1;
5242 break;
5243 case 1:
5244 numchan = 2;
5245 break;
5246 case 2:
5247 numchan = 4;
5248 break;
5249 case 3:
5250 numchan = 8;
5251 break;
5252 case 4:
5253 numchan = 3;
5254 break;
5255 case 5:
5256 numchan = 6;
5257 break;
5258 case 6:
5259 numchan = 10;
5260 break;
5261 case 7:
5262 numchan = 12;
5263 break;
5264 case 8:
5265 numchan = 16;
5266 break;
5267 }
5268 rdev->mc.vram_width = numchan * chansize;
5269 /* Could aper size report 0 ? */
5270 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5271 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5272 /* size in MB on si */
5273 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5274 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5275 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5276 si_vram_gtt_location(rdev, &rdev->mc);
5277 radeon_update_bandwidth_info(rdev);
5278
5279 return 0;
5280 }
5281
5282 /*
5283 * GART
5284 * VMID 0 is the physical GPU addresses as used by the kernel.
5285 * VMIDs 1-15 are used for userspace clients and are handled
5286 * by the radeon vm/hsa code.
5287 */
5288 /**
5289 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5290 *
5291 * @rdev: radeon_device pointer
5292 *
5293 * Flush the TLB for the VMID 0 page table (CIK).
5294 */
5295 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5296 {
5297 /* flush hdp cache */
5298 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5299
5300 /* bits 0-15 are the VM contexts0-15 */
5301 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5302 }
5303
5304 /**
5305 * cik_pcie_gart_enable - gart enable
5306 *
5307 * @rdev: radeon_device pointer
5308 *
5309 * This sets up the TLBs, programs the page tables for VMID0,
5310 * sets up the hw for VMIDs 1-15 which are allocated on
5311 * demand, and sets up the global locations for the LDS, GDS,
5312 * and GPUVM for FSA64 clients (CIK).
5313 * Returns 0 for success, errors for failure.
5314 */
5315 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5316 {
5317 int r, i;
5318
5319 if (rdev->gart.robj == NULL) {
5320 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5321 return -EINVAL;
5322 }
5323 r = radeon_gart_table_vram_pin(rdev);
5324 if (r)
5325 return r;
5326 radeon_gart_restore(rdev);
5327 /* Setup TLB control */
5328 WREG32(MC_VM_MX_L1_TLB_CNTL,
5329 (0xA << 7) |
5330 ENABLE_L1_TLB |
5331 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5332 ENABLE_ADVANCED_DRIVER_MODEL |
5333 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5334 /* Setup L2 cache */
5335 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5336 ENABLE_L2_FRAGMENT_PROCESSING |
5337 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5338 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5339 EFFECTIVE_L2_QUEUE_SIZE(7) |
5340 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5341 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5342 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5343 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5344 /* setup context0 */
5345 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5346 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5347 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5348 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5349 (u32)(rdev->dummy_page.addr >> 12));
5350 WREG32(VM_CONTEXT0_CNTL2, 0);
5351 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5352 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5353
5354 WREG32(0x15D4, 0);
5355 WREG32(0x15D8, 0);
5356 WREG32(0x15DC, 0);
5357
5358 /* empty context1-15 */
5359 /* FIXME start with 4G, once using 2 level pt switch to full
5360 * vm size space
5361 */
5362 /* set vm size, must be a multiple of 4 */
5363 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5364 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5365 for (i = 1; i < 16; i++) {
5366 if (i < 8)
5367 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5368 rdev->gart.table_addr >> 12);
5369 else
5370 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5371 rdev->gart.table_addr >> 12);
5372 }
5373
5374 /* enable context1-15 */
5375 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5376 (u32)(rdev->dummy_page.addr >> 12));
5377 WREG32(VM_CONTEXT1_CNTL2, 4);
5378 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5379 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5380 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5381 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5382 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5383 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5384 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5385 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5386 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5387 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5388 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5389 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5390 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5391
5392 if (rdev->family == CHIP_KAVERI) {
5393 u32 tmp = RREG32(CHUB_CONTROL);
5394 tmp &= ~BYPASS_VM;
5395 WREG32(CHUB_CONTROL, tmp);
5396 }
5397
5398 /* XXX SH_MEM regs */
5399 /* where to put LDS, scratch, GPUVM in FSA64 space */
5400 mutex_lock(&rdev->srbm_mutex);
5401 for (i = 0; i < 16; i++) {
5402 cik_srbm_select(rdev, 0, 0, 0, i);
5403 /* CP and shaders */
5404 WREG32(SH_MEM_CONFIG, 0);
5405 WREG32(SH_MEM_APE1_BASE, 1);
5406 WREG32(SH_MEM_APE1_LIMIT, 0);
5407 WREG32(SH_MEM_BASES, 0);
5408 /* SDMA GFX */
5409 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5410 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5411 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5412 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5413 /* XXX SDMA RLC - todo */
5414 }
5415 cik_srbm_select(rdev, 0, 0, 0, 0);
5416 mutex_unlock(&rdev->srbm_mutex);
5417
5418 cik_pcie_gart_tlb_flush(rdev);
5419 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5420 (unsigned)(rdev->mc.gtt_size >> 20),
5421 (unsigned long long)rdev->gart.table_addr);
5422 rdev->gart.ready = true;
5423 return 0;
5424 }
5425
5426 /**
5427 * cik_pcie_gart_disable - gart disable
5428 *
5429 * @rdev: radeon_device pointer
5430 *
5431 * This disables all VM page table (CIK).
5432 */
5433 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5434 {
5435 /* Disable all tables */
5436 WREG32(VM_CONTEXT0_CNTL, 0);
5437 WREG32(VM_CONTEXT1_CNTL, 0);
5438 /* Setup TLB control */
5439 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5440 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5441 /* Setup L2 cache */
5442 WREG32(VM_L2_CNTL,
5443 ENABLE_L2_FRAGMENT_PROCESSING |
5444 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5445 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5446 EFFECTIVE_L2_QUEUE_SIZE(7) |
5447 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5448 WREG32(VM_L2_CNTL2, 0);
5449 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5450 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5451 radeon_gart_table_vram_unpin(rdev);
5452 }
5453
5454 /**
5455 * cik_pcie_gart_fini - vm fini callback
5456 *
5457 * @rdev: radeon_device pointer
5458 *
5459 * Tears down the driver GART/VM setup (CIK).
5460 */
5461 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5462 {
5463 cik_pcie_gart_disable(rdev);
5464 radeon_gart_table_vram_free(rdev);
5465 radeon_gart_fini(rdev);
5466 }
5467
5468 /* vm parser */
5469 /**
5470 * cik_ib_parse - vm ib_parse callback
5471 *
5472 * @rdev: radeon_device pointer
5473 * @ib: indirect buffer pointer
5474 *
5475 * CIK uses hw IB checking so this is a nop (CIK).
5476 */
5477 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5478 {
5479 return 0;
5480 }
5481
5482 /*
5483 * vm
5484 * VMID 0 is the physical GPU addresses as used by the kernel.
5485 * VMIDs 1-15 are used for userspace clients and are handled
5486 * by the radeon vm/hsa code.
5487 */
5488 /**
5489 * cik_vm_init - cik vm init callback
5490 *
5491 * @rdev: radeon_device pointer
5492 *
5493 * Inits cik specific vm parameters (number of VMs, base of vram for
5494 * VMIDs 1-15) (CIK).
5495 * Returns 0 for success.
5496 */
5497 int cik_vm_init(struct radeon_device *rdev)
5498 {
5499 /* number of VMs */
5500 rdev->vm_manager.nvm = 16;
5501 /* base offset of vram pages */
5502 if (rdev->flags & RADEON_IS_IGP) {
5503 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5504 tmp <<= 22;
5505 rdev->vm_manager.vram_base_offset = tmp;
5506 } else
5507 rdev->vm_manager.vram_base_offset = 0;
5508
5509 return 0;
5510 }
5511
5512 /**
5513 * cik_vm_fini - cik vm fini callback
5514 *
5515 * @rdev: radeon_device pointer
5516 *
5517 * Tear down any asic specific VM setup (CIK).
5518 */
5519 void cik_vm_fini(struct radeon_device *rdev)
5520 {
5521 }
5522
5523 /**
5524 * cik_vm_decode_fault - print human readable fault info
5525 *
5526 * @rdev: radeon_device pointer
5527 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5528 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5529 *
5530 * Print human readable fault information (CIK).
5531 */
5532 static void cik_vm_decode_fault(struct radeon_device *rdev,
5533 u32 status, u32 addr, u32 mc_client)
5534 {
5535 u32 mc_id;
5536 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5537 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5538 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5539 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5540
5541 if (rdev->family == CHIP_HAWAII)
5542 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5543 else
5544 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5545
5546 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5547 protections, vmid, addr,
5548 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5549 block, mc_client, mc_id);
5550 }
5551
5552 /**
5553 * cik_vm_flush - cik vm flush using the CP
5554 *
5555 * @rdev: radeon_device pointer
5556 *
5557 * Update the page table base and flush the VM TLB
5558 * using the CP (CIK).
5559 */
5560 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5561 {
5562 struct radeon_ring *ring = &rdev->ring[ridx];
5563
5564 if (vm == NULL)
5565 return;
5566
5567 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5568 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5569 WRITE_DATA_DST_SEL(0)));
5570 if (vm->id < 8) {
5571 radeon_ring_write(ring,
5572 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5573 } else {
5574 radeon_ring_write(ring,
5575 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5576 }
5577 radeon_ring_write(ring, 0);
5578 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5579
5580 /* update SH_MEM_* regs */
5581 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5582 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5583 WRITE_DATA_DST_SEL(0)));
5584 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5585 radeon_ring_write(ring, 0);
5586 radeon_ring_write(ring, VMID(vm->id));
5587
5588 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5589 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5590 WRITE_DATA_DST_SEL(0)));
5591 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5592 radeon_ring_write(ring, 0);
5593
5594 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5595 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5596 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5597 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5598
5599 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5600 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5601 WRITE_DATA_DST_SEL(0)));
5602 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5603 radeon_ring_write(ring, 0);
5604 radeon_ring_write(ring, VMID(0));
5605
5606 /* HDP flush */
5607 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5608
5609 /* bits 0-15 are the VM contexts0-15 */
5610 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5611 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5612 WRITE_DATA_DST_SEL(0)));
5613 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5614 radeon_ring_write(ring, 0);
5615 radeon_ring_write(ring, 1 << vm->id);
5616
5617 /* compute doesn't have PFP */
5618 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5619 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5620 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5621 radeon_ring_write(ring, 0x0);
5622 }
5623 }
5624
5625 /*
5626 * RLC
5627 * The RLC is a multi-purpose microengine that handles a
5628 * variety of functions, the most important of which is
5629 * the interrupt controller.
5630 */
5631 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5632 bool enable)
5633 {
5634 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5635
5636 if (enable)
5637 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5638 else
5639 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5640 WREG32(CP_INT_CNTL_RING0, tmp);
5641 }
5642
5643 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5644 {
5645 u32 tmp;
5646
5647 tmp = RREG32(RLC_LB_CNTL);
5648 if (enable)
5649 tmp |= LOAD_BALANCE_ENABLE;
5650 else
5651 tmp &= ~LOAD_BALANCE_ENABLE;
5652 WREG32(RLC_LB_CNTL, tmp);
5653 }
5654
5655 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5656 {
5657 u32 i, j, k;
5658 u32 mask;
5659
5660 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5661 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5662 cik_select_se_sh(rdev, i, j);
5663 for (k = 0; k < rdev->usec_timeout; k++) {
5664 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5665 break;
5666 udelay(1);
5667 }
5668 }
5669 }
5670 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5671
5672 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5673 for (k = 0; k < rdev->usec_timeout; k++) {
5674 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5675 break;
5676 udelay(1);
5677 }
5678 }
5679
5680 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5681 {
5682 u32 tmp;
5683
5684 tmp = RREG32(RLC_CNTL);
5685 if (tmp != rlc)
5686 WREG32(RLC_CNTL, rlc);
5687 }
5688
5689 static u32 cik_halt_rlc(struct radeon_device *rdev)
5690 {
5691 u32 data, orig;
5692
5693 orig = data = RREG32(RLC_CNTL);
5694
5695 if (data & RLC_ENABLE) {
5696 u32 i;
5697
5698 data &= ~RLC_ENABLE;
5699 WREG32(RLC_CNTL, data);
5700
5701 for (i = 0; i < rdev->usec_timeout; i++) {
5702 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5703 break;
5704 udelay(1);
5705 }
5706
5707 cik_wait_for_rlc_serdes(rdev);
5708 }
5709
5710 return orig;
5711 }
5712
5713 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5714 {
5715 u32 tmp, i, mask;
5716
5717 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5718 WREG32(RLC_GPR_REG2, tmp);
5719
5720 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5721 for (i = 0; i < rdev->usec_timeout; i++) {
5722 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5723 break;
5724 udelay(1);
5725 }
5726
5727 for (i = 0; i < rdev->usec_timeout; i++) {
5728 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5729 break;
5730 udelay(1);
5731 }
5732 }
5733
5734 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5735 {
5736 u32 tmp;
5737
5738 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5739 WREG32(RLC_GPR_REG2, tmp);
5740 }
5741
5742 /**
5743 * cik_rlc_stop - stop the RLC ME
5744 *
5745 * @rdev: radeon_device pointer
5746 *
5747 * Halt the RLC ME (MicroEngine) (CIK).
5748 */
5749 static void cik_rlc_stop(struct radeon_device *rdev)
5750 {
5751 WREG32(RLC_CNTL, 0);
5752
5753 cik_enable_gui_idle_interrupt(rdev, false);
5754
5755 cik_wait_for_rlc_serdes(rdev);
5756 }
5757
5758 /**
5759 * cik_rlc_start - start the RLC ME
5760 *
5761 * @rdev: radeon_device pointer
5762 *
5763 * Unhalt the RLC ME (MicroEngine) (CIK).
5764 */
5765 static void cik_rlc_start(struct radeon_device *rdev)
5766 {
5767 WREG32(RLC_CNTL, RLC_ENABLE);
5768
5769 cik_enable_gui_idle_interrupt(rdev, true);
5770
5771 udelay(50);
5772 }
5773
5774 /**
5775 * cik_rlc_resume - setup the RLC hw
5776 *
5777 * @rdev: radeon_device pointer
5778 *
5779 * Initialize the RLC registers, load the ucode,
5780 * and start the RLC (CIK).
5781 * Returns 0 for success, -EINVAL if the ucode is not available.
5782 */
5783 static int cik_rlc_resume(struct radeon_device *rdev)
5784 {
5785 u32 i, size, tmp;
5786 const __be32 *fw_data;
5787
5788 if (!rdev->rlc_fw)
5789 return -EINVAL;
5790
5791 switch (rdev->family) {
5792 case CHIP_BONAIRE:
5793 case CHIP_HAWAII:
5794 default:
5795 size = BONAIRE_RLC_UCODE_SIZE;
5796 break;
5797 case CHIP_KAVERI:
5798 size = KV_RLC_UCODE_SIZE;
5799 break;
5800 case CHIP_KABINI:
5801 size = KB_RLC_UCODE_SIZE;
5802 break;
5803 }
5804
5805 cik_rlc_stop(rdev);
5806
5807 /* disable CG */
5808 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5809 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5810
5811 si_rlc_reset(rdev);
5812
5813 cik_init_pg(rdev);
5814
5815 cik_init_cg(rdev);
5816
5817 WREG32(RLC_LB_CNTR_INIT, 0);
5818 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5819
5820 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5821 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5822 WREG32(RLC_LB_PARAMS, 0x00600408);
5823 WREG32(RLC_LB_CNTL, 0x80000004);
5824
5825 WREG32(RLC_MC_CNTL, 0);
5826 WREG32(RLC_UCODE_CNTL, 0);
5827
5828 fw_data = (const __be32 *)rdev->rlc_fw->data;
5829 WREG32(RLC_GPM_UCODE_ADDR, 0);
5830 for (i = 0; i < size; i++)
5831 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5832 WREG32(RLC_GPM_UCODE_ADDR, 0);
5833
5834 /* XXX - find out what chips support lbpw */
5835 cik_enable_lbpw(rdev, false);
5836
5837 if (rdev->family == CHIP_BONAIRE)
5838 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5839
5840 cik_rlc_start(rdev);
5841
5842 return 0;
5843 }
5844
5845 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5846 {
5847 u32 data, orig, tmp, tmp2;
5848
5849 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5850
5851 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5852 cik_enable_gui_idle_interrupt(rdev, true);
5853
5854 tmp = cik_halt_rlc(rdev);
5855
5856 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5857 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5858 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5859 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5860 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5861
5862 cik_update_rlc(rdev, tmp);
5863
5864 data |= CGCG_EN | CGLS_EN;
5865 } else {
5866 cik_enable_gui_idle_interrupt(rdev, false);
5867
5868 RREG32(CB_CGTT_SCLK_CTRL);
5869 RREG32(CB_CGTT_SCLK_CTRL);
5870 RREG32(CB_CGTT_SCLK_CTRL);
5871 RREG32(CB_CGTT_SCLK_CTRL);
5872
5873 data &= ~(CGCG_EN | CGLS_EN);
5874 }
5875
5876 if (orig != data)
5877 WREG32(RLC_CGCG_CGLS_CTRL, data);
5878
5879 }
5880
5881 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5882 {
5883 u32 data, orig, tmp = 0;
5884
5885 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5886 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5887 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5888 orig = data = RREG32(CP_MEM_SLP_CNTL);
5889 data |= CP_MEM_LS_EN;
5890 if (orig != data)
5891 WREG32(CP_MEM_SLP_CNTL, data);
5892 }
5893 }
5894
5895 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5896 data &= 0xfffffffd;
5897 if (orig != data)
5898 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5899
5900 tmp = cik_halt_rlc(rdev);
5901
5902 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5903 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5904 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5905 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5906 WREG32(RLC_SERDES_WR_CTRL, data);
5907
5908 cik_update_rlc(rdev, tmp);
5909
5910 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5911 orig = data = RREG32(CGTS_SM_CTRL_REG);
5912 data &= ~SM_MODE_MASK;
5913 data |= SM_MODE(0x2);
5914 data |= SM_MODE_ENABLE;
5915 data &= ~CGTS_OVERRIDE;
5916 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5917 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5918 data &= ~CGTS_LS_OVERRIDE;
5919 data &= ~ON_MONITOR_ADD_MASK;
5920 data |= ON_MONITOR_ADD_EN;
5921 data |= ON_MONITOR_ADD(0x96);
5922 if (orig != data)
5923 WREG32(CGTS_SM_CTRL_REG, data);
5924 }
5925 } else {
5926 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5927 data |= 0x00000002;
5928 if (orig != data)
5929 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5930
5931 data = RREG32(RLC_MEM_SLP_CNTL);
5932 if (data & RLC_MEM_LS_EN) {
5933 data &= ~RLC_MEM_LS_EN;
5934 WREG32(RLC_MEM_SLP_CNTL, data);
5935 }
5936
5937 data = RREG32(CP_MEM_SLP_CNTL);
5938 if (data & CP_MEM_LS_EN) {
5939 data &= ~CP_MEM_LS_EN;
5940 WREG32(CP_MEM_SLP_CNTL, data);
5941 }
5942
5943 orig = data = RREG32(CGTS_SM_CTRL_REG);
5944 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5945 if (orig != data)
5946 WREG32(CGTS_SM_CTRL_REG, data);
5947
5948 tmp = cik_halt_rlc(rdev);
5949
5950 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5951 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5952 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5953 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5954 WREG32(RLC_SERDES_WR_CTRL, data);
5955
5956 cik_update_rlc(rdev, tmp);
5957 }
5958 }
5959
5960 static const u32 mc_cg_registers[] =
5961 {
5962 MC_HUB_MISC_HUB_CG,
5963 MC_HUB_MISC_SIP_CG,
5964 MC_HUB_MISC_VM_CG,
5965 MC_XPB_CLK_GAT,
5966 ATC_MISC_CG,
5967 MC_CITF_MISC_WR_CG,
5968 MC_CITF_MISC_RD_CG,
5969 MC_CITF_MISC_VM_CG,
5970 VM_L2_CG,
5971 };
5972
5973 static void cik_enable_mc_ls(struct radeon_device *rdev,
5974 bool enable)
5975 {
5976 int i;
5977 u32 orig, data;
5978
5979 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5980 orig = data = RREG32(mc_cg_registers[i]);
5981 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5982 data |= MC_LS_ENABLE;
5983 else
5984 data &= ~MC_LS_ENABLE;
5985 if (data != orig)
5986 WREG32(mc_cg_registers[i], data);
5987 }
5988 }
5989
5990 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5991 bool enable)
5992 {
5993 int i;
5994 u32 orig, data;
5995
5996 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5997 orig = data = RREG32(mc_cg_registers[i]);
5998 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5999 data |= MC_CG_ENABLE;
6000 else
6001 data &= ~MC_CG_ENABLE;
6002 if (data != orig)
6003 WREG32(mc_cg_registers[i], data);
6004 }
6005 }
6006
6007 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6008 bool enable)
6009 {
6010 u32 orig, data;
6011
6012 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6013 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6014 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6015 } else {
6016 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6017 data |= 0xff000000;
6018 if (data != orig)
6019 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6020
6021 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6022 data |= 0xff000000;
6023 if (data != orig)
6024 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6025 }
6026 }
6027
6028 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6029 bool enable)
6030 {
6031 u32 orig, data;
6032
6033 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6034 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6035 data |= 0x100;
6036 if (orig != data)
6037 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6038
6039 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6040 data |= 0x100;
6041 if (orig != data)
6042 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6043 } else {
6044 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6045 data &= ~0x100;
6046 if (orig != data)
6047 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6048
6049 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6050 data &= ~0x100;
6051 if (orig != data)
6052 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6053 }
6054 }
6055
6056 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6057 bool enable)
6058 {
6059 u32 orig, data;
6060
6061 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6062 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6063 data = 0xfff;
6064 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6065
6066 orig = data = RREG32(UVD_CGC_CTRL);
6067 data |= DCM;
6068 if (orig != data)
6069 WREG32(UVD_CGC_CTRL, data);
6070 } else {
6071 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6072 data &= ~0xfff;
6073 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6074
6075 orig = data = RREG32(UVD_CGC_CTRL);
6076 data &= ~DCM;
6077 if (orig != data)
6078 WREG32(UVD_CGC_CTRL, data);
6079 }
6080 }
6081
6082 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6083 bool enable)
6084 {
6085 u32 orig, data;
6086
6087 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6088
6089 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6090 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6091 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6092 else
6093 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6094 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6095
6096 if (orig != data)
6097 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6098 }
6099
6100 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6101 bool enable)
6102 {
6103 u32 orig, data;
6104
6105 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6106
6107 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6108 data &= ~CLOCK_GATING_DIS;
6109 else
6110 data |= CLOCK_GATING_DIS;
6111
6112 if (orig != data)
6113 WREG32(HDP_HOST_PATH_CNTL, data);
6114 }
6115
6116 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6117 bool enable)
6118 {
6119 u32 orig, data;
6120
6121 orig = data = RREG32(HDP_MEM_POWER_LS);
6122
6123 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6124 data |= HDP_LS_ENABLE;
6125 else
6126 data &= ~HDP_LS_ENABLE;
6127
6128 if (orig != data)
6129 WREG32(HDP_MEM_POWER_LS, data);
6130 }
6131
6132 void cik_update_cg(struct radeon_device *rdev,
6133 u32 block, bool enable)
6134 {
6135
6136 if (block & RADEON_CG_BLOCK_GFX) {
6137 cik_enable_gui_idle_interrupt(rdev, false);
6138 /* order matters! */
6139 if (enable) {
6140 cik_enable_mgcg(rdev, true);
6141 cik_enable_cgcg(rdev, true);
6142 } else {
6143 cik_enable_cgcg(rdev, false);
6144 cik_enable_mgcg(rdev, false);
6145 }
6146 cik_enable_gui_idle_interrupt(rdev, true);
6147 }
6148
6149 if (block & RADEON_CG_BLOCK_MC) {
6150 if (!(rdev->flags & RADEON_IS_IGP)) {
6151 cik_enable_mc_mgcg(rdev, enable);
6152 cik_enable_mc_ls(rdev, enable);
6153 }
6154 }
6155
6156 if (block & RADEON_CG_BLOCK_SDMA) {
6157 cik_enable_sdma_mgcg(rdev, enable);
6158 cik_enable_sdma_mgls(rdev, enable);
6159 }
6160
6161 if (block & RADEON_CG_BLOCK_BIF) {
6162 cik_enable_bif_mgls(rdev, enable);
6163 }
6164
6165 if (block & RADEON_CG_BLOCK_UVD) {
6166 if (rdev->has_uvd)
6167 cik_enable_uvd_mgcg(rdev, enable);
6168 }
6169
6170 if (block & RADEON_CG_BLOCK_HDP) {
6171 cik_enable_hdp_mgcg(rdev, enable);
6172 cik_enable_hdp_ls(rdev, enable);
6173 }
6174
6175 if (block & RADEON_CG_BLOCK_VCE) {
6176 vce_v2_0_enable_mgcg(rdev, enable);
6177 }
6178 }
6179
6180 static void cik_init_cg(struct radeon_device *rdev)
6181 {
6182
6183 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6184
6185 if (rdev->has_uvd)
6186 si_init_uvd_internal_cg(rdev);
6187
6188 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6189 RADEON_CG_BLOCK_SDMA |
6190 RADEON_CG_BLOCK_BIF |
6191 RADEON_CG_BLOCK_UVD |
6192 RADEON_CG_BLOCK_HDP), true);
6193 }
6194
6195 static void cik_fini_cg(struct radeon_device *rdev)
6196 {
6197 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6198 RADEON_CG_BLOCK_SDMA |
6199 RADEON_CG_BLOCK_BIF |
6200 RADEON_CG_BLOCK_UVD |
6201 RADEON_CG_BLOCK_HDP), false);
6202
6203 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6204 }
6205
6206 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6207 bool enable)
6208 {
6209 u32 data, orig;
6210
6211 orig = data = RREG32(RLC_PG_CNTL);
6212 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6213 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6214 else
6215 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6216 if (orig != data)
6217 WREG32(RLC_PG_CNTL, data);
6218 }
6219
6220 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6221 bool enable)
6222 {
6223 u32 data, orig;
6224
6225 orig = data = RREG32(RLC_PG_CNTL);
6226 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6227 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6228 else
6229 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6230 if (orig != data)
6231 WREG32(RLC_PG_CNTL, data);
6232 }
6233
6234 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6235 {
6236 u32 data, orig;
6237
6238 orig = data = RREG32(RLC_PG_CNTL);
6239 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6240 data &= ~DISABLE_CP_PG;
6241 else
6242 data |= DISABLE_CP_PG;
6243 if (orig != data)
6244 WREG32(RLC_PG_CNTL, data);
6245 }
6246
6247 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6248 {
6249 u32 data, orig;
6250
6251 orig = data = RREG32(RLC_PG_CNTL);
6252 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6253 data &= ~DISABLE_GDS_PG;
6254 else
6255 data |= DISABLE_GDS_PG;
6256 if (orig != data)
6257 WREG32(RLC_PG_CNTL, data);
6258 }
6259
6260 #define CP_ME_TABLE_SIZE 96
6261 #define CP_ME_TABLE_OFFSET 2048
6262 #define CP_MEC_TABLE_OFFSET 4096
6263
6264 void cik_init_cp_pg_table(struct radeon_device *rdev)
6265 {
6266 const __be32 *fw_data;
6267 volatile u32 *dst_ptr;
6268 int me, i, max_me = 4;
6269 u32 bo_offset = 0;
6270 u32 table_offset;
6271
6272 if (rdev->family == CHIP_KAVERI)
6273 max_me = 5;
6274
6275 if (rdev->rlc.cp_table_ptr == NULL)
6276 return;
6277
6278 /* write the cp table buffer */
6279 dst_ptr = rdev->rlc.cp_table_ptr;
6280 for (me = 0; me < max_me; me++) {
6281 if (me == 0) {
6282 fw_data = (const __be32 *)rdev->ce_fw->data;
6283 table_offset = CP_ME_TABLE_OFFSET;
6284 } else if (me == 1) {
6285 fw_data = (const __be32 *)rdev->pfp_fw->data;
6286 table_offset = CP_ME_TABLE_OFFSET;
6287 } else if (me == 2) {
6288 fw_data = (const __be32 *)rdev->me_fw->data;
6289 table_offset = CP_ME_TABLE_OFFSET;
6290 } else {
6291 fw_data = (const __be32 *)rdev->mec_fw->data;
6292 table_offset = CP_MEC_TABLE_OFFSET;
6293 }
6294
6295 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6296 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6297 }
6298 bo_offset += CP_ME_TABLE_SIZE;
6299 }
6300 }
6301
6302 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6303 bool enable)
6304 {
6305 u32 data, orig;
6306
6307 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6308 orig = data = RREG32(RLC_PG_CNTL);
6309 data |= GFX_PG_ENABLE;
6310 if (orig != data)
6311 WREG32(RLC_PG_CNTL, data);
6312
6313 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6314 data |= AUTO_PG_EN;
6315 if (orig != data)
6316 WREG32(RLC_AUTO_PG_CTRL, data);
6317 } else {
6318 orig = data = RREG32(RLC_PG_CNTL);
6319 data &= ~GFX_PG_ENABLE;
6320 if (orig != data)
6321 WREG32(RLC_PG_CNTL, data);
6322
6323 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6324 data &= ~AUTO_PG_EN;
6325 if (orig != data)
6326 WREG32(RLC_AUTO_PG_CTRL, data);
6327
6328 data = RREG32(DB_RENDER_CONTROL);
6329 }
6330 }
6331
6332 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6333 {
6334 u32 mask = 0, tmp, tmp1;
6335 int i;
6336
6337 cik_select_se_sh(rdev, se, sh);
6338 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6339 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6340 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6341
6342 tmp &= 0xffff0000;
6343
6344 tmp |= tmp1;
6345 tmp >>= 16;
6346
6347 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6348 mask <<= 1;
6349 mask |= 1;
6350 }
6351
6352 return (~tmp) & mask;
6353 }
6354
6355 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6356 {
6357 u32 i, j, k, active_cu_number = 0;
6358 u32 mask, counter, cu_bitmap;
6359 u32 tmp = 0;
6360
6361 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6362 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6363 mask = 1;
6364 cu_bitmap = 0;
6365 counter = 0;
6366 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6367 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6368 if (counter < 2)
6369 cu_bitmap |= mask;
6370 counter ++;
6371 }
6372 mask <<= 1;
6373 }
6374
6375 active_cu_number += counter;
6376 tmp |= (cu_bitmap << (i * 16 + j * 8));
6377 }
6378 }
6379
6380 WREG32(RLC_PG_AO_CU_MASK, tmp);
6381
6382 tmp = RREG32(RLC_MAX_PG_CU);
6383 tmp &= ~MAX_PU_CU_MASK;
6384 tmp |= MAX_PU_CU(active_cu_number);
6385 WREG32(RLC_MAX_PG_CU, tmp);
6386 }
6387
6388 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6389 bool enable)
6390 {
6391 u32 data, orig;
6392
6393 orig = data = RREG32(RLC_PG_CNTL);
6394 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6395 data |= STATIC_PER_CU_PG_ENABLE;
6396 else
6397 data &= ~STATIC_PER_CU_PG_ENABLE;
6398 if (orig != data)
6399 WREG32(RLC_PG_CNTL, data);
6400 }
6401
6402 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6403 bool enable)
6404 {
6405 u32 data, orig;
6406
6407 orig = data = RREG32(RLC_PG_CNTL);
6408 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6409 data |= DYN_PER_CU_PG_ENABLE;
6410 else
6411 data &= ~DYN_PER_CU_PG_ENABLE;
6412 if (orig != data)
6413 WREG32(RLC_PG_CNTL, data);
6414 }
6415
6416 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6417 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6418
6419 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6420 {
6421 u32 data, orig;
6422 u32 i;
6423
6424 if (rdev->rlc.cs_data) {
6425 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6426 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6427 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6428 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6429 } else {
6430 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6431 for (i = 0; i < 3; i++)
6432 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6433 }
6434 if (rdev->rlc.reg_list) {
6435 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6436 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6437 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6438 }
6439
6440 orig = data = RREG32(RLC_PG_CNTL);
6441 data |= GFX_PG_SRC;
6442 if (orig != data)
6443 WREG32(RLC_PG_CNTL, data);
6444
6445 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6446 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6447
6448 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6449 data &= ~IDLE_POLL_COUNT_MASK;
6450 data |= IDLE_POLL_COUNT(0x60);
6451 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6452
6453 data = 0x10101010;
6454 WREG32(RLC_PG_DELAY, data);
6455
6456 data = RREG32(RLC_PG_DELAY_2);
6457 data &= ~0xff;
6458 data |= 0x3;
6459 WREG32(RLC_PG_DELAY_2, data);
6460
6461 data = RREG32(RLC_AUTO_PG_CTRL);
6462 data &= ~GRBM_REG_SGIT_MASK;
6463 data |= GRBM_REG_SGIT(0x700);
6464 WREG32(RLC_AUTO_PG_CTRL, data);
6465
6466 }
6467
6468 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6469 {
6470 cik_enable_gfx_cgpg(rdev, enable);
6471 cik_enable_gfx_static_mgpg(rdev, enable);
6472 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6473 }
6474
6475 u32 cik_get_csb_size(struct radeon_device *rdev)
6476 {
6477 u32 count = 0;
6478 const struct cs_section_def *sect = NULL;
6479 const struct cs_extent_def *ext = NULL;
6480
6481 if (rdev->rlc.cs_data == NULL)
6482 return 0;
6483
6484 /* begin clear state */
6485 count += 2;
6486 /* context control state */
6487 count += 3;
6488
6489 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6490 for (ext = sect->section; ext->extent != NULL; ++ext) {
6491 if (sect->id == SECT_CONTEXT)
6492 count += 2 + ext->reg_count;
6493 else
6494 return 0;
6495 }
6496 }
6497 /* pa_sc_raster_config/pa_sc_raster_config1 */
6498 count += 4;
6499 /* end clear state */
6500 count += 2;
6501 /* clear state */
6502 count += 2;
6503
6504 return count;
6505 }
6506
6507 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6508 {
6509 u32 count = 0, i;
6510 const struct cs_section_def *sect = NULL;
6511 const struct cs_extent_def *ext = NULL;
6512
6513 if (rdev->rlc.cs_data == NULL)
6514 return;
6515 if (buffer == NULL)
6516 return;
6517
6518 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6519 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6520
6521 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6522 buffer[count++] = cpu_to_le32(0x80000000);
6523 buffer[count++] = cpu_to_le32(0x80000000);
6524
6525 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6526 for (ext = sect->section; ext->extent != NULL; ++ext) {
6527 if (sect->id == SECT_CONTEXT) {
6528 buffer[count++] =
6529 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6530 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6531 for (i = 0; i < ext->reg_count; i++)
6532 buffer[count++] = cpu_to_le32(ext->extent[i]);
6533 } else {
6534 return;
6535 }
6536 }
6537 }
6538
6539 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6540 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6541 switch (rdev->family) {
6542 case CHIP_BONAIRE:
6543 buffer[count++] = cpu_to_le32(0x16000012);
6544 buffer[count++] = cpu_to_le32(0x00000000);
6545 break;
6546 case CHIP_KAVERI:
6547 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6548 buffer[count++] = cpu_to_le32(0x00000000);
6549 break;
6550 case CHIP_KABINI:
6551 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6552 buffer[count++] = cpu_to_le32(0x00000000);
6553 break;
6554 case CHIP_HAWAII:
6555 buffer[count++] = cpu_to_le32(0x3a00161a);
6556 buffer[count++] = cpu_to_le32(0x0000002e);
6557 break;
6558 default:
6559 buffer[count++] = cpu_to_le32(0x00000000);
6560 buffer[count++] = cpu_to_le32(0x00000000);
6561 break;
6562 }
6563
6564 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6565 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6566
6567 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6568 buffer[count++] = cpu_to_le32(0);
6569 }
6570
6571 static void cik_init_pg(struct radeon_device *rdev)
6572 {
6573 if (rdev->pg_flags) {
6574 cik_enable_sck_slowdown_on_pu(rdev, true);
6575 cik_enable_sck_slowdown_on_pd(rdev, true);
6576 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6577 cik_init_gfx_cgpg(rdev);
6578 cik_enable_cp_pg(rdev, true);
6579 cik_enable_gds_pg(rdev, true);
6580 }
6581 cik_init_ao_cu_mask(rdev);
6582 cik_update_gfx_pg(rdev, true);
6583 }
6584 }
6585
6586 static void cik_fini_pg(struct radeon_device *rdev)
6587 {
6588 if (rdev->pg_flags) {
6589 cik_update_gfx_pg(rdev, false);
6590 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6591 cik_enable_cp_pg(rdev, false);
6592 cik_enable_gds_pg(rdev, false);
6593 }
6594 }
6595 }
6596
6597 /*
6598 * Interrupts
6599 * Starting with r6xx, interrupts are handled via a ring buffer.
6600 * Ring buffers are areas of GPU accessible memory that the GPU
6601 * writes interrupt vectors into and the host reads vectors out of.
6602 * There is a rptr (read pointer) that determines where the
6603 * host is currently reading, and a wptr (write pointer)
6604 * which determines where the GPU has written. When the
6605 * pointers are equal, the ring is idle. When the GPU
6606 * writes vectors to the ring buffer, it increments the
6607 * wptr. When there is an interrupt, the host then starts
6608 * fetching commands and processing them until the pointers are
6609 * equal again at which point it updates the rptr.
6610 */
6611
6612 /**
6613 * cik_enable_interrupts - Enable the interrupt ring buffer
6614 *
6615 * @rdev: radeon_device pointer
6616 *
6617 * Enable the interrupt ring buffer (CIK).
6618 */
6619 static void cik_enable_interrupts(struct radeon_device *rdev)
6620 {
6621 u32 ih_cntl = RREG32(IH_CNTL);
6622 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6623
6624 ih_cntl |= ENABLE_INTR;
6625 ih_rb_cntl |= IH_RB_ENABLE;
6626 WREG32(IH_CNTL, ih_cntl);
6627 WREG32(IH_RB_CNTL, ih_rb_cntl);
6628 rdev->ih.enabled = true;
6629 }
6630
6631 /**
6632 * cik_disable_interrupts - Disable the interrupt ring buffer
6633 *
6634 * @rdev: radeon_device pointer
6635 *
6636 * Disable the interrupt ring buffer (CIK).
6637 */
6638 static void cik_disable_interrupts(struct radeon_device *rdev)
6639 {
6640 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6641 u32 ih_cntl = RREG32(IH_CNTL);
6642
6643 ih_rb_cntl &= ~IH_RB_ENABLE;
6644 ih_cntl &= ~ENABLE_INTR;
6645 WREG32(IH_RB_CNTL, ih_rb_cntl);
6646 WREG32(IH_CNTL, ih_cntl);
6647 /* set rptr, wptr to 0 */
6648 WREG32(IH_RB_RPTR, 0);
6649 WREG32(IH_RB_WPTR, 0);
6650 rdev->ih.enabled = false;
6651 rdev->ih.rptr = 0;
6652 }
6653
6654 /**
6655 * cik_disable_interrupt_state - Disable all interrupt sources
6656 *
6657 * @rdev: radeon_device pointer
6658 *
6659 * Clear all interrupt enable bits used by the driver (CIK).
6660 */
6661 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6662 {
6663 u32 tmp;
6664
6665 /* gfx ring */
6666 tmp = RREG32(CP_INT_CNTL_RING0) &
6667 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6668 WREG32(CP_INT_CNTL_RING0, tmp);
6669 /* sdma */
6670 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6671 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6672 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6673 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6674 /* compute queues */
6675 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6676 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6677 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6678 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6679 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6680 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6681 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6682 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6683 /* grbm */
6684 WREG32(GRBM_INT_CNTL, 0);
6685 /* vline/vblank, etc. */
6686 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6687 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6688 if (rdev->num_crtc >= 4) {
6689 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6690 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6691 }
6692 if (rdev->num_crtc >= 6) {
6693 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6694 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6695 }
6696
6697 /* dac hotplug */
6698 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6699
6700 /* digital hotplug */
6701 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6702 WREG32(DC_HPD1_INT_CONTROL, tmp);
6703 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6704 WREG32(DC_HPD2_INT_CONTROL, tmp);
6705 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6706 WREG32(DC_HPD3_INT_CONTROL, tmp);
6707 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6708 WREG32(DC_HPD4_INT_CONTROL, tmp);
6709 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6710 WREG32(DC_HPD5_INT_CONTROL, tmp);
6711 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6712 WREG32(DC_HPD6_INT_CONTROL, tmp);
6713
6714 }
6715
6716 /**
6717 * cik_irq_init - init and enable the interrupt ring
6718 *
6719 * @rdev: radeon_device pointer
6720 *
6721 * Allocate a ring buffer for the interrupt controller,
6722 * enable the RLC, disable interrupts, enable the IH
6723 * ring buffer and enable it (CIK).
6724 * Called at device load and reume.
6725 * Returns 0 for success, errors for failure.
6726 */
6727 static int cik_irq_init(struct radeon_device *rdev)
6728 {
6729 int ret = 0;
6730 int rb_bufsz;
6731 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6732
6733 /* allocate ring */
6734 ret = r600_ih_ring_alloc(rdev);
6735 if (ret)
6736 return ret;
6737
6738 /* disable irqs */
6739 cik_disable_interrupts(rdev);
6740
6741 /* init rlc */
6742 ret = cik_rlc_resume(rdev);
6743 if (ret) {
6744 r600_ih_ring_fini(rdev);
6745 return ret;
6746 }
6747
6748 /* setup interrupt control */
6749 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6750 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6751 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6752 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6753 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6754 */
6755 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6756 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6757 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6758 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6759
6760 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6761 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6762
6763 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6764 IH_WPTR_OVERFLOW_CLEAR |
6765 (rb_bufsz << 1));
6766
6767 if (rdev->wb.enabled)
6768 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6769
6770 /* set the writeback address whether it's enabled or not */
6771 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6772 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6773
6774 WREG32(IH_RB_CNTL, ih_rb_cntl);
6775
6776 /* set rptr, wptr to 0 */
6777 WREG32(IH_RB_RPTR, 0);
6778 WREG32(IH_RB_WPTR, 0);
6779
6780 /* Default settings for IH_CNTL (disabled at first) */
6781 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6782 /* RPTR_REARM only works if msi's are enabled */
6783 if (rdev->msi_enabled)
6784 ih_cntl |= RPTR_REARM;
6785 WREG32(IH_CNTL, ih_cntl);
6786
6787 /* force the active interrupt state to all disabled */
6788 cik_disable_interrupt_state(rdev);
6789
6790 pci_set_master(rdev->pdev);
6791
6792 /* enable irqs */
6793 cik_enable_interrupts(rdev);
6794
6795 return ret;
6796 }
6797
6798 /**
6799 * cik_irq_set - enable/disable interrupt sources
6800 *
6801 * @rdev: radeon_device pointer
6802 *
6803 * Enable interrupt sources on the GPU (vblanks, hpd,
6804 * etc.) (CIK).
6805 * Returns 0 for success, errors for failure.
6806 */
6807 int cik_irq_set(struct radeon_device *rdev)
6808 {
6809 u32 cp_int_cntl;
6810 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6811 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6812 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6813 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6814 u32 grbm_int_cntl = 0;
6815 u32 dma_cntl, dma_cntl1;
6816 u32 thermal_int;
6817
6818 if (!rdev->irq.installed) {
6819 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6820 return -EINVAL;
6821 }
6822 /* don't enable anything if the ih is disabled */
6823 if (!rdev->ih.enabled) {
6824 cik_disable_interrupts(rdev);
6825 /* force the active interrupt state to all disabled */
6826 cik_disable_interrupt_state(rdev);
6827 return 0;
6828 }
6829
6830 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6831 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6832 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6833
6834 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6835 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6836 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6837 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6838 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6839 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6840
6841 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6842 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6843
6844 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6845 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6846 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6847 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6848 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6849 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6850 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6851 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6852
6853 if (rdev->flags & RADEON_IS_IGP)
6854 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6855 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6856 else
6857 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6858 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6859
6860 /* enable CP interrupts on all rings */
6861 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6862 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6863 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6864 }
6865 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6866 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6867 DRM_DEBUG("si_irq_set: sw int cp1\n");
6868 if (ring->me == 1) {
6869 switch (ring->pipe) {
6870 case 0:
6871 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6872 break;
6873 case 1:
6874 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6875 break;
6876 case 2:
6877 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6878 break;
6879 case 3:
6880 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6881 break;
6882 default:
6883 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6884 break;
6885 }
6886 } else if (ring->me == 2) {
6887 switch (ring->pipe) {
6888 case 0:
6889 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6890 break;
6891 case 1:
6892 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6893 break;
6894 case 2:
6895 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6896 break;
6897 case 3:
6898 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6899 break;
6900 default:
6901 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6902 break;
6903 }
6904 } else {
6905 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6906 }
6907 }
6908 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6909 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6910 DRM_DEBUG("si_irq_set: sw int cp2\n");
6911 if (ring->me == 1) {
6912 switch (ring->pipe) {
6913 case 0:
6914 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6915 break;
6916 case 1:
6917 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6918 break;
6919 case 2:
6920 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6921 break;
6922 case 3:
6923 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6924 break;
6925 default:
6926 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6927 break;
6928 }
6929 } else if (ring->me == 2) {
6930 switch (ring->pipe) {
6931 case 0:
6932 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6933 break;
6934 case 1:
6935 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6936 break;
6937 case 2:
6938 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6939 break;
6940 case 3:
6941 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6942 break;
6943 default:
6944 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6945 break;
6946 }
6947 } else {
6948 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6949 }
6950 }
6951
6952 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6953 DRM_DEBUG("cik_irq_set: sw int dma\n");
6954 dma_cntl |= TRAP_ENABLE;
6955 }
6956
6957 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6958 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6959 dma_cntl1 |= TRAP_ENABLE;
6960 }
6961
6962 if (rdev->irq.crtc_vblank_int[0] ||
6963 atomic_read(&rdev->irq.pflip[0])) {
6964 DRM_DEBUG("cik_irq_set: vblank 0\n");
6965 crtc1 |= VBLANK_INTERRUPT_MASK;
6966 }
6967 if (rdev->irq.crtc_vblank_int[1] ||
6968 atomic_read(&rdev->irq.pflip[1])) {
6969 DRM_DEBUG("cik_irq_set: vblank 1\n");
6970 crtc2 |= VBLANK_INTERRUPT_MASK;
6971 }
6972 if (rdev->irq.crtc_vblank_int[2] ||
6973 atomic_read(&rdev->irq.pflip[2])) {
6974 DRM_DEBUG("cik_irq_set: vblank 2\n");
6975 crtc3 |= VBLANK_INTERRUPT_MASK;
6976 }
6977 if (rdev->irq.crtc_vblank_int[3] ||
6978 atomic_read(&rdev->irq.pflip[3])) {
6979 DRM_DEBUG("cik_irq_set: vblank 3\n");
6980 crtc4 |= VBLANK_INTERRUPT_MASK;
6981 }
6982 if (rdev->irq.crtc_vblank_int[4] ||
6983 atomic_read(&rdev->irq.pflip[4])) {
6984 DRM_DEBUG("cik_irq_set: vblank 4\n");
6985 crtc5 |= VBLANK_INTERRUPT_MASK;
6986 }
6987 if (rdev->irq.crtc_vblank_int[5] ||
6988 atomic_read(&rdev->irq.pflip[5])) {
6989 DRM_DEBUG("cik_irq_set: vblank 5\n");
6990 crtc6 |= VBLANK_INTERRUPT_MASK;
6991 }
6992 if (rdev->irq.hpd[0]) {
6993 DRM_DEBUG("cik_irq_set: hpd 1\n");
6994 hpd1 |= DC_HPDx_INT_EN;
6995 }
6996 if (rdev->irq.hpd[1]) {
6997 DRM_DEBUG("cik_irq_set: hpd 2\n");
6998 hpd2 |= DC_HPDx_INT_EN;
6999 }
7000 if (rdev->irq.hpd[2]) {
7001 DRM_DEBUG("cik_irq_set: hpd 3\n");
7002 hpd3 |= DC_HPDx_INT_EN;
7003 }
7004 if (rdev->irq.hpd[3]) {
7005 DRM_DEBUG("cik_irq_set: hpd 4\n");
7006 hpd4 |= DC_HPDx_INT_EN;
7007 }
7008 if (rdev->irq.hpd[4]) {
7009 DRM_DEBUG("cik_irq_set: hpd 5\n");
7010 hpd5 |= DC_HPDx_INT_EN;
7011 }
7012 if (rdev->irq.hpd[5]) {
7013 DRM_DEBUG("cik_irq_set: hpd 6\n");
7014 hpd6 |= DC_HPDx_INT_EN;
7015 }
7016
7017 if (rdev->irq.dpm_thermal) {
7018 DRM_DEBUG("dpm thermal\n");
7019 if (rdev->flags & RADEON_IS_IGP)
7020 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7021 else
7022 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7023 }
7024
7025 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7026
7027 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7028 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7029
7030 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7031 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7032 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7033 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7034 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7035 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7036 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7037 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7038
7039 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7040
7041 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7042 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7043 if (rdev->num_crtc >= 4) {
7044 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7045 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7046 }
7047 if (rdev->num_crtc >= 6) {
7048 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7049 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7050 }
7051
7052 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7053 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7054 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7055 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7056 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7057 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7058
7059 if (rdev->flags & RADEON_IS_IGP)
7060 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7061 else
7062 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7063
7064 return 0;
7065 }
7066
7067 /**
7068 * cik_irq_ack - ack interrupt sources
7069 *
7070 * @rdev: radeon_device pointer
7071 *
7072 * Ack interrupt sources on the GPU (vblanks, hpd,
7073 * etc.) (CIK). Certain interrupts sources are sw
7074 * generated and do not require an explicit ack.
7075 */
7076 static inline void cik_irq_ack(struct radeon_device *rdev)
7077 {
7078 u32 tmp;
7079
7080 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7081 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7082 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7083 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7084 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7085 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7086 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7087
7088 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7089 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7090 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7091 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7092 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7093 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7094 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7095 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7096
7097 if (rdev->num_crtc >= 4) {
7098 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7099 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7100 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7101 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7102 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7103 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7104 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7105 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7106 }
7107
7108 if (rdev->num_crtc >= 6) {
7109 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7110 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7111 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7112 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7113 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7114 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7115 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7116 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7117 }
7118
7119 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7120 tmp = RREG32(DC_HPD1_INT_CONTROL);
7121 tmp |= DC_HPDx_INT_ACK;
7122 WREG32(DC_HPD1_INT_CONTROL, tmp);
7123 }
7124 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7125 tmp = RREG32(DC_HPD2_INT_CONTROL);
7126 tmp |= DC_HPDx_INT_ACK;
7127 WREG32(DC_HPD2_INT_CONTROL, tmp);
7128 }
7129 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7130 tmp = RREG32(DC_HPD3_INT_CONTROL);
7131 tmp |= DC_HPDx_INT_ACK;
7132 WREG32(DC_HPD3_INT_CONTROL, tmp);
7133 }
7134 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7135 tmp = RREG32(DC_HPD4_INT_CONTROL);
7136 tmp |= DC_HPDx_INT_ACK;
7137 WREG32(DC_HPD4_INT_CONTROL, tmp);
7138 }
7139 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7140 tmp = RREG32(DC_HPD5_INT_CONTROL);
7141 tmp |= DC_HPDx_INT_ACK;
7142 WREG32(DC_HPD5_INT_CONTROL, tmp);
7143 }
7144 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7145 tmp = RREG32(DC_HPD5_INT_CONTROL);
7146 tmp |= DC_HPDx_INT_ACK;
7147 WREG32(DC_HPD6_INT_CONTROL, tmp);
7148 }
7149 }
7150
7151 /**
7152 * cik_irq_disable - disable interrupts
7153 *
7154 * @rdev: radeon_device pointer
7155 *
7156 * Disable interrupts on the hw (CIK).
7157 */
7158 static void cik_irq_disable(struct radeon_device *rdev)
7159 {
7160 cik_disable_interrupts(rdev);
7161 /* Wait and acknowledge irq */
7162 mdelay(1);
7163 cik_irq_ack(rdev);
7164 cik_disable_interrupt_state(rdev);
7165 }
7166
7167 /**
7168 * cik_irq_disable - disable interrupts for suspend
7169 *
7170 * @rdev: radeon_device pointer
7171 *
7172 * Disable interrupts and stop the RLC (CIK).
7173 * Used for suspend.
7174 */
7175 static void cik_irq_suspend(struct radeon_device *rdev)
7176 {
7177 cik_irq_disable(rdev);
7178 cik_rlc_stop(rdev);
7179 }
7180
7181 /**
7182 * cik_irq_fini - tear down interrupt support
7183 *
7184 * @rdev: radeon_device pointer
7185 *
7186 * Disable interrupts on the hw and free the IH ring
7187 * buffer (CIK).
7188 * Used for driver unload.
7189 */
7190 static void cik_irq_fini(struct radeon_device *rdev)
7191 {
7192 cik_irq_suspend(rdev);
7193 r600_ih_ring_fini(rdev);
7194 }
7195
7196 /**
7197 * cik_get_ih_wptr - get the IH ring buffer wptr
7198 *
7199 * @rdev: radeon_device pointer
7200 *
7201 * Get the IH ring buffer wptr from either the register
7202 * or the writeback memory buffer (CIK). Also check for
7203 * ring buffer overflow and deal with it.
7204 * Used by cik_irq_process().
7205 * Returns the value of the wptr.
7206 */
7207 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7208 {
7209 u32 wptr, tmp;
7210
7211 if (rdev->wb.enabled)
7212 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7213 else
7214 wptr = RREG32(IH_RB_WPTR);
7215
7216 if (wptr & RB_OVERFLOW) {
7217 /* When a ring buffer overflow happen start parsing interrupt
7218 * from the last not overwritten vector (wptr + 16). Hopefully
7219 * this should allow us to catchup.
7220 */
7221 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7222 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7223 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7224 tmp = RREG32(IH_RB_CNTL);
7225 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7226 WREG32(IH_RB_CNTL, tmp);
7227 }
7228 return (wptr & rdev->ih.ptr_mask);
7229 }
7230
7231 /* CIK IV Ring
7232 * Each IV ring entry is 128 bits:
7233 * [7:0] - interrupt source id
7234 * [31:8] - reserved
7235 * [59:32] - interrupt source data
7236 * [63:60] - reserved
7237 * [71:64] - RINGID
7238 * CP:
7239 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7240 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7241 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7242 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7243 * PIPE_ID - ME0 0=3D
7244 * - ME1&2 compute dispatcher (4 pipes each)
7245 * SDMA:
7246 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7247 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7248 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7249 * [79:72] - VMID
7250 * [95:80] - PASID
7251 * [127:96] - reserved
7252 */
7253 /**
7254 * cik_irq_process - interrupt handler
7255 *
7256 * @rdev: radeon_device pointer
7257 *
7258 * Interrupt hander (CIK). Walk the IH ring,
7259 * ack interrupts and schedule work to handle
7260 * interrupt events.
7261 * Returns irq process return code.
7262 */
7263 int cik_irq_process(struct radeon_device *rdev)
7264 {
7265 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7266 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7267 u32 wptr;
7268 u32 rptr;
7269 u32 src_id, src_data, ring_id;
7270 u8 me_id, pipe_id, queue_id;
7271 u32 ring_index;
7272 bool queue_hotplug = false;
7273 bool queue_reset = false;
7274 u32 addr, status, mc_client;
7275 bool queue_thermal = false;
7276
7277 if (!rdev->ih.enabled || rdev->shutdown)
7278 return IRQ_NONE;
7279
7280 wptr = cik_get_ih_wptr(rdev);
7281
7282 restart_ih:
7283 /* is somebody else already processing irqs? */
7284 if (atomic_xchg(&rdev->ih.lock, 1))
7285 return IRQ_NONE;
7286
7287 rptr = rdev->ih.rptr;
7288 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7289
7290 /* Order reading of wptr vs. reading of IH ring data */
7291 rmb();
7292
7293 /* display interrupts */
7294 cik_irq_ack(rdev);
7295
7296 while (rptr != wptr) {
7297 /* wptr/rptr are in bytes! */
7298 ring_index = rptr / 4;
7299 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7300 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7301 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7302
7303 switch (src_id) {
7304 case 1: /* D1 vblank/vline */
7305 switch (src_data) {
7306 case 0: /* D1 vblank */
7307 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7308 if (rdev->irq.crtc_vblank_int[0]) {
7309 drm_handle_vblank(rdev->ddev, 0);
7310 rdev->pm.vblank_sync = true;
7311 wake_up(&rdev->irq.vblank_queue);
7312 }
7313 if (atomic_read(&rdev->irq.pflip[0]))
7314 radeon_crtc_handle_flip(rdev, 0);
7315 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7316 DRM_DEBUG("IH: D1 vblank\n");
7317 }
7318 break;
7319 case 1: /* D1 vline */
7320 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7321 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7322 DRM_DEBUG("IH: D1 vline\n");
7323 }
7324 break;
7325 default:
7326 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7327 break;
7328 }
7329 break;
7330 case 2: /* D2 vblank/vline */
7331 switch (src_data) {
7332 case 0: /* D2 vblank */
7333 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7334 if (rdev->irq.crtc_vblank_int[1]) {
7335 drm_handle_vblank(rdev->ddev, 1);
7336 rdev->pm.vblank_sync = true;
7337 wake_up(&rdev->irq.vblank_queue);
7338 }
7339 if (atomic_read(&rdev->irq.pflip[1]))
7340 radeon_crtc_handle_flip(rdev, 1);
7341 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7342 DRM_DEBUG("IH: D2 vblank\n");
7343 }
7344 break;
7345 case 1: /* D2 vline */
7346 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7347 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7348 DRM_DEBUG("IH: D2 vline\n");
7349 }
7350 break;
7351 default:
7352 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7353 break;
7354 }
7355 break;
7356 case 3: /* D3 vblank/vline */
7357 switch (src_data) {
7358 case 0: /* D3 vblank */
7359 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7360 if (rdev->irq.crtc_vblank_int[2]) {
7361 drm_handle_vblank(rdev->ddev, 2);
7362 rdev->pm.vblank_sync = true;
7363 wake_up(&rdev->irq.vblank_queue);
7364 }
7365 if (atomic_read(&rdev->irq.pflip[2]))
7366 radeon_crtc_handle_flip(rdev, 2);
7367 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7368 DRM_DEBUG("IH: D3 vblank\n");
7369 }
7370 break;
7371 case 1: /* D3 vline */
7372 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7373 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7374 DRM_DEBUG("IH: D3 vline\n");
7375 }
7376 break;
7377 default:
7378 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7379 break;
7380 }
7381 break;
7382 case 4: /* D4 vblank/vline */
7383 switch (src_data) {
7384 case 0: /* D4 vblank */
7385 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7386 if (rdev->irq.crtc_vblank_int[3]) {
7387 drm_handle_vblank(rdev->ddev, 3);
7388 rdev->pm.vblank_sync = true;
7389 wake_up(&rdev->irq.vblank_queue);
7390 }
7391 if (atomic_read(&rdev->irq.pflip[3]))
7392 radeon_crtc_handle_flip(rdev, 3);
7393 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7394 DRM_DEBUG("IH: D4 vblank\n");
7395 }
7396 break;
7397 case 1: /* D4 vline */
7398 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7399 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7400 DRM_DEBUG("IH: D4 vline\n");
7401 }
7402 break;
7403 default:
7404 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7405 break;
7406 }
7407 break;
7408 case 5: /* D5 vblank/vline */
7409 switch (src_data) {
7410 case 0: /* D5 vblank */
7411 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7412 if (rdev->irq.crtc_vblank_int[4]) {
7413 drm_handle_vblank(rdev->ddev, 4);
7414 rdev->pm.vblank_sync = true;
7415 wake_up(&rdev->irq.vblank_queue);
7416 }
7417 if (atomic_read(&rdev->irq.pflip[4]))
7418 radeon_crtc_handle_flip(rdev, 4);
7419 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7420 DRM_DEBUG("IH: D5 vblank\n");
7421 }
7422 break;
7423 case 1: /* D5 vline */
7424 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7425 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7426 DRM_DEBUG("IH: D5 vline\n");
7427 }
7428 break;
7429 default:
7430 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7431 break;
7432 }
7433 break;
7434 case 6: /* D6 vblank/vline */
7435 switch (src_data) {
7436 case 0: /* D6 vblank */
7437 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7438 if (rdev->irq.crtc_vblank_int[5]) {
7439 drm_handle_vblank(rdev->ddev, 5);
7440 rdev->pm.vblank_sync = true;
7441 wake_up(&rdev->irq.vblank_queue);
7442 }
7443 if (atomic_read(&rdev->irq.pflip[5]))
7444 radeon_crtc_handle_flip(rdev, 5);
7445 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7446 DRM_DEBUG("IH: D6 vblank\n");
7447 }
7448 break;
7449 case 1: /* D6 vline */
7450 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7451 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7452 DRM_DEBUG("IH: D6 vline\n");
7453 }
7454 break;
7455 default:
7456 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7457 break;
7458 }
7459 break;
7460 case 42: /* HPD hotplug */
7461 switch (src_data) {
7462 case 0:
7463 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7464 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7465 queue_hotplug = true;
7466 DRM_DEBUG("IH: HPD1\n");
7467 }
7468 break;
7469 case 1:
7470 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7471 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7472 queue_hotplug = true;
7473 DRM_DEBUG("IH: HPD2\n");
7474 }
7475 break;
7476 case 2:
7477 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7478 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7479 queue_hotplug = true;
7480 DRM_DEBUG("IH: HPD3\n");
7481 }
7482 break;
7483 case 3:
7484 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7485 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7486 queue_hotplug = true;
7487 DRM_DEBUG("IH: HPD4\n");
7488 }
7489 break;
7490 case 4:
7491 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7492 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7493 queue_hotplug = true;
7494 DRM_DEBUG("IH: HPD5\n");
7495 }
7496 break;
7497 case 5:
7498 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7499 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7500 queue_hotplug = true;
7501 DRM_DEBUG("IH: HPD6\n");
7502 }
7503 break;
7504 default:
7505 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7506 break;
7507 }
7508 break;
7509 case 124: /* UVD */
7510 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7511 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7512 break;
7513 case 146:
7514 case 147:
7515 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7516 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7517 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7518 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7519 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7520 addr);
7521 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7522 status);
7523 cik_vm_decode_fault(rdev, status, addr, mc_client);
7524 /* reset addr and status */
7525 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7526 break;
7527 case 167: /* VCE */
7528 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7529 switch (src_data) {
7530 case 0:
7531 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7532 break;
7533 case 1:
7534 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7535 break;
7536 default:
7537 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7538 break;
7539 }
7540 break;
7541 case 176: /* GFX RB CP_INT */
7542 case 177: /* GFX IB CP_INT */
7543 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7544 break;
7545 case 181: /* CP EOP event */
7546 DRM_DEBUG("IH: CP EOP\n");
7547 /* XXX check the bitfield order! */
7548 me_id = (ring_id & 0x60) >> 5;
7549 pipe_id = (ring_id & 0x18) >> 3;
7550 queue_id = (ring_id & 0x7) >> 0;
7551 switch (me_id) {
7552 case 0:
7553 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7554 break;
7555 case 1:
7556 case 2:
7557 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7558 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7559 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7560 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7561 break;
7562 }
7563 break;
7564 case 184: /* CP Privileged reg access */
7565 DRM_ERROR("Illegal register access in command stream\n");
7566 /* XXX check the bitfield order! */
7567 me_id = (ring_id & 0x60) >> 5;
7568 pipe_id = (ring_id & 0x18) >> 3;
7569 queue_id = (ring_id & 0x7) >> 0;
7570 switch (me_id) {
7571 case 0:
7572 /* This results in a full GPU reset, but all we need to do is soft
7573 * reset the CP for gfx
7574 */
7575 queue_reset = true;
7576 break;
7577 case 1:
7578 /* XXX compute */
7579 queue_reset = true;
7580 break;
7581 case 2:
7582 /* XXX compute */
7583 queue_reset = true;
7584 break;
7585 }
7586 break;
7587 case 185: /* CP Privileged inst */
7588 DRM_ERROR("Illegal instruction in command stream\n");
7589 /* XXX check the bitfield order! */
7590 me_id = (ring_id & 0x60) >> 5;
7591 pipe_id = (ring_id & 0x18) >> 3;
7592 queue_id = (ring_id & 0x7) >> 0;
7593 switch (me_id) {
7594 case 0:
7595 /* This results in a full GPU reset, but all we need to do is soft
7596 * reset the CP for gfx
7597 */
7598 queue_reset = true;
7599 break;
7600 case 1:
7601 /* XXX compute */
7602 queue_reset = true;
7603 break;
7604 case 2:
7605 /* XXX compute */
7606 queue_reset = true;
7607 break;
7608 }
7609 break;
7610 case 224: /* SDMA trap event */
7611 /* XXX check the bitfield order! */
7612 me_id = (ring_id & 0x3) >> 0;
7613 queue_id = (ring_id & 0xc) >> 2;
7614 DRM_DEBUG("IH: SDMA trap\n");
7615 switch (me_id) {
7616 case 0:
7617 switch (queue_id) {
7618 case 0:
7619 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7620 break;
7621 case 1:
7622 /* XXX compute */
7623 break;
7624 case 2:
7625 /* XXX compute */
7626 break;
7627 }
7628 break;
7629 case 1:
7630 switch (queue_id) {
7631 case 0:
7632 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7633 break;
7634 case 1:
7635 /* XXX compute */
7636 break;
7637 case 2:
7638 /* XXX compute */
7639 break;
7640 }
7641 break;
7642 }
7643 break;
7644 case 230: /* thermal low to high */
7645 DRM_DEBUG("IH: thermal low to high\n");
7646 rdev->pm.dpm.thermal.high_to_low = false;
7647 queue_thermal = true;
7648 break;
7649 case 231: /* thermal high to low */
7650 DRM_DEBUG("IH: thermal high to low\n");
7651 rdev->pm.dpm.thermal.high_to_low = true;
7652 queue_thermal = true;
7653 break;
7654 case 233: /* GUI IDLE */
7655 DRM_DEBUG("IH: GUI idle\n");
7656 break;
7657 case 241: /* SDMA Privileged inst */
7658 case 247: /* SDMA Privileged inst */
7659 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7660 /* XXX check the bitfield order! */
7661 me_id = (ring_id & 0x3) >> 0;
7662 queue_id = (ring_id & 0xc) >> 2;
7663 switch (me_id) {
7664 case 0:
7665 switch (queue_id) {
7666 case 0:
7667 queue_reset = true;
7668 break;
7669 case 1:
7670 /* XXX compute */
7671 queue_reset = true;
7672 break;
7673 case 2:
7674 /* XXX compute */
7675 queue_reset = true;
7676 break;
7677 }
7678 break;
7679 case 1:
7680 switch (queue_id) {
7681 case 0:
7682 queue_reset = true;
7683 break;
7684 case 1:
7685 /* XXX compute */
7686 queue_reset = true;
7687 break;
7688 case 2:
7689 /* XXX compute */
7690 queue_reset = true;
7691 break;
7692 }
7693 break;
7694 }
7695 break;
7696 default:
7697 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7698 break;
7699 }
7700
7701 /* wptr/rptr are in bytes! */
7702 rptr += 16;
7703 rptr &= rdev->ih.ptr_mask;
7704 }
7705 if (queue_hotplug)
7706 schedule_work(&rdev->hotplug_work);
7707 if (queue_reset)
7708 schedule_work(&rdev->reset_work);
7709 if (queue_thermal)
7710 schedule_work(&rdev->pm.dpm.thermal.work);
7711 rdev->ih.rptr = rptr;
7712 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7713 atomic_set(&rdev->ih.lock, 0);
7714
7715 /* make sure wptr hasn't changed while processing */
7716 wptr = cik_get_ih_wptr(rdev);
7717 if (wptr != rptr)
7718 goto restart_ih;
7719
7720 return IRQ_HANDLED;
7721 }
7722
7723 /*
7724 * startup/shutdown callbacks
7725 */
7726 /**
7727 * cik_startup - program the asic to a functional state
7728 *
7729 * @rdev: radeon_device pointer
7730 *
7731 * Programs the asic to a functional state (CIK).
7732 * Called by cik_init() and cik_resume().
7733 * Returns 0 for success, error for failure.
7734 */
7735 static int cik_startup(struct radeon_device *rdev)
7736 {
7737 struct radeon_ring *ring;
7738 int r;
7739
7740 /* enable pcie gen2/3 link */
7741 cik_pcie_gen3_enable(rdev);
7742 /* enable aspm */
7743 cik_program_aspm(rdev);
7744
7745 /* scratch needs to be initialized before MC */
7746 r = r600_vram_scratch_init(rdev);
7747 if (r)
7748 return r;
7749
7750 cik_mc_program(rdev);
7751
7752 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7753 r = ci_mc_load_microcode(rdev);
7754 if (r) {
7755 DRM_ERROR("Failed to load MC firmware!\n");
7756 return r;
7757 }
7758 }
7759
7760 r = cik_pcie_gart_enable(rdev);
7761 if (r)
7762 return r;
7763 cik_gpu_init(rdev);
7764
7765 /* allocate rlc buffers */
7766 if (rdev->flags & RADEON_IS_IGP) {
7767 if (rdev->family == CHIP_KAVERI) {
7768 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7769 rdev->rlc.reg_list_size =
7770 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7771 } else {
7772 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7773 rdev->rlc.reg_list_size =
7774 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7775 }
7776 }
7777 rdev->rlc.cs_data = ci_cs_data;
7778 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7779 r = sumo_rlc_init(rdev);
7780 if (r) {
7781 DRM_ERROR("Failed to init rlc BOs!\n");
7782 return r;
7783 }
7784
7785 /* allocate wb buffer */
7786 r = radeon_wb_init(rdev);
7787 if (r)
7788 return r;
7789
7790 /* allocate mec buffers */
7791 r = cik_mec_init(rdev);
7792 if (r) {
7793 DRM_ERROR("Failed to init MEC BOs!\n");
7794 return r;
7795 }
7796
7797 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7798 if (r) {
7799 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7800 return r;
7801 }
7802
7803 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7804 if (r) {
7805 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7806 return r;
7807 }
7808
7809 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7810 if (r) {
7811 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7812 return r;
7813 }
7814
7815 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7816 if (r) {
7817 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7818 return r;
7819 }
7820
7821 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7822 if (r) {
7823 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7824 return r;
7825 }
7826
7827 r = radeon_uvd_resume(rdev);
7828 if (!r) {
7829 r = uvd_v4_2_resume(rdev);
7830 if (!r) {
7831 r = radeon_fence_driver_start_ring(rdev,
7832 R600_RING_TYPE_UVD_INDEX);
7833 if (r)
7834 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7835 }
7836 }
7837 if (r)
7838 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7839
7840 r = radeon_vce_resume(rdev);
7841 if (!r) {
7842 r = vce_v2_0_resume(rdev);
7843 if (!r)
7844 r = radeon_fence_driver_start_ring(rdev,
7845 TN_RING_TYPE_VCE1_INDEX);
7846 if (!r)
7847 r = radeon_fence_driver_start_ring(rdev,
7848 TN_RING_TYPE_VCE2_INDEX);
7849 }
7850 if (r) {
7851 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7852 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7853 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7854 }
7855
7856 /* Enable IRQ */
7857 if (!rdev->irq.installed) {
7858 r = radeon_irq_kms_init(rdev);
7859 if (r)
7860 return r;
7861 }
7862
7863 r = cik_irq_init(rdev);
7864 if (r) {
7865 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7866 radeon_irq_kms_fini(rdev);
7867 return r;
7868 }
7869 cik_irq_set(rdev);
7870
7871 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7872 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7873 PACKET3(PACKET3_NOP, 0x3FFF));
7874 if (r)
7875 return r;
7876
7877 /* set up the compute queues */
7878 /* type-2 packets are deprecated on MEC, use type-3 instead */
7879 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7880 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7881 PACKET3(PACKET3_NOP, 0x3FFF));
7882 if (r)
7883 return r;
7884 ring->me = 1; /* first MEC */
7885 ring->pipe = 0; /* first pipe */
7886 ring->queue = 0; /* first queue */
7887 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7888
7889 /* type-2 packets are deprecated on MEC, use type-3 instead */
7890 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7891 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7892 PACKET3(PACKET3_NOP, 0x3FFF));
7893 if (r)
7894 return r;
7895 /* dGPU only have 1 MEC */
7896 ring->me = 1; /* first MEC */
7897 ring->pipe = 0; /* first pipe */
7898 ring->queue = 1; /* second queue */
7899 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7900
7901 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7902 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7903 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7904 if (r)
7905 return r;
7906
7907 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7908 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7909 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7910 if (r)
7911 return r;
7912
7913 r = cik_cp_resume(rdev);
7914 if (r)
7915 return r;
7916
7917 r = cik_sdma_resume(rdev);
7918 if (r)
7919 return r;
7920
7921 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7922 if (ring->ring_size) {
7923 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7924 RADEON_CP_PACKET2);
7925 if (!r)
7926 r = uvd_v1_0_init(rdev);
7927 if (r)
7928 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7929 }
7930
7931 r = -ENOENT;
7932
7933 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7934 if (ring->ring_size)
7935 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7936 VCE_CMD_NO_OP);
7937
7938 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7939 if (ring->ring_size)
7940 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7941 VCE_CMD_NO_OP);
7942
7943 if (!r)
7944 r = vce_v1_0_init(rdev);
7945 else if (r != -ENOENT)
7946 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7947
7948 r = radeon_ib_pool_init(rdev);
7949 if (r) {
7950 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7951 return r;
7952 }
7953
7954 r = radeon_vm_manager_init(rdev);
7955 if (r) {
7956 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7957 return r;
7958 }
7959
7960 r = dce6_audio_init(rdev);
7961 if (r)
7962 return r;
7963
7964 return 0;
7965 }
7966
7967 /**
7968 * cik_resume - resume the asic to a functional state
7969 *
7970 * @rdev: radeon_device pointer
7971 *
7972 * Programs the asic to a functional state (CIK).
7973 * Called at resume.
7974 * Returns 0 for success, error for failure.
7975 */
7976 int cik_resume(struct radeon_device *rdev)
7977 {
7978 int r;
7979
7980 /* post card */
7981 atom_asic_init(rdev->mode_info.atom_context);
7982
7983 /* init golden registers */
7984 cik_init_golden_registers(rdev);
7985
7986 if (rdev->pm.pm_method == PM_METHOD_DPM)
7987 radeon_pm_resume(rdev);
7988
7989 rdev->accel_working = true;
7990 r = cik_startup(rdev);
7991 if (r) {
7992 DRM_ERROR("cik startup failed on resume\n");
7993 rdev->accel_working = false;
7994 return r;
7995 }
7996
7997 return r;
7998
7999 }
8000
8001 /**
8002 * cik_suspend - suspend the asic
8003 *
8004 * @rdev: radeon_device pointer
8005 *
8006 * Bring the chip into a state suitable for suspend (CIK).
8007 * Called at suspend.
8008 * Returns 0 for success.
8009 */
8010 int cik_suspend(struct radeon_device *rdev)
8011 {
8012 radeon_pm_suspend(rdev);
8013 dce6_audio_fini(rdev);
8014 radeon_vm_manager_fini(rdev);
8015 cik_cp_enable(rdev, false);
8016 cik_sdma_enable(rdev, false);
8017 uvd_v1_0_fini(rdev);
8018 radeon_uvd_suspend(rdev);
8019 radeon_vce_suspend(rdev);
8020 cik_fini_pg(rdev);
8021 cik_fini_cg(rdev);
8022 cik_irq_suspend(rdev);
8023 radeon_wb_disable(rdev);
8024 cik_pcie_gart_disable(rdev);
8025 return 0;
8026 }
8027
8028 /* Plan is to move initialization in that function and use
8029 * helper function so that radeon_device_init pretty much
8030 * do nothing more than calling asic specific function. This
8031 * should also allow to remove a bunch of callback function
8032 * like vram_info.
8033 */
8034 /**
8035 * cik_init - asic specific driver and hw init
8036 *
8037 * @rdev: radeon_device pointer
8038 *
8039 * Setup asic specific driver variables and program the hw
8040 * to a functional state (CIK).
8041 * Called at driver startup.
8042 * Returns 0 for success, errors for failure.
8043 */
8044 int cik_init(struct radeon_device *rdev)
8045 {
8046 struct radeon_ring *ring;
8047 int r;
8048
8049 /* Read BIOS */
8050 if (!radeon_get_bios(rdev)) {
8051 if (ASIC_IS_AVIVO(rdev))
8052 return -EINVAL;
8053 }
8054 /* Must be an ATOMBIOS */
8055 if (!rdev->is_atom_bios) {
8056 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8057 return -EINVAL;
8058 }
8059 r = radeon_atombios_init(rdev);
8060 if (r)
8061 return r;
8062
8063 /* Post card if necessary */
8064 if (!radeon_card_posted(rdev)) {
8065 if (!rdev->bios) {
8066 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8067 return -EINVAL;
8068 }
8069 DRM_INFO("GPU not posted. posting now...\n");
8070 atom_asic_init(rdev->mode_info.atom_context);
8071 }
8072 /* init golden registers */
8073 cik_init_golden_registers(rdev);
8074 /* Initialize scratch registers */
8075 cik_scratch_init(rdev);
8076 /* Initialize surface registers */
8077 radeon_surface_init(rdev);
8078 /* Initialize clocks */
8079 radeon_get_clock_info(rdev->ddev);
8080
8081 /* Fence driver */
8082 r = radeon_fence_driver_init(rdev);
8083 if (r)
8084 return r;
8085
8086 /* initialize memory controller */
8087 r = cik_mc_init(rdev);
8088 if (r)
8089 return r;
8090 /* Memory manager */
8091 r = radeon_bo_init(rdev);
8092 if (r)
8093 return r;
8094
8095 if (rdev->flags & RADEON_IS_IGP) {
8096 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8097 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8098 r = cik_init_microcode(rdev);
8099 if (r) {
8100 DRM_ERROR("Failed to load firmware!\n");
8101 return r;
8102 }
8103 }
8104 } else {
8105 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8106 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8107 !rdev->mc_fw) {
8108 r = cik_init_microcode(rdev);
8109 if (r) {
8110 DRM_ERROR("Failed to load firmware!\n");
8111 return r;
8112 }
8113 }
8114 }
8115
8116 /* Initialize power management */
8117 radeon_pm_init(rdev);
8118
8119 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8120 ring->ring_obj = NULL;
8121 r600_ring_init(rdev, ring, 1024 * 1024);
8122
8123 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8124 ring->ring_obj = NULL;
8125 r600_ring_init(rdev, ring, 1024 * 1024);
8126 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8127 if (r)
8128 return r;
8129
8130 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8131 ring->ring_obj = NULL;
8132 r600_ring_init(rdev, ring, 1024 * 1024);
8133 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8134 if (r)
8135 return r;
8136
8137 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8138 ring->ring_obj = NULL;
8139 r600_ring_init(rdev, ring, 256 * 1024);
8140
8141 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8142 ring->ring_obj = NULL;
8143 r600_ring_init(rdev, ring, 256 * 1024);
8144
8145 r = radeon_uvd_init(rdev);
8146 if (!r) {
8147 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8148 ring->ring_obj = NULL;
8149 r600_ring_init(rdev, ring, 4096);
8150 }
8151
8152 r = radeon_vce_init(rdev);
8153 if (!r) {
8154 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8155 ring->ring_obj = NULL;
8156 r600_ring_init(rdev, ring, 4096);
8157
8158 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8159 ring->ring_obj = NULL;
8160 r600_ring_init(rdev, ring, 4096);
8161 }
8162
8163 rdev->ih.ring_obj = NULL;
8164 r600_ih_ring_init(rdev, 64 * 1024);
8165
8166 r = r600_pcie_gart_init(rdev);
8167 if (r)
8168 return r;
8169
8170 rdev->accel_working = true;
8171 r = cik_startup(rdev);
8172 if (r) {
8173 dev_err(rdev->dev, "disabling GPU acceleration\n");
8174 cik_cp_fini(rdev);
8175 cik_sdma_fini(rdev);
8176 cik_irq_fini(rdev);
8177 sumo_rlc_fini(rdev);
8178 cik_mec_fini(rdev);
8179 radeon_wb_fini(rdev);
8180 radeon_ib_pool_fini(rdev);
8181 radeon_vm_manager_fini(rdev);
8182 radeon_irq_kms_fini(rdev);
8183 cik_pcie_gart_fini(rdev);
8184 rdev->accel_working = false;
8185 }
8186
8187 /* Don't start up if the MC ucode is missing.
8188 * The default clocks and voltages before the MC ucode
8189 * is loaded are not suffient for advanced operations.
8190 */
8191 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8192 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8193 return -EINVAL;
8194 }
8195
8196 return 0;
8197 }
8198
8199 /**
8200 * cik_fini - asic specific driver and hw fini
8201 *
8202 * @rdev: radeon_device pointer
8203 *
8204 * Tear down the asic specific driver variables and program the hw
8205 * to an idle state (CIK).
8206 * Called at driver unload.
8207 */
8208 void cik_fini(struct radeon_device *rdev)
8209 {
8210 radeon_pm_fini(rdev);
8211 cik_cp_fini(rdev);
8212 cik_sdma_fini(rdev);
8213 cik_fini_pg(rdev);
8214 cik_fini_cg(rdev);
8215 cik_irq_fini(rdev);
8216 sumo_rlc_fini(rdev);
8217 cik_mec_fini(rdev);
8218 radeon_wb_fini(rdev);
8219 radeon_vm_manager_fini(rdev);
8220 radeon_ib_pool_fini(rdev);
8221 radeon_irq_kms_fini(rdev);
8222 uvd_v1_0_fini(rdev);
8223 radeon_uvd_fini(rdev);
8224 radeon_vce_fini(rdev);
8225 cik_pcie_gart_fini(rdev);
8226 r600_vram_scratch_fini(rdev);
8227 radeon_gem_fini(rdev);
8228 radeon_fence_driver_fini(rdev);
8229 radeon_bo_fini(rdev);
8230 radeon_atombios_fini(rdev);
8231 kfree(rdev->bios);
8232 rdev->bios = NULL;
8233 }
8234
8235 void dce8_program_fmt(struct drm_encoder *encoder)
8236 {
8237 struct drm_device *dev = encoder->dev;
8238 struct radeon_device *rdev = dev->dev_private;
8239 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8240 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8241 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8242 int bpc = 0;
8243 u32 tmp = 0;
8244 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8245
8246 if (connector) {
8247 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8248 bpc = radeon_get_monitor_bpc(connector);
8249 dither = radeon_connector->dither;
8250 }
8251
8252 /* LVDS/eDP FMT is set up by atom */
8253 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8254 return;
8255
8256 /* not needed for analog */
8257 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8258 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8259 return;
8260
8261 if (bpc == 0)
8262 return;
8263
8264 switch (bpc) {
8265 case 6:
8266 if (dither == RADEON_FMT_DITHER_ENABLE)
8267 /* XXX sort out optimal dither settings */
8268 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8269 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8270 else
8271 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8272 break;
8273 case 8:
8274 if (dither == RADEON_FMT_DITHER_ENABLE)
8275 /* XXX sort out optimal dither settings */
8276 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8277 FMT_RGB_RANDOM_ENABLE |
8278 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8279 else
8280 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8281 break;
8282 case 10:
8283 if (dither == RADEON_FMT_DITHER_ENABLE)
8284 /* XXX sort out optimal dither settings */
8285 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8286 FMT_RGB_RANDOM_ENABLE |
8287 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8288 else
8289 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8290 break;
8291 default:
8292 /* not needed */
8293 break;
8294 }
8295
8296 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8297 }
8298
8299 /* display watermark setup */
8300 /**
8301 * dce8_line_buffer_adjust - Set up the line buffer
8302 *
8303 * @rdev: radeon_device pointer
8304 * @radeon_crtc: the selected display controller
8305 * @mode: the current display mode on the selected display
8306 * controller
8307 *
8308 * Setup up the line buffer allocation for
8309 * the selected display controller (CIK).
8310 * Returns the line buffer size in pixels.
8311 */
8312 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8313 struct radeon_crtc *radeon_crtc,
8314 struct drm_display_mode *mode)
8315 {
8316 u32 tmp, buffer_alloc, i;
8317 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8318 /*
8319 * Line Buffer Setup
8320 * There are 6 line buffers, one for each display controllers.
8321 * There are 3 partitions per LB. Select the number of partitions
8322 * to enable based on the display width. For display widths larger
8323 * than 4096, you need use to use 2 display controllers and combine
8324 * them using the stereo blender.
8325 */
8326 if (radeon_crtc->base.enabled && mode) {
8327 if (mode->crtc_hdisplay < 1920) {
8328 tmp = 1;
8329 buffer_alloc = 2;
8330 } else if (mode->crtc_hdisplay < 2560) {
8331 tmp = 2;
8332 buffer_alloc = 2;
8333 } else if (mode->crtc_hdisplay < 4096) {
8334 tmp = 0;
8335 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8336 } else {
8337 DRM_DEBUG_KMS("Mode too big for LB!\n");
8338 tmp = 0;
8339 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8340 }
8341 } else {
8342 tmp = 1;
8343 buffer_alloc = 0;
8344 }
8345
8346 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8347 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8348
8349 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8350 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8351 for (i = 0; i < rdev->usec_timeout; i++) {
8352 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8353 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8354 break;
8355 udelay(1);
8356 }
8357
8358 if (radeon_crtc->base.enabled && mode) {
8359 switch (tmp) {
8360 case 0:
8361 default:
8362 return 4096 * 2;
8363 case 1:
8364 return 1920 * 2;
8365 case 2:
8366 return 2560 * 2;
8367 }
8368 }
8369
8370 /* controller not enabled, so no lb used */
8371 return 0;
8372 }
8373
8374 /**
8375 * cik_get_number_of_dram_channels - get the number of dram channels
8376 *
8377 * @rdev: radeon_device pointer
8378 *
8379 * Look up the number of video ram channels (CIK).
8380 * Used for display watermark bandwidth calculations
8381 * Returns the number of dram channels
8382 */
8383 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8384 {
8385 u32 tmp = RREG32(MC_SHARED_CHMAP);
8386
8387 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8388 case 0:
8389 default:
8390 return 1;
8391 case 1:
8392 return 2;
8393 case 2:
8394 return 4;
8395 case 3:
8396 return 8;
8397 case 4:
8398 return 3;
8399 case 5:
8400 return 6;
8401 case 6:
8402 return 10;
8403 case 7:
8404 return 12;
8405 case 8:
8406 return 16;
8407 }
8408 }
8409
8410 struct dce8_wm_params {
8411 u32 dram_channels; /* number of dram channels */
8412 u32 yclk; /* bandwidth per dram data pin in kHz */
8413 u32 sclk; /* engine clock in kHz */
8414 u32 disp_clk; /* display clock in kHz */
8415 u32 src_width; /* viewport width */
8416 u32 active_time; /* active display time in ns */
8417 u32 blank_time; /* blank time in ns */
8418 bool interlaced; /* mode is interlaced */
8419 fixed20_12 vsc; /* vertical scale ratio */
8420 u32 num_heads; /* number of active crtcs */
8421 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8422 u32 lb_size; /* line buffer allocated to pipe */
8423 u32 vtaps; /* vertical scaler taps */
8424 };
8425
8426 /**
8427 * dce8_dram_bandwidth - get the dram bandwidth
8428 *
8429 * @wm: watermark calculation data
8430 *
8431 * Calculate the raw dram bandwidth (CIK).
8432 * Used for display watermark bandwidth calculations
8433 * Returns the dram bandwidth in MBytes/s
8434 */
8435 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8436 {
8437 /* Calculate raw DRAM Bandwidth */
8438 fixed20_12 dram_efficiency; /* 0.7 */
8439 fixed20_12 yclk, dram_channels, bandwidth;
8440 fixed20_12 a;
8441
8442 a.full = dfixed_const(1000);
8443 yclk.full = dfixed_const(wm->yclk);
8444 yclk.full = dfixed_div(yclk, a);
8445 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8446 a.full = dfixed_const(10);
8447 dram_efficiency.full = dfixed_const(7);
8448 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8449 bandwidth.full = dfixed_mul(dram_channels, yclk);
8450 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8451
8452 return dfixed_trunc(bandwidth);
8453 }
8454
8455 /**
8456 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8457 *
8458 * @wm: watermark calculation data
8459 *
8460 * Calculate the dram bandwidth used for display (CIK).
8461 * Used for display watermark bandwidth calculations
8462 * Returns the dram bandwidth for display in MBytes/s
8463 */
8464 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8465 {
8466 /* Calculate DRAM Bandwidth and the part allocated to display. */
8467 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8468 fixed20_12 yclk, dram_channels, bandwidth;
8469 fixed20_12 a;
8470
8471 a.full = dfixed_const(1000);
8472 yclk.full = dfixed_const(wm->yclk);
8473 yclk.full = dfixed_div(yclk, a);
8474 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8475 a.full = dfixed_const(10);
8476 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8477 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8478 bandwidth.full = dfixed_mul(dram_channels, yclk);
8479 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8480
8481 return dfixed_trunc(bandwidth);
8482 }
8483
8484 /**
8485 * dce8_data_return_bandwidth - get the data return bandwidth
8486 *
8487 * @wm: watermark calculation data
8488 *
8489 * Calculate the data return bandwidth used for display (CIK).
8490 * Used for display watermark bandwidth calculations
8491 * Returns the data return bandwidth in MBytes/s
8492 */
8493 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8494 {
8495 /* Calculate the display Data return Bandwidth */
8496 fixed20_12 return_efficiency; /* 0.8 */
8497 fixed20_12 sclk, bandwidth;
8498 fixed20_12 a;
8499
8500 a.full = dfixed_const(1000);
8501 sclk.full = dfixed_const(wm->sclk);
8502 sclk.full = dfixed_div(sclk, a);
8503 a.full = dfixed_const(10);
8504 return_efficiency.full = dfixed_const(8);
8505 return_efficiency.full = dfixed_div(return_efficiency, a);
8506 a.full = dfixed_const(32);
8507 bandwidth.full = dfixed_mul(a, sclk);
8508 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8509
8510 return dfixed_trunc(bandwidth);
8511 }
8512
8513 /**
8514 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8515 *
8516 * @wm: watermark calculation data
8517 *
8518 * Calculate the dmif bandwidth used for display (CIK).
8519 * Used for display watermark bandwidth calculations
8520 * Returns the dmif bandwidth in MBytes/s
8521 */
8522 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8523 {
8524 /* Calculate the DMIF Request Bandwidth */
8525 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8526 fixed20_12 disp_clk, bandwidth;
8527 fixed20_12 a, b;
8528
8529 a.full = dfixed_const(1000);
8530 disp_clk.full = dfixed_const(wm->disp_clk);
8531 disp_clk.full = dfixed_div(disp_clk, a);
8532 a.full = dfixed_const(32);
8533 b.full = dfixed_mul(a, disp_clk);
8534
8535 a.full = dfixed_const(10);
8536 disp_clk_request_efficiency.full = dfixed_const(8);
8537 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8538
8539 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8540
8541 return dfixed_trunc(bandwidth);
8542 }
8543
8544 /**
8545 * dce8_available_bandwidth - get the min available bandwidth
8546 *
8547 * @wm: watermark calculation data
8548 *
8549 * Calculate the min available bandwidth used for display (CIK).
8550 * Used for display watermark bandwidth calculations
8551 * Returns the min available bandwidth in MBytes/s
8552 */
8553 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8554 {
8555 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8556 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8557 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8558 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8559
8560 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8561 }
8562
8563 /**
8564 * dce8_average_bandwidth - get the average available bandwidth
8565 *
8566 * @wm: watermark calculation data
8567 *
8568 * Calculate the average available bandwidth used for display (CIK).
8569 * Used for display watermark bandwidth calculations
8570 * Returns the average available bandwidth in MBytes/s
8571 */
8572 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8573 {
8574 /* Calculate the display mode Average Bandwidth
8575 * DisplayMode should contain the source and destination dimensions,
8576 * timing, etc.
8577 */
8578 fixed20_12 bpp;
8579 fixed20_12 line_time;
8580 fixed20_12 src_width;
8581 fixed20_12 bandwidth;
8582 fixed20_12 a;
8583
8584 a.full = dfixed_const(1000);
8585 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8586 line_time.full = dfixed_div(line_time, a);
8587 bpp.full = dfixed_const(wm->bytes_per_pixel);
8588 src_width.full = dfixed_const(wm->src_width);
8589 bandwidth.full = dfixed_mul(src_width, bpp);
8590 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8591 bandwidth.full = dfixed_div(bandwidth, line_time);
8592
8593 return dfixed_trunc(bandwidth);
8594 }
8595
8596 /**
8597 * dce8_latency_watermark - get the latency watermark
8598 *
8599 * @wm: watermark calculation data
8600 *
8601 * Calculate the latency watermark (CIK).
8602 * Used for display watermark bandwidth calculations
8603 * Returns the latency watermark in ns
8604 */
8605 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8606 {
8607 /* First calculate the latency in ns */
8608 u32 mc_latency = 2000; /* 2000 ns. */
8609 u32 available_bandwidth = dce8_available_bandwidth(wm);
8610 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8611 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8612 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8613 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8614 (wm->num_heads * cursor_line_pair_return_time);
8615 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8616 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8617 u32 tmp, dmif_size = 12288;
8618 fixed20_12 a, b, c;
8619
8620 if (wm->num_heads == 0)
8621 return 0;
8622
8623 a.full = dfixed_const(2);
8624 b.full = dfixed_const(1);
8625 if ((wm->vsc.full > a.full) ||
8626 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8627 (wm->vtaps >= 5) ||
8628 ((wm->vsc.full >= a.full) && wm->interlaced))
8629 max_src_lines_per_dst_line = 4;
8630 else
8631 max_src_lines_per_dst_line = 2;
8632
8633 a.full = dfixed_const(available_bandwidth);
8634 b.full = dfixed_const(wm->num_heads);
8635 a.full = dfixed_div(a, b);
8636
8637 b.full = dfixed_const(mc_latency + 512);
8638 c.full = dfixed_const(wm->disp_clk);
8639 b.full = dfixed_div(b, c);
8640
8641 c.full = dfixed_const(dmif_size);
8642 b.full = dfixed_div(c, b);
8643
8644 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8645
8646 b.full = dfixed_const(1000);
8647 c.full = dfixed_const(wm->disp_clk);
8648 b.full = dfixed_div(c, b);
8649 c.full = dfixed_const(wm->bytes_per_pixel);
8650 b.full = dfixed_mul(b, c);
8651
8652 lb_fill_bw = min(tmp, dfixed_trunc(b));
8653
8654 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8655 b.full = dfixed_const(1000);
8656 c.full = dfixed_const(lb_fill_bw);
8657 b.full = dfixed_div(c, b);
8658 a.full = dfixed_div(a, b);
8659 line_fill_time = dfixed_trunc(a);
8660
8661 if (line_fill_time < wm->active_time)
8662 return latency;
8663 else
8664 return latency + (line_fill_time - wm->active_time);
8665
8666 }
8667
8668 /**
8669 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8670 * average and available dram bandwidth
8671 *
8672 * @wm: watermark calculation data
8673 *
8674 * Check if the display average bandwidth fits in the display
8675 * dram bandwidth (CIK).
8676 * Used for display watermark bandwidth calculations
8677 * Returns true if the display fits, false if not.
8678 */
8679 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8680 {
8681 if (dce8_average_bandwidth(wm) <=
8682 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8683 return true;
8684 else
8685 return false;
8686 }
8687
8688 /**
8689 * dce8_average_bandwidth_vs_available_bandwidth - check
8690 * average and available bandwidth
8691 *
8692 * @wm: watermark calculation data
8693 *
8694 * Check if the display average bandwidth fits in the display
8695 * available bandwidth (CIK).
8696 * Used for display watermark bandwidth calculations
8697 * Returns true if the display fits, false if not.
8698 */
8699 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8700 {
8701 if (dce8_average_bandwidth(wm) <=
8702 (dce8_available_bandwidth(wm) / wm->num_heads))
8703 return true;
8704 else
8705 return false;
8706 }
8707
8708 /**
8709 * dce8_check_latency_hiding - check latency hiding
8710 *
8711 * @wm: watermark calculation data
8712 *
8713 * Check latency hiding (CIK).
8714 * Used for display watermark bandwidth calculations
8715 * Returns true if the display fits, false if not.
8716 */
8717 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8718 {
8719 u32 lb_partitions = wm->lb_size / wm->src_width;
8720 u32 line_time = wm->active_time + wm->blank_time;
8721 u32 latency_tolerant_lines;
8722 u32 latency_hiding;
8723 fixed20_12 a;
8724
8725 a.full = dfixed_const(1);
8726 if (wm->vsc.full > a.full)
8727 latency_tolerant_lines = 1;
8728 else {
8729 if (lb_partitions <= (wm->vtaps + 1))
8730 latency_tolerant_lines = 1;
8731 else
8732 latency_tolerant_lines = 2;
8733 }
8734
8735 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8736
8737 if (dce8_latency_watermark(wm) <= latency_hiding)
8738 return true;
8739 else
8740 return false;
8741 }
8742
8743 /**
8744 * dce8_program_watermarks - program display watermarks
8745 *
8746 * @rdev: radeon_device pointer
8747 * @radeon_crtc: the selected display controller
8748 * @lb_size: line buffer size
8749 * @num_heads: number of display controllers in use
8750 *
8751 * Calculate and program the display watermarks for the
8752 * selected display controller (CIK).
8753 */
8754 static void dce8_program_watermarks(struct radeon_device *rdev,
8755 struct radeon_crtc *radeon_crtc,
8756 u32 lb_size, u32 num_heads)
8757 {
8758 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8759 struct dce8_wm_params wm_low, wm_high;
8760 u32 pixel_period;
8761 u32 line_time = 0;
8762 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8763 u32 tmp, wm_mask;
8764
8765 if (radeon_crtc->base.enabled && num_heads && mode) {
8766 pixel_period = 1000000 / (u32)mode->clock;
8767 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8768
8769 /* watermark for high clocks */
8770 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8771 rdev->pm.dpm_enabled) {
8772 wm_high.yclk =
8773 radeon_dpm_get_mclk(rdev, false) * 10;
8774 wm_high.sclk =
8775 radeon_dpm_get_sclk(rdev, false) * 10;
8776 } else {
8777 wm_high.yclk = rdev->pm.current_mclk * 10;
8778 wm_high.sclk = rdev->pm.current_sclk * 10;
8779 }
8780
8781 wm_high.disp_clk = mode->clock;
8782 wm_high.src_width = mode->crtc_hdisplay;
8783 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8784 wm_high.blank_time = line_time - wm_high.active_time;
8785 wm_high.interlaced = false;
8786 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8787 wm_high.interlaced = true;
8788 wm_high.vsc = radeon_crtc->vsc;
8789 wm_high.vtaps = 1;
8790 if (radeon_crtc->rmx_type != RMX_OFF)
8791 wm_high.vtaps = 2;
8792 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8793 wm_high.lb_size = lb_size;
8794 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8795 wm_high.num_heads = num_heads;
8796
8797 /* set for high clocks */
8798 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8799
8800 /* possibly force display priority to high */
8801 /* should really do this at mode validation time... */
8802 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8803 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8804 !dce8_check_latency_hiding(&wm_high) ||
8805 (rdev->disp_priority == 2)) {
8806 DRM_DEBUG_KMS("force priority to high\n");
8807 }
8808
8809 /* watermark for low clocks */
8810 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8811 rdev->pm.dpm_enabled) {
8812 wm_low.yclk =
8813 radeon_dpm_get_mclk(rdev, true) * 10;
8814 wm_low.sclk =
8815 radeon_dpm_get_sclk(rdev, true) * 10;
8816 } else {
8817 wm_low.yclk = rdev->pm.current_mclk * 10;
8818 wm_low.sclk = rdev->pm.current_sclk * 10;
8819 }
8820
8821 wm_low.disp_clk = mode->clock;
8822 wm_low.src_width = mode->crtc_hdisplay;
8823 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8824 wm_low.blank_time = line_time - wm_low.active_time;
8825 wm_low.interlaced = false;
8826 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8827 wm_low.interlaced = true;
8828 wm_low.vsc = radeon_crtc->vsc;
8829 wm_low.vtaps = 1;
8830 if (radeon_crtc->rmx_type != RMX_OFF)
8831 wm_low.vtaps = 2;
8832 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8833 wm_low.lb_size = lb_size;
8834 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8835 wm_low.num_heads = num_heads;
8836
8837 /* set for low clocks */
8838 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8839
8840 /* possibly force display priority to high */
8841 /* should really do this at mode validation time... */
8842 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8843 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8844 !dce8_check_latency_hiding(&wm_low) ||
8845 (rdev->disp_priority == 2)) {
8846 DRM_DEBUG_KMS("force priority to high\n");
8847 }
8848 }
8849
8850 /* select wm A */
8851 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8852 tmp = wm_mask;
8853 tmp &= ~LATENCY_WATERMARK_MASK(3);
8854 tmp |= LATENCY_WATERMARK_MASK(1);
8855 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8856 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8857 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8858 LATENCY_HIGH_WATERMARK(line_time)));
8859 /* select wm B */
8860 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8861 tmp &= ~LATENCY_WATERMARK_MASK(3);
8862 tmp |= LATENCY_WATERMARK_MASK(2);
8863 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8864 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8865 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8866 LATENCY_HIGH_WATERMARK(line_time)));
8867 /* restore original selection */
8868 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8869
8870 /* save values for DPM */
8871 radeon_crtc->line_time = line_time;
8872 radeon_crtc->wm_high = latency_watermark_a;
8873 radeon_crtc->wm_low = latency_watermark_b;
8874 }
8875
8876 /**
8877 * dce8_bandwidth_update - program display watermarks
8878 *
8879 * @rdev: radeon_device pointer
8880 *
8881 * Calculate and program the display watermarks and line
8882 * buffer allocation (CIK).
8883 */
8884 void dce8_bandwidth_update(struct radeon_device *rdev)
8885 {
8886 struct drm_display_mode *mode = NULL;
8887 u32 num_heads = 0, lb_size;
8888 int i;
8889
8890 radeon_update_display_priority(rdev);
8891
8892 for (i = 0; i < rdev->num_crtc; i++) {
8893 if (rdev->mode_info.crtcs[i]->base.enabled)
8894 num_heads++;
8895 }
8896 for (i = 0; i < rdev->num_crtc; i++) {
8897 mode = &rdev->mode_info.crtcs[i]->base.mode;
8898 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8899 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8900 }
8901 }
8902
8903 /**
8904 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8905 *
8906 * @rdev: radeon_device pointer
8907 *
8908 * Fetches a GPU clock counter snapshot (SI).
8909 * Returns the 64 bit clock counter snapshot.
8910 */
8911 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8912 {
8913 uint64_t clock;
8914
8915 mutex_lock(&rdev->gpu_clock_mutex);
8916 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8917 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8918 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8919 mutex_unlock(&rdev->gpu_clock_mutex);
8920 return clock;
8921 }
8922
8923 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8924 u32 cntl_reg, u32 status_reg)
8925 {
8926 int r, i;
8927 struct atom_clock_dividers dividers;
8928 uint32_t tmp;
8929
8930 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8931 clock, false, &dividers);
8932 if (r)
8933 return r;
8934
8935 tmp = RREG32_SMC(cntl_reg);
8936 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8937 tmp |= dividers.post_divider;
8938 WREG32_SMC(cntl_reg, tmp);
8939
8940 for (i = 0; i < 100; i++) {
8941 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8942 break;
8943 mdelay(10);
8944 }
8945 if (i == 100)
8946 return -ETIMEDOUT;
8947
8948 return 0;
8949 }
8950
8951 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8952 {
8953 int r = 0;
8954
8955 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8956 if (r)
8957 return r;
8958
8959 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8960 return r;
8961 }
8962
8963 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
8964 {
8965 int r, i;
8966 struct atom_clock_dividers dividers;
8967 u32 tmp;
8968
8969 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8970 ecclk, false, &dividers);
8971 if (r)
8972 return r;
8973
8974 for (i = 0; i < 100; i++) {
8975 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8976 break;
8977 mdelay(10);
8978 }
8979 if (i == 100)
8980 return -ETIMEDOUT;
8981
8982 tmp = RREG32_SMC(CG_ECLK_CNTL);
8983 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
8984 tmp |= dividers.post_divider;
8985 WREG32_SMC(CG_ECLK_CNTL, tmp);
8986
8987 for (i = 0; i < 100; i++) {
8988 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8989 break;
8990 mdelay(10);
8991 }
8992 if (i == 100)
8993 return -ETIMEDOUT;
8994
8995 return 0;
8996 }
8997
8998 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8999 {
9000 struct pci_dev *root = rdev->pdev->bus->self;
9001 int bridge_pos, gpu_pos;
9002 u32 speed_cntl, mask, current_data_rate;
9003 int ret, i;
9004 u16 tmp16;
9005
9006 if (radeon_pcie_gen2 == 0)
9007 return;
9008
9009 if (rdev->flags & RADEON_IS_IGP)
9010 return;
9011
9012 if (!(rdev->flags & RADEON_IS_PCIE))
9013 return;
9014
9015 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9016 if (ret != 0)
9017 return;
9018
9019 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9020 return;
9021
9022 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9023 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9024 LC_CURRENT_DATA_RATE_SHIFT;
9025 if (mask & DRM_PCIE_SPEED_80) {
9026 if (current_data_rate == 2) {
9027 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9028 return;
9029 }
9030 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9031 } else if (mask & DRM_PCIE_SPEED_50) {
9032 if (current_data_rate == 1) {
9033 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9034 return;
9035 }
9036 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9037 }
9038
9039 bridge_pos = pci_pcie_cap(root);
9040 if (!bridge_pos)
9041 return;
9042
9043 gpu_pos = pci_pcie_cap(rdev->pdev);
9044 if (!gpu_pos)
9045 return;
9046
9047 if (mask & DRM_PCIE_SPEED_80) {
9048 /* re-try equalization if gen3 is not already enabled */
9049 if (current_data_rate != 2) {
9050 u16 bridge_cfg, gpu_cfg;
9051 u16 bridge_cfg2, gpu_cfg2;
9052 u32 max_lw, current_lw, tmp;
9053
9054 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9055 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9056
9057 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9058 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9059
9060 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9061 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9062
9063 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9064 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9065 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9066
9067 if (current_lw < max_lw) {
9068 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9069 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9070 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9071 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9072 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9073 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9074 }
9075 }
9076
9077 for (i = 0; i < 10; i++) {
9078 /* check status */
9079 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9080 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9081 break;
9082
9083 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9084 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9085
9086 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9087 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9088
9089 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9090 tmp |= LC_SET_QUIESCE;
9091 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9092
9093 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9094 tmp |= LC_REDO_EQ;
9095 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9096
9097 mdelay(100);
9098
9099 /* linkctl */
9100 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9101 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9102 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9103 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9104
9105 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9106 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9107 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9108 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9109
9110 /* linkctl2 */
9111 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9112 tmp16 &= ~((1 << 4) | (7 << 9));
9113 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9114 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9115
9116 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9117 tmp16 &= ~((1 << 4) | (7 << 9));
9118 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9119 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9120
9121 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9122 tmp &= ~LC_SET_QUIESCE;
9123 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9124 }
9125 }
9126 }
9127
9128 /* set the link speed */
9129 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9130 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9131 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9132
9133 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9134 tmp16 &= ~0xf;
9135 if (mask & DRM_PCIE_SPEED_80)
9136 tmp16 |= 3; /* gen3 */
9137 else if (mask & DRM_PCIE_SPEED_50)
9138 tmp16 |= 2; /* gen2 */
9139 else
9140 tmp16 |= 1; /* gen1 */
9141 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9142
9143 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9144 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9145 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9146
9147 for (i = 0; i < rdev->usec_timeout; i++) {
9148 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9149 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9150 break;
9151 udelay(1);
9152 }
9153 }
9154
9155 static void cik_program_aspm(struct radeon_device *rdev)
9156 {
9157 u32 data, orig;
9158 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9159 bool disable_clkreq = false;
9160
9161 if (radeon_aspm == 0)
9162 return;
9163
9164 /* XXX double check IGPs */
9165 if (rdev->flags & RADEON_IS_IGP)
9166 return;
9167
9168 if (!(rdev->flags & RADEON_IS_PCIE))
9169 return;
9170
9171 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9172 data &= ~LC_XMIT_N_FTS_MASK;
9173 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9174 if (orig != data)
9175 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9176
9177 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9178 data |= LC_GO_TO_RECOVERY;
9179 if (orig != data)
9180 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9181
9182 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9183 data |= P_IGNORE_EDB_ERR;
9184 if (orig != data)
9185 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9186
9187 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9188 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9189 data |= LC_PMI_TO_L1_DIS;
9190 if (!disable_l0s)
9191 data |= LC_L0S_INACTIVITY(7);
9192
9193 if (!disable_l1) {
9194 data |= LC_L1_INACTIVITY(7);
9195 data &= ~LC_PMI_TO_L1_DIS;
9196 if (orig != data)
9197 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9198
9199 if (!disable_plloff_in_l1) {
9200 bool clk_req_support;
9201
9202 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9203 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9204 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9205 if (orig != data)
9206 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9207
9208 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9209 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9210 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9211 if (orig != data)
9212 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9213
9214 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9215 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9216 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9217 if (orig != data)
9218 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9219
9220 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9221 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9222 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9223 if (orig != data)
9224 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9225
9226 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9227 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9228 data |= LC_DYN_LANES_PWR_STATE(3);
9229 if (orig != data)
9230 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9231
9232 if (!disable_clkreq) {
9233 struct pci_dev *root = rdev->pdev->bus->self;
9234 u32 lnkcap;
9235
9236 clk_req_support = false;
9237 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9238 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9239 clk_req_support = true;
9240 } else {
9241 clk_req_support = false;
9242 }
9243
9244 if (clk_req_support) {
9245 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9246 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9247 if (orig != data)
9248 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9249
9250 orig = data = RREG32_SMC(THM_CLK_CNTL);
9251 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9252 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9253 if (orig != data)
9254 WREG32_SMC(THM_CLK_CNTL, data);
9255
9256 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9257 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9258 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9259 if (orig != data)
9260 WREG32_SMC(MISC_CLK_CTRL, data);
9261
9262 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9263 data &= ~BCLK_AS_XCLK;
9264 if (orig != data)
9265 WREG32_SMC(CG_CLKPIN_CNTL, data);
9266
9267 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9268 data &= ~FORCE_BIF_REFCLK_EN;
9269 if (orig != data)
9270 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9271
9272 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9273 data &= ~MPLL_CLKOUT_SEL_MASK;
9274 data |= MPLL_CLKOUT_SEL(4);
9275 if (orig != data)
9276 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9277 }
9278 }
9279 } else {
9280 if (orig != data)
9281 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9282 }
9283
9284 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9285 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9286 if (orig != data)
9287 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9288
9289 if (!disable_l0s) {
9290 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9291 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9292 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9293 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9294 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9295 data &= ~LC_L0S_INACTIVITY_MASK;
9296 if (orig != data)
9297 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9298 }
9299 }
9300 }
9301 }
Linux kernel - Deliverables
This page took 0.698691 seconds and 6 git commands to generate.