projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/radeon: fix typo in PG flags
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
35
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
49 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
50 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51 MODULE_FIRMWARE("radeon/KABINI_me.bin");
52 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
55 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
56
57 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58 extern void r600_ih_ring_fini(struct radeon_device *rdev);
59 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
61 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
62 extern void sumo_rlc_fini(struct radeon_device *rdev);
63 extern int sumo_rlc_init(struct radeon_device *rdev);
64 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
65 extern void si_rlc_reset(struct radeon_device *rdev);
66 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
67 extern int cik_sdma_resume(struct radeon_device *rdev);
68 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69 extern void cik_sdma_fini(struct radeon_device *rdev);
70 extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
71 struct radeon_ib *ib,
72 uint64_t pe,
73 uint64_t addr, unsigned count,
74 uint32_t incr, uint32_t flags);
75 static void cik_rlc_stop(struct radeon_device *rdev);
76 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
77 static void cik_program_aspm(struct radeon_device *rdev);
78 static void cik_init_pg(struct radeon_device *rdev);
79 static void cik_init_cg(struct radeon_device *rdev);
80 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
81 bool enable);
82
83 /* get temperature in millidegrees */
84 int ci_get_temp(struct radeon_device *rdev)
85 {
86 u32 temp;
87 int actual_temp = 0;
88
89 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
90 CTF_TEMP_SHIFT;
91
92 if (temp & 0x200)
93 actual_temp = 255;
94 else
95 actual_temp = temp & 0x1ff;
96
97 actual_temp = actual_temp * 1000;
98
99 return actual_temp;
100 }
101
102 /* get temperature in millidegrees */
103 int kv_get_temp(struct radeon_device *rdev)
104 {
105 u32 temp;
106 int actual_temp = 0;
107
108 temp = RREG32_SMC(0xC0300E0C);
109
110 if (temp)
111 actual_temp = (temp / 8) - 49;
112 else
113 actual_temp = 0;
114
115 actual_temp = actual_temp * 1000;
116
117 return actual_temp;
118 }
119
120 /*
121 * Indirect registers accessor
122 */
123 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
124 {
125 unsigned long flags;
126 u32 r;
127
128 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
129 WREG32(PCIE_INDEX, reg);
130 (void)RREG32(PCIE_INDEX);
131 r = RREG32(PCIE_DATA);
132 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
133 return r;
134 }
135
136 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
137 {
138 unsigned long flags;
139
140 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
141 WREG32(PCIE_INDEX, reg);
142 (void)RREG32(PCIE_INDEX);
143 WREG32(PCIE_DATA, v);
144 (void)RREG32(PCIE_DATA);
145 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
146 }
147
148 static const u32 spectre_rlc_save_restore_register_list[] =
149 {
150 (0x0e00 << 16) | (0xc12c >> 2),
151 0x00000000,
152 (0x0e00 << 16) | (0xc140 >> 2),
153 0x00000000,
154 (0x0e00 << 16) | (0xc150 >> 2),
155 0x00000000,
156 (0x0e00 << 16) | (0xc15c >> 2),
157 0x00000000,
158 (0x0e00 << 16) | (0xc168 >> 2),
159 0x00000000,
160 (0x0e00 << 16) | (0xc170 >> 2),
161 0x00000000,
162 (0x0e00 << 16) | (0xc178 >> 2),
163 0x00000000,
164 (0x0e00 << 16) | (0xc204 >> 2),
165 0x00000000,
166 (0x0e00 << 16) | (0xc2b4 >> 2),
167 0x00000000,
168 (0x0e00 << 16) | (0xc2b8 >> 2),
169 0x00000000,
170 (0x0e00 << 16) | (0xc2bc >> 2),
171 0x00000000,
172 (0x0e00 << 16) | (0xc2c0 >> 2),
173 0x00000000,
174 (0x0e00 << 16) | (0x8228 >> 2),
175 0x00000000,
176 (0x0e00 << 16) | (0x829c >> 2),
177 0x00000000,
178 (0x0e00 << 16) | (0x869c >> 2),
179 0x00000000,
180 (0x0600 << 16) | (0x98f4 >> 2),
181 0x00000000,
182 (0x0e00 << 16) | (0x98f8 >> 2),
183 0x00000000,
184 (0x0e00 << 16) | (0x9900 >> 2),
185 0x00000000,
186 (0x0e00 << 16) | (0xc260 >> 2),
187 0x00000000,
188 (0x0e00 << 16) | (0x90e8 >> 2),
189 0x00000000,
190 (0x0e00 << 16) | (0x3c000 >> 2),
191 0x00000000,
192 (0x0e00 << 16) | (0x3c00c >> 2),
193 0x00000000,
194 (0x0e00 << 16) | (0x8c1c >> 2),
195 0x00000000,
196 (0x0e00 << 16) | (0x9700 >> 2),
197 0x00000000,
198 (0x0e00 << 16) | (0xcd20 >> 2),
199 0x00000000,
200 (0x4e00 << 16) | (0xcd20 >> 2),
201 0x00000000,
202 (0x5e00 << 16) | (0xcd20 >> 2),
203 0x00000000,
204 (0x6e00 << 16) | (0xcd20 >> 2),
205 0x00000000,
206 (0x7e00 << 16) | (0xcd20 >> 2),
207 0x00000000,
208 (0x8e00 << 16) | (0xcd20 >> 2),
209 0x00000000,
210 (0x9e00 << 16) | (0xcd20 >> 2),
211 0x00000000,
212 (0xae00 << 16) | (0xcd20 >> 2),
213 0x00000000,
214 (0xbe00 << 16) | (0xcd20 >> 2),
215 0x00000000,
216 (0x0e00 << 16) | (0x89bc >> 2),
217 0x00000000,
218 (0x0e00 << 16) | (0x8900 >> 2),
219 0x00000000,
220 0x3,
221 (0x0e00 << 16) | (0xc130 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc134 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc1fc >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc208 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc264 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc268 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc26c >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0xc270 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0xc274 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xc278 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0xc27c >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0xc280 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc284 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc288 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0xc28c >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0xc290 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0xc294 >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0xc298 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xc29c >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xc2a0 >> 2),
260 0x00000000,
261 (0x0e00 << 16) | (0xc2a4 >> 2),
262 0x00000000,
263 (0x0e00 << 16) | (0xc2a8 >> 2),
264 0x00000000,
265 (0x0e00 << 16) | (0xc2ac >> 2),
266 0x00000000,
267 (0x0e00 << 16) | (0xc2b0 >> 2),
268 0x00000000,
269 (0x0e00 << 16) | (0x301d0 >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0x30238 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0x30250 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x30254 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x30258 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x3025c >> 2),
280 0x00000000,
281 (0x4e00 << 16) | (0xc900 >> 2),
282 0x00000000,
283 (0x5e00 << 16) | (0xc900 >> 2),
284 0x00000000,
285 (0x6e00 << 16) | (0xc900 >> 2),
286 0x00000000,
287 (0x7e00 << 16) | (0xc900 >> 2),
288 0x00000000,
289 (0x8e00 << 16) | (0xc900 >> 2),
290 0x00000000,
291 (0x9e00 << 16) | (0xc900 >> 2),
292 0x00000000,
293 (0xae00 << 16) | (0xc900 >> 2),
294 0x00000000,
295 (0xbe00 << 16) | (0xc900 >> 2),
296 0x00000000,
297 (0x4e00 << 16) | (0xc904 >> 2),
298 0x00000000,
299 (0x5e00 << 16) | (0xc904 >> 2),
300 0x00000000,
301 (0x6e00 << 16) | (0xc904 >> 2),
302 0x00000000,
303 (0x7e00 << 16) | (0xc904 >> 2),
304 0x00000000,
305 (0x8e00 << 16) | (0xc904 >> 2),
306 0x00000000,
307 (0x9e00 << 16) | (0xc904 >> 2),
308 0x00000000,
309 (0xae00 << 16) | (0xc904 >> 2),
310 0x00000000,
311 (0xbe00 << 16) | (0xc904 >> 2),
312 0x00000000,
313 (0x4e00 << 16) | (0xc908 >> 2),
314 0x00000000,
315 (0x5e00 << 16) | (0xc908 >> 2),
316 0x00000000,
317 (0x6e00 << 16) | (0xc908 >> 2),
318 0x00000000,
319 (0x7e00 << 16) | (0xc908 >> 2),
320 0x00000000,
321 (0x8e00 << 16) | (0xc908 >> 2),
322 0x00000000,
323 (0x9e00 << 16) | (0xc908 >> 2),
324 0x00000000,
325 (0xae00 << 16) | (0xc908 >> 2),
326 0x00000000,
327 (0xbe00 << 16) | (0xc908 >> 2),
328 0x00000000,
329 (0x4e00 << 16) | (0xc90c >> 2),
330 0x00000000,
331 (0x5e00 << 16) | (0xc90c >> 2),
332 0x00000000,
333 (0x6e00 << 16) | (0xc90c >> 2),
334 0x00000000,
335 (0x7e00 << 16) | (0xc90c >> 2),
336 0x00000000,
337 (0x8e00 << 16) | (0xc90c >> 2),
338 0x00000000,
339 (0x9e00 << 16) | (0xc90c >> 2),
340 0x00000000,
341 (0xae00 << 16) | (0xc90c >> 2),
342 0x00000000,
343 (0xbe00 << 16) | (0xc90c >> 2),
344 0x00000000,
345 (0x4e00 << 16) | (0xc910 >> 2),
346 0x00000000,
347 (0x5e00 << 16) | (0xc910 >> 2),
348 0x00000000,
349 (0x6e00 << 16) | (0xc910 >> 2),
350 0x00000000,
351 (0x7e00 << 16) | (0xc910 >> 2),
352 0x00000000,
353 (0x8e00 << 16) | (0xc910 >> 2),
354 0x00000000,
355 (0x9e00 << 16) | (0xc910 >> 2),
356 0x00000000,
357 (0xae00 << 16) | (0xc910 >> 2),
358 0x00000000,
359 (0xbe00 << 16) | (0xc910 >> 2),
360 0x00000000,
361 (0x0e00 << 16) | (0xc99c >> 2),
362 0x00000000,
363 (0x0e00 << 16) | (0x9834 >> 2),
364 0x00000000,
365 (0x0000 << 16) | (0x30f00 >> 2),
366 0x00000000,
367 (0x0001 << 16) | (0x30f00 >> 2),
368 0x00000000,
369 (0x0000 << 16) | (0x30f04 >> 2),
370 0x00000000,
371 (0x0001 << 16) | (0x30f04 >> 2),
372 0x00000000,
373 (0x0000 << 16) | (0x30f08 >> 2),
374 0x00000000,
375 (0x0001 << 16) | (0x30f08 >> 2),
376 0x00000000,
377 (0x0000 << 16) | (0x30f0c >> 2),
378 0x00000000,
379 (0x0001 << 16) | (0x30f0c >> 2),
380 0x00000000,
381 (0x0600 << 16) | (0x9b7c >> 2),
382 0x00000000,
383 (0x0e00 << 16) | (0x8a14 >> 2),
384 0x00000000,
385 (0x0e00 << 16) | (0x8a18 >> 2),
386 0x00000000,
387 (0x0600 << 16) | (0x30a00 >> 2),
388 0x00000000,
389 (0x0e00 << 16) | (0x8bf0 >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0x8bcc >> 2),
392 0x00000000,
393 (0x0e00 << 16) | (0x8b24 >> 2),
394 0x00000000,
395 (0x0e00 << 16) | (0x30a04 >> 2),
396 0x00000000,
397 (0x0600 << 16) | (0x30a10 >> 2),
398 0x00000000,
399 (0x0600 << 16) | (0x30a14 >> 2),
400 0x00000000,
401 (0x0600 << 16) | (0x30a18 >> 2),
402 0x00000000,
403 (0x0600 << 16) | (0x30a2c >> 2),
404 0x00000000,
405 (0x0e00 << 16) | (0xc700 >> 2),
406 0x00000000,
407 (0x0e00 << 16) | (0xc704 >> 2),
408 0x00000000,
409 (0x0e00 << 16) | (0xc708 >> 2),
410 0x00000000,
411 (0x0e00 << 16) | (0xc768 >> 2),
412 0x00000000,
413 (0x0400 << 16) | (0xc770 >> 2),
414 0x00000000,
415 (0x0400 << 16) | (0xc774 >> 2),
416 0x00000000,
417 (0x0400 << 16) | (0xc778 >> 2),
418 0x00000000,
419 (0x0400 << 16) | (0xc77c >> 2),
420 0x00000000,
421 (0x0400 << 16) | (0xc780 >> 2),
422 0x00000000,
423 (0x0400 << 16) | (0xc784 >> 2),
424 0x00000000,
425 (0x0400 << 16) | (0xc788 >> 2),
426 0x00000000,
427 (0x0400 << 16) | (0xc78c >> 2),
428 0x00000000,
429 (0x0400 << 16) | (0xc798 >> 2),
430 0x00000000,
431 (0x0400 << 16) | (0xc79c >> 2),
432 0x00000000,
433 (0x0400 << 16) | (0xc7a0 >> 2),
434 0x00000000,
435 (0x0400 << 16) | (0xc7a4 >> 2),
436 0x00000000,
437 (0x0400 << 16) | (0xc7a8 >> 2),
438 0x00000000,
439 (0x0400 << 16) | (0xc7ac >> 2),
440 0x00000000,
441 (0x0400 << 16) | (0xc7b0 >> 2),
442 0x00000000,
443 (0x0400 << 16) | (0xc7b4 >> 2),
444 0x00000000,
445 (0x0e00 << 16) | (0x9100 >> 2),
446 0x00000000,
447 (0x0e00 << 16) | (0x3c010 >> 2),
448 0x00000000,
449 (0x0e00 << 16) | (0x92a8 >> 2),
450 0x00000000,
451 (0x0e00 << 16) | (0x92ac >> 2),
452 0x00000000,
453 (0x0e00 << 16) | (0x92b4 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0x92b8 >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x92bc >> 2),
458 0x00000000,
459 (0x0e00 << 16) | (0x92c0 >> 2),
460 0x00000000,
461 (0x0e00 << 16) | (0x92c4 >> 2),
462 0x00000000,
463 (0x0e00 << 16) | (0x92c8 >> 2),
464 0x00000000,
465 (0x0e00 << 16) | (0x92cc >> 2),
466 0x00000000,
467 (0x0e00 << 16) | (0x92d0 >> 2),
468 0x00000000,
469 (0x0e00 << 16) | (0x8c00 >> 2),
470 0x00000000,
471 (0x0e00 << 16) | (0x8c04 >> 2),
472 0x00000000,
473 (0x0e00 << 16) | (0x8c20 >> 2),
474 0x00000000,
475 (0x0e00 << 16) | (0x8c38 >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8c3c >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0xae00 >> 2),
480 0x00000000,
481 (0x0e00 << 16) | (0x9604 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0xac08 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0xac0c >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0xac10 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0xac14 >> 2),
490 0x00000000,
491 (0x0e00 << 16) | (0xac58 >> 2),
492 0x00000000,
493 (0x0e00 << 16) | (0xac68 >> 2),
494 0x00000000,
495 (0x0e00 << 16) | (0xac6c >> 2),
496 0x00000000,
497 (0x0e00 << 16) | (0xac70 >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xac74 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xac78 >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xac7c >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xac80 >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0xac84 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0xac88 >> 2),
510 0x00000000,
511 (0x0e00 << 16) | (0xac8c >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0x970c >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0x9714 >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0x9718 >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0x971c >> 2),
520 0x00000000,
521 (0x0e00 << 16) | (0x31068 >> 2),
522 0x00000000,
523 (0x4e00 << 16) | (0x31068 >> 2),
524 0x00000000,
525 (0x5e00 << 16) | (0x31068 >> 2),
526 0x00000000,
527 (0x6e00 << 16) | (0x31068 >> 2),
528 0x00000000,
529 (0x7e00 << 16) | (0x31068 >> 2),
530 0x00000000,
531 (0x8e00 << 16) | (0x31068 >> 2),
532 0x00000000,
533 (0x9e00 << 16) | (0x31068 >> 2),
534 0x00000000,
535 (0xae00 << 16) | (0x31068 >> 2),
536 0x00000000,
537 (0xbe00 << 16) | (0x31068 >> 2),
538 0x00000000,
539 (0x0e00 << 16) | (0xcd10 >> 2),
540 0x00000000,
541 (0x0e00 << 16) | (0xcd14 >> 2),
542 0x00000000,
543 (0x0e00 << 16) | (0x88b0 >> 2),
544 0x00000000,
545 (0x0e00 << 16) | (0x88b4 >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0x88b8 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0x88bc >> 2),
550 0x00000000,
551 (0x0400 << 16) | (0x89c0 >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x88c4 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x88c8 >> 2),
556 0x00000000,
557 (0x0e00 << 16) | (0x88d0 >> 2),
558 0x00000000,
559 (0x0e00 << 16) | (0x88d4 >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x88d8 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x8980 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x30938 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x3093c >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x30940 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x89a0 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x30900 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x30904 >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x89b4 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x3c210 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x3c214 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x3c218 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x8904 >> 2),
586 0x00000000,
587 0x5,
588 (0x0e00 << 16) | (0x8c28 >> 2),
589 (0x0e00 << 16) | (0x8c2c >> 2),
590 (0x0e00 << 16) | (0x8c30 >> 2),
591 (0x0e00 << 16) | (0x8c34 >> 2),
592 (0x0e00 << 16) | (0x9600 >> 2),
593 };
594
595 static const u32 kalindi_rlc_save_restore_register_list[] =
596 {
597 (0x0e00 << 16) | (0xc12c >> 2),
598 0x00000000,
599 (0x0e00 << 16) | (0xc140 >> 2),
600 0x00000000,
601 (0x0e00 << 16) | (0xc150 >> 2),
602 0x00000000,
603 (0x0e00 << 16) | (0xc15c >> 2),
604 0x00000000,
605 (0x0e00 << 16) | (0xc168 >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xc170 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xc204 >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xc2b4 >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xc2b8 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xc2bc >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xc2c0 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0x8228 >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0x829c >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0x869c >> 2),
624 0x00000000,
625 (0x0600 << 16) | (0x98f4 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0x98f8 >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0x9900 >> 2),
630 0x00000000,
631 (0x0e00 << 16) | (0xc260 >> 2),
632 0x00000000,
633 (0x0e00 << 16) | (0x90e8 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0x3c000 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x3c00c >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0x8c1c >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x9700 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0xcd20 >> 2),
644 0x00000000,
645 (0x4e00 << 16) | (0xcd20 >> 2),
646 0x00000000,
647 (0x5e00 << 16) | (0xcd20 >> 2),
648 0x00000000,
649 (0x6e00 << 16) | (0xcd20 >> 2),
650 0x00000000,
651 (0x7e00 << 16) | (0xcd20 >> 2),
652 0x00000000,
653 (0x0e00 << 16) | (0x89bc >> 2),
654 0x00000000,
655 (0x0e00 << 16) | (0x8900 >> 2),
656 0x00000000,
657 0x3,
658 (0x0e00 << 16) | (0xc130 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc134 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc1fc >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc208 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc264 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc268 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc26c >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc270 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc274 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc28c >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc290 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0xc294 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0xc298 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xc2a0 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0xc2a4 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0xc2a8 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc2ac >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x301d0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30238 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30250 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x30254 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x30258 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3025c >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xc900 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xc900 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xc900 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xc900 >> 2),
711 0x00000000,
712 (0x4e00 << 16) | (0xc904 >> 2),
713 0x00000000,
714 (0x5e00 << 16) | (0xc904 >> 2),
715 0x00000000,
716 (0x6e00 << 16) | (0xc904 >> 2),
717 0x00000000,
718 (0x7e00 << 16) | (0xc904 >> 2),
719 0x00000000,
720 (0x4e00 << 16) | (0xc908 >> 2),
721 0x00000000,
722 (0x5e00 << 16) | (0xc908 >> 2),
723 0x00000000,
724 (0x6e00 << 16) | (0xc908 >> 2),
725 0x00000000,
726 (0x7e00 << 16) | (0xc908 >> 2),
727 0x00000000,
728 (0x4e00 << 16) | (0xc90c >> 2),
729 0x00000000,
730 (0x5e00 << 16) | (0xc90c >> 2),
731 0x00000000,
732 (0x6e00 << 16) | (0xc90c >> 2),
733 0x00000000,
734 (0x7e00 << 16) | (0xc90c >> 2),
735 0x00000000,
736 (0x4e00 << 16) | (0xc910 >> 2),
737 0x00000000,
738 (0x5e00 << 16) | (0xc910 >> 2),
739 0x00000000,
740 (0x6e00 << 16) | (0xc910 >> 2),
741 0x00000000,
742 (0x7e00 << 16) | (0xc910 >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0xc99c >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x9834 >> 2),
747 0x00000000,
748 (0x0000 << 16) | (0x30f00 >> 2),
749 0x00000000,
750 (0x0000 << 16) | (0x30f04 >> 2),
751 0x00000000,
752 (0x0000 << 16) | (0x30f08 >> 2),
753 0x00000000,
754 (0x0000 << 16) | (0x30f0c >> 2),
755 0x00000000,
756 (0x0600 << 16) | (0x9b7c >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x8a14 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x8a18 >> 2),
761 0x00000000,
762 (0x0600 << 16) | (0x30a00 >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x8bf0 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0x8bcc >> 2),
767 0x00000000,
768 (0x0e00 << 16) | (0x8b24 >> 2),
769 0x00000000,
770 (0x0e00 << 16) | (0x30a04 >> 2),
771 0x00000000,
772 (0x0600 << 16) | (0x30a10 >> 2),
773 0x00000000,
774 (0x0600 << 16) | (0x30a14 >> 2),
775 0x00000000,
776 (0x0600 << 16) | (0x30a18 >> 2),
777 0x00000000,
778 (0x0600 << 16) | (0x30a2c >> 2),
779 0x00000000,
780 (0x0e00 << 16) | (0xc700 >> 2),
781 0x00000000,
782 (0x0e00 << 16) | (0xc704 >> 2),
783 0x00000000,
784 (0x0e00 << 16) | (0xc708 >> 2),
785 0x00000000,
786 (0x0e00 << 16) | (0xc768 >> 2),
787 0x00000000,
788 (0x0400 << 16) | (0xc770 >> 2),
789 0x00000000,
790 (0x0400 << 16) | (0xc774 >> 2),
791 0x00000000,
792 (0x0400 << 16) | (0xc798 >> 2),
793 0x00000000,
794 (0x0400 << 16) | (0xc79c >> 2),
795 0x00000000,
796 (0x0e00 << 16) | (0x9100 >> 2),
797 0x00000000,
798 (0x0e00 << 16) | (0x3c010 >> 2),
799 0x00000000,
800 (0x0e00 << 16) | (0x8c00 >> 2),
801 0x00000000,
802 (0x0e00 << 16) | (0x8c04 >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0x8c20 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0x8c38 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0x8c3c >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0xae00 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0x9604 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0xac08 >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0xac0c >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0xac10 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0xac14 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0xac58 >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0xac68 >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0xac6c >> 2),
827 0x00000000,
828 (0x0e00 << 16) | (0xac70 >> 2),
829 0x00000000,
830 (0x0e00 << 16) | (0xac74 >> 2),
831 0x00000000,
832 (0x0e00 << 16) | (0xac78 >> 2),
833 0x00000000,
834 (0x0e00 << 16) | (0xac7c >> 2),
835 0x00000000,
836 (0x0e00 << 16) | (0xac80 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xac84 >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0xac88 >> 2),
841 0x00000000,
842 (0x0e00 << 16) | (0xac8c >> 2),
843 0x00000000,
844 (0x0e00 << 16) | (0x970c >> 2),
845 0x00000000,
846 (0x0e00 << 16) | (0x9714 >> 2),
847 0x00000000,
848 (0x0e00 << 16) | (0x9718 >> 2),
849 0x00000000,
850 (0x0e00 << 16) | (0x971c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x31068 >> 2),
853 0x00000000,
854 (0x4e00 << 16) | (0x31068 >> 2),
855 0x00000000,
856 (0x5e00 << 16) | (0x31068 >> 2),
857 0x00000000,
858 (0x6e00 << 16) | (0x31068 >> 2),
859 0x00000000,
860 (0x7e00 << 16) | (0x31068 >> 2),
861 0x00000000,
862 (0x0e00 << 16) | (0xcd10 >> 2),
863 0x00000000,
864 (0x0e00 << 16) | (0xcd14 >> 2),
865 0x00000000,
866 (0x0e00 << 16) | (0x88b0 >> 2),
867 0x00000000,
868 (0x0e00 << 16) | (0x88b4 >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0x88b8 >> 2),
871 0x00000000,
872 (0x0e00 << 16) | (0x88bc >> 2),
873 0x00000000,
874 (0x0400 << 16) | (0x89c0 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0x88c4 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0x88c8 >> 2),
879 0x00000000,
880 (0x0e00 << 16) | (0x88d0 >> 2),
881 0x00000000,
882 (0x0e00 << 16) | (0x88d4 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x88d8 >> 2),
885 0x00000000,
886 (0x0e00 << 16) | (0x8980 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x30938 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x3093c >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x30940 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x89a0 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x30900 >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x30904 >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x89b4 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x3e1fc >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0x3c210 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x3c214 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0x3c218 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0x8904 >> 2),
911 0x00000000,
912 0x5,
913 (0x0e00 << 16) | (0x8c28 >> 2),
914 (0x0e00 << 16) | (0x8c2c >> 2),
915 (0x0e00 << 16) | (0x8c30 >> 2),
916 (0x0e00 << 16) | (0x8c34 >> 2),
917 (0x0e00 << 16) | (0x9600 >> 2),
918 };
919
920 static const u32 bonaire_golden_spm_registers[] =
921 {
922 0x30800, 0xe0ffffff, 0xe0000000
923 };
924
925 static const u32 bonaire_golden_common_registers[] =
926 {
927 0xc770, 0xffffffff, 0x00000800,
928 0xc774, 0xffffffff, 0x00000800,
929 0xc798, 0xffffffff, 0x00007fbf,
930 0xc79c, 0xffffffff, 0x00007faf
931 };
932
933 static const u32 bonaire_golden_registers[] =
934 {
935 0x3354, 0x00000333, 0x00000333,
936 0x3350, 0x000c0fc0, 0x00040200,
937 0x9a10, 0x00010000, 0x00058208,
938 0x3c000, 0xffff1fff, 0x00140000,
939 0x3c200, 0xfdfc0fff, 0x00000100,
940 0x3c234, 0x40000000, 0x40000200,
941 0x9830, 0xffffffff, 0x00000000,
942 0x9834, 0xf00fffff, 0x00000400,
943 0x9838, 0x0002021c, 0x00020200,
944 0xc78, 0x00000080, 0x00000000,
945 0x5bb0, 0x000000f0, 0x00000070,
946 0x5bc0, 0xf0311fff, 0x80300000,
947 0x98f8, 0x73773777, 0x12010001,
948 0x350c, 0x00810000, 0x408af000,
949 0x7030, 0x31000111, 0x00000011,
950 0x2f48, 0x73773777, 0x12010001,
951 0x220c, 0x00007fb6, 0x0021a1b1,
952 0x2210, 0x00007fb6, 0x002021b1,
953 0x2180, 0x00007fb6, 0x00002191,
954 0x2218, 0x00007fb6, 0x002121b1,
955 0x221c, 0x00007fb6, 0x002021b1,
956 0x21dc, 0x00007fb6, 0x00002191,
957 0x21e0, 0x00007fb6, 0x00002191,
958 0x3628, 0x0000003f, 0x0000000a,
959 0x362c, 0x0000003f, 0x0000000a,
960 0x2ae4, 0x00073ffe, 0x000022a2,
961 0x240c, 0x000007ff, 0x00000000,
962 0x8a14, 0xf000003f, 0x00000007,
963 0x8bf0, 0x00002001, 0x00000001,
964 0x8b24, 0xffffffff, 0x00ffffff,
965 0x30a04, 0x0000ff0f, 0x00000000,
966 0x28a4c, 0x07ffffff, 0x06000000,
967 0x4d8, 0x00000fff, 0x00000100,
968 0x3e78, 0x00000001, 0x00000002,
969 0x9100, 0x03000000, 0x0362c688,
970 0x8c00, 0x000000ff, 0x00000001,
971 0xe40, 0x00001fff, 0x00001fff,
972 0x9060, 0x0000007f, 0x00000020,
973 0x9508, 0x00010000, 0x00010000,
974 0xac14, 0x000003ff, 0x000000f3,
975 0xac0c, 0xffffffff, 0x00001032
976 };
977
978 static const u32 bonaire_mgcg_cgcg_init[] =
979 {
980 0xc420, 0xffffffff, 0xfffffffc,
981 0x30800, 0xffffffff, 0xe0000000,
982 0x3c2a0, 0xffffffff, 0x00000100,
983 0x3c208, 0xffffffff, 0x00000100,
984 0x3c2c0, 0xffffffff, 0xc0000100,
985 0x3c2c8, 0xffffffff, 0xc0000100,
986 0x3c2c4, 0xffffffff, 0xc0000100,
987 0x55e4, 0xffffffff, 0x00600100,
988 0x3c280, 0xffffffff, 0x00000100,
989 0x3c214, 0xffffffff, 0x06000100,
990 0x3c220, 0xffffffff, 0x00000100,
991 0x3c218, 0xffffffff, 0x06000100,
992 0x3c204, 0xffffffff, 0x00000100,
993 0x3c2e0, 0xffffffff, 0x00000100,
994 0x3c224, 0xffffffff, 0x00000100,
995 0x3c200, 0xffffffff, 0x00000100,
996 0x3c230, 0xffffffff, 0x00000100,
997 0x3c234, 0xffffffff, 0x00000100,
998 0x3c250, 0xffffffff, 0x00000100,
999 0x3c254, 0xffffffff, 0x00000100,
1000 0x3c258, 0xffffffff, 0x00000100,
1001 0x3c25c, 0xffffffff, 0x00000100,
1002 0x3c260, 0xffffffff, 0x00000100,
1003 0x3c27c, 0xffffffff, 0x00000100,
1004 0x3c278, 0xffffffff, 0x00000100,
1005 0x3c210, 0xffffffff, 0x06000100,
1006 0x3c290, 0xffffffff, 0x00000100,
1007 0x3c274, 0xffffffff, 0x00000100,
1008 0x3c2b4, 0xffffffff, 0x00000100,
1009 0x3c2b0, 0xffffffff, 0x00000100,
1010 0x3c270, 0xffffffff, 0x00000100,
1011 0x30800, 0xffffffff, 0xe0000000,
1012 0x3c020, 0xffffffff, 0x00010000,
1013 0x3c024, 0xffffffff, 0x00030002,
1014 0x3c028, 0xffffffff, 0x00040007,
1015 0x3c02c, 0xffffffff, 0x00060005,
1016 0x3c030, 0xffffffff, 0x00090008,
1017 0x3c034, 0xffffffff, 0x00010000,
1018 0x3c038, 0xffffffff, 0x00030002,
1019 0x3c03c, 0xffffffff, 0x00040007,
1020 0x3c040, 0xffffffff, 0x00060005,
1021 0x3c044, 0xffffffff, 0x00090008,
1022 0x3c048, 0xffffffff, 0x00010000,
1023 0x3c04c, 0xffffffff, 0x00030002,
1024 0x3c050, 0xffffffff, 0x00040007,
1025 0x3c054, 0xffffffff, 0x00060005,
1026 0x3c058, 0xffffffff, 0x00090008,
1027 0x3c05c, 0xffffffff, 0x00010000,
1028 0x3c060, 0xffffffff, 0x00030002,
1029 0x3c064, 0xffffffff, 0x00040007,
1030 0x3c068, 0xffffffff, 0x00060005,
1031 0x3c06c, 0xffffffff, 0x00090008,
1032 0x3c070, 0xffffffff, 0x00010000,
1033 0x3c074, 0xffffffff, 0x00030002,
1034 0x3c078, 0xffffffff, 0x00040007,
1035 0x3c07c, 0xffffffff, 0x00060005,
1036 0x3c080, 0xffffffff, 0x00090008,
1037 0x3c084, 0xffffffff, 0x00010000,
1038 0x3c088, 0xffffffff, 0x00030002,
1039 0x3c08c, 0xffffffff, 0x00040007,
1040 0x3c090, 0xffffffff, 0x00060005,
1041 0x3c094, 0xffffffff, 0x00090008,
1042 0x3c098, 0xffffffff, 0x00010000,
1043 0x3c09c, 0xffffffff, 0x00030002,
1044 0x3c0a0, 0xffffffff, 0x00040007,
1045 0x3c0a4, 0xffffffff, 0x00060005,
1046 0x3c0a8, 0xffffffff, 0x00090008,
1047 0x3c000, 0xffffffff, 0x96e00200,
1048 0x8708, 0xffffffff, 0x00900100,
1049 0xc424, 0xffffffff, 0x0020003f,
1050 0x38, 0xffffffff, 0x0140001c,
1051 0x3c, 0x000f0000, 0x000f0000,
1052 0x220, 0xffffffff, 0xC060000C,
1053 0x224, 0xc0000fff, 0x00000100,
1054 0xf90, 0xffffffff, 0x00000100,
1055 0xf98, 0x00000101, 0x00000000,
1056 0x20a8, 0xffffffff, 0x00000104,
1057 0x55e4, 0xff000fff, 0x00000100,
1058 0x30cc, 0xc0000fff, 0x00000104,
1059 0xc1e4, 0x00000001, 0x00000001,
1060 0xd00c, 0xff000ff0, 0x00000100,
1061 0xd80c, 0xff000ff0, 0x00000100
1062 };
1063
1064 static const u32 spectre_golden_spm_registers[] =
1065 {
1066 0x30800, 0xe0ffffff, 0xe0000000
1067 };
1068
1069 static const u32 spectre_golden_common_registers[] =
1070 {
1071 0xc770, 0xffffffff, 0x00000800,
1072 0xc774, 0xffffffff, 0x00000800,
1073 0xc798, 0xffffffff, 0x00007fbf,
1074 0xc79c, 0xffffffff, 0x00007faf
1075 };
1076
1077 static const u32 spectre_golden_registers[] =
1078 {
1079 0x3c000, 0xffff1fff, 0x96940200,
1080 0x3c00c, 0xffff0001, 0xff000000,
1081 0x3c200, 0xfffc0fff, 0x00000100,
1082 0x6ed8, 0x00010101, 0x00010000,
1083 0x9834, 0xf00fffff, 0x00000400,
1084 0x9838, 0xfffffffc, 0x00020200,
1085 0x5bb0, 0x000000f0, 0x00000070,
1086 0x5bc0, 0xf0311fff, 0x80300000,
1087 0x98f8, 0x73773777, 0x12010001,
1088 0x9b7c, 0x00ff0000, 0x00fc0000,
1089 0x2f48, 0x73773777, 0x12010001,
1090 0x8a14, 0xf000003f, 0x00000007,
1091 0x8b24, 0xffffffff, 0x00ffffff,
1092 0x28350, 0x3f3f3fff, 0x00000082,
1093 0x28355, 0x0000003f, 0x00000000,
1094 0x3e78, 0x00000001, 0x00000002,
1095 0x913c, 0xffff03df, 0x00000004,
1096 0xc768, 0x00000008, 0x00000008,
1097 0x8c00, 0x000008ff, 0x00000800,
1098 0x9508, 0x00010000, 0x00010000,
1099 0xac0c, 0xffffffff, 0x54763210,
1100 0x214f8, 0x01ff01ff, 0x00000002,
1101 0x21498, 0x007ff800, 0x00200000,
1102 0x2015c, 0xffffffff, 0x00000f40,
1103 0x30934, 0xffffffff, 0x00000001
1104 };
1105
1106 static const u32 spectre_mgcg_cgcg_init[] =
1107 {
1108 0xc420, 0xffffffff, 0xfffffffc,
1109 0x30800, 0xffffffff, 0xe0000000,
1110 0x3c2a0, 0xffffffff, 0x00000100,
1111 0x3c208, 0xffffffff, 0x00000100,
1112 0x3c2c0, 0xffffffff, 0x00000100,
1113 0x3c2c8, 0xffffffff, 0x00000100,
1114 0x3c2c4, 0xffffffff, 0x00000100,
1115 0x55e4, 0xffffffff, 0x00600100,
1116 0x3c280, 0xffffffff, 0x00000100,
1117 0x3c214, 0xffffffff, 0x06000100,
1118 0x3c220, 0xffffffff, 0x00000100,
1119 0x3c218, 0xffffffff, 0x06000100,
1120 0x3c204, 0xffffffff, 0x00000100,
1121 0x3c2e0, 0xffffffff, 0x00000100,
1122 0x3c224, 0xffffffff, 0x00000100,
1123 0x3c200, 0xffffffff, 0x00000100,
1124 0x3c230, 0xffffffff, 0x00000100,
1125 0x3c234, 0xffffffff, 0x00000100,
1126 0x3c250, 0xffffffff, 0x00000100,
1127 0x3c254, 0xffffffff, 0x00000100,
1128 0x3c258, 0xffffffff, 0x00000100,
1129 0x3c25c, 0xffffffff, 0x00000100,
1130 0x3c260, 0xffffffff, 0x00000100,
1131 0x3c27c, 0xffffffff, 0x00000100,
1132 0x3c278, 0xffffffff, 0x00000100,
1133 0x3c210, 0xffffffff, 0x06000100,
1134 0x3c290, 0xffffffff, 0x00000100,
1135 0x3c274, 0xffffffff, 0x00000100,
1136 0x3c2b4, 0xffffffff, 0x00000100,
1137 0x3c2b0, 0xffffffff, 0x00000100,
1138 0x3c270, 0xffffffff, 0x00000100,
1139 0x30800, 0xffffffff, 0xe0000000,
1140 0x3c020, 0xffffffff, 0x00010000,
1141 0x3c024, 0xffffffff, 0x00030002,
1142 0x3c028, 0xffffffff, 0x00040007,
1143 0x3c02c, 0xffffffff, 0x00060005,
1144 0x3c030, 0xffffffff, 0x00090008,
1145 0x3c034, 0xffffffff, 0x00010000,
1146 0x3c038, 0xffffffff, 0x00030002,
1147 0x3c03c, 0xffffffff, 0x00040007,
1148 0x3c040, 0xffffffff, 0x00060005,
1149 0x3c044, 0xffffffff, 0x00090008,
1150 0x3c048, 0xffffffff, 0x00010000,
1151 0x3c04c, 0xffffffff, 0x00030002,
1152 0x3c050, 0xffffffff, 0x00040007,
1153 0x3c054, 0xffffffff, 0x00060005,
1154 0x3c058, 0xffffffff, 0x00090008,
1155 0x3c05c, 0xffffffff, 0x00010000,
1156 0x3c060, 0xffffffff, 0x00030002,
1157 0x3c064, 0xffffffff, 0x00040007,
1158 0x3c068, 0xffffffff, 0x00060005,
1159 0x3c06c, 0xffffffff, 0x00090008,
1160 0x3c070, 0xffffffff, 0x00010000,
1161 0x3c074, 0xffffffff, 0x00030002,
1162 0x3c078, 0xffffffff, 0x00040007,
1163 0x3c07c, 0xffffffff, 0x00060005,
1164 0x3c080, 0xffffffff, 0x00090008,
1165 0x3c084, 0xffffffff, 0x00010000,
1166 0x3c088, 0xffffffff, 0x00030002,
1167 0x3c08c, 0xffffffff, 0x00040007,
1168 0x3c090, 0xffffffff, 0x00060005,
1169 0x3c094, 0xffffffff, 0x00090008,
1170 0x3c098, 0xffffffff, 0x00010000,
1171 0x3c09c, 0xffffffff, 0x00030002,
1172 0x3c0a0, 0xffffffff, 0x00040007,
1173 0x3c0a4, 0xffffffff, 0x00060005,
1174 0x3c0a8, 0xffffffff, 0x00090008,
1175 0x3c0ac, 0xffffffff, 0x00010000,
1176 0x3c0b0, 0xffffffff, 0x00030002,
1177 0x3c0b4, 0xffffffff, 0x00040007,
1178 0x3c0b8, 0xffffffff, 0x00060005,
1179 0x3c0bc, 0xffffffff, 0x00090008,
1180 0x3c000, 0xffffffff, 0x96e00200,
1181 0x8708, 0xffffffff, 0x00900100,
1182 0xc424, 0xffffffff, 0x0020003f,
1183 0x38, 0xffffffff, 0x0140001c,
1184 0x3c, 0x000f0000, 0x000f0000,
1185 0x220, 0xffffffff, 0xC060000C,
1186 0x224, 0xc0000fff, 0x00000100,
1187 0xf90, 0xffffffff, 0x00000100,
1188 0xf98, 0x00000101, 0x00000000,
1189 0x20a8, 0xffffffff, 0x00000104,
1190 0x55e4, 0xff000fff, 0x00000100,
1191 0x30cc, 0xc0000fff, 0x00000104,
1192 0xc1e4, 0x00000001, 0x00000001,
1193 0xd00c, 0xff000ff0, 0x00000100,
1194 0xd80c, 0xff000ff0, 0x00000100
1195 };
1196
1197 static const u32 kalindi_golden_spm_registers[] =
1198 {
1199 0x30800, 0xe0ffffff, 0xe0000000
1200 };
1201
1202 static const u32 kalindi_golden_common_registers[] =
1203 {
1204 0xc770, 0xffffffff, 0x00000800,
1205 0xc774, 0xffffffff, 0x00000800,
1206 0xc798, 0xffffffff, 0x00007fbf,
1207 0xc79c, 0xffffffff, 0x00007faf
1208 };
1209
1210 static const u32 kalindi_golden_registers[] =
1211 {
1212 0x3c000, 0xffffdfff, 0x6e944040,
1213 0x55e4, 0xff607fff, 0xfc000100,
1214 0x3c220, 0xff000fff, 0x00000100,
1215 0x3c224, 0xff000fff, 0x00000100,
1216 0x3c200, 0xfffc0fff, 0x00000100,
1217 0x6ed8, 0x00010101, 0x00010000,
1218 0x9830, 0xffffffff, 0x00000000,
1219 0x9834, 0xf00fffff, 0x00000400,
1220 0x5bb0, 0x000000f0, 0x00000070,
1221 0x5bc0, 0xf0311fff, 0x80300000,
1222 0x98f8, 0x73773777, 0x12010001,
1223 0x98fc, 0xffffffff, 0x00000010,
1224 0x9b7c, 0x00ff0000, 0x00fc0000,
1225 0x8030, 0x00001f0f, 0x0000100a,
1226 0x2f48, 0x73773777, 0x12010001,
1227 0x2408, 0x000fffff, 0x000c007f,
1228 0x8a14, 0xf000003f, 0x00000007,
1229 0x8b24, 0x3fff3fff, 0x00ffcfff,
1230 0x30a04, 0x0000ff0f, 0x00000000,
1231 0x28a4c, 0x07ffffff, 0x06000000,
1232 0x4d8, 0x00000fff, 0x00000100,
1233 0x3e78, 0x00000001, 0x00000002,
1234 0xc768, 0x00000008, 0x00000008,
1235 0x8c00, 0x000000ff, 0x00000003,
1236 0x214f8, 0x01ff01ff, 0x00000002,
1237 0x21498, 0x007ff800, 0x00200000,
1238 0x2015c, 0xffffffff, 0x00000f40,
1239 0x88c4, 0x001f3ae3, 0x00000082,
1240 0x88d4, 0x0000001f, 0x00000010,
1241 0x30934, 0xffffffff, 0x00000000
1242 };
1243
1244 static const u32 kalindi_mgcg_cgcg_init[] =
1245 {
1246 0xc420, 0xffffffff, 0xfffffffc,
1247 0x30800, 0xffffffff, 0xe0000000,
1248 0x3c2a0, 0xffffffff, 0x00000100,
1249 0x3c208, 0xffffffff, 0x00000100,
1250 0x3c2c0, 0xffffffff, 0x00000100,
1251 0x3c2c8, 0xffffffff, 0x00000100,
1252 0x3c2c4, 0xffffffff, 0x00000100,
1253 0x55e4, 0xffffffff, 0x00600100,
1254 0x3c280, 0xffffffff, 0x00000100,
1255 0x3c214, 0xffffffff, 0x06000100,
1256 0x3c220, 0xffffffff, 0x00000100,
1257 0x3c218, 0xffffffff, 0x06000100,
1258 0x3c204, 0xffffffff, 0x00000100,
1259 0x3c2e0, 0xffffffff, 0x00000100,
1260 0x3c224, 0xffffffff, 0x00000100,
1261 0x3c200, 0xffffffff, 0x00000100,
1262 0x3c230, 0xffffffff, 0x00000100,
1263 0x3c234, 0xffffffff, 0x00000100,
1264 0x3c250, 0xffffffff, 0x00000100,
1265 0x3c254, 0xffffffff, 0x00000100,
1266 0x3c258, 0xffffffff, 0x00000100,
1267 0x3c25c, 0xffffffff, 0x00000100,
1268 0x3c260, 0xffffffff, 0x00000100,
1269 0x3c27c, 0xffffffff, 0x00000100,
1270 0x3c278, 0xffffffff, 0x00000100,
1271 0x3c210, 0xffffffff, 0x06000100,
1272 0x3c290, 0xffffffff, 0x00000100,
1273 0x3c274, 0xffffffff, 0x00000100,
1274 0x3c2b4, 0xffffffff, 0x00000100,
1275 0x3c2b0, 0xffffffff, 0x00000100,
1276 0x3c270, 0xffffffff, 0x00000100,
1277 0x30800, 0xffffffff, 0xe0000000,
1278 0x3c020, 0xffffffff, 0x00010000,
1279 0x3c024, 0xffffffff, 0x00030002,
1280 0x3c028, 0xffffffff, 0x00040007,
1281 0x3c02c, 0xffffffff, 0x00060005,
1282 0x3c030, 0xffffffff, 0x00090008,
1283 0x3c034, 0xffffffff, 0x00010000,
1284 0x3c038, 0xffffffff, 0x00030002,
1285 0x3c03c, 0xffffffff, 0x00040007,
1286 0x3c040, 0xffffffff, 0x00060005,
1287 0x3c044, 0xffffffff, 0x00090008,
1288 0x3c000, 0xffffffff, 0x96e00200,
1289 0x8708, 0xffffffff, 0x00900100,
1290 0xc424, 0xffffffff, 0x0020003f,
1291 0x38, 0xffffffff, 0x0140001c,
1292 0x3c, 0x000f0000, 0x000f0000,
1293 0x220, 0xffffffff, 0xC060000C,
1294 0x224, 0xc0000fff, 0x00000100,
1295 0x20a8, 0xffffffff, 0x00000104,
1296 0x55e4, 0xff000fff, 0x00000100,
1297 0x30cc, 0xc0000fff, 0x00000104,
1298 0xc1e4, 0x00000001, 0x00000001,
1299 0xd00c, 0xff000ff0, 0x00000100,
1300 0xd80c, 0xff000ff0, 0x00000100
1301 };
1302
1303 static void cik_init_golden_registers(struct radeon_device *rdev)
1304 {
1305 switch (rdev->family) {
1306 case CHIP_BONAIRE:
1307 radeon_program_register_sequence(rdev,
1308 bonaire_mgcg_cgcg_init,
1309 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1310 radeon_program_register_sequence(rdev,
1311 bonaire_golden_registers,
1312 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1313 radeon_program_register_sequence(rdev,
1314 bonaire_golden_common_registers,
1315 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1316 radeon_program_register_sequence(rdev,
1317 bonaire_golden_spm_registers,
1318 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1319 break;
1320 case CHIP_KABINI:
1321 radeon_program_register_sequence(rdev,
1322 kalindi_mgcg_cgcg_init,
1323 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1324 radeon_program_register_sequence(rdev,
1325 kalindi_golden_registers,
1326 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1327 radeon_program_register_sequence(rdev,
1328 kalindi_golden_common_registers,
1329 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1330 radeon_program_register_sequence(rdev,
1331 kalindi_golden_spm_registers,
1332 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1333 break;
1334 case CHIP_KAVERI:
1335 radeon_program_register_sequence(rdev,
1336 spectre_mgcg_cgcg_init,
1337 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1338 radeon_program_register_sequence(rdev,
1339 spectre_golden_registers,
1340 (const u32)ARRAY_SIZE(spectre_golden_registers));
1341 radeon_program_register_sequence(rdev,
1342 spectre_golden_common_registers,
1343 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1344 radeon_program_register_sequence(rdev,
1345 spectre_golden_spm_registers,
1346 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1347 break;
1348 default:
1349 break;
1350 }
1351 }
1352
1353 /**
1354 * cik_get_xclk - get the xclk
1355 *
1356 * @rdev: radeon_device pointer
1357 *
1358 * Returns the reference clock used by the gfx engine
1359 * (CIK).
1360 */
1361 u32 cik_get_xclk(struct radeon_device *rdev)
1362 {
1363 u32 reference_clock = rdev->clock.spll.reference_freq;
1364
1365 if (rdev->flags & RADEON_IS_IGP) {
1366 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1367 return reference_clock / 2;
1368 } else {
1369 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1370 return reference_clock / 4;
1371 }
1372 return reference_clock;
1373 }
1374
1375 /**
1376 * cik_mm_rdoorbell - read a doorbell dword
1377 *
1378 * @rdev: radeon_device pointer
1379 * @offset: byte offset into the aperture
1380 *
1381 * Returns the value in the doorbell aperture at the
1382 * requested offset (CIK).
1383 */
1384 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1385 {
1386 if (offset < rdev->doorbell.size) {
1387 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1388 } else {
1389 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1390 return 0;
1391 }
1392 }
1393
1394 /**
1395 * cik_mm_wdoorbell - write a doorbell dword
1396 *
1397 * @rdev: radeon_device pointer
1398 * @offset: byte offset into the aperture
1399 * @v: value to write
1400 *
1401 * Writes @v to the doorbell aperture at the
1402 * requested offset (CIK).
1403 */
1404 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1405 {
1406 if (offset < rdev->doorbell.size) {
1407 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1408 } else {
1409 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1410 }
1411 }
1412
1413 #define BONAIRE_IO_MC_REGS_SIZE 36
1414
1415 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1416 {
1417 {0x00000070, 0x04400000},
1418 {0x00000071, 0x80c01803},
1419 {0x00000072, 0x00004004},
1420 {0x00000073, 0x00000100},
1421 {0x00000074, 0x00ff0000},
1422 {0x00000075, 0x34000000},
1423 {0x00000076, 0x08000014},
1424 {0x00000077, 0x00cc08ec},
1425 {0x00000078, 0x00000400},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x04090000},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007e, 0x4408a8e8},
1430 {0x0000007f, 0x00000304},
1431 {0x00000080, 0x00000000},
1432 {0x00000082, 0x00000001},
1433 {0x00000083, 0x00000002},
1434 {0x00000084, 0xf3e4f400},
1435 {0x00000085, 0x052024e3},
1436 {0x00000087, 0x00000000},
1437 {0x00000088, 0x01000000},
1438 {0x0000008a, 0x1c0a0000},
1439 {0x0000008b, 0xff010000},
1440 {0x0000008d, 0xffffefff},
1441 {0x0000008e, 0xfff3efff},
1442 {0x0000008f, 0xfff3efbf},
1443 {0x00000092, 0xf7ffffff},
1444 {0x00000093, 0xffffff7f},
1445 {0x00000095, 0x00101101},
1446 {0x00000096, 0x00000fff},
1447 {0x00000097, 0x00116fff},
1448 {0x00000098, 0x60010000},
1449 {0x00000099, 0x10010000},
1450 {0x0000009a, 0x00006000},
1451 {0x0000009b, 0x00001000},
1452 {0x0000009f, 0x00b48000}
1453 };
1454
1455 /**
1456 * cik_srbm_select - select specific register instances
1457 *
1458 * @rdev: radeon_device pointer
1459 * @me: selected ME (micro engine)
1460 * @pipe: pipe
1461 * @queue: queue
1462 * @vmid: VMID
1463 *
1464 * Switches the currently active registers instances. Some
1465 * registers are instanced per VMID, others are instanced per
1466 * me/pipe/queue combination.
1467 */
1468 static void cik_srbm_select(struct radeon_device *rdev,
1469 u32 me, u32 pipe, u32 queue, u32 vmid)
1470 {
1471 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1472 MEID(me & 0x3) |
1473 VMID(vmid & 0xf) |
1474 QUEUEID(queue & 0x7));
1475 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1476 }
1477
1478 /* ucode loading */
1479 /**
1480 * ci_mc_load_microcode - load MC ucode into the hw
1481 *
1482 * @rdev: radeon_device pointer
1483 *
1484 * Load the GDDR MC ucode into the hw (CIK).
1485 * Returns 0 on success, error on failure.
1486 */
1487 static int ci_mc_load_microcode(struct radeon_device *rdev)
1488 {
1489 const __be32 *fw_data;
1490 u32 running, blackout = 0;
1491 u32 *io_mc_regs;
1492 int i, ucode_size, regs_size;
1493
1494 if (!rdev->mc_fw)
1495 return -EINVAL;
1496
1497 switch (rdev->family) {
1498 case CHIP_BONAIRE:
1499 default:
1500 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1501 ucode_size = CIK_MC_UCODE_SIZE;
1502 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1503 break;
1504 }
1505
1506 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1507
1508 if (running == 0) {
1509 if (running) {
1510 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1511 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1512 }
1513
1514 /* reset the engine and set to writable */
1515 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1517
1518 /* load mc io regs */
1519 for (i = 0; i < regs_size; i++) {
1520 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1521 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1522 }
1523 /* load the MC ucode */
1524 fw_data = (const __be32 *)rdev->mc_fw->data;
1525 for (i = 0; i < ucode_size; i++)
1526 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1527
1528 /* put the engine back into the active state */
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1530 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1531 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1532
1533 /* wait for training to complete */
1534 for (i = 0; i < rdev->usec_timeout; i++) {
1535 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1536 break;
1537 udelay(1);
1538 }
1539 for (i = 0; i < rdev->usec_timeout; i++) {
1540 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1541 break;
1542 udelay(1);
1543 }
1544
1545 if (running)
1546 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1547 }
1548
1549 return 0;
1550 }
1551
1552 /**
1553 * cik_init_microcode - load ucode images from disk
1554 *
1555 * @rdev: radeon_device pointer
1556 *
1557 * Use the firmware interface to load the ucode images into
1558 * the driver (not loaded into hw).
1559 * Returns 0 on success, error on failure.
1560 */
1561 static int cik_init_microcode(struct radeon_device *rdev)
1562 {
1563 const char *chip_name;
1564 size_t pfp_req_size, me_req_size, ce_req_size,
1565 mec_req_size, rlc_req_size, mc_req_size,
1566 sdma_req_size, smc_req_size;
1567 char fw_name[30];
1568 int err;
1569
1570 DRM_DEBUG("\n");
1571
1572 switch (rdev->family) {
1573 case CHIP_BONAIRE:
1574 chip_name = "BONAIRE";
1575 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1576 me_req_size = CIK_ME_UCODE_SIZE * 4;
1577 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1578 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1579 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1580 mc_req_size = CIK_MC_UCODE_SIZE * 4;
1581 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1582 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1583 break;
1584 case CHIP_KAVERI:
1585 chip_name = "KAVERI";
1586 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1587 me_req_size = CIK_ME_UCODE_SIZE * 4;
1588 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1589 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1590 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1591 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1592 break;
1593 case CHIP_KABINI:
1594 chip_name = "KABINI";
1595 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1596 me_req_size = CIK_ME_UCODE_SIZE * 4;
1597 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1598 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1599 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1600 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1601 break;
1602 default: BUG();
1603 }
1604
1605 DRM_INFO("Loading %s Microcode\n", chip_name);
1606
1607 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1608 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1609 if (err)
1610 goto out;
1611 if (rdev->pfp_fw->size != pfp_req_size) {
1612 printk(KERN_ERR
1613 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1614 rdev->pfp_fw->size, fw_name);
1615 err = -EINVAL;
1616 goto out;
1617 }
1618
1619 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1620 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1621 if (err)
1622 goto out;
1623 if (rdev->me_fw->size != me_req_size) {
1624 printk(KERN_ERR
1625 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1626 rdev->me_fw->size, fw_name);
1627 err = -EINVAL;
1628 }
1629
1630 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1631 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1632 if (err)
1633 goto out;
1634 if (rdev->ce_fw->size != ce_req_size) {
1635 printk(KERN_ERR
1636 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1637 rdev->ce_fw->size, fw_name);
1638 err = -EINVAL;
1639 }
1640
1641 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1642 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1643 if (err)
1644 goto out;
1645 if (rdev->mec_fw->size != mec_req_size) {
1646 printk(KERN_ERR
1647 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1648 rdev->mec_fw->size, fw_name);
1649 err = -EINVAL;
1650 }
1651
1652 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1653 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1654 if (err)
1655 goto out;
1656 if (rdev->rlc_fw->size != rlc_req_size) {
1657 printk(KERN_ERR
1658 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1659 rdev->rlc_fw->size, fw_name);
1660 err = -EINVAL;
1661 }
1662
1663 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1664 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1665 if (err)
1666 goto out;
1667 if (rdev->sdma_fw->size != sdma_req_size) {
1668 printk(KERN_ERR
1669 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1670 rdev->sdma_fw->size, fw_name);
1671 err = -EINVAL;
1672 }
1673
1674 /* No SMC, MC ucode on APUs */
1675 if (!(rdev->flags & RADEON_IS_IGP)) {
1676 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1677 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1678 if (err)
1679 goto out;
1680 if (rdev->mc_fw->size != mc_req_size) {
1681 printk(KERN_ERR
1682 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1683 rdev->mc_fw->size, fw_name);
1684 err = -EINVAL;
1685 }
1686
1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1688 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1689 if (err) {
1690 printk(KERN_ERR
1691 "smc: error loading firmware \"%s\"\n",
1692 fw_name);
1693 release_firmware(rdev->smc_fw);
1694 rdev->smc_fw = NULL;
1695 } else if (rdev->smc_fw->size != smc_req_size) {
1696 printk(KERN_ERR
1697 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1698 rdev->smc_fw->size, fw_name);
1699 err = -EINVAL;
1700 }
1701 }
1702
1703 out:
1704 if (err) {
1705 if (err != -EINVAL)
1706 printk(KERN_ERR
1707 "cik_cp: Failed to load firmware \"%s\"\n",
1708 fw_name);
1709 release_firmware(rdev->pfp_fw);
1710 rdev->pfp_fw = NULL;
1711 release_firmware(rdev->me_fw);
1712 rdev->me_fw = NULL;
1713 release_firmware(rdev->ce_fw);
1714 rdev->ce_fw = NULL;
1715 release_firmware(rdev->rlc_fw);
1716 rdev->rlc_fw = NULL;
1717 release_firmware(rdev->mc_fw);
1718 rdev->mc_fw = NULL;
1719 release_firmware(rdev->smc_fw);
1720 rdev->smc_fw = NULL;
1721 }
1722 return err;
1723 }
1724
1725 /*
1726 * Core functions
1727 */
1728 /**
1729 * cik_tiling_mode_table_init - init the hw tiling table
1730 *
1731 * @rdev: radeon_device pointer
1732 *
1733 * Starting with SI, the tiling setup is done globally in a
1734 * set of 32 tiling modes. Rather than selecting each set of
1735 * parameters per surface as on older asics, we just select
1736 * which index in the tiling table we want to use, and the
1737 * surface uses those parameters (CIK).
1738 */
1739 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1740 {
1741 const u32 num_tile_mode_states = 32;
1742 const u32 num_secondary_tile_mode_states = 16;
1743 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1744 u32 num_pipe_configs;
1745 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1746 rdev->config.cik.max_shader_engines;
1747
1748 switch (rdev->config.cik.mem_row_size_in_kb) {
1749 case 1:
1750 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1751 break;
1752 case 2:
1753 default:
1754 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1755 break;
1756 case 4:
1757 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1758 break;
1759 }
1760
1761 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1762 if (num_pipe_configs > 8)
1763 num_pipe_configs = 8; /* ??? */
1764
1765 if (num_pipe_configs == 8) {
1766 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767 switch (reg_offset) {
1768 case 0:
1769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1771 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1773 break;
1774 case 1:
1775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1777 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1779 break;
1780 case 2:
1781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1783 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1785 break;
1786 case 3:
1787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1789 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1790 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1791 break;
1792 case 4:
1793 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1795 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1796 TILE_SPLIT(split_equal_to_row_size));
1797 break;
1798 case 5:
1799 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1800 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1801 break;
1802 case 6:
1803 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1805 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1806 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1807 break;
1808 case 7:
1809 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1812 TILE_SPLIT(split_equal_to_row_size));
1813 break;
1814 case 8:
1815 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1816 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1817 break;
1818 case 9:
1819 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1821 break;
1822 case 10:
1823 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1824 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1825 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1826 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1827 break;
1828 case 11:
1829 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1830 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1833 break;
1834 case 12:
1835 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1837 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1839 break;
1840 case 13:
1841 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1842 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1843 break;
1844 case 14:
1845 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1846 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1847 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1848 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1849 break;
1850 case 16:
1851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1852 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1855 break;
1856 case 17:
1857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1859 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1861 break;
1862 case 27:
1863 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1865 break;
1866 case 28:
1867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1868 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1869 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1870 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1871 break;
1872 case 29:
1873 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1874 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1875 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1877 break;
1878 case 30:
1879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1881 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1883 break;
1884 default:
1885 gb_tile_moden = 0;
1886 break;
1887 }
1888 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1889 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1890 }
1891 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1892 switch (reg_offset) {
1893 case 0:
1894 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1897 NUM_BANKS(ADDR_SURF_16_BANK));
1898 break;
1899 case 1:
1900 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1901 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1902 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1903 NUM_BANKS(ADDR_SURF_16_BANK));
1904 break;
1905 case 2:
1906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1909 NUM_BANKS(ADDR_SURF_16_BANK));
1910 break;
1911 case 3:
1912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1915 NUM_BANKS(ADDR_SURF_16_BANK));
1916 break;
1917 case 4:
1918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1921 NUM_BANKS(ADDR_SURF_8_BANK));
1922 break;
1923 case 5:
1924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1927 NUM_BANKS(ADDR_SURF_4_BANK));
1928 break;
1929 case 6:
1930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1933 NUM_BANKS(ADDR_SURF_2_BANK));
1934 break;
1935 case 8:
1936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1939 NUM_BANKS(ADDR_SURF_16_BANK));
1940 break;
1941 case 9:
1942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1945 NUM_BANKS(ADDR_SURF_16_BANK));
1946 break;
1947 case 10:
1948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1951 NUM_BANKS(ADDR_SURF_16_BANK));
1952 break;
1953 case 11:
1954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1957 NUM_BANKS(ADDR_SURF_16_BANK));
1958 break;
1959 case 12:
1960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1963 NUM_BANKS(ADDR_SURF_8_BANK));
1964 break;
1965 case 13:
1966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1969 NUM_BANKS(ADDR_SURF_4_BANK));
1970 break;
1971 case 14:
1972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1975 NUM_BANKS(ADDR_SURF_2_BANK));
1976 break;
1977 default:
1978 gb_tile_moden = 0;
1979 break;
1980 }
1981 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1982 }
1983 } else if (num_pipe_configs == 4) {
1984 if (num_rbs == 4) {
1985 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1986 switch (reg_offset) {
1987 case 0:
1988 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1989 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1990 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1991 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1992 break;
1993 case 1:
1994 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1995 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1996 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1997 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1998 break;
1999 case 2:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2004 break;
2005 case 3:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2010 break;
2011 case 4:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2015 TILE_SPLIT(split_equal_to_row_size));
2016 break;
2017 case 5:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2020 break;
2021 case 6:
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2025 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2026 break;
2027 case 7:
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2031 TILE_SPLIT(split_equal_to_row_size));
2032 break;
2033 case 8:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2035 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2036 break;
2037 case 9:
2038 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2039 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2040 break;
2041 case 10:
2042 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2044 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2045 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2046 break;
2047 case 11:
2048 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2049 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2050 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2051 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2052 break;
2053 case 12:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 13:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2062 break;
2063 case 14:
2064 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2065 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2066 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2067 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2068 break;
2069 case 16:
2070 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2071 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2072 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2073 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2074 break;
2075 case 17:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 27:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2084 break;
2085 case 28:
2086 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2088 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2090 break;
2091 case 29:
2092 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2094 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2096 break;
2097 case 30:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 default:
2104 gb_tile_moden = 0;
2105 break;
2106 }
2107 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2108 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2109 }
2110 } else if (num_rbs < 4) {
2111 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2112 switch (reg_offset) {
2113 case 0:
2114 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2115 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2116 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2117 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2118 break;
2119 case 1:
2120 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2122 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2124 break;
2125 case 2:
2126 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2127 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2128 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2129 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2130 break;
2131 case 3:
2132 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2133 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2134 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2135 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2136 break;
2137 case 4:
2138 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2140 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2141 TILE_SPLIT(split_equal_to_row_size));
2142 break;
2143 case 5:
2144 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2146 break;
2147 case 6:
2148 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2150 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2151 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2152 break;
2153 case 7:
2154 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2155 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2156 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2157 TILE_SPLIT(split_equal_to_row_size));
2158 break;
2159 case 8:
2160 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2161 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2162 break;
2163 case 9:
2164 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2166 break;
2167 case 10:
2168 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2170 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2172 break;
2173 case 11:
2174 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2175 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2176 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2178 break;
2179 case 12:
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2184 break;
2185 case 13:
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2188 break;
2189 case 14:
2190 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2191 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2192 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2193 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2194 break;
2195 case 16:
2196 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2198 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2200 break;
2201 case 17:
2202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2204 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2206 break;
2207 case 27:
2208 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2209 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2210 break;
2211 case 28:
2212 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2213 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2214 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2215 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2216 break;
2217 case 29:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2222 break;
2223 case 30:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2228 break;
2229 default:
2230 gb_tile_moden = 0;
2231 break;
2232 }
2233 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2234 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2235 }
2236 }
2237 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2238 switch (reg_offset) {
2239 case 0:
2240 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2241 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2242 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2243 NUM_BANKS(ADDR_SURF_16_BANK));
2244 break;
2245 case 1:
2246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2249 NUM_BANKS(ADDR_SURF_16_BANK));
2250 break;
2251 case 2:
2252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2255 NUM_BANKS(ADDR_SURF_16_BANK));
2256 break;
2257 case 3:
2258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2261 NUM_BANKS(ADDR_SURF_16_BANK));
2262 break;
2263 case 4:
2264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2267 NUM_BANKS(ADDR_SURF_16_BANK));
2268 break;
2269 case 5:
2270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2273 NUM_BANKS(ADDR_SURF_8_BANK));
2274 break;
2275 case 6:
2276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2279 NUM_BANKS(ADDR_SURF_4_BANK));
2280 break;
2281 case 8:
2282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2285 NUM_BANKS(ADDR_SURF_16_BANK));
2286 break;
2287 case 9:
2288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2291 NUM_BANKS(ADDR_SURF_16_BANK));
2292 break;
2293 case 10:
2294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2297 NUM_BANKS(ADDR_SURF_16_BANK));
2298 break;
2299 case 11:
2300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2303 NUM_BANKS(ADDR_SURF_16_BANK));
2304 break;
2305 case 12:
2306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2309 NUM_BANKS(ADDR_SURF_16_BANK));
2310 break;
2311 case 13:
2312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2315 NUM_BANKS(ADDR_SURF_8_BANK));
2316 break;
2317 case 14:
2318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2321 NUM_BANKS(ADDR_SURF_4_BANK));
2322 break;
2323 default:
2324 gb_tile_moden = 0;
2325 break;
2326 }
2327 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2328 }
2329 } else if (num_pipe_configs == 2) {
2330 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2331 switch (reg_offset) {
2332 case 0:
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P2) |
2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2337 break;
2338 case 1:
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P2) |
2342 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2343 break;
2344 case 2:
2345 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2347 PIPE_CONFIG(ADDR_SURF_P2) |
2348 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2349 break;
2350 case 3:
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P2) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2355 break;
2356 case 4:
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P2) |
2360 TILE_SPLIT(split_equal_to_row_size));
2361 break;
2362 case 5:
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2364 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2365 break;
2366 case 6:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P2) |
2370 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2371 break;
2372 case 7:
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P2) |
2376 TILE_SPLIT(split_equal_to_row_size));
2377 break;
2378 case 8:
2379 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2380 break;
2381 case 9:
2382 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2384 break;
2385 case 10:
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P2) |
2389 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2390 break;
2391 case 11:
2392 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P2) |
2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2396 break;
2397 case 12:
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P2) |
2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2402 break;
2403 case 13:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2406 break;
2407 case 14:
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P2) |
2411 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2412 break;
2413 case 16:
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P2) |
2417 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2418 break;
2419 case 17:
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P2) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 break;
2425 case 27:
2426 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2428 break;
2429 case 28:
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P2) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434 break;
2435 case 29:
2436 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P2) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440 break;
2441 case 30:
2442 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P2) |
2445 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2446 break;
2447 default:
2448 gb_tile_moden = 0;
2449 break;
2450 }
2451 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2452 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2453 }
2454 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2455 switch (reg_offset) {
2456 case 0:
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2461 break;
2462 case 1:
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2467 break;
2468 case 2:
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2472 NUM_BANKS(ADDR_SURF_16_BANK));
2473 break;
2474 case 3:
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2478 NUM_BANKS(ADDR_SURF_16_BANK));
2479 break;
2480 case 4:
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2484 NUM_BANKS(ADDR_SURF_16_BANK));
2485 break;
2486 case 5:
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2491 break;
2492 case 6:
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_8_BANK));
2497 break;
2498 case 8:
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2503 break;
2504 case 9:
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2508 NUM_BANKS(ADDR_SURF_16_BANK));
2509 break;
2510 case 10:
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2514 NUM_BANKS(ADDR_SURF_16_BANK));
2515 break;
2516 case 11:
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2520 NUM_BANKS(ADDR_SURF_16_BANK));
2521 break;
2522 case 12:
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2526 NUM_BANKS(ADDR_SURF_16_BANK));
2527 break;
2528 case 13:
2529 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2531 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2532 NUM_BANKS(ADDR_SURF_16_BANK));
2533 break;
2534 case 14:
2535 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2538 NUM_BANKS(ADDR_SURF_8_BANK));
2539 break;
2540 default:
2541 gb_tile_moden = 0;
2542 break;
2543 }
2544 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2545 }
2546 } else
2547 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2548 }
2549
2550 /**
2551 * cik_select_se_sh - select which SE, SH to address
2552 *
2553 * @rdev: radeon_device pointer
2554 * @se_num: shader engine to address
2555 * @sh_num: sh block to address
2556 *
2557 * Select which SE, SH combinations to address. Certain
2558 * registers are instanced per SE or SH. 0xffffffff means
2559 * broadcast to all SEs or SHs (CIK).
2560 */
2561 static void cik_select_se_sh(struct radeon_device *rdev,
2562 u32 se_num, u32 sh_num)
2563 {
2564 u32 data = INSTANCE_BROADCAST_WRITES;
2565
2566 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2567 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2568 else if (se_num == 0xffffffff)
2569 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2570 else if (sh_num == 0xffffffff)
2571 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2572 else
2573 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2574 WREG32(GRBM_GFX_INDEX, data);
2575 }
2576
2577 /**
2578 * cik_create_bitmask - create a bitmask
2579 *
2580 * @bit_width: length of the mask
2581 *
2582 * create a variable length bit mask (CIK).
2583 * Returns the bitmask.
2584 */
2585 static u32 cik_create_bitmask(u32 bit_width)
2586 {
2587 u32 i, mask = 0;
2588
2589 for (i = 0; i < bit_width; i++) {
2590 mask <<= 1;
2591 mask |= 1;
2592 }
2593 return mask;
2594 }
2595
2596 /**
2597 * cik_select_se_sh - select which SE, SH to address
2598 *
2599 * @rdev: radeon_device pointer
2600 * @max_rb_num: max RBs (render backends) for the asic
2601 * @se_num: number of SEs (shader engines) for the asic
2602 * @sh_per_se: number of SH blocks per SE for the asic
2603 *
2604 * Calculates the bitmask of disabled RBs (CIK).
2605 * Returns the disabled RB bitmask.
2606 */
2607 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2608 u32 max_rb_num, u32 se_num,
2609 u32 sh_per_se)
2610 {
2611 u32 data, mask;
2612
2613 data = RREG32(CC_RB_BACKEND_DISABLE);
2614 if (data & 1)
2615 data &= BACKEND_DISABLE_MASK;
2616 else
2617 data = 0;
2618 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2619
2620 data >>= BACKEND_DISABLE_SHIFT;
2621
2622 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2623
2624 return data & mask;
2625 }
2626
2627 /**
2628 * cik_setup_rb - setup the RBs on the asic
2629 *
2630 * @rdev: radeon_device pointer
2631 * @se_num: number of SEs (shader engines) for the asic
2632 * @sh_per_se: number of SH blocks per SE for the asic
2633 * @max_rb_num: max RBs (render backends) for the asic
2634 *
2635 * Configures per-SE/SH RB registers (CIK).
2636 */
2637 static void cik_setup_rb(struct radeon_device *rdev,
2638 u32 se_num, u32 sh_per_se,
2639 u32 max_rb_num)
2640 {
2641 int i, j;
2642 u32 data, mask;
2643 u32 disabled_rbs = 0;
2644 u32 enabled_rbs = 0;
2645
2646 for (i = 0; i < se_num; i++) {
2647 for (j = 0; j < sh_per_se; j++) {
2648 cik_select_se_sh(rdev, i, j);
2649 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2650 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2651 }
2652 }
2653 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2654
2655 mask = 1;
2656 for (i = 0; i < max_rb_num; i++) {
2657 if (!(disabled_rbs & mask))
2658 enabled_rbs |= mask;
2659 mask <<= 1;
2660 }
2661
2662 for (i = 0; i < se_num; i++) {
2663 cik_select_se_sh(rdev, i, 0xffffffff);
2664 data = 0;
2665 for (j = 0; j < sh_per_se; j++) {
2666 switch (enabled_rbs & 3) {
2667 case 1:
2668 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2669 break;
2670 case 2:
2671 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2672 break;
2673 case 3:
2674 default:
2675 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2676 break;
2677 }
2678 enabled_rbs >>= 2;
2679 }
2680 WREG32(PA_SC_RASTER_CONFIG, data);
2681 }
2682 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2683 }
2684
2685 /**
2686 * cik_gpu_init - setup the 3D engine
2687 *
2688 * @rdev: radeon_device pointer
2689 *
2690 * Configures the 3D engine and tiling configuration
2691 * registers so that the 3D engine is usable.
2692 */
2693 static void cik_gpu_init(struct radeon_device *rdev)
2694 {
2695 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2696 u32 mc_shared_chmap, mc_arb_ramcfg;
2697 u32 hdp_host_path_cntl;
2698 u32 tmp;
2699 int i, j;
2700
2701 switch (rdev->family) {
2702 case CHIP_BONAIRE:
2703 rdev->config.cik.max_shader_engines = 2;
2704 rdev->config.cik.max_tile_pipes = 4;
2705 rdev->config.cik.max_cu_per_sh = 7;
2706 rdev->config.cik.max_sh_per_se = 1;
2707 rdev->config.cik.max_backends_per_se = 2;
2708 rdev->config.cik.max_texture_channel_caches = 4;
2709 rdev->config.cik.max_gprs = 256;
2710 rdev->config.cik.max_gs_threads = 32;
2711 rdev->config.cik.max_hw_contexts = 8;
2712
2713 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2714 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2715 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2716 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2717 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2718 break;
2719 case CHIP_KAVERI:
2720 rdev->config.cik.max_shader_engines = 1;
2721 rdev->config.cik.max_tile_pipes = 4;
2722 if ((rdev->pdev->device == 0x1304) ||
2723 (rdev->pdev->device == 0x1305) ||
2724 (rdev->pdev->device == 0x130C) ||
2725 (rdev->pdev->device == 0x130F) ||
2726 (rdev->pdev->device == 0x1310) ||
2727 (rdev->pdev->device == 0x1311) ||
2728 (rdev->pdev->device == 0x131C)) {
2729 rdev->config.cik.max_cu_per_sh = 8;
2730 rdev->config.cik.max_backends_per_se = 2;
2731 } else if ((rdev->pdev->device == 0x1309) ||
2732 (rdev->pdev->device == 0x130A) ||
2733 (rdev->pdev->device == 0x130D) ||
2734 (rdev->pdev->device == 0x1313) ||
2735 (rdev->pdev->device == 0x131D)) {
2736 rdev->config.cik.max_cu_per_sh = 6;
2737 rdev->config.cik.max_backends_per_se = 2;
2738 } else if ((rdev->pdev->device == 0x1306) ||
2739 (rdev->pdev->device == 0x1307) ||
2740 (rdev->pdev->device == 0x130B) ||
2741 (rdev->pdev->device == 0x130E) ||
2742 (rdev->pdev->device == 0x1315) ||
2743 (rdev->pdev->device == 0x131B)) {
2744 rdev->config.cik.max_cu_per_sh = 4;
2745 rdev->config.cik.max_backends_per_se = 1;
2746 } else {
2747 rdev->config.cik.max_cu_per_sh = 3;
2748 rdev->config.cik.max_backends_per_se = 1;
2749 }
2750 rdev->config.cik.max_sh_per_se = 1;
2751 rdev->config.cik.max_texture_channel_caches = 4;
2752 rdev->config.cik.max_gprs = 256;
2753 rdev->config.cik.max_gs_threads = 16;
2754 rdev->config.cik.max_hw_contexts = 8;
2755
2756 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2757 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2758 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2759 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2760 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2761 break;
2762 case CHIP_KABINI:
2763 default:
2764 rdev->config.cik.max_shader_engines = 1;
2765 rdev->config.cik.max_tile_pipes = 2;
2766 rdev->config.cik.max_cu_per_sh = 2;
2767 rdev->config.cik.max_sh_per_se = 1;
2768 rdev->config.cik.max_backends_per_se = 1;
2769 rdev->config.cik.max_texture_channel_caches = 2;
2770 rdev->config.cik.max_gprs = 256;
2771 rdev->config.cik.max_gs_threads = 16;
2772 rdev->config.cik.max_hw_contexts = 8;
2773
2774 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2775 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2776 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2777 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2778 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2779 break;
2780 }
2781
2782 /* Initialize HDP */
2783 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2784 WREG32((0x2c14 + j), 0x00000000);
2785 WREG32((0x2c18 + j), 0x00000000);
2786 WREG32((0x2c1c + j), 0x00000000);
2787 WREG32((0x2c20 + j), 0x00000000);
2788 WREG32((0x2c24 + j), 0x00000000);
2789 }
2790
2791 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2792
2793 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2794
2795 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2796 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2797
2798 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2799 rdev->config.cik.mem_max_burst_length_bytes = 256;
2800 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2801 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2802 if (rdev->config.cik.mem_row_size_in_kb > 4)
2803 rdev->config.cik.mem_row_size_in_kb = 4;
2804 /* XXX use MC settings? */
2805 rdev->config.cik.shader_engine_tile_size = 32;
2806 rdev->config.cik.num_gpus = 1;
2807 rdev->config.cik.multi_gpu_tile_size = 64;
2808
2809 /* fix up row size */
2810 gb_addr_config &= ~ROW_SIZE_MASK;
2811 switch (rdev->config.cik.mem_row_size_in_kb) {
2812 case 1:
2813 default:
2814 gb_addr_config |= ROW_SIZE(0);
2815 break;
2816 case 2:
2817 gb_addr_config |= ROW_SIZE(1);
2818 break;
2819 case 4:
2820 gb_addr_config |= ROW_SIZE(2);
2821 break;
2822 }
2823
2824 /* setup tiling info dword. gb_addr_config is not adequate since it does
2825 * not have bank info, so create a custom tiling dword.
2826 * bits 3:0 num_pipes
2827 * bits 7:4 num_banks
2828 * bits 11:8 group_size
2829 * bits 15:12 row_size
2830 */
2831 rdev->config.cik.tile_config = 0;
2832 switch (rdev->config.cik.num_tile_pipes) {
2833 case 1:
2834 rdev->config.cik.tile_config |= (0 << 0);
2835 break;
2836 case 2:
2837 rdev->config.cik.tile_config |= (1 << 0);
2838 break;
2839 case 4:
2840 rdev->config.cik.tile_config |= (2 << 0);
2841 break;
2842 case 8:
2843 default:
2844 /* XXX what about 12? */
2845 rdev->config.cik.tile_config |= (3 << 0);
2846 break;
2847 }
2848 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
2849 rdev->config.cik.tile_config |= 1 << 4;
2850 else
2851 rdev->config.cik.tile_config |= 0 << 4;
2852 rdev->config.cik.tile_config |=
2853 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2854 rdev->config.cik.tile_config |=
2855 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2856
2857 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2858 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2859 WREG32(DMIF_ADDR_CALC, gb_addr_config);
2860 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2861 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
2862 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2863 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2864 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2865
2866 cik_tiling_mode_table_init(rdev);
2867
2868 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2869 rdev->config.cik.max_sh_per_se,
2870 rdev->config.cik.max_backends_per_se);
2871
2872 /* set HW defaults for 3D engine */
2873 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2874
2875 WREG32(SX_DEBUG_1, 0x20);
2876
2877 WREG32(TA_CNTL_AUX, 0x00010000);
2878
2879 tmp = RREG32(SPI_CONFIG_CNTL);
2880 tmp |= 0x03000000;
2881 WREG32(SPI_CONFIG_CNTL, tmp);
2882
2883 WREG32(SQ_CONFIG, 1);
2884
2885 WREG32(DB_DEBUG, 0);
2886
2887 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2888 tmp |= 0x00000400;
2889 WREG32(DB_DEBUG2, tmp);
2890
2891 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2892 tmp |= 0x00020200;
2893 WREG32(DB_DEBUG3, tmp);
2894
2895 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2896 tmp |= 0x00018208;
2897 WREG32(CB_HW_CONTROL, tmp);
2898
2899 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2900
2901 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2902 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2903 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2904 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2905
2906 WREG32(VGT_NUM_INSTANCES, 1);
2907
2908 WREG32(CP_PERFMON_CNTL, 0);
2909
2910 WREG32(SQ_CONFIG, 0);
2911
2912 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2913 FORCE_EOV_MAX_REZ_CNT(255)));
2914
2915 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2916 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2917
2918 WREG32(VGT_GS_VERTEX_REUSE, 16);
2919 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2920
2921 tmp = RREG32(HDP_MISC_CNTL);
2922 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2923 WREG32(HDP_MISC_CNTL, tmp);
2924
2925 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2926 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2927
2928 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2929 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2930
2931 udelay(50);
2932 }
2933
2934 /*
2935 * GPU scratch registers helpers function.
2936 */
2937 /**
2938 * cik_scratch_init - setup driver info for CP scratch regs
2939 *
2940 * @rdev: radeon_device pointer
2941 *
2942 * Set up the number and offset of the CP scratch registers.
2943 * NOTE: use of CP scratch registers is a legacy inferface and
2944 * is not used by default on newer asics (r6xx+). On newer asics,
2945 * memory buffers are used for fences rather than scratch regs.
2946 */
2947 static void cik_scratch_init(struct radeon_device *rdev)
2948 {
2949 int i;
2950
2951 rdev->scratch.num_reg = 7;
2952 rdev->scratch.reg_base = SCRATCH_REG0;
2953 for (i = 0; i < rdev->scratch.num_reg; i++) {
2954 rdev->scratch.free[i] = true;
2955 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2956 }
2957 }
2958
2959 /**
2960 * cik_ring_test - basic gfx ring test
2961 *
2962 * @rdev: radeon_device pointer
2963 * @ring: radeon_ring structure holding ring information
2964 *
2965 * Allocate a scratch register and write to it using the gfx ring (CIK).
2966 * Provides a basic gfx ring test to verify that the ring is working.
2967 * Used by cik_cp_gfx_resume();
2968 * Returns 0 on success, error on failure.
2969 */
2970 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2971 {
2972 uint32_t scratch;
2973 uint32_t tmp = 0;
2974 unsigned i;
2975 int r;
2976
2977 r = radeon_scratch_get(rdev, &scratch);
2978 if (r) {
2979 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2980 return r;
2981 }
2982 WREG32(scratch, 0xCAFEDEAD);
2983 r = radeon_ring_lock(rdev, ring, 3);
2984 if (r) {
2985 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2986 radeon_scratch_free(rdev, scratch);
2987 return r;
2988 }
2989 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2990 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2991 radeon_ring_write(ring, 0xDEADBEEF);
2992 radeon_ring_unlock_commit(rdev, ring);
2993
2994 for (i = 0; i < rdev->usec_timeout; i++) {
2995 tmp = RREG32(scratch);
2996 if (tmp == 0xDEADBEEF)
2997 break;
2998 DRM_UDELAY(1);
2999 }
3000 if (i < rdev->usec_timeout) {
3001 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3002 } else {
3003 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3004 ring->idx, scratch, tmp);
3005 r = -EINVAL;
3006 }
3007 radeon_scratch_free(rdev, scratch);
3008 return r;
3009 }
3010
3011 /**
3012 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3013 *
3014 * @rdev: radeon_device pointer
3015 * @fence: radeon fence object
3016 *
3017 * Emits a fence sequnce number on the gfx ring and flushes
3018 * GPU caches.
3019 */
3020 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3021 struct radeon_fence *fence)
3022 {
3023 struct radeon_ring *ring = &rdev->ring[fence->ring];
3024 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3025
3026 /* EVENT_WRITE_EOP - flush caches, send int */
3027 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3028 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3029 EOP_TC_ACTION_EN |
3030 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3031 EVENT_INDEX(5)));
3032 radeon_ring_write(ring, addr & 0xfffffffc);
3033 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3034 radeon_ring_write(ring, fence->seq);
3035 radeon_ring_write(ring, 0);
3036 /* HDP flush */
3037 /* We should be using the new WAIT_REG_MEM special op packet here
3038 * but it causes the CP to hang
3039 */
3040 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3041 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3042 WRITE_DATA_DST_SEL(0)));
3043 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3044 radeon_ring_write(ring, 0);
3045 radeon_ring_write(ring, 0);
3046 }
3047
3048 /**
3049 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3050 *
3051 * @rdev: radeon_device pointer
3052 * @fence: radeon fence object
3053 *
3054 * Emits a fence sequnce number on the compute ring and flushes
3055 * GPU caches.
3056 */
3057 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3058 struct radeon_fence *fence)
3059 {
3060 struct radeon_ring *ring = &rdev->ring[fence->ring];
3061 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3062
3063 /* RELEASE_MEM - flush caches, send int */
3064 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3065 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3066 EOP_TC_ACTION_EN |
3067 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3068 EVENT_INDEX(5)));
3069 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3070 radeon_ring_write(ring, addr & 0xfffffffc);
3071 radeon_ring_write(ring, upper_32_bits(addr));
3072 radeon_ring_write(ring, fence->seq);
3073 radeon_ring_write(ring, 0);
3074 /* HDP flush */
3075 /* We should be using the new WAIT_REG_MEM special op packet here
3076 * but it causes the CP to hang
3077 */
3078 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3079 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3080 WRITE_DATA_DST_SEL(0)));
3081 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3082 radeon_ring_write(ring, 0);
3083 radeon_ring_write(ring, 0);
3084 }
3085
3086 void cik_semaphore_ring_emit(struct radeon_device *rdev,
3087 struct radeon_ring *ring,
3088 struct radeon_semaphore *semaphore,
3089 bool emit_wait)
3090 {
3091 uint64_t addr = semaphore->gpu_addr;
3092 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3093
3094 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3095 radeon_ring_write(ring, addr & 0xffffffff);
3096 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3097 }
3098
3099 /*
3100 * IB stuff
3101 */
3102 /**
3103 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3104 *
3105 * @rdev: radeon_device pointer
3106 * @ib: radeon indirect buffer object
3107 *
3108 * Emits an DE (drawing engine) or CE (constant engine) IB
3109 * on the gfx ring. IBs are usually generated by userspace
3110 * acceleration drivers and submitted to the kernel for
3111 * sheduling on the ring. This function schedules the IB
3112 * on the gfx ring for execution by the GPU.
3113 */
3114 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3115 {
3116 struct radeon_ring *ring = &rdev->ring[ib->ring];
3117 u32 header, control = INDIRECT_BUFFER_VALID;
3118
3119 if (ib->is_const_ib) {
3120 /* set switch buffer packet before const IB */
3121 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3122 radeon_ring_write(ring, 0);
3123
3124 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3125 } else {
3126 u32 next_rptr;
3127 if (ring->rptr_save_reg) {
3128 next_rptr = ring->wptr + 3 + 4;
3129 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3130 radeon_ring_write(ring, ((ring->rptr_save_reg -
3131 PACKET3_SET_UCONFIG_REG_START) >> 2));
3132 radeon_ring_write(ring, next_rptr);
3133 } else if (rdev->wb.enabled) {
3134 next_rptr = ring->wptr + 5 + 4;
3135 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3136 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3137 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3138 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3139 radeon_ring_write(ring, next_rptr);
3140 }
3141
3142 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3143 }
3144
3145 control |= ib->length_dw |
3146 (ib->vm ? (ib->vm->id << 24) : 0);
3147
3148 radeon_ring_write(ring, header);
3149 radeon_ring_write(ring,
3150 #ifdef __BIG_ENDIAN
3151 (2 << 0) |
3152 #endif
3153 (ib->gpu_addr & 0xFFFFFFFC));
3154 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3155 radeon_ring_write(ring, control);
3156 }
3157
3158 /**
3159 * cik_ib_test - basic gfx ring IB test
3160 *
3161 * @rdev: radeon_device pointer
3162 * @ring: radeon_ring structure holding ring information
3163 *
3164 * Allocate an IB and execute it on the gfx ring (CIK).
3165 * Provides a basic gfx ring test to verify that IBs are working.
3166 * Returns 0 on success, error on failure.
3167 */
3168 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3169 {
3170 struct radeon_ib ib;
3171 uint32_t scratch;
3172 uint32_t tmp = 0;
3173 unsigned i;
3174 int r;
3175
3176 r = radeon_scratch_get(rdev, &scratch);
3177 if (r) {
3178 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3179 return r;
3180 }
3181 WREG32(scratch, 0xCAFEDEAD);
3182 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3183 if (r) {
3184 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3185 return r;
3186 }
3187 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3188 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3189 ib.ptr[2] = 0xDEADBEEF;
3190 ib.length_dw = 3;
3191 r = radeon_ib_schedule(rdev, &ib, NULL);
3192 if (r) {
3193 radeon_scratch_free(rdev, scratch);
3194 radeon_ib_free(rdev, &ib);
3195 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3196 return r;
3197 }
3198 r = radeon_fence_wait(ib.fence, false);
3199 if (r) {
3200 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3201 return r;
3202 }
3203 for (i = 0; i < rdev->usec_timeout; i++) {
3204 tmp = RREG32(scratch);
3205 if (tmp == 0xDEADBEEF)
3206 break;
3207 DRM_UDELAY(1);
3208 }
3209 if (i < rdev->usec_timeout) {
3210 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3211 } else {
3212 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3213 scratch, tmp);
3214 r = -EINVAL;
3215 }
3216 radeon_scratch_free(rdev, scratch);
3217 radeon_ib_free(rdev, &ib);
3218 return r;
3219 }
3220
3221 /*
3222 * CP.
3223 * On CIK, gfx and compute now have independant command processors.
3224 *
3225 * GFX
3226 * Gfx consists of a single ring and can process both gfx jobs and
3227 * compute jobs. The gfx CP consists of three microengines (ME):
3228 * PFP - Pre-Fetch Parser
3229 * ME - Micro Engine
3230 * CE - Constant Engine
3231 * The PFP and ME make up what is considered the Drawing Engine (DE).
3232 * The CE is an asynchronous engine used for updating buffer desciptors
3233 * used by the DE so that they can be loaded into cache in parallel
3234 * while the DE is processing state update packets.
3235 *
3236 * Compute
3237 * The compute CP consists of two microengines (ME):
3238 * MEC1 - Compute MicroEngine 1
3239 * MEC2 - Compute MicroEngine 2
3240 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3241 * The queues are exposed to userspace and are programmed directly
3242 * by the compute runtime.
3243 */
3244 /**
3245 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3246 *
3247 * @rdev: radeon_device pointer
3248 * @enable: enable or disable the MEs
3249 *
3250 * Halts or unhalts the gfx MEs.
3251 */
3252 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3253 {
3254 if (enable)
3255 WREG32(CP_ME_CNTL, 0);
3256 else {
3257 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3258 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3259 }
3260 udelay(50);
3261 }
3262
3263 /**
3264 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3265 *
3266 * @rdev: radeon_device pointer
3267 *
3268 * Loads the gfx PFP, ME, and CE ucode.
3269 * Returns 0 for success, -EINVAL if the ucode is not available.
3270 */
3271 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3272 {
3273 const __be32 *fw_data;
3274 int i;
3275
3276 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3277 return -EINVAL;
3278
3279 cik_cp_gfx_enable(rdev, false);
3280
3281 /* PFP */
3282 fw_data = (const __be32 *)rdev->pfp_fw->data;
3283 WREG32(CP_PFP_UCODE_ADDR, 0);
3284 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3285 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3286 WREG32(CP_PFP_UCODE_ADDR, 0);
3287
3288 /* CE */
3289 fw_data = (const __be32 *)rdev->ce_fw->data;
3290 WREG32(CP_CE_UCODE_ADDR, 0);
3291 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3292 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3293 WREG32(CP_CE_UCODE_ADDR, 0);
3294
3295 /* ME */
3296 fw_data = (const __be32 *)rdev->me_fw->data;
3297 WREG32(CP_ME_RAM_WADDR, 0);
3298 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3299 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3300 WREG32(CP_ME_RAM_WADDR, 0);
3301
3302 WREG32(CP_PFP_UCODE_ADDR, 0);
3303 WREG32(CP_CE_UCODE_ADDR, 0);
3304 WREG32(CP_ME_RAM_WADDR, 0);
3305 WREG32(CP_ME_RAM_RADDR, 0);
3306 return 0;
3307 }
3308
3309 /**
3310 * cik_cp_gfx_start - start the gfx ring
3311 *
3312 * @rdev: radeon_device pointer
3313 *
3314 * Enables the ring and loads the clear state context and other
3315 * packets required to init the ring.
3316 * Returns 0 for success, error for failure.
3317 */
3318 static int cik_cp_gfx_start(struct radeon_device *rdev)
3319 {
3320 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3321 int r, i;
3322
3323 /* init the CP */
3324 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3325 WREG32(CP_ENDIAN_SWAP, 0);
3326 WREG32(CP_DEVICE_ID, 1);
3327
3328 cik_cp_gfx_enable(rdev, true);
3329
3330 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3331 if (r) {
3332 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3333 return r;
3334 }
3335
3336 /* init the CE partitions. CE only used for gfx on CIK */
3337 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3338 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3339 radeon_ring_write(ring, 0xc000);
3340 radeon_ring_write(ring, 0xc000);
3341
3342 /* setup clear context state */
3343 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3344 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3345
3346 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3347 radeon_ring_write(ring, 0x80000000);
3348 radeon_ring_write(ring, 0x80000000);
3349
3350 for (i = 0; i < cik_default_size; i++)
3351 radeon_ring_write(ring, cik_default_state[i]);
3352
3353 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3354 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3355
3356 /* set clear context state */
3357 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3358 radeon_ring_write(ring, 0);
3359
3360 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3361 radeon_ring_write(ring, 0x00000316);
3362 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3363 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3364
3365 radeon_ring_unlock_commit(rdev, ring);
3366
3367 return 0;
3368 }
3369
3370 /**
3371 * cik_cp_gfx_fini - stop the gfx ring
3372 *
3373 * @rdev: radeon_device pointer
3374 *
3375 * Stop the gfx ring and tear down the driver ring
3376 * info.
3377 */
3378 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3379 {
3380 cik_cp_gfx_enable(rdev, false);
3381 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3382 }
3383
3384 /**
3385 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3386 *
3387 * @rdev: radeon_device pointer
3388 *
3389 * Program the location and size of the gfx ring buffer
3390 * and test it to make sure it's working.
3391 * Returns 0 for success, error for failure.
3392 */
3393 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3394 {
3395 struct radeon_ring *ring;
3396 u32 tmp;
3397 u32 rb_bufsz;
3398 u64 rb_addr;
3399 int r;
3400
3401 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3402 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3403
3404 /* Set the write pointer delay */
3405 WREG32(CP_RB_WPTR_DELAY, 0);
3406
3407 /* set the RB to use vmid 0 */
3408 WREG32(CP_RB_VMID, 0);
3409
3410 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3411
3412 /* ring 0 - compute and gfx */
3413 /* Set ring buffer size */
3414 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3415 rb_bufsz = order_base_2(ring->ring_size / 8);
3416 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3417 #ifdef __BIG_ENDIAN
3418 tmp |= BUF_SWAP_32BIT;
3419 #endif
3420 WREG32(CP_RB0_CNTL, tmp);
3421
3422 /* Initialize the ring buffer's read and write pointers */
3423 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3424 ring->wptr = 0;
3425 WREG32(CP_RB0_WPTR, ring->wptr);
3426
3427 /* set the wb address wether it's enabled or not */
3428 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3429 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3430
3431 /* scratch register shadowing is no longer supported */
3432 WREG32(SCRATCH_UMSK, 0);
3433
3434 if (!rdev->wb.enabled)
3435 tmp |= RB_NO_UPDATE;
3436
3437 mdelay(1);
3438 WREG32(CP_RB0_CNTL, tmp);
3439
3440 rb_addr = ring->gpu_addr >> 8;
3441 WREG32(CP_RB0_BASE, rb_addr);
3442 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3443
3444 ring->rptr = RREG32(CP_RB0_RPTR);
3445
3446 /* start the ring */
3447 cik_cp_gfx_start(rdev);
3448 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3449 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3450 if (r) {
3451 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3452 return r;
3453 }
3454 return 0;
3455 }
3456
3457 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3458 struct radeon_ring *ring)
3459 {
3460 u32 rptr;
3461
3462
3463
3464 if (rdev->wb.enabled) {
3465 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3466 } else {
3467 mutex_lock(&rdev->srbm_mutex);
3468 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3469 rptr = RREG32(CP_HQD_PQ_RPTR);
3470 cik_srbm_select(rdev, 0, 0, 0, 0);
3471 mutex_unlock(&rdev->srbm_mutex);
3472 }
3473
3474 return rptr;
3475 }
3476
3477 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3478 struct radeon_ring *ring)
3479 {
3480 u32 wptr;
3481
3482 if (rdev->wb.enabled) {
3483 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3484 } else {
3485 mutex_lock(&rdev->srbm_mutex);
3486 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3487 wptr = RREG32(CP_HQD_PQ_WPTR);
3488 cik_srbm_select(rdev, 0, 0, 0, 0);
3489 mutex_unlock(&rdev->srbm_mutex);
3490 }
3491
3492 return wptr;
3493 }
3494
3495 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3496 struct radeon_ring *ring)
3497 {
3498 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3499 WDOORBELL32(ring->doorbell_offset, ring->wptr);
3500 }
3501
3502 /**
3503 * cik_cp_compute_enable - enable/disable the compute CP MEs
3504 *
3505 * @rdev: radeon_device pointer
3506 * @enable: enable or disable the MEs
3507 *
3508 * Halts or unhalts the compute MEs.
3509 */
3510 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3511 {
3512 if (enable)
3513 WREG32(CP_MEC_CNTL, 0);
3514 else
3515 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3516 udelay(50);
3517 }
3518
3519 /**
3520 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3521 *
3522 * @rdev: radeon_device pointer
3523 *
3524 * Loads the compute MEC1&2 ucode.
3525 * Returns 0 for success, -EINVAL if the ucode is not available.
3526 */
3527 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3528 {
3529 const __be32 *fw_data;
3530 int i;
3531
3532 if (!rdev->mec_fw)
3533 return -EINVAL;
3534
3535 cik_cp_compute_enable(rdev, false);
3536
3537 /* MEC1 */
3538 fw_data = (const __be32 *)rdev->mec_fw->data;
3539 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3540 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3541 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3542 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3543
3544 if (rdev->family == CHIP_KAVERI) {
3545 /* MEC2 */
3546 fw_data = (const __be32 *)rdev->mec_fw->data;
3547 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3548 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3549 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3550 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3551 }
3552
3553 return 0;
3554 }
3555
3556 /**
3557 * cik_cp_compute_start - start the compute queues
3558 *
3559 * @rdev: radeon_device pointer
3560 *
3561 * Enable the compute queues.
3562 * Returns 0 for success, error for failure.
3563 */
3564 static int cik_cp_compute_start(struct radeon_device *rdev)
3565 {
3566 cik_cp_compute_enable(rdev, true);
3567
3568 return 0;
3569 }
3570
3571 /**
3572 * cik_cp_compute_fini - stop the compute queues
3573 *
3574 * @rdev: radeon_device pointer
3575 *
3576 * Stop the compute queues and tear down the driver queue
3577 * info.
3578 */
3579 static void cik_cp_compute_fini(struct radeon_device *rdev)
3580 {
3581 int i, idx, r;
3582
3583 cik_cp_compute_enable(rdev, false);
3584
3585 for (i = 0; i < 2; i++) {
3586 if (i == 0)
3587 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3588 else
3589 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3590
3591 if (rdev->ring[idx].mqd_obj) {
3592 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3593 if (unlikely(r != 0))
3594 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3595
3596 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3597 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3598
3599 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3600 rdev->ring[idx].mqd_obj = NULL;
3601 }
3602 }
3603 }
3604
3605 static void cik_mec_fini(struct radeon_device *rdev)
3606 {
3607 int r;
3608
3609 if (rdev->mec.hpd_eop_obj) {
3610 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3611 if (unlikely(r != 0))
3612 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3613 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3614 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3615
3616 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3617 rdev->mec.hpd_eop_obj = NULL;
3618 }
3619 }
3620
3621 #define MEC_HPD_SIZE 2048
3622
3623 static int cik_mec_init(struct radeon_device *rdev)
3624 {
3625 int r;
3626 u32 *hpd;
3627
3628 /*
3629 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3630 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3631 */
3632 if (rdev->family == CHIP_KAVERI)
3633 rdev->mec.num_mec = 2;
3634 else
3635 rdev->mec.num_mec = 1;
3636 rdev->mec.num_pipe = 4;
3637 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3638
3639 if (rdev->mec.hpd_eop_obj == NULL) {
3640 r = radeon_bo_create(rdev,
3641 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3642 PAGE_SIZE, true,
3643 RADEON_GEM_DOMAIN_GTT, NULL,
3644 &rdev->mec.hpd_eop_obj);
3645 if (r) {
3646 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3647 return r;
3648 }
3649 }
3650
3651 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3652 if (unlikely(r != 0)) {
3653 cik_mec_fini(rdev);
3654 return r;
3655 }
3656 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3657 &rdev->mec.hpd_eop_gpu_addr);
3658 if (r) {
3659 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3660 cik_mec_fini(rdev);
3661 return r;
3662 }
3663 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3664 if (r) {
3665 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3666 cik_mec_fini(rdev);
3667 return r;
3668 }
3669
3670 /* clear memory. Not sure if this is required or not */
3671 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3672
3673 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3674 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3675
3676 return 0;
3677 }
3678
3679 struct hqd_registers
3680 {
3681 u32 cp_mqd_base_addr;
3682 u32 cp_mqd_base_addr_hi;
3683 u32 cp_hqd_active;
3684 u32 cp_hqd_vmid;
3685 u32 cp_hqd_persistent_state;
3686 u32 cp_hqd_pipe_priority;
3687 u32 cp_hqd_queue_priority;
3688 u32 cp_hqd_quantum;
3689 u32 cp_hqd_pq_base;
3690 u32 cp_hqd_pq_base_hi;
3691 u32 cp_hqd_pq_rptr;
3692 u32 cp_hqd_pq_rptr_report_addr;
3693 u32 cp_hqd_pq_rptr_report_addr_hi;
3694 u32 cp_hqd_pq_wptr_poll_addr;
3695 u32 cp_hqd_pq_wptr_poll_addr_hi;
3696 u32 cp_hqd_pq_doorbell_control;
3697 u32 cp_hqd_pq_wptr;
3698 u32 cp_hqd_pq_control;
3699 u32 cp_hqd_ib_base_addr;
3700 u32 cp_hqd_ib_base_addr_hi;
3701 u32 cp_hqd_ib_rptr;
3702 u32 cp_hqd_ib_control;
3703 u32 cp_hqd_iq_timer;
3704 u32 cp_hqd_iq_rptr;
3705 u32 cp_hqd_dequeue_request;
3706 u32 cp_hqd_dma_offload;
3707 u32 cp_hqd_sema_cmd;
3708 u32 cp_hqd_msg_type;
3709 u32 cp_hqd_atomic0_preop_lo;
3710 u32 cp_hqd_atomic0_preop_hi;
3711 u32 cp_hqd_atomic1_preop_lo;
3712 u32 cp_hqd_atomic1_preop_hi;
3713 u32 cp_hqd_hq_scheduler0;
3714 u32 cp_hqd_hq_scheduler1;
3715 u32 cp_mqd_control;
3716 };
3717
3718 struct bonaire_mqd
3719 {
3720 u32 header;
3721 u32 dispatch_initiator;
3722 u32 dimensions[3];
3723 u32 start_idx[3];
3724 u32 num_threads[3];
3725 u32 pipeline_stat_enable;
3726 u32 perf_counter_enable;
3727 u32 pgm[2];
3728 u32 tba[2];
3729 u32 tma[2];
3730 u32 pgm_rsrc[2];
3731 u32 vmid;
3732 u32 resource_limits;
3733 u32 static_thread_mgmt01[2];
3734 u32 tmp_ring_size;
3735 u32 static_thread_mgmt23[2];
3736 u32 restart[3];
3737 u32 thread_trace_enable;
3738 u32 reserved1;
3739 u32 user_data[16];
3740 u32 vgtcs_invoke_count[2];
3741 struct hqd_registers queue_state;
3742 u32 dequeue_cntr;
3743 u32 interrupt_queue[64];
3744 };
3745
3746 /**
3747 * cik_cp_compute_resume - setup the compute queue registers
3748 *
3749 * @rdev: radeon_device pointer
3750 *
3751 * Program the compute queues and test them to make sure they
3752 * are working.
3753 * Returns 0 for success, error for failure.
3754 */
3755 static int cik_cp_compute_resume(struct radeon_device *rdev)
3756 {
3757 int r, i, idx;
3758 u32 tmp;
3759 bool use_doorbell = true;
3760 u64 hqd_gpu_addr;
3761 u64 mqd_gpu_addr;
3762 u64 eop_gpu_addr;
3763 u64 wb_gpu_addr;
3764 u32 *buf;
3765 struct bonaire_mqd *mqd;
3766
3767 r = cik_cp_compute_start(rdev);
3768 if (r)
3769 return r;
3770
3771 /* fix up chicken bits */
3772 tmp = RREG32(CP_CPF_DEBUG);
3773 tmp |= (1 << 23);
3774 WREG32(CP_CPF_DEBUG, tmp);
3775
3776 /* init the pipes */
3777 mutex_lock(&rdev->srbm_mutex);
3778 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3779 int me = (i < 4) ? 1 : 2;
3780 int pipe = (i < 4) ? i : (i - 4);
3781
3782 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3783
3784 cik_srbm_select(rdev, me, pipe, 0, 0);
3785
3786 /* write the EOP addr */
3787 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3788 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3789
3790 /* set the VMID assigned */
3791 WREG32(CP_HPD_EOP_VMID, 0);
3792
3793 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3794 tmp = RREG32(CP_HPD_EOP_CONTROL);
3795 tmp &= ~EOP_SIZE_MASK;
3796 tmp |= order_base_2(MEC_HPD_SIZE / 8);
3797 WREG32(CP_HPD_EOP_CONTROL, tmp);
3798 }
3799 cik_srbm_select(rdev, 0, 0, 0, 0);
3800 mutex_unlock(&rdev->srbm_mutex);
3801
3802 /* init the queues. Just two for now. */
3803 for (i = 0; i < 2; i++) {
3804 if (i == 0)
3805 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3806 else
3807 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3808
3809 if (rdev->ring[idx].mqd_obj == NULL) {
3810 r = radeon_bo_create(rdev,
3811 sizeof(struct bonaire_mqd),
3812 PAGE_SIZE, true,
3813 RADEON_GEM_DOMAIN_GTT, NULL,
3814 &rdev->ring[idx].mqd_obj);
3815 if (r) {
3816 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3817 return r;
3818 }
3819 }
3820
3821 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3822 if (unlikely(r != 0)) {
3823 cik_cp_compute_fini(rdev);
3824 return r;
3825 }
3826 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3827 &mqd_gpu_addr);
3828 if (r) {
3829 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3830 cik_cp_compute_fini(rdev);
3831 return r;
3832 }
3833 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3834 if (r) {
3835 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3836 cik_cp_compute_fini(rdev);
3837 return r;
3838 }
3839
3840 /* doorbell offset */
3841 rdev->ring[idx].doorbell_offset =
3842 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3843
3844 /* init the mqd struct */
3845 memset(buf, 0, sizeof(struct bonaire_mqd));
3846
3847 mqd = (struct bonaire_mqd *)buf;
3848 mqd->header = 0xC0310800;
3849 mqd->static_thread_mgmt01[0] = 0xffffffff;
3850 mqd->static_thread_mgmt01[1] = 0xffffffff;
3851 mqd->static_thread_mgmt23[0] = 0xffffffff;
3852 mqd->static_thread_mgmt23[1] = 0xffffffff;
3853
3854 mutex_lock(&rdev->srbm_mutex);
3855 cik_srbm_select(rdev, rdev->ring[idx].me,
3856 rdev->ring[idx].pipe,
3857 rdev->ring[idx].queue, 0);
3858
3859 /* disable wptr polling */
3860 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3861 tmp &= ~WPTR_POLL_EN;
3862 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3863
3864 /* enable doorbell? */
3865 mqd->queue_state.cp_hqd_pq_doorbell_control =
3866 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3867 if (use_doorbell)
3868 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3869 else
3870 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3871 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3872 mqd->queue_state.cp_hqd_pq_doorbell_control);
3873
3874 /* disable the queue if it's active */
3875 mqd->queue_state.cp_hqd_dequeue_request = 0;
3876 mqd->queue_state.cp_hqd_pq_rptr = 0;
3877 mqd->queue_state.cp_hqd_pq_wptr= 0;
3878 if (RREG32(CP_HQD_ACTIVE) & 1) {
3879 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3880 for (i = 0; i < rdev->usec_timeout; i++) {
3881 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3882 break;
3883 udelay(1);
3884 }
3885 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3886 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3887 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3888 }
3889
3890 /* set the pointer to the MQD */
3891 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3892 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3893 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3894 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3895 /* set MQD vmid to 0 */
3896 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3897 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3898 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3899
3900 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3901 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3902 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3903 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3904 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3905 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3906
3907 /* set up the HQD, this is similar to CP_RB0_CNTL */
3908 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3909 mqd->queue_state.cp_hqd_pq_control &=
3910 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3911
3912 mqd->queue_state.cp_hqd_pq_control |=
3913 order_base_2(rdev->ring[idx].ring_size / 8);
3914 mqd->queue_state.cp_hqd_pq_control |=
3915 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
3916 #ifdef __BIG_ENDIAN
3917 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3918 #endif
3919 mqd->queue_state.cp_hqd_pq_control &=
3920 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3921 mqd->queue_state.cp_hqd_pq_control |=
3922 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3923 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3924
3925 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3926 if (i == 0)
3927 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3928 else
3929 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3930 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3931 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3932 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3933 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3934 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3935
3936 /* set the wb address wether it's enabled or not */
3937 if (i == 0)
3938 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3939 else
3940 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3941 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3942 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3943 upper_32_bits(wb_gpu_addr) & 0xffff;
3944 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3945 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3946 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3947 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3948
3949 /* enable the doorbell if requested */
3950 if (use_doorbell) {
3951 mqd->queue_state.cp_hqd_pq_doorbell_control =
3952 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3953 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3954 mqd->queue_state.cp_hqd_pq_doorbell_control |=
3955 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3956 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3957 mqd->queue_state.cp_hqd_pq_doorbell_control &=
3958 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3959
3960 } else {
3961 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3962 }
3963 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3964 mqd->queue_state.cp_hqd_pq_doorbell_control);
3965
3966 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3967 rdev->ring[idx].wptr = 0;
3968 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3969 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3970 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3971 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3972
3973 /* set the vmid for the queue */
3974 mqd->queue_state.cp_hqd_vmid = 0;
3975 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3976
3977 /* activate the queue */
3978 mqd->queue_state.cp_hqd_active = 1;
3979 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3980
3981 cik_srbm_select(rdev, 0, 0, 0, 0);
3982 mutex_unlock(&rdev->srbm_mutex);
3983
3984 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3985 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3986
3987 rdev->ring[idx].ready = true;
3988 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3989 if (r)
3990 rdev->ring[idx].ready = false;
3991 }
3992
3993 return 0;
3994 }
3995
3996 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3997 {
3998 cik_cp_gfx_enable(rdev, enable);
3999 cik_cp_compute_enable(rdev, enable);
4000 }
4001
4002 static int cik_cp_load_microcode(struct radeon_device *rdev)
4003 {
4004 int r;
4005
4006 r = cik_cp_gfx_load_microcode(rdev);
4007 if (r)
4008 return r;
4009 r = cik_cp_compute_load_microcode(rdev);
4010 if (r)
4011 return r;
4012
4013 return 0;
4014 }
4015
4016 static void cik_cp_fini(struct radeon_device *rdev)
4017 {
4018 cik_cp_gfx_fini(rdev);
4019 cik_cp_compute_fini(rdev);
4020 }
4021
4022 static int cik_cp_resume(struct radeon_device *rdev)
4023 {
4024 int r;
4025
4026 cik_enable_gui_idle_interrupt(rdev, false);
4027
4028 r = cik_cp_load_microcode(rdev);
4029 if (r)
4030 return r;
4031
4032 r = cik_cp_gfx_resume(rdev);
4033 if (r)
4034 return r;
4035 r = cik_cp_compute_resume(rdev);
4036 if (r)
4037 return r;
4038
4039 cik_enable_gui_idle_interrupt(rdev, true);
4040
4041 return 0;
4042 }
4043
4044 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4045 {
4046 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4047 RREG32(GRBM_STATUS));
4048 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4049 RREG32(GRBM_STATUS2));
4050 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4051 RREG32(GRBM_STATUS_SE0));
4052 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4053 RREG32(GRBM_STATUS_SE1));
4054 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4055 RREG32(GRBM_STATUS_SE2));
4056 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4057 RREG32(GRBM_STATUS_SE3));
4058 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4059 RREG32(SRBM_STATUS));
4060 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4061 RREG32(SRBM_STATUS2));
4062 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4063 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4064 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4065 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4066 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4067 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4068 RREG32(CP_STALLED_STAT1));
4069 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4070 RREG32(CP_STALLED_STAT2));
4071 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4072 RREG32(CP_STALLED_STAT3));
4073 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4074 RREG32(CP_CPF_BUSY_STAT));
4075 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4076 RREG32(CP_CPF_STALLED_STAT1));
4077 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4078 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4079 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4080 RREG32(CP_CPC_STALLED_STAT1));
4081 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4082 }
4083
4084 /**
4085 * cik_gpu_check_soft_reset - check which blocks are busy
4086 *
4087 * @rdev: radeon_device pointer
4088 *
4089 * Check which blocks are busy and return the relevant reset
4090 * mask to be used by cik_gpu_soft_reset().
4091 * Returns a mask of the blocks to be reset.
4092 */
4093 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4094 {
4095 u32 reset_mask = 0;
4096 u32 tmp;
4097
4098 /* GRBM_STATUS */
4099 tmp = RREG32(GRBM_STATUS);
4100 if (tmp & (PA_BUSY | SC_BUSY |
4101 BCI_BUSY | SX_BUSY |
4102 TA_BUSY | VGT_BUSY |
4103 DB_BUSY | CB_BUSY |
4104 GDS_BUSY | SPI_BUSY |
4105 IA_BUSY | IA_BUSY_NO_DMA))
4106 reset_mask |= RADEON_RESET_GFX;
4107
4108 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4109 reset_mask |= RADEON_RESET_CP;
4110
4111 /* GRBM_STATUS2 */
4112 tmp = RREG32(GRBM_STATUS2);
4113 if (tmp & RLC_BUSY)
4114 reset_mask |= RADEON_RESET_RLC;
4115
4116 /* SDMA0_STATUS_REG */
4117 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4118 if (!(tmp & SDMA_IDLE))
4119 reset_mask |= RADEON_RESET_DMA;
4120
4121 /* SDMA1_STATUS_REG */
4122 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4123 if (!(tmp & SDMA_IDLE))
4124 reset_mask |= RADEON_RESET_DMA1;
4125
4126 /* SRBM_STATUS2 */
4127 tmp = RREG32(SRBM_STATUS2);
4128 if (tmp & SDMA_BUSY)
4129 reset_mask |= RADEON_RESET_DMA;
4130
4131 if (tmp & SDMA1_BUSY)
4132 reset_mask |= RADEON_RESET_DMA1;
4133
4134 /* SRBM_STATUS */
4135 tmp = RREG32(SRBM_STATUS);
4136
4137 if (tmp & IH_BUSY)
4138 reset_mask |= RADEON_RESET_IH;
4139
4140 if (tmp & SEM_BUSY)
4141 reset_mask |= RADEON_RESET_SEM;
4142
4143 if (tmp & GRBM_RQ_PENDING)
4144 reset_mask |= RADEON_RESET_GRBM;
4145
4146 if (tmp & VMC_BUSY)
4147 reset_mask |= RADEON_RESET_VMC;
4148
4149 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4150 MCC_BUSY | MCD_BUSY))
4151 reset_mask |= RADEON_RESET_MC;
4152
4153 if (evergreen_is_display_hung(rdev))
4154 reset_mask |= RADEON_RESET_DISPLAY;
4155
4156 /* Skip MC reset as it's mostly likely not hung, just busy */
4157 if (reset_mask & RADEON_RESET_MC) {
4158 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4159 reset_mask &= ~RADEON_RESET_MC;
4160 }
4161
4162 return reset_mask;
4163 }
4164
4165 /**
4166 * cik_gpu_soft_reset - soft reset GPU
4167 *
4168 * @rdev: radeon_device pointer
4169 * @reset_mask: mask of which blocks to reset
4170 *
4171 * Soft reset the blocks specified in @reset_mask.
4172 */
4173 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4174 {
4175 struct evergreen_mc_save save;
4176 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4177 u32 tmp;
4178
4179 if (reset_mask == 0)
4180 return;
4181
4182 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4183
4184 cik_print_gpu_status_regs(rdev);
4185 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4186 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4187 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4188 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4189
4190 /* stop the rlc */
4191 cik_rlc_stop(rdev);
4192
4193 /* Disable GFX parsing/prefetching */
4194 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4195
4196 /* Disable MEC parsing/prefetching */
4197 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4198
4199 if (reset_mask & RADEON_RESET_DMA) {
4200 /* sdma0 */
4201 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4202 tmp |= SDMA_HALT;
4203 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4204 }
4205 if (reset_mask & RADEON_RESET_DMA1) {
4206 /* sdma1 */
4207 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4208 tmp |= SDMA_HALT;
4209 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4210 }
4211
4212 evergreen_mc_stop(rdev, &save);
4213 if (evergreen_mc_wait_for_idle(rdev)) {
4214 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4215 }
4216
4217 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4218 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4219
4220 if (reset_mask & RADEON_RESET_CP) {
4221 grbm_soft_reset |= SOFT_RESET_CP;
4222
4223 srbm_soft_reset |= SOFT_RESET_GRBM;
4224 }
4225
4226 if (reset_mask & RADEON_RESET_DMA)
4227 srbm_soft_reset |= SOFT_RESET_SDMA;
4228
4229 if (reset_mask & RADEON_RESET_DMA1)
4230 srbm_soft_reset |= SOFT_RESET_SDMA1;
4231
4232 if (reset_mask & RADEON_RESET_DISPLAY)
4233 srbm_soft_reset |= SOFT_RESET_DC;
4234
4235 if (reset_mask & RADEON_RESET_RLC)
4236 grbm_soft_reset |= SOFT_RESET_RLC;
4237
4238 if (reset_mask & RADEON_RESET_SEM)
4239 srbm_soft_reset |= SOFT_RESET_SEM;
4240
4241 if (reset_mask & RADEON_RESET_IH)
4242 srbm_soft_reset |= SOFT_RESET_IH;
4243
4244 if (reset_mask & RADEON_RESET_GRBM)
4245 srbm_soft_reset |= SOFT_RESET_GRBM;
4246
4247 if (reset_mask & RADEON_RESET_VMC)
4248 srbm_soft_reset |= SOFT_RESET_VMC;
4249
4250 if (!(rdev->flags & RADEON_IS_IGP)) {
4251 if (reset_mask & RADEON_RESET_MC)
4252 srbm_soft_reset |= SOFT_RESET_MC;
4253 }
4254
4255 if (grbm_soft_reset) {
4256 tmp = RREG32(GRBM_SOFT_RESET);
4257 tmp |= grbm_soft_reset;
4258 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4259 WREG32(GRBM_SOFT_RESET, tmp);
4260 tmp = RREG32(GRBM_SOFT_RESET);
4261
4262 udelay(50);
4263
4264 tmp &= ~grbm_soft_reset;
4265 WREG32(GRBM_SOFT_RESET, tmp);
4266 tmp = RREG32(GRBM_SOFT_RESET);
4267 }
4268
4269 if (srbm_soft_reset) {
4270 tmp = RREG32(SRBM_SOFT_RESET);
4271 tmp |= srbm_soft_reset;
4272 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4273 WREG32(SRBM_SOFT_RESET, tmp);
4274 tmp = RREG32(SRBM_SOFT_RESET);
4275
4276 udelay(50);
4277
4278 tmp &= ~srbm_soft_reset;
4279 WREG32(SRBM_SOFT_RESET, tmp);
4280 tmp = RREG32(SRBM_SOFT_RESET);
4281 }
4282
4283 /* Wait a little for things to settle down */
4284 udelay(50);
4285
4286 evergreen_mc_resume(rdev, &save);
4287 udelay(50);
4288
4289 cik_print_gpu_status_regs(rdev);
4290 }
4291
4292 /**
4293 * cik_asic_reset - soft reset GPU
4294 *
4295 * @rdev: radeon_device pointer
4296 *
4297 * Look up which blocks are hung and attempt
4298 * to reset them.
4299 * Returns 0 for success.
4300 */
4301 int cik_asic_reset(struct radeon_device *rdev)
4302 {
4303 u32 reset_mask;
4304
4305 reset_mask = cik_gpu_check_soft_reset(rdev);
4306
4307 if (reset_mask)
4308 r600_set_bios_scratch_engine_hung(rdev, true);
4309
4310 cik_gpu_soft_reset(rdev, reset_mask);
4311
4312 reset_mask = cik_gpu_check_soft_reset(rdev);
4313
4314 if (!reset_mask)
4315 r600_set_bios_scratch_engine_hung(rdev, false);
4316
4317 return 0;
4318 }
4319
4320 /**
4321 * cik_gfx_is_lockup - check if the 3D engine is locked up
4322 *
4323 * @rdev: radeon_device pointer
4324 * @ring: radeon_ring structure holding ring information
4325 *
4326 * Check if the 3D engine is locked up (CIK).
4327 * Returns true if the engine is locked, false if not.
4328 */
4329 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4330 {
4331 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4332
4333 if (!(reset_mask & (RADEON_RESET_GFX |
4334 RADEON_RESET_COMPUTE |
4335 RADEON_RESET_CP))) {
4336 radeon_ring_lockup_update(ring);
4337 return false;
4338 }
4339 /* force CP activities */
4340 radeon_ring_force_activity(rdev, ring);
4341 return radeon_ring_test_lockup(rdev, ring);
4342 }
4343
4344 /* MC */
4345 /**
4346 * cik_mc_program - program the GPU memory controller
4347 *
4348 * @rdev: radeon_device pointer
4349 *
4350 * Set the location of vram, gart, and AGP in the GPU's
4351 * physical address space (CIK).
4352 */
4353 static void cik_mc_program(struct radeon_device *rdev)
4354 {
4355 struct evergreen_mc_save save;
4356 u32 tmp;
4357 int i, j;
4358
4359 /* Initialize HDP */
4360 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4361 WREG32((0x2c14 + j), 0x00000000);
4362 WREG32((0x2c18 + j), 0x00000000);
4363 WREG32((0x2c1c + j), 0x00000000);
4364 WREG32((0x2c20 + j), 0x00000000);
4365 WREG32((0x2c24 + j), 0x00000000);
4366 }
4367 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4368
4369 evergreen_mc_stop(rdev, &save);
4370 if (radeon_mc_wait_for_idle(rdev)) {
4371 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4372 }
4373 /* Lockout access through VGA aperture*/
4374 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4375 /* Update configuration */
4376 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4377 rdev->mc.vram_start >> 12);
4378 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4379 rdev->mc.vram_end >> 12);
4380 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4381 rdev->vram_scratch.gpu_addr >> 12);
4382 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4383 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4384 WREG32(MC_VM_FB_LOCATION, tmp);
4385 /* XXX double check these! */
4386 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4387 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4388 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4389 WREG32(MC_VM_AGP_BASE, 0);
4390 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4391 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4392 if (radeon_mc_wait_for_idle(rdev)) {
4393 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4394 }
4395 evergreen_mc_resume(rdev, &save);
4396 /* we need to own VRAM, so turn off the VGA renderer here
4397 * to stop it overwriting our objects */
4398 rv515_vga_render_disable(rdev);
4399 }
4400
4401 /**
4402 * cik_mc_init - initialize the memory controller driver params
4403 *
4404 * @rdev: radeon_device pointer
4405 *
4406 * Look up the amount of vram, vram width, and decide how to place
4407 * vram and gart within the GPU's physical address space (CIK).
4408 * Returns 0 for success.
4409 */
4410 static int cik_mc_init(struct radeon_device *rdev)
4411 {
4412 u32 tmp;
4413 int chansize, numchan;
4414
4415 /* Get VRAM informations */
4416 rdev->mc.vram_is_ddr = true;
4417 tmp = RREG32(MC_ARB_RAMCFG);
4418 if (tmp & CHANSIZE_MASK) {
4419 chansize = 64;
4420 } else {
4421 chansize = 32;
4422 }
4423 tmp = RREG32(MC_SHARED_CHMAP);
4424 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4425 case 0:
4426 default:
4427 numchan = 1;
4428 break;
4429 case 1:
4430 numchan = 2;
4431 break;
4432 case 2:
4433 numchan = 4;
4434 break;
4435 case 3:
4436 numchan = 8;
4437 break;
4438 case 4:
4439 numchan = 3;
4440 break;
4441 case 5:
4442 numchan = 6;
4443 break;
4444 case 6:
4445 numchan = 10;
4446 break;
4447 case 7:
4448 numchan = 12;
4449 break;
4450 case 8:
4451 numchan = 16;
4452 break;
4453 }
4454 rdev->mc.vram_width = numchan * chansize;
4455 /* Could aper size report 0 ? */
4456 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4457 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4458 /* size in MB on si */
4459 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4460 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4461 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4462 si_vram_gtt_location(rdev, &rdev->mc);
4463 radeon_update_bandwidth_info(rdev);
4464
4465 return 0;
4466 }
4467
4468 /*
4469 * GART
4470 * VMID 0 is the physical GPU addresses as used by the kernel.
4471 * VMIDs 1-15 are used for userspace clients and are handled
4472 * by the radeon vm/hsa code.
4473 */
4474 /**
4475 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4476 *
4477 * @rdev: radeon_device pointer
4478 *
4479 * Flush the TLB for the VMID 0 page table (CIK).
4480 */
4481 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4482 {
4483 /* flush hdp cache */
4484 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4485
4486 /* bits 0-15 are the VM contexts0-15 */
4487 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4488 }
4489
4490 /**
4491 * cik_pcie_gart_enable - gart enable
4492 *
4493 * @rdev: radeon_device pointer
4494 *
4495 * This sets up the TLBs, programs the page tables for VMID0,
4496 * sets up the hw for VMIDs 1-15 which are allocated on
4497 * demand, and sets up the global locations for the LDS, GDS,
4498 * and GPUVM for FSA64 clients (CIK).
4499 * Returns 0 for success, errors for failure.
4500 */
4501 static int cik_pcie_gart_enable(struct radeon_device *rdev)
4502 {
4503 int r, i;
4504
4505 if (rdev->gart.robj == NULL) {
4506 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4507 return -EINVAL;
4508 }
4509 r = radeon_gart_table_vram_pin(rdev);
4510 if (r)
4511 return r;
4512 radeon_gart_restore(rdev);
4513 /* Setup TLB control */
4514 WREG32(MC_VM_MX_L1_TLB_CNTL,
4515 (0xA << 7) |
4516 ENABLE_L1_TLB |
4517 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4518 ENABLE_ADVANCED_DRIVER_MODEL |
4519 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4520 /* Setup L2 cache */
4521 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4522 ENABLE_L2_FRAGMENT_PROCESSING |
4523 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4524 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4525 EFFECTIVE_L2_QUEUE_SIZE(7) |
4526 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4527 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4528 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4529 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4530 /* setup context0 */
4531 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4532 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4533 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4534 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4535 (u32)(rdev->dummy_page.addr >> 12));
4536 WREG32(VM_CONTEXT0_CNTL2, 0);
4537 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4538 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4539
4540 WREG32(0x15D4, 0);
4541 WREG32(0x15D8, 0);
4542 WREG32(0x15DC, 0);
4543
4544 /* empty context1-15 */
4545 /* FIXME start with 4G, once using 2 level pt switch to full
4546 * vm size space
4547 */
4548 /* set vm size, must be a multiple of 4 */
4549 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4550 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4551 for (i = 1; i < 16; i++) {
4552 if (i < 8)
4553 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4554 rdev->gart.table_addr >> 12);
4555 else
4556 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4557 rdev->gart.table_addr >> 12);
4558 }
4559
4560 /* enable context1-15 */
4561 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4562 (u32)(rdev->dummy_page.addr >> 12));
4563 WREG32(VM_CONTEXT1_CNTL2, 4);
4564 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4565 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4566 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4567 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4568 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4569 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4570 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4571 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4572 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4573 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4574 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4575 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4576 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4577
4578 /* TC cache setup ??? */
4579 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4580 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4581 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4582
4583 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4584 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4585 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4586 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4587 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4588
4589 WREG32(TC_CFG_L1_VOLATILE, 0);
4590 WREG32(TC_CFG_L2_VOLATILE, 0);
4591
4592 if (rdev->family == CHIP_KAVERI) {
4593 u32 tmp = RREG32(CHUB_CONTROL);
4594 tmp &= ~BYPASS_VM;
4595 WREG32(CHUB_CONTROL, tmp);
4596 }
4597
4598 /* XXX SH_MEM regs */
4599 /* where to put LDS, scratch, GPUVM in FSA64 space */
4600 mutex_lock(&rdev->srbm_mutex);
4601 for (i = 0; i < 16; i++) {
4602 cik_srbm_select(rdev, 0, 0, 0, i);
4603 /* CP and shaders */
4604 WREG32(SH_MEM_CONFIG, 0);
4605 WREG32(SH_MEM_APE1_BASE, 1);
4606 WREG32(SH_MEM_APE1_LIMIT, 0);
4607 WREG32(SH_MEM_BASES, 0);
4608 /* SDMA GFX */
4609 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4610 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4611 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4612 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4613 /* XXX SDMA RLC - todo */
4614 }
4615 cik_srbm_select(rdev, 0, 0, 0, 0);
4616 mutex_unlock(&rdev->srbm_mutex);
4617
4618 cik_pcie_gart_tlb_flush(rdev);
4619 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4620 (unsigned)(rdev->mc.gtt_size >> 20),
4621 (unsigned long long)rdev->gart.table_addr);
4622 rdev->gart.ready = true;
4623 return 0;
4624 }
4625
4626 /**
4627 * cik_pcie_gart_disable - gart disable
4628 *
4629 * @rdev: radeon_device pointer
4630 *
4631 * This disables all VM page table (CIK).
4632 */
4633 static void cik_pcie_gart_disable(struct radeon_device *rdev)
4634 {
4635 /* Disable all tables */
4636 WREG32(VM_CONTEXT0_CNTL, 0);
4637 WREG32(VM_CONTEXT1_CNTL, 0);
4638 /* Setup TLB control */
4639 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4640 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4641 /* Setup L2 cache */
4642 WREG32(VM_L2_CNTL,
4643 ENABLE_L2_FRAGMENT_PROCESSING |
4644 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4645 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4646 EFFECTIVE_L2_QUEUE_SIZE(7) |
4647 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4648 WREG32(VM_L2_CNTL2, 0);
4649 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4650 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4651 radeon_gart_table_vram_unpin(rdev);
4652 }
4653
4654 /**
4655 * cik_pcie_gart_fini - vm fini callback
4656 *
4657 * @rdev: radeon_device pointer
4658 *
4659 * Tears down the driver GART/VM setup (CIK).
4660 */
4661 static void cik_pcie_gart_fini(struct radeon_device *rdev)
4662 {
4663 cik_pcie_gart_disable(rdev);
4664 radeon_gart_table_vram_free(rdev);
4665 radeon_gart_fini(rdev);
4666 }
4667
4668 /* vm parser */
4669 /**
4670 * cik_ib_parse - vm ib_parse callback
4671 *
4672 * @rdev: radeon_device pointer
4673 * @ib: indirect buffer pointer
4674 *
4675 * CIK uses hw IB checking so this is a nop (CIK).
4676 */
4677 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4678 {
4679 return 0;
4680 }
4681
4682 /*
4683 * vm
4684 * VMID 0 is the physical GPU addresses as used by the kernel.
4685 * VMIDs 1-15 are used for userspace clients and are handled
4686 * by the radeon vm/hsa code.
4687 */
4688 /**
4689 * cik_vm_init - cik vm init callback
4690 *
4691 * @rdev: radeon_device pointer
4692 *
4693 * Inits cik specific vm parameters (number of VMs, base of vram for
4694 * VMIDs 1-15) (CIK).
4695 * Returns 0 for success.
4696 */
4697 int cik_vm_init(struct radeon_device *rdev)
4698 {
4699 /* number of VMs */
4700 rdev->vm_manager.nvm = 16;
4701 /* base offset of vram pages */
4702 if (rdev->flags & RADEON_IS_IGP) {
4703 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4704 tmp <<= 22;
4705 rdev->vm_manager.vram_base_offset = tmp;
4706 } else
4707 rdev->vm_manager.vram_base_offset = 0;
4708
4709 return 0;
4710 }
4711
4712 /**
4713 * cik_vm_fini - cik vm fini callback
4714 *
4715 * @rdev: radeon_device pointer
4716 *
4717 * Tear down any asic specific VM setup (CIK).
4718 */
4719 void cik_vm_fini(struct radeon_device *rdev)
4720 {
4721 }
4722
4723 /**
4724 * cik_vm_decode_fault - print human readable fault info
4725 *
4726 * @rdev: radeon_device pointer
4727 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4728 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4729 *
4730 * Print human readable fault information (CIK).
4731 */
4732 static void cik_vm_decode_fault(struct radeon_device *rdev,
4733 u32 status, u32 addr, u32 mc_client)
4734 {
4735 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4736 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4737 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4738 char *block = (char *)&mc_client;
4739
4740 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4741 protections, vmid, addr,
4742 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4743 block, mc_id);
4744 }
4745
4746 /**
4747 * cik_vm_flush - cik vm flush using the CP
4748 *
4749 * @rdev: radeon_device pointer
4750 *
4751 * Update the page table base and flush the VM TLB
4752 * using the CP (CIK).
4753 */
4754 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4755 {
4756 struct radeon_ring *ring = &rdev->ring[ridx];
4757
4758 if (vm == NULL)
4759 return;
4760
4761 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4762 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4763 WRITE_DATA_DST_SEL(0)));
4764 if (vm->id < 8) {
4765 radeon_ring_write(ring,
4766 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4767 } else {
4768 radeon_ring_write(ring,
4769 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4770 }
4771 radeon_ring_write(ring, 0);
4772 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4773
4774 /* update SH_MEM_* regs */
4775 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4776 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4777 WRITE_DATA_DST_SEL(0)));
4778 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4779 radeon_ring_write(ring, 0);
4780 radeon_ring_write(ring, VMID(vm->id));
4781
4782 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4783 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4784 WRITE_DATA_DST_SEL(0)));
4785 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4786 radeon_ring_write(ring, 0);
4787
4788 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4789 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4790 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4791 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4792
4793 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4794 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4795 WRITE_DATA_DST_SEL(0)));
4796 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4797 radeon_ring_write(ring, 0);
4798 radeon_ring_write(ring, VMID(0));
4799
4800 /* HDP flush */
4801 /* We should be using the WAIT_REG_MEM packet here like in
4802 * cik_fence_ring_emit(), but it causes the CP to hang in this
4803 * context...
4804 */
4805 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4806 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4807 WRITE_DATA_DST_SEL(0)));
4808 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4809 radeon_ring_write(ring, 0);
4810 radeon_ring_write(ring, 0);
4811
4812 /* bits 0-15 are the VM contexts0-15 */
4813 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4814 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4815 WRITE_DATA_DST_SEL(0)));
4816 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4817 radeon_ring_write(ring, 0);
4818 radeon_ring_write(ring, 1 << vm->id);
4819
4820 /* compute doesn't have PFP */
4821 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4822 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4823 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4824 radeon_ring_write(ring, 0x0);
4825 }
4826 }
4827
4828 /**
4829 * cik_vm_set_page - update the page tables using sDMA
4830 *
4831 * @rdev: radeon_device pointer
4832 * @ib: indirect buffer to fill with commands
4833 * @pe: addr of the page entry
4834 * @addr: dst addr to write into pe
4835 * @count: number of page entries to update
4836 * @incr: increase next addr by incr bytes
4837 * @flags: access flags
4838 *
4839 * Update the page tables using CP or sDMA (CIK).
4840 */
4841 void cik_vm_set_page(struct radeon_device *rdev,
4842 struct radeon_ib *ib,
4843 uint64_t pe,
4844 uint64_t addr, unsigned count,
4845 uint32_t incr, uint32_t flags)
4846 {
4847 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4848 uint64_t value;
4849 unsigned ndw;
4850
4851 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4852 /* CP */
4853 while (count) {
4854 ndw = 2 + count * 2;
4855 if (ndw > 0x3FFE)
4856 ndw = 0x3FFE;
4857
4858 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4859 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4860 WRITE_DATA_DST_SEL(1));
4861 ib->ptr[ib->length_dw++] = pe;
4862 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4863 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4864 if (flags & RADEON_VM_PAGE_SYSTEM) {
4865 value = radeon_vm_map_gart(rdev, addr);
4866 value &= 0xFFFFFFFFFFFFF000ULL;
4867 } else if (flags & RADEON_VM_PAGE_VALID) {
4868 value = addr;
4869 } else {
4870 value = 0;
4871 }
4872 addr += incr;
4873 value |= r600_flags;
4874 ib->ptr[ib->length_dw++] = value;
4875 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4876 }
4877 }
4878 } else {
4879 /* DMA */
4880 cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4881 }
4882 }
4883
4884 /*
4885 * RLC
4886 * The RLC is a multi-purpose microengine that handles a
4887 * variety of functions, the most important of which is
4888 * the interrupt controller.
4889 */
4890 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4891 bool enable)
4892 {
4893 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4894
4895 if (enable)
4896 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4897 else
4898 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4899 WREG32(CP_INT_CNTL_RING0, tmp);
4900 }
4901
4902 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
4903 {
4904 u32 tmp;
4905
4906 tmp = RREG32(RLC_LB_CNTL);
4907 if (enable)
4908 tmp |= LOAD_BALANCE_ENABLE;
4909 else
4910 tmp &= ~LOAD_BALANCE_ENABLE;
4911 WREG32(RLC_LB_CNTL, tmp);
4912 }
4913
4914 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4915 {
4916 u32 i, j, k;
4917 u32 mask;
4918
4919 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4920 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4921 cik_select_se_sh(rdev, i, j);
4922 for (k = 0; k < rdev->usec_timeout; k++) {
4923 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4924 break;
4925 udelay(1);
4926 }
4927 }
4928 }
4929 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4930
4931 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4932 for (k = 0; k < rdev->usec_timeout; k++) {
4933 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4934 break;
4935 udelay(1);
4936 }
4937 }
4938
4939 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
4940 {
4941 u32 tmp;
4942
4943 tmp = RREG32(RLC_CNTL);
4944 if (tmp != rlc)
4945 WREG32(RLC_CNTL, rlc);
4946 }
4947
4948 static u32 cik_halt_rlc(struct radeon_device *rdev)
4949 {
4950 u32 data, orig;
4951
4952 orig = data = RREG32(RLC_CNTL);
4953
4954 if (data & RLC_ENABLE) {
4955 u32 i;
4956
4957 data &= ~RLC_ENABLE;
4958 WREG32(RLC_CNTL, data);
4959
4960 for (i = 0; i < rdev->usec_timeout; i++) {
4961 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
4962 break;
4963 udelay(1);
4964 }
4965
4966 cik_wait_for_rlc_serdes(rdev);
4967 }
4968
4969 return orig;
4970 }
4971
4972 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
4973 {
4974 u32 tmp, i, mask;
4975
4976 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
4977 WREG32(RLC_GPR_REG2, tmp);
4978
4979 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
4980 for (i = 0; i < rdev->usec_timeout; i++) {
4981 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
4982 break;
4983 udelay(1);
4984 }
4985
4986 for (i = 0; i < rdev->usec_timeout; i++) {
4987 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
4988 break;
4989 udelay(1);
4990 }
4991 }
4992
4993 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
4994 {
4995 u32 tmp;
4996
4997 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
4998 WREG32(RLC_GPR_REG2, tmp);
4999 }
5000
5001 /**
5002 * cik_rlc_stop - stop the RLC ME
5003 *
5004 * @rdev: radeon_device pointer
5005 *
5006 * Halt the RLC ME (MicroEngine) (CIK).
5007 */
5008 static void cik_rlc_stop(struct radeon_device *rdev)
5009 {
5010 WREG32(RLC_CNTL, 0);
5011
5012 cik_enable_gui_idle_interrupt(rdev, false);
5013
5014 cik_wait_for_rlc_serdes(rdev);
5015 }
5016
5017 /**
5018 * cik_rlc_start - start the RLC ME
5019 *
5020 * @rdev: radeon_device pointer
5021 *
5022 * Unhalt the RLC ME (MicroEngine) (CIK).
5023 */
5024 static void cik_rlc_start(struct radeon_device *rdev)
5025 {
5026 WREG32(RLC_CNTL, RLC_ENABLE);
5027
5028 cik_enable_gui_idle_interrupt(rdev, true);
5029
5030 udelay(50);
5031 }
5032
5033 /**
5034 * cik_rlc_resume - setup the RLC hw
5035 *
5036 * @rdev: radeon_device pointer
5037 *
5038 * Initialize the RLC registers, load the ucode,
5039 * and start the RLC (CIK).
5040 * Returns 0 for success, -EINVAL if the ucode is not available.
5041 */
5042 static int cik_rlc_resume(struct radeon_device *rdev)
5043 {
5044 u32 i, size, tmp;
5045 const __be32 *fw_data;
5046
5047 if (!rdev->rlc_fw)
5048 return -EINVAL;
5049
5050 switch (rdev->family) {
5051 case CHIP_BONAIRE:
5052 default:
5053 size = BONAIRE_RLC_UCODE_SIZE;
5054 break;
5055 case CHIP_KAVERI:
5056 size = KV_RLC_UCODE_SIZE;
5057 break;
5058 case CHIP_KABINI:
5059 size = KB_RLC_UCODE_SIZE;
5060 break;
5061 }
5062
5063 cik_rlc_stop(rdev);
5064
5065 /* disable CG */
5066 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5067 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5068
5069 si_rlc_reset(rdev);
5070
5071 cik_init_pg(rdev);
5072
5073 cik_init_cg(rdev);
5074
5075 WREG32(RLC_LB_CNTR_INIT, 0);
5076 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5077
5078 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5079 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5080 WREG32(RLC_LB_PARAMS, 0x00600408);
5081 WREG32(RLC_LB_CNTL, 0x80000004);
5082
5083 WREG32(RLC_MC_CNTL, 0);
5084 WREG32(RLC_UCODE_CNTL, 0);
5085
5086 fw_data = (const __be32 *)rdev->rlc_fw->data;
5087 WREG32(RLC_GPM_UCODE_ADDR, 0);
5088 for (i = 0; i < size; i++)
5089 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5090 WREG32(RLC_GPM_UCODE_ADDR, 0);
5091
5092 /* XXX - find out what chips support lbpw */
5093 cik_enable_lbpw(rdev, false);
5094
5095 if (rdev->family == CHIP_BONAIRE)
5096 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5097
5098 cik_rlc_start(rdev);
5099
5100 return 0;
5101 }
5102
5103 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5104 {
5105 u32 data, orig, tmp, tmp2;
5106
5107 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5108
5109 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5110 cik_enable_gui_idle_interrupt(rdev, true);
5111
5112 tmp = cik_halt_rlc(rdev);
5113
5114 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5115 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5116 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5117 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5118 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5119
5120 cik_update_rlc(rdev, tmp);
5121
5122 data |= CGCG_EN | CGLS_EN;
5123 } else {
5124 cik_enable_gui_idle_interrupt(rdev, false);
5125
5126 RREG32(CB_CGTT_SCLK_CTRL);
5127 RREG32(CB_CGTT_SCLK_CTRL);
5128 RREG32(CB_CGTT_SCLK_CTRL);
5129 RREG32(CB_CGTT_SCLK_CTRL);
5130
5131 data &= ~(CGCG_EN | CGLS_EN);
5132 }
5133
5134 if (orig != data)
5135 WREG32(RLC_CGCG_CGLS_CTRL, data);
5136
5137 }
5138
5139 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5140 {
5141 u32 data, orig, tmp = 0;
5142
5143 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5144 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5145 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5146 orig = data = RREG32(CP_MEM_SLP_CNTL);
5147 data |= CP_MEM_LS_EN;
5148 if (orig != data)
5149 WREG32(CP_MEM_SLP_CNTL, data);
5150 }
5151 }
5152
5153 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5154 data &= 0xfffffffd;
5155 if (orig != data)
5156 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5157
5158 tmp = cik_halt_rlc(rdev);
5159
5160 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5161 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5162 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5163 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5164 WREG32(RLC_SERDES_WR_CTRL, data);
5165
5166 cik_update_rlc(rdev, tmp);
5167
5168 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5169 orig = data = RREG32(CGTS_SM_CTRL_REG);
5170 data &= ~SM_MODE_MASK;
5171 data |= SM_MODE(0x2);
5172 data |= SM_MODE_ENABLE;
5173 data &= ~CGTS_OVERRIDE;
5174 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5175 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5176 data &= ~CGTS_LS_OVERRIDE;
5177 data &= ~ON_MONITOR_ADD_MASK;
5178 data |= ON_MONITOR_ADD_EN;
5179 data |= ON_MONITOR_ADD(0x96);
5180 if (orig != data)
5181 WREG32(CGTS_SM_CTRL_REG, data);
5182 }
5183 } else {
5184 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5185 data |= 0x00000002;
5186 if (orig != data)
5187 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5188
5189 data = RREG32(RLC_MEM_SLP_CNTL);
5190 if (data & RLC_MEM_LS_EN) {
5191 data &= ~RLC_MEM_LS_EN;
5192 WREG32(RLC_MEM_SLP_CNTL, data);
5193 }
5194
5195 data = RREG32(CP_MEM_SLP_CNTL);
5196 if (data & CP_MEM_LS_EN) {
5197 data &= ~CP_MEM_LS_EN;
5198 WREG32(CP_MEM_SLP_CNTL, data);
5199 }
5200
5201 orig = data = RREG32(CGTS_SM_CTRL_REG);
5202 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5203 if (orig != data)
5204 WREG32(CGTS_SM_CTRL_REG, data);
5205
5206 tmp = cik_halt_rlc(rdev);
5207
5208 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5209 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5210 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5211 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5212 WREG32(RLC_SERDES_WR_CTRL, data);
5213
5214 cik_update_rlc(rdev, tmp);
5215 }
5216 }
5217
5218 static const u32 mc_cg_registers[] =
5219 {
5220 MC_HUB_MISC_HUB_CG,
5221 MC_HUB_MISC_SIP_CG,
5222 MC_HUB_MISC_VM_CG,
5223 MC_XPB_CLK_GAT,
5224 ATC_MISC_CG,
5225 MC_CITF_MISC_WR_CG,
5226 MC_CITF_MISC_RD_CG,
5227 MC_CITF_MISC_VM_CG,
5228 VM_L2_CG,
5229 };
5230
5231 static void cik_enable_mc_ls(struct radeon_device *rdev,
5232 bool enable)
5233 {
5234 int i;
5235 u32 orig, data;
5236
5237 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5238 orig = data = RREG32(mc_cg_registers[i]);
5239 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5240 data |= MC_LS_ENABLE;
5241 else
5242 data &= ~MC_LS_ENABLE;
5243 if (data != orig)
5244 WREG32(mc_cg_registers[i], data);
5245 }
5246 }
5247
5248 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5249 bool enable)
5250 {
5251 int i;
5252 u32 orig, data;
5253
5254 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5255 orig = data = RREG32(mc_cg_registers[i]);
5256 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5257 data |= MC_CG_ENABLE;
5258 else
5259 data &= ~MC_CG_ENABLE;
5260 if (data != orig)
5261 WREG32(mc_cg_registers[i], data);
5262 }
5263 }
5264
5265 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5266 bool enable)
5267 {
5268 u32 orig, data;
5269
5270 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5271 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5272 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5273 } else {
5274 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5275 data |= 0xff000000;
5276 if (data != orig)
5277 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5278
5279 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5280 data |= 0xff000000;
5281 if (data != orig)
5282 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5283 }
5284 }
5285
5286 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5287 bool enable)
5288 {
5289 u32 orig, data;
5290
5291 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
5292 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5293 data |= 0x100;
5294 if (orig != data)
5295 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5296
5297 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5298 data |= 0x100;
5299 if (orig != data)
5300 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5301 } else {
5302 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5303 data &= ~0x100;
5304 if (orig != data)
5305 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5306
5307 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5308 data &= ~0x100;
5309 if (orig != data)
5310 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5311 }
5312 }
5313
5314 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5315 bool enable)
5316 {
5317 u32 orig, data;
5318
5319 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5320 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5321 data = 0xfff;
5322 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5323
5324 orig = data = RREG32(UVD_CGC_CTRL);
5325 data |= DCM;
5326 if (orig != data)
5327 WREG32(UVD_CGC_CTRL, data);
5328 } else {
5329 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5330 data &= ~0xfff;
5331 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5332
5333 orig = data = RREG32(UVD_CGC_CTRL);
5334 data &= ~DCM;
5335 if (orig != data)
5336 WREG32(UVD_CGC_CTRL, data);
5337 }
5338 }
5339
5340 static void cik_enable_bif_mgls(struct radeon_device *rdev,
5341 bool enable)
5342 {
5343 u32 orig, data;
5344
5345 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5346
5347 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5348 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5349 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5350 else
5351 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5352 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5353
5354 if (orig != data)
5355 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5356 }
5357
5358 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5359 bool enable)
5360 {
5361 u32 orig, data;
5362
5363 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5364
5365 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5366 data &= ~CLOCK_GATING_DIS;
5367 else
5368 data |= CLOCK_GATING_DIS;
5369
5370 if (orig != data)
5371 WREG32(HDP_HOST_PATH_CNTL, data);
5372 }
5373
5374 static void cik_enable_hdp_ls(struct radeon_device *rdev,
5375 bool enable)
5376 {
5377 u32 orig, data;
5378
5379 orig = data = RREG32(HDP_MEM_POWER_LS);
5380
5381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5382 data |= HDP_LS_ENABLE;
5383 else
5384 data &= ~HDP_LS_ENABLE;
5385
5386 if (orig != data)
5387 WREG32(HDP_MEM_POWER_LS, data);
5388 }
5389
5390 void cik_update_cg(struct radeon_device *rdev,
5391 u32 block, bool enable)
5392 {
5393
5394 if (block & RADEON_CG_BLOCK_GFX) {
5395 cik_enable_gui_idle_interrupt(rdev, false);
5396 /* order matters! */
5397 if (enable) {
5398 cik_enable_mgcg(rdev, true);
5399 cik_enable_cgcg(rdev, true);
5400 } else {
5401 cik_enable_cgcg(rdev, false);
5402 cik_enable_mgcg(rdev, false);
5403 }
5404 cik_enable_gui_idle_interrupt(rdev, true);
5405 }
5406
5407 if (block & RADEON_CG_BLOCK_MC) {
5408 if (!(rdev->flags & RADEON_IS_IGP)) {
5409 cik_enable_mc_mgcg(rdev, enable);
5410 cik_enable_mc_ls(rdev, enable);
5411 }
5412 }
5413
5414 if (block & RADEON_CG_BLOCK_SDMA) {
5415 cik_enable_sdma_mgcg(rdev, enable);
5416 cik_enable_sdma_mgls(rdev, enable);
5417 }
5418
5419 if (block & RADEON_CG_BLOCK_BIF) {
5420 cik_enable_bif_mgls(rdev, enable);
5421 }
5422
5423 if (block & RADEON_CG_BLOCK_UVD) {
5424 if (rdev->has_uvd)
5425 cik_enable_uvd_mgcg(rdev, enable);
5426 }
5427
5428 if (block & RADEON_CG_BLOCK_HDP) {
5429 cik_enable_hdp_mgcg(rdev, enable);
5430 cik_enable_hdp_ls(rdev, enable);
5431 }
5432 }
5433
5434 static void cik_init_cg(struct radeon_device *rdev)
5435 {
5436
5437 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
5438
5439 if (rdev->has_uvd)
5440 si_init_uvd_internal_cg(rdev);
5441
5442 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5443 RADEON_CG_BLOCK_SDMA |
5444 RADEON_CG_BLOCK_BIF |
5445 RADEON_CG_BLOCK_UVD |
5446 RADEON_CG_BLOCK_HDP), true);
5447 }
5448
5449 static void cik_fini_cg(struct radeon_device *rdev)
5450 {
5451 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5452 RADEON_CG_BLOCK_SDMA |
5453 RADEON_CG_BLOCK_BIF |
5454 RADEON_CG_BLOCK_UVD |
5455 RADEON_CG_BLOCK_HDP), false);
5456
5457 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5458 }
5459
5460 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5461 bool enable)
5462 {
5463 u32 data, orig;
5464
5465 orig = data = RREG32(RLC_PG_CNTL);
5466 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5467 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5468 else
5469 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5470 if (orig != data)
5471 WREG32(RLC_PG_CNTL, data);
5472 }
5473
5474 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5475 bool enable)
5476 {
5477 u32 data, orig;
5478
5479 orig = data = RREG32(RLC_PG_CNTL);
5480 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5481 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5482 else
5483 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5484 if (orig != data)
5485 WREG32(RLC_PG_CNTL, data);
5486 }
5487
5488 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5489 {
5490 u32 data, orig;
5491
5492 orig = data = RREG32(RLC_PG_CNTL);
5493 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
5494 data &= ~DISABLE_CP_PG;
5495 else
5496 data |= DISABLE_CP_PG;
5497 if (orig != data)
5498 WREG32(RLC_PG_CNTL, data);
5499 }
5500
5501 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5502 {
5503 u32 data, orig;
5504
5505 orig = data = RREG32(RLC_PG_CNTL);
5506 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
5507 data &= ~DISABLE_GDS_PG;
5508 else
5509 data |= DISABLE_GDS_PG;
5510 if (orig != data)
5511 WREG32(RLC_PG_CNTL, data);
5512 }
5513
5514 #define CP_ME_TABLE_SIZE 96
5515 #define CP_ME_TABLE_OFFSET 2048
5516 #define CP_MEC_TABLE_OFFSET 4096
5517
5518 void cik_init_cp_pg_table(struct radeon_device *rdev)
5519 {
5520 const __be32 *fw_data;
5521 volatile u32 *dst_ptr;
5522 int me, i, max_me = 4;
5523 u32 bo_offset = 0;
5524 u32 table_offset;
5525
5526 if (rdev->family == CHIP_KAVERI)
5527 max_me = 5;
5528
5529 if (rdev->rlc.cp_table_ptr == NULL)
5530 return;
5531
5532 /* write the cp table buffer */
5533 dst_ptr = rdev->rlc.cp_table_ptr;
5534 for (me = 0; me < max_me; me++) {
5535 if (me == 0) {
5536 fw_data = (const __be32 *)rdev->ce_fw->data;
5537 table_offset = CP_ME_TABLE_OFFSET;
5538 } else if (me == 1) {
5539 fw_data = (const __be32 *)rdev->pfp_fw->data;
5540 table_offset = CP_ME_TABLE_OFFSET;
5541 } else if (me == 2) {
5542 fw_data = (const __be32 *)rdev->me_fw->data;
5543 table_offset = CP_ME_TABLE_OFFSET;
5544 } else {
5545 fw_data = (const __be32 *)rdev->mec_fw->data;
5546 table_offset = CP_MEC_TABLE_OFFSET;
5547 }
5548
5549 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
5550 dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
5551 }
5552 bo_offset += CP_ME_TABLE_SIZE;
5553 }
5554 }
5555
5556 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5557 bool enable)
5558 {
5559 u32 data, orig;
5560
5561 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5562 orig = data = RREG32(RLC_PG_CNTL);
5563 data |= GFX_PG_ENABLE;
5564 if (orig != data)
5565 WREG32(RLC_PG_CNTL, data);
5566
5567 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5568 data |= AUTO_PG_EN;
5569 if (orig != data)
5570 WREG32(RLC_AUTO_PG_CTRL, data);
5571 } else {
5572 orig = data = RREG32(RLC_PG_CNTL);
5573 data &= ~GFX_PG_ENABLE;
5574 if (orig != data)
5575 WREG32(RLC_PG_CNTL, data);
5576
5577 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5578 data &= ~AUTO_PG_EN;
5579 if (orig != data)
5580 WREG32(RLC_AUTO_PG_CTRL, data);
5581
5582 data = RREG32(DB_RENDER_CONTROL);
5583 }
5584 }
5585
5586 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5587 {
5588 u32 mask = 0, tmp, tmp1;
5589 int i;
5590
5591 cik_select_se_sh(rdev, se, sh);
5592 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5593 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5594 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5595
5596 tmp &= 0xffff0000;
5597
5598 tmp |= tmp1;
5599 tmp >>= 16;
5600
5601 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5602 mask <<= 1;
5603 mask |= 1;
5604 }
5605
5606 return (~tmp) & mask;
5607 }
5608
5609 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
5610 {
5611 u32 i, j, k, active_cu_number = 0;
5612 u32 mask, counter, cu_bitmap;
5613 u32 tmp = 0;
5614
5615 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5616 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5617 mask = 1;
5618 cu_bitmap = 0;
5619 counter = 0;
5620 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5621 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5622 if (counter < 2)
5623 cu_bitmap |= mask;
5624 counter ++;
5625 }
5626 mask <<= 1;
5627 }
5628
5629 active_cu_number += counter;
5630 tmp |= (cu_bitmap << (i * 16 + j * 8));
5631 }
5632 }
5633
5634 WREG32(RLC_PG_AO_CU_MASK, tmp);
5635
5636 tmp = RREG32(RLC_MAX_PG_CU);
5637 tmp &= ~MAX_PU_CU_MASK;
5638 tmp |= MAX_PU_CU(active_cu_number);
5639 WREG32(RLC_MAX_PG_CU, tmp);
5640 }
5641
5642 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5643 bool enable)
5644 {
5645 u32 data, orig;
5646
5647 orig = data = RREG32(RLC_PG_CNTL);
5648 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
5649 data |= STATIC_PER_CU_PG_ENABLE;
5650 else
5651 data &= ~STATIC_PER_CU_PG_ENABLE;
5652 if (orig != data)
5653 WREG32(RLC_PG_CNTL, data);
5654 }
5655
5656 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5657 bool enable)
5658 {
5659 u32 data, orig;
5660
5661 orig = data = RREG32(RLC_PG_CNTL);
5662 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
5663 data |= DYN_PER_CU_PG_ENABLE;
5664 else
5665 data &= ~DYN_PER_CU_PG_ENABLE;
5666 if (orig != data)
5667 WREG32(RLC_PG_CNTL, data);
5668 }
5669
5670 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5671 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
5672
5673 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5674 {
5675 u32 data, orig;
5676 u32 i;
5677
5678 if (rdev->rlc.cs_data) {
5679 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5680 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
5681 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
5682 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
5683 } else {
5684 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5685 for (i = 0; i < 3; i++)
5686 WREG32(RLC_GPM_SCRATCH_DATA, 0);
5687 }
5688 if (rdev->rlc.reg_list) {
5689 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5690 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5691 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
5692 }
5693
5694 orig = data = RREG32(RLC_PG_CNTL);
5695 data |= GFX_PG_SRC;
5696 if (orig != data)
5697 WREG32(RLC_PG_CNTL, data);
5698
5699 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5700 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
5701
5702 data = RREG32(CP_RB_WPTR_POLL_CNTL);
5703 data &= ~IDLE_POLL_COUNT_MASK;
5704 data |= IDLE_POLL_COUNT(0x60);
5705 WREG32(CP_RB_WPTR_POLL_CNTL, data);
5706
5707 data = 0x10101010;
5708 WREG32(RLC_PG_DELAY, data);
5709
5710 data = RREG32(RLC_PG_DELAY_2);
5711 data &= ~0xff;
5712 data |= 0x3;
5713 WREG32(RLC_PG_DELAY_2, data);
5714
5715 data = RREG32(RLC_AUTO_PG_CTRL);
5716 data &= ~GRBM_REG_SGIT_MASK;
5717 data |= GRBM_REG_SGIT(0x700);
5718 WREG32(RLC_AUTO_PG_CTRL, data);
5719
5720 }
5721
5722 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
5723 {
5724 cik_enable_gfx_cgpg(rdev, enable);
5725 cik_enable_gfx_static_mgpg(rdev, enable);
5726 cik_enable_gfx_dynamic_mgpg(rdev, enable);
5727 }
5728
5729 u32 cik_get_csb_size(struct radeon_device *rdev)
5730 {
5731 u32 count = 0;
5732 const struct cs_section_def *sect = NULL;
5733 const struct cs_extent_def *ext = NULL;
5734
5735 if (rdev->rlc.cs_data == NULL)
5736 return 0;
5737
5738 /* begin clear state */
5739 count += 2;
5740 /* context control state */
5741 count += 3;
5742
5743 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5744 for (ext = sect->section; ext->extent != NULL; ++ext) {
5745 if (sect->id == SECT_CONTEXT)
5746 count += 2 + ext->reg_count;
5747 else
5748 return 0;
5749 }
5750 }
5751 /* pa_sc_raster_config/pa_sc_raster_config1 */
5752 count += 4;
5753 /* end clear state */
5754 count += 2;
5755 /* clear state */
5756 count += 2;
5757
5758 return count;
5759 }
5760
5761 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5762 {
5763 u32 count = 0, i;
5764 const struct cs_section_def *sect = NULL;
5765 const struct cs_extent_def *ext = NULL;
5766
5767 if (rdev->rlc.cs_data == NULL)
5768 return;
5769 if (buffer == NULL)
5770 return;
5771
5772 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5773 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5774
5775 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5776 buffer[count++] = 0x80000000;
5777 buffer[count++] = 0x80000000;
5778
5779 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5780 for (ext = sect->section; ext->extent != NULL; ++ext) {
5781 if (sect->id == SECT_CONTEXT) {
5782 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5783 buffer[count++] = ext->reg_index - 0xa000;
5784 for (i = 0; i < ext->reg_count; i++)
5785 buffer[count++] = ext->extent[i];
5786 } else {
5787 return;
5788 }
5789 }
5790 }
5791
5792 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
5793 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5794 switch (rdev->family) {
5795 case CHIP_BONAIRE:
5796 buffer[count++] = 0x16000012;
5797 buffer[count++] = 0x00000000;
5798 break;
5799 case CHIP_KAVERI:
5800 buffer[count++] = 0x00000000; /* XXX */
5801 buffer[count++] = 0x00000000;
5802 break;
5803 case CHIP_KABINI:
5804 buffer[count++] = 0x00000000; /* XXX */
5805 buffer[count++] = 0x00000000;
5806 break;
5807 default:
5808 buffer[count++] = 0x00000000;
5809 buffer[count++] = 0x00000000;
5810 break;
5811 }
5812
5813 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5814 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5815
5816 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5817 buffer[count++] = 0;
5818 }
5819
5820 static void cik_init_pg(struct radeon_device *rdev)
5821 {
5822 if (rdev->pg_flags) {
5823 cik_enable_sck_slowdown_on_pu(rdev, true);
5824 cik_enable_sck_slowdown_on_pd(rdev, true);
5825 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5826 cik_init_gfx_cgpg(rdev);
5827 cik_enable_cp_pg(rdev, true);
5828 cik_enable_gds_pg(rdev, true);
5829 }
5830 cik_init_ao_cu_mask(rdev);
5831 cik_update_gfx_pg(rdev, true);
5832 }
5833 }
5834
5835 static void cik_fini_pg(struct radeon_device *rdev)
5836 {
5837 if (rdev->pg_flags) {
5838 cik_update_gfx_pg(rdev, false);
5839 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5840 cik_enable_cp_pg(rdev, false);
5841 cik_enable_gds_pg(rdev, false);
5842 }
5843 }
5844 }
5845
5846 /*
5847 * Interrupts
5848 * Starting with r6xx, interrupts are handled via a ring buffer.
5849 * Ring buffers are areas of GPU accessible memory that the GPU
5850 * writes interrupt vectors into and the host reads vectors out of.
5851 * There is a rptr (read pointer) that determines where the
5852 * host is currently reading, and a wptr (write pointer)
5853 * which determines where the GPU has written. When the
5854 * pointers are equal, the ring is idle. When the GPU
5855 * writes vectors to the ring buffer, it increments the
5856 * wptr. When there is an interrupt, the host then starts
5857 * fetching commands and processing them until the pointers are
5858 * equal again at which point it updates the rptr.
5859 */
5860
5861 /**
5862 * cik_enable_interrupts - Enable the interrupt ring buffer
5863 *
5864 * @rdev: radeon_device pointer
5865 *
5866 * Enable the interrupt ring buffer (CIK).
5867 */
5868 static void cik_enable_interrupts(struct radeon_device *rdev)
5869 {
5870 u32 ih_cntl = RREG32(IH_CNTL);
5871 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5872
5873 ih_cntl |= ENABLE_INTR;
5874 ih_rb_cntl |= IH_RB_ENABLE;
5875 WREG32(IH_CNTL, ih_cntl);
5876 WREG32(IH_RB_CNTL, ih_rb_cntl);
5877 rdev->ih.enabled = true;
5878 }
5879
5880 /**
5881 * cik_disable_interrupts - Disable the interrupt ring buffer
5882 *
5883 * @rdev: radeon_device pointer
5884 *
5885 * Disable the interrupt ring buffer (CIK).
5886 */
5887 static void cik_disable_interrupts(struct radeon_device *rdev)
5888 {
5889 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5890 u32 ih_cntl = RREG32(IH_CNTL);
5891
5892 ih_rb_cntl &= ~IH_RB_ENABLE;
5893 ih_cntl &= ~ENABLE_INTR;
5894 WREG32(IH_RB_CNTL, ih_rb_cntl);
5895 WREG32(IH_CNTL, ih_cntl);
5896 /* set rptr, wptr to 0 */
5897 WREG32(IH_RB_RPTR, 0);
5898 WREG32(IH_RB_WPTR, 0);
5899 rdev->ih.enabled = false;
5900 rdev->ih.rptr = 0;
5901 }
5902
5903 /**
5904 * cik_disable_interrupt_state - Disable all interrupt sources
5905 *
5906 * @rdev: radeon_device pointer
5907 *
5908 * Clear all interrupt enable bits used by the driver (CIK).
5909 */
5910 static void cik_disable_interrupt_state(struct radeon_device *rdev)
5911 {
5912 u32 tmp;
5913
5914 /* gfx ring */
5915 tmp = RREG32(CP_INT_CNTL_RING0) &
5916 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5917 WREG32(CP_INT_CNTL_RING0, tmp);
5918 /* sdma */
5919 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5920 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5921 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5922 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5923 /* compute queues */
5924 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
5925 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
5926 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
5927 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
5928 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
5929 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
5930 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
5931 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
5932 /* grbm */
5933 WREG32(GRBM_INT_CNTL, 0);
5934 /* vline/vblank, etc. */
5935 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5936 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5937 if (rdev->num_crtc >= 4) {
5938 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5939 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5940 }
5941 if (rdev->num_crtc >= 6) {
5942 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5943 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5944 }
5945
5946 /* dac hotplug */
5947 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5948
5949 /* digital hotplug */
5950 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5951 WREG32(DC_HPD1_INT_CONTROL, tmp);
5952 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5953 WREG32(DC_HPD2_INT_CONTROL, tmp);
5954 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5955 WREG32(DC_HPD3_INT_CONTROL, tmp);
5956 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5957 WREG32(DC_HPD4_INT_CONTROL, tmp);
5958 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5959 WREG32(DC_HPD5_INT_CONTROL, tmp);
5960 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5961 WREG32(DC_HPD6_INT_CONTROL, tmp);
5962
5963 }
5964
5965 /**
5966 * cik_irq_init - init and enable the interrupt ring
5967 *
5968 * @rdev: radeon_device pointer
5969 *
5970 * Allocate a ring buffer for the interrupt controller,
5971 * enable the RLC, disable interrupts, enable the IH
5972 * ring buffer and enable it (CIK).
5973 * Called at device load and reume.
5974 * Returns 0 for success, errors for failure.
5975 */
5976 static int cik_irq_init(struct radeon_device *rdev)
5977 {
5978 int ret = 0;
5979 int rb_bufsz;
5980 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5981
5982 /* allocate ring */
5983 ret = r600_ih_ring_alloc(rdev);
5984 if (ret)
5985 return ret;
5986
5987 /* disable irqs */
5988 cik_disable_interrupts(rdev);
5989
5990 /* init rlc */
5991 ret = cik_rlc_resume(rdev);
5992 if (ret) {
5993 r600_ih_ring_fini(rdev);
5994 return ret;
5995 }
5996
5997 /* setup interrupt control */
5998 /* XXX this should actually be a bus address, not an MC address. same on older asics */
5999 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6000 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6001 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6002 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6003 */
6004 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6005 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6006 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6007 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6008
6009 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6010 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6011
6012 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6013 IH_WPTR_OVERFLOW_CLEAR |
6014 (rb_bufsz << 1));
6015
6016 if (rdev->wb.enabled)
6017 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6018
6019 /* set the writeback address whether it's enabled or not */
6020 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6021 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6022
6023 WREG32(IH_RB_CNTL, ih_rb_cntl);
6024
6025 /* set rptr, wptr to 0 */
6026 WREG32(IH_RB_RPTR, 0);
6027 WREG32(IH_RB_WPTR, 0);
6028
6029 /* Default settings for IH_CNTL (disabled at first) */
6030 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6031 /* RPTR_REARM only works if msi's are enabled */
6032 if (rdev->msi_enabled)
6033 ih_cntl |= RPTR_REARM;
6034 WREG32(IH_CNTL, ih_cntl);
6035
6036 /* force the active interrupt state to all disabled */
6037 cik_disable_interrupt_state(rdev);
6038
6039 pci_set_master(rdev->pdev);
6040
6041 /* enable irqs */
6042 cik_enable_interrupts(rdev);
6043
6044 return ret;
6045 }
6046
6047 /**
6048 * cik_irq_set - enable/disable interrupt sources
6049 *
6050 * @rdev: radeon_device pointer
6051 *
6052 * Enable interrupt sources on the GPU (vblanks, hpd,
6053 * etc.) (CIK).
6054 * Returns 0 for success, errors for failure.
6055 */
6056 int cik_irq_set(struct radeon_device *rdev)
6057 {
6058 u32 cp_int_cntl;
6059 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6060 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6061 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6062 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6063 u32 grbm_int_cntl = 0;
6064 u32 dma_cntl, dma_cntl1;
6065 u32 thermal_int;
6066
6067 if (!rdev->irq.installed) {
6068 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6069 return -EINVAL;
6070 }
6071 /* don't enable anything if the ih is disabled */
6072 if (!rdev->ih.enabled) {
6073 cik_disable_interrupts(rdev);
6074 /* force the active interrupt state to all disabled */
6075 cik_disable_interrupt_state(rdev);
6076 return 0;
6077 }
6078
6079 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6080 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6081 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6082
6083 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6084 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6085 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6086 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6087 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6088 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6089
6090 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6091 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6092
6093 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6094 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6095 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6096 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6097 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6098 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6099 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6100 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6101
6102 if (rdev->flags & RADEON_IS_IGP)
6103 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6104 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6105 else
6106 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6107 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6108
6109 /* enable CP interrupts on all rings */
6110 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6111 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6112 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6113 }
6114 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6115 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6116 DRM_DEBUG("si_irq_set: sw int cp1\n");
6117 if (ring->me == 1) {
6118 switch (ring->pipe) {
6119 case 0:
6120 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6121 break;
6122 case 1:
6123 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6124 break;
6125 case 2:
6126 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6127 break;
6128 case 3:
6129 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6130 break;
6131 default:
6132 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6133 break;
6134 }
6135 } else if (ring->me == 2) {
6136 switch (ring->pipe) {
6137 case 0:
6138 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6139 break;
6140 case 1:
6141 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6142 break;
6143 case 2:
6144 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6145 break;
6146 case 3:
6147 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6148 break;
6149 default:
6150 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6151 break;
6152 }
6153 } else {
6154 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6155 }
6156 }
6157 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6158 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6159 DRM_DEBUG("si_irq_set: sw int cp2\n");
6160 if (ring->me == 1) {
6161 switch (ring->pipe) {
6162 case 0:
6163 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6164 break;
6165 case 1:
6166 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6167 break;
6168 case 2:
6169 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6170 break;
6171 case 3:
6172 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6173 break;
6174 default:
6175 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6176 break;
6177 }
6178 } else if (ring->me == 2) {
6179 switch (ring->pipe) {
6180 case 0:
6181 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6182 break;
6183 case 1:
6184 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6185 break;
6186 case 2:
6187 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6188 break;
6189 case 3:
6190 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6191 break;
6192 default:
6193 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6194 break;
6195 }
6196 } else {
6197 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6198 }
6199 }
6200
6201 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6202 DRM_DEBUG("cik_irq_set: sw int dma\n");
6203 dma_cntl |= TRAP_ENABLE;
6204 }
6205
6206 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6207 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6208 dma_cntl1 |= TRAP_ENABLE;
6209 }
6210
6211 if (rdev->irq.crtc_vblank_int[0] ||
6212 atomic_read(&rdev->irq.pflip[0])) {
6213 DRM_DEBUG("cik_irq_set: vblank 0\n");
6214 crtc1 |= VBLANK_INTERRUPT_MASK;
6215 }
6216 if (rdev->irq.crtc_vblank_int[1] ||
6217 atomic_read(&rdev->irq.pflip[1])) {
6218 DRM_DEBUG("cik_irq_set: vblank 1\n");
6219 crtc2 |= VBLANK_INTERRUPT_MASK;
6220 }
6221 if (rdev->irq.crtc_vblank_int[2] ||
6222 atomic_read(&rdev->irq.pflip[2])) {
6223 DRM_DEBUG("cik_irq_set: vblank 2\n");
6224 crtc3 |= VBLANK_INTERRUPT_MASK;
6225 }
6226 if (rdev->irq.crtc_vblank_int[3] ||
6227 atomic_read(&rdev->irq.pflip[3])) {
6228 DRM_DEBUG("cik_irq_set: vblank 3\n");
6229 crtc4 |= VBLANK_INTERRUPT_MASK;
6230 }
6231 if (rdev->irq.crtc_vblank_int[4] ||
6232 atomic_read(&rdev->irq.pflip[4])) {
6233 DRM_DEBUG("cik_irq_set: vblank 4\n");
6234 crtc5 |= VBLANK_INTERRUPT_MASK;
6235 }
6236 if (rdev->irq.crtc_vblank_int[5] ||
6237 atomic_read(&rdev->irq.pflip[5])) {
6238 DRM_DEBUG("cik_irq_set: vblank 5\n");
6239 crtc6 |= VBLANK_INTERRUPT_MASK;
6240 }
6241 if (rdev->irq.hpd[0]) {
6242 DRM_DEBUG("cik_irq_set: hpd 1\n");
6243 hpd1 |= DC_HPDx_INT_EN;
6244 }
6245 if (rdev->irq.hpd[1]) {
6246 DRM_DEBUG("cik_irq_set: hpd 2\n");
6247 hpd2 |= DC_HPDx_INT_EN;
6248 }
6249 if (rdev->irq.hpd[2]) {
6250 DRM_DEBUG("cik_irq_set: hpd 3\n");
6251 hpd3 |= DC_HPDx_INT_EN;
6252 }
6253 if (rdev->irq.hpd[3]) {
6254 DRM_DEBUG("cik_irq_set: hpd 4\n");
6255 hpd4 |= DC_HPDx_INT_EN;
6256 }
6257 if (rdev->irq.hpd[4]) {
6258 DRM_DEBUG("cik_irq_set: hpd 5\n");
6259 hpd5 |= DC_HPDx_INT_EN;
6260 }
6261 if (rdev->irq.hpd[5]) {
6262 DRM_DEBUG("cik_irq_set: hpd 6\n");
6263 hpd6 |= DC_HPDx_INT_EN;
6264 }
6265
6266 if (rdev->irq.dpm_thermal) {
6267 DRM_DEBUG("dpm thermal\n");
6268 if (rdev->flags & RADEON_IS_IGP)
6269 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6270 else
6271 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6272 }
6273
6274 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6275
6276 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6277 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6278
6279 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6280 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6281 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6282 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6283 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6284 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6285 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6286 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6287
6288 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6289
6290 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6291 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6292 if (rdev->num_crtc >= 4) {
6293 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6294 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6295 }
6296 if (rdev->num_crtc >= 6) {
6297 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6298 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6299 }
6300
6301 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6302 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6303 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6304 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6305 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6306 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6307
6308 if (rdev->flags & RADEON_IS_IGP)
6309 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6310 else
6311 WREG32_SMC(CG_THERMAL_INT, thermal_int);
6312
6313 return 0;
6314 }
6315
6316 /**
6317 * cik_irq_ack - ack interrupt sources
6318 *
6319 * @rdev: radeon_device pointer
6320 *
6321 * Ack interrupt sources on the GPU (vblanks, hpd,
6322 * etc.) (CIK). Certain interrupts sources are sw
6323 * generated and do not require an explicit ack.
6324 */
6325 static inline void cik_irq_ack(struct radeon_device *rdev)
6326 {
6327 u32 tmp;
6328
6329 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6330 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6331 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6332 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6333 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6334 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6335 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6336
6337 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6338 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6339 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6340 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6341 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6342 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6343 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6344 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6345
6346 if (rdev->num_crtc >= 4) {
6347 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6348 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6349 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6350 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6351 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6352 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6353 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6354 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6355 }
6356
6357 if (rdev->num_crtc >= 6) {
6358 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6359 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6360 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6361 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6362 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6364 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6366 }
6367
6368 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6369 tmp = RREG32(DC_HPD1_INT_CONTROL);
6370 tmp |= DC_HPDx_INT_ACK;
6371 WREG32(DC_HPD1_INT_CONTROL, tmp);
6372 }
6373 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6374 tmp = RREG32(DC_HPD2_INT_CONTROL);
6375 tmp |= DC_HPDx_INT_ACK;
6376 WREG32(DC_HPD2_INT_CONTROL, tmp);
6377 }
6378 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6379 tmp = RREG32(DC_HPD3_INT_CONTROL);
6380 tmp |= DC_HPDx_INT_ACK;
6381 WREG32(DC_HPD3_INT_CONTROL, tmp);
6382 }
6383 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6384 tmp = RREG32(DC_HPD4_INT_CONTROL);
6385 tmp |= DC_HPDx_INT_ACK;
6386 WREG32(DC_HPD4_INT_CONTROL, tmp);
6387 }
6388 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6389 tmp = RREG32(DC_HPD5_INT_CONTROL);
6390 tmp |= DC_HPDx_INT_ACK;
6391 WREG32(DC_HPD5_INT_CONTROL, tmp);
6392 }
6393 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6394 tmp = RREG32(DC_HPD5_INT_CONTROL);
6395 tmp |= DC_HPDx_INT_ACK;
6396 WREG32(DC_HPD6_INT_CONTROL, tmp);
6397 }
6398 }
6399
6400 /**
6401 * cik_irq_disable - disable interrupts
6402 *
6403 * @rdev: radeon_device pointer
6404 *
6405 * Disable interrupts on the hw (CIK).
6406 */
6407 static void cik_irq_disable(struct radeon_device *rdev)
6408 {
6409 cik_disable_interrupts(rdev);
6410 /* Wait and acknowledge irq */
6411 mdelay(1);
6412 cik_irq_ack(rdev);
6413 cik_disable_interrupt_state(rdev);
6414 }
6415
6416 /**
6417 * cik_irq_disable - disable interrupts for suspend
6418 *
6419 * @rdev: radeon_device pointer
6420 *
6421 * Disable interrupts and stop the RLC (CIK).
6422 * Used for suspend.
6423 */
6424 static void cik_irq_suspend(struct radeon_device *rdev)
6425 {
6426 cik_irq_disable(rdev);
6427 cik_rlc_stop(rdev);
6428 }
6429
6430 /**
6431 * cik_irq_fini - tear down interrupt support
6432 *
6433 * @rdev: radeon_device pointer
6434 *
6435 * Disable interrupts on the hw and free the IH ring
6436 * buffer (CIK).
6437 * Used for driver unload.
6438 */
6439 static void cik_irq_fini(struct radeon_device *rdev)
6440 {
6441 cik_irq_suspend(rdev);
6442 r600_ih_ring_fini(rdev);
6443 }
6444
6445 /**
6446 * cik_get_ih_wptr - get the IH ring buffer wptr
6447 *
6448 * @rdev: radeon_device pointer
6449 *
6450 * Get the IH ring buffer wptr from either the register
6451 * or the writeback memory buffer (CIK). Also check for
6452 * ring buffer overflow and deal with it.
6453 * Used by cik_irq_process().
6454 * Returns the value of the wptr.
6455 */
6456 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6457 {
6458 u32 wptr, tmp;
6459
6460 if (rdev->wb.enabled)
6461 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6462 else
6463 wptr = RREG32(IH_RB_WPTR);
6464
6465 if (wptr & RB_OVERFLOW) {
6466 /* When a ring buffer overflow happen start parsing interrupt
6467 * from the last not overwritten vector (wptr + 16). Hopefully
6468 * this should allow us to catchup.
6469 */
6470 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6471 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6472 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6473 tmp = RREG32(IH_RB_CNTL);
6474 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6475 WREG32(IH_RB_CNTL, tmp);
6476 }
6477 return (wptr & rdev->ih.ptr_mask);
6478 }
6479
6480 /* CIK IV Ring
6481 * Each IV ring entry is 128 bits:
6482 * [7:0] - interrupt source id
6483 * [31:8] - reserved
6484 * [59:32] - interrupt source data
6485 * [63:60] - reserved
6486 * [71:64] - RINGID
6487 * CP:
6488 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
6489 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6490 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6491 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6492 * PIPE_ID - ME0 0=3D
6493 * - ME1&2 compute dispatcher (4 pipes each)
6494 * SDMA:
6495 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6496 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6497 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
6498 * [79:72] - VMID
6499 * [95:80] - PASID
6500 * [127:96] - reserved
6501 */
6502 /**
6503 * cik_irq_process - interrupt handler
6504 *
6505 * @rdev: radeon_device pointer
6506 *
6507 * Interrupt hander (CIK). Walk the IH ring,
6508 * ack interrupts and schedule work to handle
6509 * interrupt events.
6510 * Returns irq process return code.
6511 */
6512 int cik_irq_process(struct radeon_device *rdev)
6513 {
6514 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6515 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6516 u32 wptr;
6517 u32 rptr;
6518 u32 src_id, src_data, ring_id;
6519 u8 me_id, pipe_id, queue_id;
6520 u32 ring_index;
6521 bool queue_hotplug = false;
6522 bool queue_reset = false;
6523 u32 addr, status, mc_client;
6524 bool queue_thermal = false;
6525
6526 if (!rdev->ih.enabled || rdev->shutdown)
6527 return IRQ_NONE;
6528
6529 wptr = cik_get_ih_wptr(rdev);
6530
6531 restart_ih:
6532 /* is somebody else already processing irqs? */
6533 if (atomic_xchg(&rdev->ih.lock, 1))
6534 return IRQ_NONE;
6535
6536 rptr = rdev->ih.rptr;
6537 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6538
6539 /* Order reading of wptr vs. reading of IH ring data */
6540 rmb();
6541
6542 /* display interrupts */
6543 cik_irq_ack(rdev);
6544
6545 while (rptr != wptr) {
6546 /* wptr/rptr are in bytes! */
6547 ring_index = rptr / 4;
6548 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6549 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6550 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6551
6552 switch (src_id) {
6553 case 1: /* D1 vblank/vline */
6554 switch (src_data) {
6555 case 0: /* D1 vblank */
6556 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6557 if (rdev->irq.crtc_vblank_int[0]) {
6558 drm_handle_vblank(rdev->ddev, 0);
6559 rdev->pm.vblank_sync = true;
6560 wake_up(&rdev->irq.vblank_queue);
6561 }
6562 if (atomic_read(&rdev->irq.pflip[0]))
6563 radeon_crtc_handle_flip(rdev, 0);
6564 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6565 DRM_DEBUG("IH: D1 vblank\n");
6566 }
6567 break;
6568 case 1: /* D1 vline */
6569 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6570 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6571 DRM_DEBUG("IH: D1 vline\n");
6572 }
6573 break;
6574 default:
6575 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6576 break;
6577 }
6578 break;
6579 case 2: /* D2 vblank/vline */
6580 switch (src_data) {
6581 case 0: /* D2 vblank */
6582 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6583 if (rdev->irq.crtc_vblank_int[1]) {
6584 drm_handle_vblank(rdev->ddev, 1);
6585 rdev->pm.vblank_sync = true;
6586 wake_up(&rdev->irq.vblank_queue);
6587 }
6588 if (atomic_read(&rdev->irq.pflip[1]))
6589 radeon_crtc_handle_flip(rdev, 1);
6590 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6591 DRM_DEBUG("IH: D2 vblank\n");
6592 }
6593 break;
6594 case 1: /* D2 vline */
6595 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6596 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6597 DRM_DEBUG("IH: D2 vline\n");
6598 }
6599 break;
6600 default:
6601 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6602 break;
6603 }
6604 break;
6605 case 3: /* D3 vblank/vline */
6606 switch (src_data) {
6607 case 0: /* D3 vblank */
6608 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6609 if (rdev->irq.crtc_vblank_int[2]) {
6610 drm_handle_vblank(rdev->ddev, 2);
6611 rdev->pm.vblank_sync = true;
6612 wake_up(&rdev->irq.vblank_queue);
6613 }
6614 if (atomic_read(&rdev->irq.pflip[2]))
6615 radeon_crtc_handle_flip(rdev, 2);
6616 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6617 DRM_DEBUG("IH: D3 vblank\n");
6618 }
6619 break;
6620 case 1: /* D3 vline */
6621 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6622 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6623 DRM_DEBUG("IH: D3 vline\n");
6624 }
6625 break;
6626 default:
6627 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6628 break;
6629 }
6630 break;
6631 case 4: /* D4 vblank/vline */
6632 switch (src_data) {
6633 case 0: /* D4 vblank */
6634 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6635 if (rdev->irq.crtc_vblank_int[3]) {
6636 drm_handle_vblank(rdev->ddev, 3);
6637 rdev->pm.vblank_sync = true;
6638 wake_up(&rdev->irq.vblank_queue);
6639 }
6640 if (atomic_read(&rdev->irq.pflip[3]))
6641 radeon_crtc_handle_flip(rdev, 3);
6642 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6643 DRM_DEBUG("IH: D4 vblank\n");
6644 }
6645 break;
6646 case 1: /* D4 vline */
6647 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6648 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6649 DRM_DEBUG("IH: D4 vline\n");
6650 }
6651 break;
6652 default:
6653 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6654 break;
6655 }
6656 break;
6657 case 5: /* D5 vblank/vline */
6658 switch (src_data) {
6659 case 0: /* D5 vblank */
6660 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6661 if (rdev->irq.crtc_vblank_int[4]) {
6662 drm_handle_vblank(rdev->ddev, 4);
6663 rdev->pm.vblank_sync = true;
6664 wake_up(&rdev->irq.vblank_queue);
6665 }
6666 if (atomic_read(&rdev->irq.pflip[4]))
6667 radeon_crtc_handle_flip(rdev, 4);
6668 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6669 DRM_DEBUG("IH: D5 vblank\n");
6670 }
6671 break;
6672 case 1: /* D5 vline */
6673 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6674 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6675 DRM_DEBUG("IH: D5 vline\n");
6676 }
6677 break;
6678 default:
6679 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6680 break;
6681 }
6682 break;
6683 case 6: /* D6 vblank/vline */
6684 switch (src_data) {
6685 case 0: /* D6 vblank */
6686 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6687 if (rdev->irq.crtc_vblank_int[5]) {
6688 drm_handle_vblank(rdev->ddev, 5);
6689 rdev->pm.vblank_sync = true;
6690 wake_up(&rdev->irq.vblank_queue);
6691 }
6692 if (atomic_read(&rdev->irq.pflip[5]))
6693 radeon_crtc_handle_flip(rdev, 5);
6694 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6695 DRM_DEBUG("IH: D6 vblank\n");
6696 }
6697 break;
6698 case 1: /* D6 vline */
6699 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6700 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6701 DRM_DEBUG("IH: D6 vline\n");
6702 }
6703 break;
6704 default:
6705 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6706 break;
6707 }
6708 break;
6709 case 42: /* HPD hotplug */
6710 switch (src_data) {
6711 case 0:
6712 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6713 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6714 queue_hotplug = true;
6715 DRM_DEBUG("IH: HPD1\n");
6716 }
6717 break;
6718 case 1:
6719 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6720 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6721 queue_hotplug = true;
6722 DRM_DEBUG("IH: HPD2\n");
6723 }
6724 break;
6725 case 2:
6726 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6727 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6728 queue_hotplug = true;
6729 DRM_DEBUG("IH: HPD3\n");
6730 }
6731 break;
6732 case 3:
6733 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6734 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6735 queue_hotplug = true;
6736 DRM_DEBUG("IH: HPD4\n");
6737 }
6738 break;
6739 case 4:
6740 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6741 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6742 queue_hotplug = true;
6743 DRM_DEBUG("IH: HPD5\n");
6744 }
6745 break;
6746 case 5:
6747 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6748 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6749 queue_hotplug = true;
6750 DRM_DEBUG("IH: HPD6\n");
6751 }
6752 break;
6753 default:
6754 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6755 break;
6756 }
6757 break;
6758 case 124: /* UVD */
6759 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6760 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6761 break;
6762 case 146:
6763 case 147:
6764 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6765 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6766 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
6767 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6768 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6769 addr);
6770 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6771 status);
6772 cik_vm_decode_fault(rdev, status, addr, mc_client);
6773 /* reset addr and status */
6774 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6775 break;
6776 case 176: /* GFX RB CP_INT */
6777 case 177: /* GFX IB CP_INT */
6778 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6779 break;
6780 case 181: /* CP EOP event */
6781 DRM_DEBUG("IH: CP EOP\n");
6782 /* XXX check the bitfield order! */
6783 me_id = (ring_id & 0x60) >> 5;
6784 pipe_id = (ring_id & 0x18) >> 3;
6785 queue_id = (ring_id & 0x7) >> 0;
6786 switch (me_id) {
6787 case 0:
6788 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6789 break;
6790 case 1:
6791 case 2:
6792 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6793 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6794 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6795 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6796 break;
6797 }
6798 break;
6799 case 184: /* CP Privileged reg access */
6800 DRM_ERROR("Illegal register access in command stream\n");
6801 /* XXX check the bitfield order! */
6802 me_id = (ring_id & 0x60) >> 5;
6803 pipe_id = (ring_id & 0x18) >> 3;
6804 queue_id = (ring_id & 0x7) >> 0;
6805 switch (me_id) {
6806 case 0:
6807 /* This results in a full GPU reset, but all we need to do is soft
6808 * reset the CP for gfx
6809 */
6810 queue_reset = true;
6811 break;
6812 case 1:
6813 /* XXX compute */
6814 queue_reset = true;
6815 break;
6816 case 2:
6817 /* XXX compute */
6818 queue_reset = true;
6819 break;
6820 }
6821 break;
6822 case 185: /* CP Privileged inst */
6823 DRM_ERROR("Illegal instruction in command stream\n");
6824 /* XXX check the bitfield order! */
6825 me_id = (ring_id & 0x60) >> 5;
6826 pipe_id = (ring_id & 0x18) >> 3;
6827 queue_id = (ring_id & 0x7) >> 0;
6828 switch (me_id) {
6829 case 0:
6830 /* This results in a full GPU reset, but all we need to do is soft
6831 * reset the CP for gfx
6832 */
6833 queue_reset = true;
6834 break;
6835 case 1:
6836 /* XXX compute */
6837 queue_reset = true;
6838 break;
6839 case 2:
6840 /* XXX compute */
6841 queue_reset = true;
6842 break;
6843 }
6844 break;
6845 case 224: /* SDMA trap event */
6846 /* XXX check the bitfield order! */
6847 me_id = (ring_id & 0x3) >> 0;
6848 queue_id = (ring_id & 0xc) >> 2;
6849 DRM_DEBUG("IH: SDMA trap\n");
6850 switch (me_id) {
6851 case 0:
6852 switch (queue_id) {
6853 case 0:
6854 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6855 break;
6856 case 1:
6857 /* XXX compute */
6858 break;
6859 case 2:
6860 /* XXX compute */
6861 break;
6862 }
6863 break;
6864 case 1:
6865 switch (queue_id) {
6866 case 0:
6867 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6868 break;
6869 case 1:
6870 /* XXX compute */
6871 break;
6872 case 2:
6873 /* XXX compute */
6874 break;
6875 }
6876 break;
6877 }
6878 break;
6879 case 230: /* thermal low to high */
6880 DRM_DEBUG("IH: thermal low to high\n");
6881 rdev->pm.dpm.thermal.high_to_low = false;
6882 queue_thermal = true;
6883 break;
6884 case 231: /* thermal high to low */
6885 DRM_DEBUG("IH: thermal high to low\n");
6886 rdev->pm.dpm.thermal.high_to_low = true;
6887 queue_thermal = true;
6888 break;
6889 case 233: /* GUI IDLE */
6890 DRM_DEBUG("IH: GUI idle\n");
6891 break;
6892 case 241: /* SDMA Privileged inst */
6893 case 247: /* SDMA Privileged inst */
6894 DRM_ERROR("Illegal instruction in SDMA command stream\n");
6895 /* XXX check the bitfield order! */
6896 me_id = (ring_id & 0x3) >> 0;
6897 queue_id = (ring_id & 0xc) >> 2;
6898 switch (me_id) {
6899 case 0:
6900 switch (queue_id) {
6901 case 0:
6902 queue_reset = true;
6903 break;
6904 case 1:
6905 /* XXX compute */
6906 queue_reset = true;
6907 break;
6908 case 2:
6909 /* XXX compute */
6910 queue_reset = true;
6911 break;
6912 }
6913 break;
6914 case 1:
6915 switch (queue_id) {
6916 case 0:
6917 queue_reset = true;
6918 break;
6919 case 1:
6920 /* XXX compute */
6921 queue_reset = true;
6922 break;
6923 case 2:
6924 /* XXX compute */
6925 queue_reset = true;
6926 break;
6927 }
6928 break;
6929 }
6930 break;
6931 default:
6932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6933 break;
6934 }
6935
6936 /* wptr/rptr are in bytes! */
6937 rptr += 16;
6938 rptr &= rdev->ih.ptr_mask;
6939 }
6940 if (queue_hotplug)
6941 schedule_work(&rdev->hotplug_work);
6942 if (queue_reset)
6943 schedule_work(&rdev->reset_work);
6944 if (queue_thermal)
6945 schedule_work(&rdev->pm.dpm.thermal.work);
6946 rdev->ih.rptr = rptr;
6947 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6948 atomic_set(&rdev->ih.lock, 0);
6949
6950 /* make sure wptr hasn't changed while processing */
6951 wptr = cik_get_ih_wptr(rdev);
6952 if (wptr != rptr)
6953 goto restart_ih;
6954
6955 return IRQ_HANDLED;
6956 }
6957
6958 /*
6959 * startup/shutdown callbacks
6960 */
6961 /**
6962 * cik_startup - program the asic to a functional state
6963 *
6964 * @rdev: radeon_device pointer
6965 *
6966 * Programs the asic to a functional state (CIK).
6967 * Called by cik_init() and cik_resume().
6968 * Returns 0 for success, error for failure.
6969 */
6970 static int cik_startup(struct radeon_device *rdev)
6971 {
6972 struct radeon_ring *ring;
6973 int r;
6974
6975 /* enable pcie gen2/3 link */
6976 cik_pcie_gen3_enable(rdev);
6977 /* enable aspm */
6978 cik_program_aspm(rdev);
6979
6980 /* scratch needs to be initialized before MC */
6981 r = r600_vram_scratch_init(rdev);
6982 if (r)
6983 return r;
6984
6985 cik_mc_program(rdev);
6986
6987 if (rdev->flags & RADEON_IS_IGP) {
6988 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6989 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
6990 r = cik_init_microcode(rdev);
6991 if (r) {
6992 DRM_ERROR("Failed to load firmware!\n");
6993 return r;
6994 }
6995 }
6996 } else {
6997 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6998 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
6999 !rdev->mc_fw) {
7000 r = cik_init_microcode(rdev);
7001 if (r) {
7002 DRM_ERROR("Failed to load firmware!\n");
7003 return r;
7004 }
7005 }
7006
7007 r = ci_mc_load_microcode(rdev);
7008 if (r) {
7009 DRM_ERROR("Failed to load MC firmware!\n");
7010 return r;
7011 }
7012 }
7013
7014 r = cik_pcie_gart_enable(rdev);
7015 if (r)
7016 return r;
7017 cik_gpu_init(rdev);
7018
7019 /* allocate rlc buffers */
7020 if (rdev->flags & RADEON_IS_IGP) {
7021 if (rdev->family == CHIP_KAVERI) {
7022 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7023 rdev->rlc.reg_list_size =
7024 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7025 } else {
7026 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7027 rdev->rlc.reg_list_size =
7028 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7029 }
7030 }
7031 rdev->rlc.cs_data = ci_cs_data;
7032 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7033 r = sumo_rlc_init(rdev);
7034 if (r) {
7035 DRM_ERROR("Failed to init rlc BOs!\n");
7036 return r;
7037 }
7038
7039 /* allocate wb buffer */
7040 r = radeon_wb_init(rdev);
7041 if (r)
7042 return r;
7043
7044 /* allocate mec buffers */
7045 r = cik_mec_init(rdev);
7046 if (r) {
7047 DRM_ERROR("Failed to init MEC BOs!\n");
7048 return r;
7049 }
7050
7051 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7052 if (r) {
7053 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7054 return r;
7055 }
7056
7057 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7058 if (r) {
7059 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7060 return r;
7061 }
7062
7063 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7064 if (r) {
7065 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7066 return r;
7067 }
7068
7069 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7070 if (r) {
7071 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7072 return r;
7073 }
7074
7075 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7076 if (r) {
7077 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7078 return r;
7079 }
7080
7081 r = radeon_uvd_resume(rdev);
7082 if (!r) {
7083 r = uvd_v4_2_resume(rdev);
7084 if (!r) {
7085 r = radeon_fence_driver_start_ring(rdev,
7086 R600_RING_TYPE_UVD_INDEX);
7087 if (r)
7088 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7089 }
7090 }
7091 if (r)
7092 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7093
7094 /* Enable IRQ */
7095 if (!rdev->irq.installed) {
7096 r = radeon_irq_kms_init(rdev);
7097 if (r)
7098 return r;
7099 }
7100
7101 r = cik_irq_init(rdev);
7102 if (r) {
7103 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7104 radeon_irq_kms_fini(rdev);
7105 return r;
7106 }
7107 cik_irq_set(rdev);
7108
7109 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7110 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7111 CP_RB0_RPTR, CP_RB0_WPTR,
7112 RADEON_CP_PACKET2);
7113 if (r)
7114 return r;
7115
7116 /* set up the compute queues */
7117 /* type-2 packets are deprecated on MEC, use type-3 instead */
7118 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7119 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7120 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7121 PACKET3(PACKET3_NOP, 0x3FFF));
7122 if (r)
7123 return r;
7124 ring->me = 1; /* first MEC */
7125 ring->pipe = 0; /* first pipe */
7126 ring->queue = 0; /* first queue */
7127 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7128
7129 /* type-2 packets are deprecated on MEC, use type-3 instead */
7130 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7131 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7132 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7133 PACKET3(PACKET3_NOP, 0x3FFF));
7134 if (r)
7135 return r;
7136 /* dGPU only have 1 MEC */
7137 ring->me = 1; /* first MEC */
7138 ring->pipe = 0; /* first pipe */
7139 ring->queue = 1; /* second queue */
7140 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7141
7142 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7143 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7144 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7145 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
7146 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7147 if (r)
7148 return r;
7149
7150 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7151 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7152 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7153 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
7154 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7155 if (r)
7156 return r;
7157
7158 r = cik_cp_resume(rdev);
7159 if (r)
7160 return r;
7161
7162 r = cik_sdma_resume(rdev);
7163 if (r)
7164 return r;
7165
7166 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7167 if (ring->ring_size) {
7168 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7169 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
7170 RADEON_CP_PACKET2);
7171 if (!r)
7172 r = uvd_v1_0_init(rdev);
7173 if (r)
7174 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7175 }
7176
7177 r = radeon_ib_pool_init(rdev);
7178 if (r) {
7179 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7180 return r;
7181 }
7182
7183 r = radeon_vm_manager_init(rdev);
7184 if (r) {
7185 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7186 return r;
7187 }
7188
7189 r = dce6_audio_init(rdev);
7190 if (r)
7191 return r;
7192
7193 return 0;
7194 }
7195
7196 /**
7197 * cik_resume - resume the asic to a functional state
7198 *
7199 * @rdev: radeon_device pointer
7200 *
7201 * Programs the asic to a functional state (CIK).
7202 * Called at resume.
7203 * Returns 0 for success, error for failure.
7204 */
7205 int cik_resume(struct radeon_device *rdev)
7206 {
7207 int r;
7208
7209 /* post card */
7210 atom_asic_init(rdev->mode_info.atom_context);
7211
7212 /* init golden registers */
7213 cik_init_golden_registers(rdev);
7214
7215 rdev->accel_working = true;
7216 r = cik_startup(rdev);
7217 if (r) {
7218 DRM_ERROR("cik startup failed on resume\n");
7219 rdev->accel_working = false;
7220 return r;
7221 }
7222
7223 return r;
7224
7225 }
7226
7227 /**
7228 * cik_suspend - suspend the asic
7229 *
7230 * @rdev: radeon_device pointer
7231 *
7232 * Bring the chip into a state suitable for suspend (CIK).
7233 * Called at suspend.
7234 * Returns 0 for success.
7235 */
7236 int cik_suspend(struct radeon_device *rdev)
7237 {
7238 dce6_audio_fini(rdev);
7239 radeon_vm_manager_fini(rdev);
7240 cik_cp_enable(rdev, false);
7241 cik_sdma_enable(rdev, false);
7242 uvd_v1_0_fini(rdev);
7243 radeon_uvd_suspend(rdev);
7244 cik_fini_pg(rdev);
7245 cik_fini_cg(rdev);
7246 cik_irq_suspend(rdev);
7247 radeon_wb_disable(rdev);
7248 cik_pcie_gart_disable(rdev);
7249 return 0;
7250 }
7251
7252 /* Plan is to move initialization in that function and use
7253 * helper function so that radeon_device_init pretty much
7254 * do nothing more than calling asic specific function. This
7255 * should also allow to remove a bunch of callback function
7256 * like vram_info.
7257 */
7258 /**
7259 * cik_init - asic specific driver and hw init
7260 *
7261 * @rdev: radeon_device pointer
7262 *
7263 * Setup asic specific driver variables and program the hw
7264 * to a functional state (CIK).
7265 * Called at driver startup.
7266 * Returns 0 for success, errors for failure.
7267 */
7268 int cik_init(struct radeon_device *rdev)
7269 {
7270 struct radeon_ring *ring;
7271 int r;
7272
7273 /* Read BIOS */
7274 if (!radeon_get_bios(rdev)) {
7275 if (ASIC_IS_AVIVO(rdev))
7276 return -EINVAL;
7277 }
7278 /* Must be an ATOMBIOS */
7279 if (!rdev->is_atom_bios) {
7280 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7281 return -EINVAL;
7282 }
7283 r = radeon_atombios_init(rdev);
7284 if (r)
7285 return r;
7286
7287 /* Post card if necessary */
7288 if (!radeon_card_posted(rdev)) {
7289 if (!rdev->bios) {
7290 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7291 return -EINVAL;
7292 }
7293 DRM_INFO("GPU not posted. posting now...\n");
7294 atom_asic_init(rdev->mode_info.atom_context);
7295 }
7296 /* init golden registers */
7297 cik_init_golden_registers(rdev);
7298 /* Initialize scratch registers */
7299 cik_scratch_init(rdev);
7300 /* Initialize surface registers */
7301 radeon_surface_init(rdev);
7302 /* Initialize clocks */
7303 radeon_get_clock_info(rdev->ddev);
7304
7305 /* Fence driver */
7306 r = radeon_fence_driver_init(rdev);
7307 if (r)
7308 return r;
7309
7310 /* initialize memory controller */
7311 r = cik_mc_init(rdev);
7312 if (r)
7313 return r;
7314 /* Memory manager */
7315 r = radeon_bo_init(rdev);
7316 if (r)
7317 return r;
7318
7319 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7320 ring->ring_obj = NULL;
7321 r600_ring_init(rdev, ring, 1024 * 1024);
7322
7323 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7324 ring->ring_obj = NULL;
7325 r600_ring_init(rdev, ring, 1024 * 1024);
7326 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7327 if (r)
7328 return r;
7329
7330 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7331 ring->ring_obj = NULL;
7332 r600_ring_init(rdev, ring, 1024 * 1024);
7333 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7334 if (r)
7335 return r;
7336
7337 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7338 ring->ring_obj = NULL;
7339 r600_ring_init(rdev, ring, 256 * 1024);
7340
7341 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7342 ring->ring_obj = NULL;
7343 r600_ring_init(rdev, ring, 256 * 1024);
7344
7345 r = radeon_uvd_init(rdev);
7346 if (!r) {
7347 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7348 ring->ring_obj = NULL;
7349 r600_ring_init(rdev, ring, 4096);
7350 }
7351
7352 rdev->ih.ring_obj = NULL;
7353 r600_ih_ring_init(rdev, 64 * 1024);
7354
7355 r = r600_pcie_gart_init(rdev);
7356 if (r)
7357 return r;
7358
7359 rdev->accel_working = true;
7360 r = cik_startup(rdev);
7361 if (r) {
7362 dev_err(rdev->dev, "disabling GPU acceleration\n");
7363 cik_cp_fini(rdev);
7364 cik_sdma_fini(rdev);
7365 cik_irq_fini(rdev);
7366 sumo_rlc_fini(rdev);
7367 cik_mec_fini(rdev);
7368 radeon_wb_fini(rdev);
7369 radeon_ib_pool_fini(rdev);
7370 radeon_vm_manager_fini(rdev);
7371 radeon_irq_kms_fini(rdev);
7372 cik_pcie_gart_fini(rdev);
7373 rdev->accel_working = false;
7374 }
7375
7376 /* Don't start up if the MC ucode is missing.
7377 * The default clocks and voltages before the MC ucode
7378 * is loaded are not suffient for advanced operations.
7379 */
7380 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7381 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7382 return -EINVAL;
7383 }
7384
7385 return 0;
7386 }
7387
7388 /**
7389 * cik_fini - asic specific driver and hw fini
7390 *
7391 * @rdev: radeon_device pointer
7392 *
7393 * Tear down the asic specific driver variables and program the hw
7394 * to an idle state (CIK).
7395 * Called at driver unload.
7396 */
7397 void cik_fini(struct radeon_device *rdev)
7398 {
7399 cik_cp_fini(rdev);
7400 cik_sdma_fini(rdev);
7401 cik_fini_pg(rdev);
7402 cik_fini_cg(rdev);
7403 cik_irq_fini(rdev);
7404 sumo_rlc_fini(rdev);
7405 cik_mec_fini(rdev);
7406 radeon_wb_fini(rdev);
7407 radeon_vm_manager_fini(rdev);
7408 radeon_ib_pool_fini(rdev);
7409 radeon_irq_kms_fini(rdev);
7410 uvd_v1_0_fini(rdev);
7411 radeon_uvd_fini(rdev);
7412 cik_pcie_gart_fini(rdev);
7413 r600_vram_scratch_fini(rdev);
7414 radeon_gem_fini(rdev);
7415 radeon_fence_driver_fini(rdev);
7416 radeon_bo_fini(rdev);
7417 radeon_atombios_fini(rdev);
7418 kfree(rdev->bios);
7419 rdev->bios = NULL;
7420 }
7421
7422 /* display watermark setup */
7423 /**
7424 * dce8_line_buffer_adjust - Set up the line buffer
7425 *
7426 * @rdev: radeon_device pointer
7427 * @radeon_crtc: the selected display controller
7428 * @mode: the current display mode on the selected display
7429 * controller
7430 *
7431 * Setup up the line buffer allocation for
7432 * the selected display controller (CIK).
7433 * Returns the line buffer size in pixels.
7434 */
7435 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7436 struct radeon_crtc *radeon_crtc,
7437 struct drm_display_mode *mode)
7438 {
7439 u32 tmp, buffer_alloc, i;
7440 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
7441 /*
7442 * Line Buffer Setup
7443 * There are 6 line buffers, one for each display controllers.
7444 * There are 3 partitions per LB. Select the number of partitions
7445 * to enable based on the display width. For display widths larger
7446 * than 4096, you need use to use 2 display controllers and combine
7447 * them using the stereo blender.
7448 */
7449 if (radeon_crtc->base.enabled && mode) {
7450 if (mode->crtc_hdisplay < 1920) {
7451 tmp = 1;
7452 buffer_alloc = 2;
7453 } else if (mode->crtc_hdisplay < 2560) {
7454 tmp = 2;
7455 buffer_alloc = 2;
7456 } else if (mode->crtc_hdisplay < 4096) {
7457 tmp = 0;
7458 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7459 } else {
7460 DRM_DEBUG_KMS("Mode too big for LB!\n");
7461 tmp = 0;
7462 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7463 }
7464 } else {
7465 tmp = 1;
7466 buffer_alloc = 0;
7467 }
7468
7469 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7470 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7471
7472 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7473 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7474 for (i = 0; i < rdev->usec_timeout; i++) {
7475 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7476 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7477 break;
7478 udelay(1);
7479 }
7480
7481 if (radeon_crtc->base.enabled && mode) {
7482 switch (tmp) {
7483 case 0:
7484 default:
7485 return 4096 * 2;
7486 case 1:
7487 return 1920 * 2;
7488 case 2:
7489 return 2560 * 2;
7490 }
7491 }
7492
7493 /* controller not enabled, so no lb used */
7494 return 0;
7495 }
7496
7497 /**
7498 * cik_get_number_of_dram_channels - get the number of dram channels
7499 *
7500 * @rdev: radeon_device pointer
7501 *
7502 * Look up the number of video ram channels (CIK).
7503 * Used for display watermark bandwidth calculations
7504 * Returns the number of dram channels
7505 */
7506 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7507 {
7508 u32 tmp = RREG32(MC_SHARED_CHMAP);
7509
7510 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7511 case 0:
7512 default:
7513 return 1;
7514 case 1:
7515 return 2;
7516 case 2:
7517 return 4;
7518 case 3:
7519 return 8;
7520 case 4:
7521 return 3;
7522 case 5:
7523 return 6;
7524 case 6:
7525 return 10;
7526 case 7:
7527 return 12;
7528 case 8:
7529 return 16;
7530 }
7531 }
7532
7533 struct dce8_wm_params {
7534 u32 dram_channels; /* number of dram channels */
7535 u32 yclk; /* bandwidth per dram data pin in kHz */
7536 u32 sclk; /* engine clock in kHz */
7537 u32 disp_clk; /* display clock in kHz */
7538 u32 src_width; /* viewport width */
7539 u32 active_time; /* active display time in ns */
7540 u32 blank_time; /* blank time in ns */
7541 bool interlaced; /* mode is interlaced */
7542 fixed20_12 vsc; /* vertical scale ratio */
7543 u32 num_heads; /* number of active crtcs */
7544 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7545 u32 lb_size; /* line buffer allocated to pipe */
7546 u32 vtaps; /* vertical scaler taps */
7547 };
7548
7549 /**
7550 * dce8_dram_bandwidth - get the dram bandwidth
7551 *
7552 * @wm: watermark calculation data
7553 *
7554 * Calculate the raw dram bandwidth (CIK).
7555 * Used for display watermark bandwidth calculations
7556 * Returns the dram bandwidth in MBytes/s
7557 */
7558 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7559 {
7560 /* Calculate raw DRAM Bandwidth */
7561 fixed20_12 dram_efficiency; /* 0.7 */
7562 fixed20_12 yclk, dram_channels, bandwidth;
7563 fixed20_12 a;
7564
7565 a.full = dfixed_const(1000);
7566 yclk.full = dfixed_const(wm->yclk);
7567 yclk.full = dfixed_div(yclk, a);
7568 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7569 a.full = dfixed_const(10);
7570 dram_efficiency.full = dfixed_const(7);
7571 dram_efficiency.full = dfixed_div(dram_efficiency, a);
7572 bandwidth.full = dfixed_mul(dram_channels, yclk);
7573 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7574
7575 return dfixed_trunc(bandwidth);
7576 }
7577
7578 /**
7579 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7580 *
7581 * @wm: watermark calculation data
7582 *
7583 * Calculate the dram bandwidth used for display (CIK).
7584 * Used for display watermark bandwidth calculations
7585 * Returns the dram bandwidth for display in MBytes/s
7586 */
7587 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7588 {
7589 /* Calculate DRAM Bandwidth and the part allocated to display. */
7590 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7591 fixed20_12 yclk, dram_channels, bandwidth;
7592 fixed20_12 a;
7593
7594 a.full = dfixed_const(1000);
7595 yclk.full = dfixed_const(wm->yclk);
7596 yclk.full = dfixed_div(yclk, a);
7597 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7598 a.full = dfixed_const(10);
7599 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7600 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7601 bandwidth.full = dfixed_mul(dram_channels, yclk);
7602 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7603
7604 return dfixed_trunc(bandwidth);
7605 }
7606
7607 /**
7608 * dce8_data_return_bandwidth - get the data return bandwidth
7609 *
7610 * @wm: watermark calculation data
7611 *
7612 * Calculate the data return bandwidth used for display (CIK).
7613 * Used for display watermark bandwidth calculations
7614 * Returns the data return bandwidth in MBytes/s
7615 */
7616 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7617 {
7618 /* Calculate the display Data return Bandwidth */
7619 fixed20_12 return_efficiency; /* 0.8 */
7620 fixed20_12 sclk, bandwidth;
7621 fixed20_12 a;
7622
7623 a.full = dfixed_const(1000);
7624 sclk.full = dfixed_const(wm->sclk);
7625 sclk.full = dfixed_div(sclk, a);
7626 a.full = dfixed_const(10);
7627 return_efficiency.full = dfixed_const(8);
7628 return_efficiency.full = dfixed_div(return_efficiency, a);
7629 a.full = dfixed_const(32);
7630 bandwidth.full = dfixed_mul(a, sclk);
7631 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7632
7633 return dfixed_trunc(bandwidth);
7634 }
7635
7636 /**
7637 * dce8_dmif_request_bandwidth - get the dmif bandwidth
7638 *
7639 * @wm: watermark calculation data
7640 *
7641 * Calculate the dmif bandwidth used for display (CIK).
7642 * Used for display watermark bandwidth calculations
7643 * Returns the dmif bandwidth in MBytes/s
7644 */
7645 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7646 {
7647 /* Calculate the DMIF Request Bandwidth */
7648 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7649 fixed20_12 disp_clk, bandwidth;
7650 fixed20_12 a, b;
7651
7652 a.full = dfixed_const(1000);
7653 disp_clk.full = dfixed_const(wm->disp_clk);
7654 disp_clk.full = dfixed_div(disp_clk, a);
7655 a.full = dfixed_const(32);
7656 b.full = dfixed_mul(a, disp_clk);
7657
7658 a.full = dfixed_const(10);
7659 disp_clk_request_efficiency.full = dfixed_const(8);
7660 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7661
7662 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7663
7664 return dfixed_trunc(bandwidth);
7665 }
7666
7667 /**
7668 * dce8_available_bandwidth - get the min available bandwidth
7669 *
7670 * @wm: watermark calculation data
7671 *
7672 * Calculate the min available bandwidth used for display (CIK).
7673 * Used for display watermark bandwidth calculations
7674 * Returns the min available bandwidth in MBytes/s
7675 */
7676 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7677 {
7678 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7679 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7680 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7681 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7682
7683 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7684 }
7685
7686 /**
7687 * dce8_average_bandwidth - get the average available bandwidth
7688 *
7689 * @wm: watermark calculation data
7690 *
7691 * Calculate the average available bandwidth used for display (CIK).
7692 * Used for display watermark bandwidth calculations
7693 * Returns the average available bandwidth in MBytes/s
7694 */
7695 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7696 {
7697 /* Calculate the display mode Average Bandwidth
7698 * DisplayMode should contain the source and destination dimensions,
7699 * timing, etc.
7700 */
7701 fixed20_12 bpp;
7702 fixed20_12 line_time;
7703 fixed20_12 src_width;
7704 fixed20_12 bandwidth;
7705 fixed20_12 a;
7706
7707 a.full = dfixed_const(1000);
7708 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7709 line_time.full = dfixed_div(line_time, a);
7710 bpp.full = dfixed_const(wm->bytes_per_pixel);
7711 src_width.full = dfixed_const(wm->src_width);
7712 bandwidth.full = dfixed_mul(src_width, bpp);
7713 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7714 bandwidth.full = dfixed_div(bandwidth, line_time);
7715
7716 return dfixed_trunc(bandwidth);
7717 }
7718
7719 /**
7720 * dce8_latency_watermark - get the latency watermark
7721 *
7722 * @wm: watermark calculation data
7723 *
7724 * Calculate the latency watermark (CIK).
7725 * Used for display watermark bandwidth calculations
7726 * Returns the latency watermark in ns
7727 */
7728 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7729 {
7730 /* First calculate the latency in ns */
7731 u32 mc_latency = 2000; /* 2000 ns. */
7732 u32 available_bandwidth = dce8_available_bandwidth(wm);
7733 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7734 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7735 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7736 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7737 (wm->num_heads * cursor_line_pair_return_time);
7738 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7739 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7740 u32 tmp, dmif_size = 12288;
7741 fixed20_12 a, b, c;
7742
7743 if (wm->num_heads == 0)
7744 return 0;
7745
7746 a.full = dfixed_const(2);
7747 b.full = dfixed_const(1);
7748 if ((wm->vsc.full > a.full) ||
7749 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7750 (wm->vtaps >= 5) ||
7751 ((wm->vsc.full >= a.full) && wm->interlaced))
7752 max_src_lines_per_dst_line = 4;
7753 else
7754 max_src_lines_per_dst_line = 2;
7755
7756 a.full = dfixed_const(available_bandwidth);
7757 b.full = dfixed_const(wm->num_heads);
7758 a.full = dfixed_div(a, b);
7759
7760 b.full = dfixed_const(mc_latency + 512);
7761 c.full = dfixed_const(wm->disp_clk);
7762 b.full = dfixed_div(b, c);
7763
7764 c.full = dfixed_const(dmif_size);
7765 b.full = dfixed_div(c, b);
7766
7767 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7768
7769 b.full = dfixed_const(1000);
7770 c.full = dfixed_const(wm->disp_clk);
7771 b.full = dfixed_div(c, b);
7772 c.full = dfixed_const(wm->bytes_per_pixel);
7773 b.full = dfixed_mul(b, c);
7774
7775 lb_fill_bw = min(tmp, dfixed_trunc(b));
7776
7777 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7778 b.full = dfixed_const(1000);
7779 c.full = dfixed_const(lb_fill_bw);
7780 b.full = dfixed_div(c, b);
7781 a.full = dfixed_div(a, b);
7782 line_fill_time = dfixed_trunc(a);
7783
7784 if (line_fill_time < wm->active_time)
7785 return latency;
7786 else
7787 return latency + (line_fill_time - wm->active_time);
7788
7789 }
7790
7791 /**
7792 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7793 * average and available dram bandwidth
7794 *
7795 * @wm: watermark calculation data
7796 *
7797 * Check if the display average bandwidth fits in the display
7798 * dram bandwidth (CIK).
7799 * Used for display watermark bandwidth calculations
7800 * Returns true if the display fits, false if not.
7801 */
7802 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7803 {
7804 if (dce8_average_bandwidth(wm) <=
7805 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7806 return true;
7807 else
7808 return false;
7809 }
7810
7811 /**
7812 * dce8_average_bandwidth_vs_available_bandwidth - check
7813 * average and available bandwidth
7814 *
7815 * @wm: watermark calculation data
7816 *
7817 * Check if the display average bandwidth fits in the display
7818 * available bandwidth (CIK).
7819 * Used for display watermark bandwidth calculations
7820 * Returns true if the display fits, false if not.
7821 */
7822 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7823 {
7824 if (dce8_average_bandwidth(wm) <=
7825 (dce8_available_bandwidth(wm) / wm->num_heads))
7826 return true;
7827 else
7828 return false;
7829 }
7830
7831 /**
7832 * dce8_check_latency_hiding - check latency hiding
7833 *
7834 * @wm: watermark calculation data
7835 *
7836 * Check latency hiding (CIK).
7837 * Used for display watermark bandwidth calculations
7838 * Returns true if the display fits, false if not.
7839 */
7840 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7841 {
7842 u32 lb_partitions = wm->lb_size / wm->src_width;
7843 u32 line_time = wm->active_time + wm->blank_time;
7844 u32 latency_tolerant_lines;
7845 u32 latency_hiding;
7846 fixed20_12 a;
7847
7848 a.full = dfixed_const(1);
7849 if (wm->vsc.full > a.full)
7850 latency_tolerant_lines = 1;
7851 else {
7852 if (lb_partitions <= (wm->vtaps + 1))
7853 latency_tolerant_lines = 1;
7854 else
7855 latency_tolerant_lines = 2;
7856 }
7857
7858 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
7859
7860 if (dce8_latency_watermark(wm) <= latency_hiding)
7861 return true;
7862 else
7863 return false;
7864 }
7865
7866 /**
7867 * dce8_program_watermarks - program display watermarks
7868 *
7869 * @rdev: radeon_device pointer
7870 * @radeon_crtc: the selected display controller
7871 * @lb_size: line buffer size
7872 * @num_heads: number of display controllers in use
7873 *
7874 * Calculate and program the display watermarks for the
7875 * selected display controller (CIK).
7876 */
7877 static void dce8_program_watermarks(struct radeon_device *rdev,
7878 struct radeon_crtc *radeon_crtc,
7879 u32 lb_size, u32 num_heads)
7880 {
7881 struct drm_display_mode *mode = &radeon_crtc->base.mode;
7882 struct dce8_wm_params wm_low, wm_high;
7883 u32 pixel_period;
7884 u32 line_time = 0;
7885 u32 latency_watermark_a = 0, latency_watermark_b = 0;
7886 u32 tmp, wm_mask;
7887
7888 if (radeon_crtc->base.enabled && num_heads && mode) {
7889 pixel_period = 1000000 / (u32)mode->clock;
7890 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
7891
7892 /* watermark for high clocks */
7893 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7894 rdev->pm.dpm_enabled) {
7895 wm_high.yclk =
7896 radeon_dpm_get_mclk(rdev, false) * 10;
7897 wm_high.sclk =
7898 radeon_dpm_get_sclk(rdev, false) * 10;
7899 } else {
7900 wm_high.yclk = rdev->pm.current_mclk * 10;
7901 wm_high.sclk = rdev->pm.current_sclk * 10;
7902 }
7903
7904 wm_high.disp_clk = mode->clock;
7905 wm_high.src_width = mode->crtc_hdisplay;
7906 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
7907 wm_high.blank_time = line_time - wm_high.active_time;
7908 wm_high.interlaced = false;
7909 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7910 wm_high.interlaced = true;
7911 wm_high.vsc = radeon_crtc->vsc;
7912 wm_high.vtaps = 1;
7913 if (radeon_crtc->rmx_type != RMX_OFF)
7914 wm_high.vtaps = 2;
7915 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
7916 wm_high.lb_size = lb_size;
7917 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
7918 wm_high.num_heads = num_heads;
7919
7920 /* set for high clocks */
7921 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
7922
7923 /* possibly force display priority to high */
7924 /* should really do this at mode validation time... */
7925 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
7926 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
7927 !dce8_check_latency_hiding(&wm_high) ||
7928 (rdev->disp_priority == 2)) {
7929 DRM_DEBUG_KMS("force priority to high\n");
7930 }
7931
7932 /* watermark for low clocks */
7933 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7934 rdev->pm.dpm_enabled) {
7935 wm_low.yclk =
7936 radeon_dpm_get_mclk(rdev, true) * 10;
7937 wm_low.sclk =
7938 radeon_dpm_get_sclk(rdev, true) * 10;
7939 } else {
7940 wm_low.yclk = rdev->pm.current_mclk * 10;
7941 wm_low.sclk = rdev->pm.current_sclk * 10;
7942 }
7943
7944 wm_low.disp_clk = mode->clock;
7945 wm_low.src_width = mode->crtc_hdisplay;
7946 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
7947 wm_low.blank_time = line_time - wm_low.active_time;
7948 wm_low.interlaced = false;
7949 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7950 wm_low.interlaced = true;
7951 wm_low.vsc = radeon_crtc->vsc;
7952 wm_low.vtaps = 1;
7953 if (radeon_crtc->rmx_type != RMX_OFF)
7954 wm_low.vtaps = 2;
7955 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
7956 wm_low.lb_size = lb_size;
7957 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
7958 wm_low.num_heads = num_heads;
7959
7960 /* set for low clocks */
7961 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
7962
7963 /* possibly force display priority to high */
7964 /* should really do this at mode validation time... */
7965 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
7966 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
7967 !dce8_check_latency_hiding(&wm_low) ||
7968 (rdev->disp_priority == 2)) {
7969 DRM_DEBUG_KMS("force priority to high\n");
7970 }
7971 }
7972
7973 /* select wm A */
7974 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7975 tmp = wm_mask;
7976 tmp &= ~LATENCY_WATERMARK_MASK(3);
7977 tmp |= LATENCY_WATERMARK_MASK(1);
7978 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7979 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7980 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
7981 LATENCY_HIGH_WATERMARK(line_time)));
7982 /* select wm B */
7983 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7984 tmp &= ~LATENCY_WATERMARK_MASK(3);
7985 tmp |= LATENCY_WATERMARK_MASK(2);
7986 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7987 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7988 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
7989 LATENCY_HIGH_WATERMARK(line_time)));
7990 /* restore original selection */
7991 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
7992
7993 /* save values for DPM */
7994 radeon_crtc->line_time = line_time;
7995 radeon_crtc->wm_high = latency_watermark_a;
7996 radeon_crtc->wm_low = latency_watermark_b;
7997 }
7998
7999 /**
8000 * dce8_bandwidth_update - program display watermarks
8001 *
8002 * @rdev: radeon_device pointer
8003 *
8004 * Calculate and program the display watermarks and line
8005 * buffer allocation (CIK).
8006 */
8007 void dce8_bandwidth_update(struct radeon_device *rdev)
8008 {
8009 struct drm_display_mode *mode = NULL;
8010 u32 num_heads = 0, lb_size;
8011 int i;
8012
8013 radeon_update_display_priority(rdev);
8014
8015 for (i = 0; i < rdev->num_crtc; i++) {
8016 if (rdev->mode_info.crtcs[i]->base.enabled)
8017 num_heads++;
8018 }
8019 for (i = 0; i < rdev->num_crtc; i++) {
8020 mode = &rdev->mode_info.crtcs[i]->base.mode;
8021 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8022 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8023 }
8024 }
8025
8026 /**
8027 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8028 *
8029 * @rdev: radeon_device pointer
8030 *
8031 * Fetches a GPU clock counter snapshot (SI).
8032 * Returns the 64 bit clock counter snapshot.
8033 */
8034 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8035 {
8036 uint64_t clock;
8037
8038 mutex_lock(&rdev->gpu_clock_mutex);
8039 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8040 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8041 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8042 mutex_unlock(&rdev->gpu_clock_mutex);
8043 return clock;
8044 }
8045
8046 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8047 u32 cntl_reg, u32 status_reg)
8048 {
8049 int r, i;
8050 struct atom_clock_dividers dividers;
8051 uint32_t tmp;
8052
8053 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8054 clock, false, &dividers);
8055 if (r)
8056 return r;
8057
8058 tmp = RREG32_SMC(cntl_reg);
8059 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8060 tmp |= dividers.post_divider;
8061 WREG32_SMC(cntl_reg, tmp);
8062
8063 for (i = 0; i < 100; i++) {
8064 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8065 break;
8066 mdelay(10);
8067 }
8068 if (i == 100)
8069 return -ETIMEDOUT;
8070
8071 return 0;
8072 }
8073
8074 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8075 {
8076 int r = 0;
8077
8078 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8079 if (r)
8080 return r;
8081
8082 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8083 return r;
8084 }
8085
8086 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8087 {
8088 struct pci_dev *root = rdev->pdev->bus->self;
8089 int bridge_pos, gpu_pos;
8090 u32 speed_cntl, mask, current_data_rate;
8091 int ret, i;
8092 u16 tmp16;
8093
8094 if (radeon_pcie_gen2 == 0)
8095 return;
8096
8097 if (rdev->flags & RADEON_IS_IGP)
8098 return;
8099
8100 if (!(rdev->flags & RADEON_IS_PCIE))
8101 return;
8102
8103 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8104 if (ret != 0)
8105 return;
8106
8107 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8108 return;
8109
8110 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8111 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8112 LC_CURRENT_DATA_RATE_SHIFT;
8113 if (mask & DRM_PCIE_SPEED_80) {
8114 if (current_data_rate == 2) {
8115 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8116 return;
8117 }
8118 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8119 } else if (mask & DRM_PCIE_SPEED_50) {
8120 if (current_data_rate == 1) {
8121 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8122 return;
8123 }
8124 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8125 }
8126
8127 bridge_pos = pci_pcie_cap(root);
8128 if (!bridge_pos)
8129 return;
8130
8131 gpu_pos = pci_pcie_cap(rdev->pdev);
8132 if (!gpu_pos)
8133 return;
8134
8135 if (mask & DRM_PCIE_SPEED_80) {
8136 /* re-try equalization if gen3 is not already enabled */
8137 if (current_data_rate != 2) {
8138 u16 bridge_cfg, gpu_cfg;
8139 u16 bridge_cfg2, gpu_cfg2;
8140 u32 max_lw, current_lw, tmp;
8141
8142 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8143 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8144
8145 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8146 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8147
8148 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8149 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8150
8151 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8152 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8153 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8154
8155 if (current_lw < max_lw) {
8156 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8157 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8158 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8159 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8160 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8161 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8162 }
8163 }
8164
8165 for (i = 0; i < 10; i++) {
8166 /* check status */
8167 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8168 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8169 break;
8170
8171 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8172 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8173
8174 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8175 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8176
8177 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8178 tmp |= LC_SET_QUIESCE;
8179 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8180
8181 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8182 tmp |= LC_REDO_EQ;
8183 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8184
8185 mdelay(100);
8186
8187 /* linkctl */
8188 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8189 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8190 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8191 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8192
8193 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8194 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8195 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8196 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8197
8198 /* linkctl2 */
8199 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8200 tmp16 &= ~((1 << 4) | (7 << 9));
8201 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8202 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8203
8204 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8205 tmp16 &= ~((1 << 4) | (7 << 9));
8206 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8207 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8208
8209 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8210 tmp &= ~LC_SET_QUIESCE;
8211 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8212 }
8213 }
8214 }
8215
8216 /* set the link speed */
8217 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8218 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8219 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8220
8221 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8222 tmp16 &= ~0xf;
8223 if (mask & DRM_PCIE_SPEED_80)
8224 tmp16 |= 3; /* gen3 */
8225 else if (mask & DRM_PCIE_SPEED_50)
8226 tmp16 |= 2; /* gen2 */
8227 else
8228 tmp16 |= 1; /* gen1 */
8229 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8230
8231 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8232 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8233 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8234
8235 for (i = 0; i < rdev->usec_timeout; i++) {
8236 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8237 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8238 break;
8239 udelay(1);
8240 }
8241 }
8242
8243 static void cik_program_aspm(struct radeon_device *rdev)
8244 {
8245 u32 data, orig;
8246 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8247 bool disable_clkreq = false;
8248
8249 if (radeon_aspm == 0)
8250 return;
8251
8252 /* XXX double check IGPs */
8253 if (rdev->flags & RADEON_IS_IGP)
8254 return;
8255
8256 if (!(rdev->flags & RADEON_IS_PCIE))
8257 return;
8258
8259 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8260 data &= ~LC_XMIT_N_FTS_MASK;
8261 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8262 if (orig != data)
8263 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8264
8265 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8266 data |= LC_GO_TO_RECOVERY;
8267 if (orig != data)
8268 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8269
8270 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8271 data |= P_IGNORE_EDB_ERR;
8272 if (orig != data)
8273 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8274
8275 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8276 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8277 data |= LC_PMI_TO_L1_DIS;
8278 if (!disable_l0s)
8279 data |= LC_L0S_INACTIVITY(7);
8280
8281 if (!disable_l1) {
8282 data |= LC_L1_INACTIVITY(7);
8283 data &= ~LC_PMI_TO_L1_DIS;
8284 if (orig != data)
8285 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8286
8287 if (!disable_plloff_in_l1) {
8288 bool clk_req_support;
8289
8290 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8291 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8292 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8293 if (orig != data)
8294 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8295
8296 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8297 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8298 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8299 if (orig != data)
8300 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8301
8302 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8303 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8304 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8305 if (orig != data)
8306 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8307
8308 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8309 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8310 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8311 if (orig != data)
8312 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8313
8314 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8315 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8316 data |= LC_DYN_LANES_PWR_STATE(3);
8317 if (orig != data)
8318 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8319
8320 if (!disable_clkreq) {
8321 struct pci_dev *root = rdev->pdev->bus->self;
8322 u32 lnkcap;
8323
8324 clk_req_support = false;
8325 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8326 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8327 clk_req_support = true;
8328 } else {
8329 clk_req_support = false;
8330 }
8331
8332 if (clk_req_support) {
8333 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8334 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8335 if (orig != data)
8336 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8337
8338 orig = data = RREG32_SMC(THM_CLK_CNTL);
8339 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8340 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8341 if (orig != data)
8342 WREG32_SMC(THM_CLK_CNTL, data);
8343
8344 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8345 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8346 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8347 if (orig != data)
8348 WREG32_SMC(MISC_CLK_CTRL, data);
8349
8350 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8351 data &= ~BCLK_AS_XCLK;
8352 if (orig != data)
8353 WREG32_SMC(CG_CLKPIN_CNTL, data);
8354
8355 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8356 data &= ~FORCE_BIF_REFCLK_EN;
8357 if (orig != data)
8358 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8359
8360 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8361 data &= ~MPLL_CLKOUT_SEL_MASK;
8362 data |= MPLL_CLKOUT_SEL(4);
8363 if (orig != data)
8364 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8365 }
8366 }
8367 } else {
8368 if (orig != data)
8369 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8370 }
8371
8372 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8373 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8374 if (orig != data)
8375 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8376
8377 if (!disable_l0s) {
8378 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8379 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8380 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8381 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8382 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8383 data &= ~LC_L0S_INACTIVITY_MASK;
8384 if (orig != data)
8385 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8386 }
8387 }
8388 }
8389 }
Linux kernel - Deliverables
This page took 0.367045 seconds and 5 git commands to generate.