projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fix init ordering for r600+
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
02c81327
AD		 44MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 49MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 50MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51MODULE_FIRMWARE("radeon/KABINI_me.bin");
52MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 55MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 56
a59781bb
AD		 57extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 59extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 61extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 62extern void sumo_rlc_fini(struct radeon_device *rdev);
63extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 64extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 65extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 66extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 67extern int cik_sdma_resume(struct radeon_device *rdev);
68extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69extern void cik_sdma_fini(struct radeon_device *rdev);
70extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
71 struct radeon_ib *ib,
72 uint64_t pe,
73 uint64_t addr, unsigned count,
74 uint32_t incr, uint32_t flags);
cc066715 75static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 76static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 77static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 78static void cik_init_pg(struct radeon_device *rdev);
79static void cik_init_cg(struct radeon_device *rdev);
6f2043ce 80
286d9cc6
AD		 81/* get temperature in millidegrees */
82int ci_get_temp(struct radeon_device *rdev)
83{
84 u32 temp;
85 int actual_temp = 0;
86
87 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
88 CTF_TEMP_SHIFT;
89
90 if (temp & 0x200)
91 actual_temp = 255;
92 else
93 actual_temp = temp & 0x1ff;
94
95 actual_temp = actual_temp * 1000;
96
97 return actual_temp;
98}
99
100/* get temperature in millidegrees */
101int kv_get_temp(struct radeon_device *rdev)
102{
103 u32 temp;
104 int actual_temp = 0;
105
106 temp = RREG32_SMC(0xC0300E0C);
107
108 if (temp)
109 actual_temp = (temp / 8) - 49;
110 else
111 actual_temp = 0;
112
113 actual_temp = actual_temp * 1000;
114
115 return actual_temp;
116}
117
6e2c3c0a
AD		 118/*
119 * Indirect registers accessor
120 */
121u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
122{
123 u32 r;
124
125 WREG32(PCIE_INDEX, reg);
126 (void)RREG32(PCIE_INDEX);
127 r = RREG32(PCIE_DATA);
128 return r;
129}
130
131void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
132{
133 WREG32(PCIE_INDEX, reg);
134 (void)RREG32(PCIE_INDEX);
135 WREG32(PCIE_DATA, v);
136 (void)RREG32(PCIE_DATA);
137}
138
22c775ce
AD		 139static const u32 spectre_rlc_save_restore_register_list[] =
140{
141 (0x0e00 << 16) | (0xc12c >> 2),
142 0x00000000,
143 (0x0e00 << 16) | (0xc140 >> 2),
144 0x00000000,
145 (0x0e00 << 16) | (0xc150 >> 2),
146 0x00000000,
147 (0x0e00 << 16) | (0xc15c >> 2),
148 0x00000000,
149 (0x0e00 << 16) | (0xc168 >> 2),
150 0x00000000,
151 (0x0e00 << 16) | (0xc170 >> 2),
152 0x00000000,
153 (0x0e00 << 16) | (0xc178 >> 2),
154 0x00000000,
155 (0x0e00 << 16) | (0xc204 >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc2b4 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc2b8 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc2bc >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc2c0 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0x8228 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0x829c >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0x869c >> 2),
170 0x00000000,
171 (0x0600 << 16) | (0x98f4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0x98f8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0x9900 >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc260 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x90e8 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x3c000 >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x3c00c >> 2),
184 0x00000000,
185 (0x0e00 << 16) | (0x8c1c >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x9700 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0xcd20 >> 2),
190 0x00000000,
191 (0x4e00 << 16) | (0xcd20 >> 2),
192 0x00000000,
193 (0x5e00 << 16) | (0xcd20 >> 2),
194 0x00000000,
195 (0x6e00 << 16) | (0xcd20 >> 2),
196 0x00000000,
197 (0x7e00 << 16) | (0xcd20 >> 2),
198 0x00000000,
199 (0x8e00 << 16) | (0xcd20 >> 2),
200 0x00000000,
201 (0x9e00 << 16) | (0xcd20 >> 2),
202 0x00000000,
203 (0xae00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0xbe00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x0e00 << 16) | (0x89bc >> 2),
208 0x00000000,
209 (0x0e00 << 16) | (0x8900 >> 2),
210 0x00000000,
211 0x3,
212 (0x0e00 << 16) | (0xc130 >> 2),
213 0x00000000,
214 (0x0e00 << 16) | (0xc134 >> 2),
215 0x00000000,
216 (0x0e00 << 16) | (0xc1fc >> 2),
217 0x00000000,
218 (0x0e00 << 16) | (0xc208 >> 2),
219 0x00000000,
220 (0x0e00 << 16) | (0xc264 >> 2),
221 0x00000000,
222 (0x0e00 << 16) | (0xc268 >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0xc26c >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0xc270 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc274 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc278 >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc27c >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc280 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc284 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc288 >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc28c >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc290 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc294 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc298 >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc29c >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc2a0 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc2a4 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc2a8 >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc2ac >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc2b0 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0x301d0 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0x30238 >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0x30250 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0x30254 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0x30258 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0x3025c >> 2),
271 0x00000000,
272 (0x4e00 << 16) | (0xc900 >> 2),
273 0x00000000,
274 (0x5e00 << 16) | (0xc900 >> 2),
275 0x00000000,
276 (0x6e00 << 16) | (0xc900 >> 2),
277 0x00000000,
278 (0x7e00 << 16) | (0xc900 >> 2),
279 0x00000000,
280 (0x8e00 << 16) | (0xc900 >> 2),
281 0x00000000,
282 (0x9e00 << 16) | (0xc900 >> 2),
283 0x00000000,
284 (0xae00 << 16) | (0xc900 >> 2),
285 0x00000000,
286 (0xbe00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x4e00 << 16) | (0xc904 >> 2),
289 0x00000000,
290 (0x5e00 << 16) | (0xc904 >> 2),
291 0x00000000,
292 (0x6e00 << 16) | (0xc904 >> 2),
293 0x00000000,
294 (0x7e00 << 16) | (0xc904 >> 2),
295 0x00000000,
296 (0x8e00 << 16) | (0xc904 >> 2),
297 0x00000000,
298 (0x9e00 << 16) | (0xc904 >> 2),
299 0x00000000,
300 (0xae00 << 16) | (0xc904 >> 2),
301 0x00000000,
302 (0xbe00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x4e00 << 16) | (0xc908 >> 2),
305 0x00000000,
306 (0x5e00 << 16) | (0xc908 >> 2),
307 0x00000000,
308 (0x6e00 << 16) | (0xc908 >> 2),
309 0x00000000,
310 (0x7e00 << 16) | (0xc908 >> 2),
311 0x00000000,
312 (0x8e00 << 16) | (0xc908 >> 2),
313 0x00000000,
314 (0x9e00 << 16) | (0xc908 >> 2),
315 0x00000000,
316 (0xae00 << 16) | (0xc908 >> 2),
317 0x00000000,
318 (0xbe00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x4e00 << 16) | (0xc90c >> 2),
321 0x00000000,
322 (0x5e00 << 16) | (0xc90c >> 2),
323 0x00000000,
324 (0x6e00 << 16) | (0xc90c >> 2),
325 0x00000000,
326 (0x7e00 << 16) | (0xc90c >> 2),
327 0x00000000,
328 (0x8e00 << 16) | (0xc90c >> 2),
329 0x00000000,
330 (0x9e00 << 16) | (0xc90c >> 2),
331 0x00000000,
332 (0xae00 << 16) | (0xc90c >> 2),
333 0x00000000,
334 (0xbe00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x4e00 << 16) | (0xc910 >> 2),
337 0x00000000,
338 (0x5e00 << 16) | (0xc910 >> 2),
339 0x00000000,
340 (0x6e00 << 16) | (0xc910 >> 2),
341 0x00000000,
342 (0x7e00 << 16) | (0xc910 >> 2),
343 0x00000000,
344 (0x8e00 << 16) | (0xc910 >> 2),
345 0x00000000,
346 (0x9e00 << 16) | (0xc910 >> 2),
347 0x00000000,
348 (0xae00 << 16) | (0xc910 >> 2),
349 0x00000000,
350 (0xbe00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc99c >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0x9834 >> 2),
355 0x00000000,
356 (0x0000 << 16) | (0x30f00 >> 2),
357 0x00000000,
358 (0x0001 << 16) | (0x30f00 >> 2),
359 0x00000000,
360 (0x0000 << 16) | (0x30f04 >> 2),
361 0x00000000,
362 (0x0001 << 16) | (0x30f04 >> 2),
363 0x00000000,
364 (0x0000 << 16) | (0x30f08 >> 2),
365 0x00000000,
366 (0x0001 << 16) | (0x30f08 >> 2),
367 0x00000000,
368 (0x0000 << 16) | (0x30f0c >> 2),
369 0x00000000,
370 (0x0001 << 16) | (0x30f0c >> 2),
371 0x00000000,
372 (0x0600 << 16) | (0x9b7c >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0x8a14 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0x8a18 >> 2),
377 0x00000000,
378 (0x0600 << 16) | (0x30a00 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0x8bf0 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0x8bcc >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0x8b24 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0x30a04 >> 2),
387 0x00000000,
388 (0x0600 << 16) | (0x30a10 >> 2),
389 0x00000000,
390 (0x0600 << 16) | (0x30a14 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a18 >> 2),
393 0x00000000,
394 (0x0600 << 16) | (0x30a2c >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0xc700 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0xc704 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0xc708 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0xc768 >> 2),
403 0x00000000,
404 (0x0400 << 16) | (0xc770 >> 2),
405 0x00000000,
406 (0x0400 << 16) | (0xc774 >> 2),
407 0x00000000,
408 (0x0400 << 16) | (0xc778 >> 2),
409 0x00000000,
410 (0x0400 << 16) | (0xc77c >> 2),
411 0x00000000,
412 (0x0400 << 16) | (0xc780 >> 2),
413 0x00000000,
414 (0x0400 << 16) | (0xc784 >> 2),
415 0x00000000,
416 (0x0400 << 16) | (0xc788 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc78c >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc798 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc79c >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc7a0 >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc7a4 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc7a8 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc7ac >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc7b0 >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc7b4 >> 2),
435 0x00000000,
436 (0x0e00 << 16) | (0x9100 >> 2),
437 0x00000000,
438 (0x0e00 << 16) | (0x3c010 >> 2),
439 0x00000000,
440 (0x0e00 << 16) | (0x92a8 >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x92ac >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x92b4 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x92b8 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x92bc >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x92c0 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x92c4 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92c8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92cc >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92d0 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x8c00 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x8c04 >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x8c20 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x8c38 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x8c3c >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xae00 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x9604 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0xac08 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0xac0c >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0xac10 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xac14 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xac58 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xac68 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0xac6c >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac70 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac74 >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac78 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac7c >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac80 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac84 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac88 >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac8c >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x970c >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x9714 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x9718 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x971c >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x31068 >> 2),
513 0x00000000,
514 (0x4e00 << 16) | (0x31068 >> 2),
515 0x00000000,
516 (0x5e00 << 16) | (0x31068 >> 2),
517 0x00000000,
518 (0x6e00 << 16) | (0x31068 >> 2),
519 0x00000000,
520 (0x7e00 << 16) | (0x31068 >> 2),
521 0x00000000,
522 (0x8e00 << 16) | (0x31068 >> 2),
523 0x00000000,
524 (0x9e00 << 16) | (0x31068 >> 2),
525 0x00000000,
526 (0xae00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0xbe00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xcd10 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xcd14 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x88b0 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x88b4 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0x88b8 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x88bc >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0x89c0 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0x88c4 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0x88c8 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88d0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88d4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88d8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x8980 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0x30938 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x3093c >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x30940 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x89a0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x30900 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x30904 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x89b4 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x3c210 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3c214 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x3c218 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x8904 >> 2),
577 0x00000000,
578 0x5,
579 (0x0e00 << 16) | (0x8c28 >> 2),
580 (0x0e00 << 16) | (0x8c2c >> 2),
581 (0x0e00 << 16) | (0x8c30 >> 2),
582 (0x0e00 << 16) | (0x8c34 >> 2),
583 (0x0e00 << 16) | (0x9600 >> 2),
584};
585
586static const u32 kalindi_rlc_save_restore_register_list[] =
587{
588 (0x0e00 << 16) | (0xc12c >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0xc140 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0xc150 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0xc15c >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0xc168 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xc170 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xc204 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xc2b4 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc2b8 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc2bc >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc2c0 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0x8228 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x829c >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x869c >> 2),
615 0x00000000,
616 (0x0600 << 16) | (0x98f4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x98f8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x9900 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc260 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x90e8 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x3c000 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x3c00c >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x8c1c >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x9700 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0xcd20 >> 2),
635 0x00000000,
636 (0x4e00 << 16) | (0xcd20 >> 2),
637 0x00000000,
638 (0x5e00 << 16) | (0xcd20 >> 2),
639 0x00000000,
640 (0x6e00 << 16) | (0xcd20 >> 2),
641 0x00000000,
642 (0x7e00 << 16) | (0xcd20 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x89bc >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x8900 >> 2),
647 0x00000000,
648 0x3,
649 (0x0e00 << 16) | (0xc130 >> 2),
650 0x00000000,
651 (0x0e00 << 16) | (0xc134 >> 2),
652 0x00000000,
653 (0x0e00 << 16) | (0xc1fc >> 2),
654 0x00000000,
655 (0x0e00 << 16) | (0xc208 >> 2),
656 0x00000000,
657 (0x0e00 << 16) | (0xc264 >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0xc268 >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0xc26c >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0xc270 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc274 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc28c >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc290 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc294 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc298 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc2a0 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc2a4 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc2a8 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc2ac >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0x301d0 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0x30238 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0x30250 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0x30254 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0x30258 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0x3025c >> 2),
694 0x00000000,
695 (0x4e00 << 16) | (0xc900 >> 2),
696 0x00000000,
697 (0x5e00 << 16) | (0xc900 >> 2),
698 0x00000000,
699 (0x6e00 << 16) | (0xc900 >> 2),
700 0x00000000,
701 (0x7e00 << 16) | (0xc900 >> 2),
702 0x00000000,
703 (0x4e00 << 16) | (0xc904 >> 2),
704 0x00000000,
705 (0x5e00 << 16) | (0xc904 >> 2),
706 0x00000000,
707 (0x6e00 << 16) | (0xc904 >> 2),
708 0x00000000,
709 (0x7e00 << 16) | (0xc904 >> 2),
710 0x00000000,
711 (0x4e00 << 16) | (0xc908 >> 2),
712 0x00000000,
713 (0x5e00 << 16) | (0xc908 >> 2),
714 0x00000000,
715 (0x6e00 << 16) | (0xc908 >> 2),
716 0x00000000,
717 (0x7e00 << 16) | (0xc908 >> 2),
718 0x00000000,
719 (0x4e00 << 16) | (0xc90c >> 2),
720 0x00000000,
721 (0x5e00 << 16) | (0xc90c >> 2),
722 0x00000000,
723 (0x6e00 << 16) | (0xc90c >> 2),
724 0x00000000,
725 (0x7e00 << 16) | (0xc90c >> 2),
726 0x00000000,
727 (0x4e00 << 16) | (0xc910 >> 2),
728 0x00000000,
729 (0x5e00 << 16) | (0xc910 >> 2),
730 0x00000000,
731 (0x6e00 << 16) | (0xc910 >> 2),
732 0x00000000,
733 (0x7e00 << 16) | (0xc910 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc99c >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0x9834 >> 2),
738 0x00000000,
739 (0x0000 << 16) | (0x30f00 >> 2),
740 0x00000000,
741 (0x0000 << 16) | (0x30f04 >> 2),
742 0x00000000,
743 (0x0000 << 16) | (0x30f08 >> 2),
744 0x00000000,
745 (0x0000 << 16) | (0x30f0c >> 2),
746 0x00000000,
747 (0x0600 << 16) | (0x9b7c >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0x8a14 >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x8a18 >> 2),
752 0x00000000,
753 (0x0600 << 16) | (0x30a00 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x8bf0 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x8bcc >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x8b24 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x30a04 >> 2),
762 0x00000000,
763 (0x0600 << 16) | (0x30a10 >> 2),
764 0x00000000,
765 (0x0600 << 16) | (0x30a14 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a18 >> 2),
768 0x00000000,
769 (0x0600 << 16) | (0x30a2c >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0xc700 >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0xc704 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0xc708 >> 2),
776 0x00000000,
777 (0x0e00 << 16) | (0xc768 >> 2),
778 0x00000000,
779 (0x0400 << 16) | (0xc770 >> 2),
780 0x00000000,
781 (0x0400 << 16) | (0xc774 >> 2),
782 0x00000000,
783 (0x0400 << 16) | (0xc798 >> 2),
784 0x00000000,
785 (0x0400 << 16) | (0xc79c >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0x9100 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0x3c010 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0x8c00 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0x8c04 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0x8c20 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0x8c38 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0x8c3c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xae00 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x9604 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xac08 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xac0c >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xac10 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xac14 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xac58 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xac68 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0xac6c >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac70 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac74 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac78 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac7c >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac80 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac84 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac88 >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac8c >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0x970c >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0x9714 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0x9718 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0x971c >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0x31068 >> 2),
844 0x00000000,
845 (0x4e00 << 16) | (0x31068 >> 2),
846 0x00000000,
847 (0x5e00 << 16) | (0x31068 >> 2),
848 0x00000000,
849 (0x6e00 << 16) | (0x31068 >> 2),
850 0x00000000,
851 (0x7e00 << 16) | (0x31068 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0xcd10 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0xcd14 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x88b0 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x88b4 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x88b8 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x88bc >> 2),
864 0x00000000,
865 (0x0400 << 16) | (0x89c0 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x88c4 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x88c8 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88d0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88d4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88d8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x8980 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x30938 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x3093c >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x30940 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x89a0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x30900 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x30904 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x89b4 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x3e1fc >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3c210 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x3c214 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x3c218 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x8904 >> 2),
902 0x00000000,
903 0x5,
904 (0x0e00 << 16) | (0x8c28 >> 2),
905 (0x0e00 << 16) | (0x8c2c >> 2),
906 (0x0e00 << 16) | (0x8c30 >> 2),
907 (0x0e00 << 16) | (0x8c34 >> 2),
908 (0x0e00 << 16) | (0x9600 >> 2),
909};
910
0aafd313
AD		 911static const u32 bonaire_golden_spm_registers[] =
912{
913 0x30800, 0xe0ffffff, 0xe0000000
914};
915
916static const u32 bonaire_golden_common_registers[] =
917{
918 0xc770, 0xffffffff, 0x00000800,
919 0xc774, 0xffffffff, 0x00000800,
920 0xc798, 0xffffffff, 0x00007fbf,
921 0xc79c, 0xffffffff, 0x00007faf
922};
923
924static const u32 bonaire_golden_registers[] =
925{
926 0x3354, 0x00000333, 0x00000333,
927 0x3350, 0x000c0fc0, 0x00040200,
928 0x9a10, 0x00010000, 0x00058208,
929 0x3c000, 0xffff1fff, 0x00140000,
930 0x3c200, 0xfdfc0fff, 0x00000100,
931 0x3c234, 0x40000000, 0x40000200,
932 0x9830, 0xffffffff, 0x00000000,
933 0x9834, 0xf00fffff, 0x00000400,
934 0x9838, 0x0002021c, 0x00020200,
935 0xc78, 0x00000080, 0x00000000,
936 0x5bb0, 0x000000f0, 0x00000070,
937 0x5bc0, 0xf0311fff, 0x80300000,
938 0x98f8, 0x73773777, 0x12010001,
939 0x350c, 0x00810000, 0x408af000,
940 0x7030, 0x31000111, 0x00000011,
941 0x2f48, 0x73773777, 0x12010001,
942 0x220c, 0x00007fb6, 0x0021a1b1,
943 0x2210, 0x00007fb6, 0x002021b1,
944 0x2180, 0x00007fb6, 0x00002191,
945 0x2218, 0x00007fb6, 0x002121b1,
946 0x221c, 0x00007fb6, 0x002021b1,
947 0x21dc, 0x00007fb6, 0x00002191,
948 0x21e0, 0x00007fb6, 0x00002191,
949 0x3628, 0x0000003f, 0x0000000a,
950 0x362c, 0x0000003f, 0x0000000a,
951 0x2ae4, 0x00073ffe, 0x000022a2,
952 0x240c, 0x000007ff, 0x00000000,
953 0x8a14, 0xf000003f, 0x00000007,
954 0x8bf0, 0x00002001, 0x00000001,
955 0x8b24, 0xffffffff, 0x00ffffff,
956 0x30a04, 0x0000ff0f, 0x00000000,
957 0x28a4c, 0x07ffffff, 0x06000000,
958 0x4d8, 0x00000fff, 0x00000100,
959 0x3e78, 0x00000001, 0x00000002,
960 0x9100, 0x03000000, 0x0362c688,
961 0x8c00, 0x000000ff, 0x00000001,
962 0xe40, 0x00001fff, 0x00001fff,
963 0x9060, 0x0000007f, 0x00000020,
964 0x9508, 0x00010000, 0x00010000,
965 0xac14, 0x000003ff, 0x000000f3,
966 0xac0c, 0xffffffff, 0x00001032
967};
968
969static const u32 bonaire_mgcg_cgcg_init[] =
970{
971 0xc420, 0xffffffff, 0xfffffffc,
972 0x30800, 0xffffffff, 0xe0000000,
973 0x3c2a0, 0xffffffff, 0x00000100,
974 0x3c208, 0xffffffff, 0x00000100,
975 0x3c2c0, 0xffffffff, 0xc0000100,
976 0x3c2c8, 0xffffffff, 0xc0000100,
977 0x3c2c4, 0xffffffff, 0xc0000100,
978 0x55e4, 0xffffffff, 0x00600100,
979 0x3c280, 0xffffffff, 0x00000100,
980 0x3c214, 0xffffffff, 0x06000100,
981 0x3c220, 0xffffffff, 0x00000100,
982 0x3c218, 0xffffffff, 0x06000100,
983 0x3c204, 0xffffffff, 0x00000100,
984 0x3c2e0, 0xffffffff, 0x00000100,
985 0x3c224, 0xffffffff, 0x00000100,
986 0x3c200, 0xffffffff, 0x00000100,
987 0x3c230, 0xffffffff, 0x00000100,
988 0x3c234, 0xffffffff, 0x00000100,
989 0x3c250, 0xffffffff, 0x00000100,
990 0x3c254, 0xffffffff, 0x00000100,
991 0x3c258, 0xffffffff, 0x00000100,
992 0x3c25c, 0xffffffff, 0x00000100,
993 0x3c260, 0xffffffff, 0x00000100,
994 0x3c27c, 0xffffffff, 0x00000100,
995 0x3c278, 0xffffffff, 0x00000100,
996 0x3c210, 0xffffffff, 0x06000100,
997 0x3c290, 0xffffffff, 0x00000100,
998 0x3c274, 0xffffffff, 0x00000100,
999 0x3c2b4, 0xffffffff, 0x00000100,
1000 0x3c2b0, 0xffffffff, 0x00000100,
1001 0x3c270, 0xffffffff, 0x00000100,
1002 0x30800, 0xffffffff, 0xe0000000,
1003 0x3c020, 0xffffffff, 0x00010000,
1004 0x3c024, 0xffffffff, 0x00030002,
1005 0x3c028, 0xffffffff, 0x00040007,
1006 0x3c02c, 0xffffffff, 0x00060005,
1007 0x3c030, 0xffffffff, 0x00090008,
1008 0x3c034, 0xffffffff, 0x00010000,
1009 0x3c038, 0xffffffff, 0x00030002,
1010 0x3c03c, 0xffffffff, 0x00040007,
1011 0x3c040, 0xffffffff, 0x00060005,
1012 0x3c044, 0xffffffff, 0x00090008,
1013 0x3c048, 0xffffffff, 0x00010000,
1014 0x3c04c, 0xffffffff, 0x00030002,
1015 0x3c050, 0xffffffff, 0x00040007,
1016 0x3c054, 0xffffffff, 0x00060005,
1017 0x3c058, 0xffffffff, 0x00090008,
1018 0x3c05c, 0xffffffff, 0x00010000,
1019 0x3c060, 0xffffffff, 0x00030002,
1020 0x3c064, 0xffffffff, 0x00040007,
1021 0x3c068, 0xffffffff, 0x00060005,
1022 0x3c06c, 0xffffffff, 0x00090008,
1023 0x3c070, 0xffffffff, 0x00010000,
1024 0x3c074, 0xffffffff, 0x00030002,
1025 0x3c078, 0xffffffff, 0x00040007,
1026 0x3c07c, 0xffffffff, 0x00060005,
1027 0x3c080, 0xffffffff, 0x00090008,
1028 0x3c084, 0xffffffff, 0x00010000,
1029 0x3c088, 0xffffffff, 0x00030002,
1030 0x3c08c, 0xffffffff, 0x00040007,
1031 0x3c090, 0xffffffff, 0x00060005,
1032 0x3c094, 0xffffffff, 0x00090008,
1033 0x3c098, 0xffffffff, 0x00010000,
1034 0x3c09c, 0xffffffff, 0x00030002,
1035 0x3c0a0, 0xffffffff, 0x00040007,
1036 0x3c0a4, 0xffffffff, 0x00060005,
1037 0x3c0a8, 0xffffffff, 0x00090008,
1038 0x3c000, 0xffffffff, 0x96e00200,
1039 0x8708, 0xffffffff, 0x00900100,
1040 0xc424, 0xffffffff, 0x0020003f,
1041 0x38, 0xffffffff, 0x0140001c,
1042 0x3c, 0x000f0000, 0x000f0000,
1043 0x220, 0xffffffff, 0xC060000C,
1044 0x224, 0xc0000fff, 0x00000100,
1045 0xf90, 0xffffffff, 0x00000100,
1046 0xf98, 0x00000101, 0x00000000,
1047 0x20a8, 0xffffffff, 0x00000104,
1048 0x55e4, 0xff000fff, 0x00000100,
1049 0x30cc, 0xc0000fff, 0x00000104,
1050 0xc1e4, 0x00000001, 0x00000001,
1051 0xd00c, 0xff000ff0, 0x00000100,
1052 0xd80c, 0xff000ff0, 0x00000100
1053};
1054
1055static const u32 spectre_golden_spm_registers[] =
1056{
1057 0x30800, 0xe0ffffff, 0xe0000000
1058};
1059
1060static const u32 spectre_golden_common_registers[] =
1061{
1062 0xc770, 0xffffffff, 0x00000800,
1063 0xc774, 0xffffffff, 0x00000800,
1064 0xc798, 0xffffffff, 0x00007fbf,
1065 0xc79c, 0xffffffff, 0x00007faf
1066};
1067
1068static const u32 spectre_golden_registers[] =
1069{
1070 0x3c000, 0xffff1fff, 0x96940200,
1071 0x3c00c, 0xffff0001, 0xff000000,
1072 0x3c200, 0xfffc0fff, 0x00000100,
1073 0x6ed8, 0x00010101, 0x00010000,
1074 0x9834, 0xf00fffff, 0x00000400,
1075 0x9838, 0xfffffffc, 0x00020200,
1076 0x5bb0, 0x000000f0, 0x00000070,
1077 0x5bc0, 0xf0311fff, 0x80300000,
1078 0x98f8, 0x73773777, 0x12010001,
1079 0x9b7c, 0x00ff0000, 0x00fc0000,
1080 0x2f48, 0x73773777, 0x12010001,
1081 0x8a14, 0xf000003f, 0x00000007,
1082 0x8b24, 0xffffffff, 0x00ffffff,
1083 0x28350, 0x3f3f3fff, 0x00000082,
1084 0x28355, 0x0000003f, 0x00000000,
1085 0x3e78, 0x00000001, 0x00000002,
1086 0x913c, 0xffff03df, 0x00000004,
1087 0xc768, 0x00000008, 0x00000008,
1088 0x8c00, 0x000008ff, 0x00000800,
1089 0x9508, 0x00010000, 0x00010000,
1090 0xac0c, 0xffffffff, 0x54763210,
1091 0x214f8, 0x01ff01ff, 0x00000002,
1092 0x21498, 0x007ff800, 0x00200000,
1093 0x2015c, 0xffffffff, 0x00000f40,
1094 0x30934, 0xffffffff, 0x00000001
1095};
1096
1097static const u32 spectre_mgcg_cgcg_init[] =
1098{
1099 0xc420, 0xffffffff, 0xfffffffc,
1100 0x30800, 0xffffffff, 0xe0000000,
1101 0x3c2a0, 0xffffffff, 0x00000100,
1102 0x3c208, 0xffffffff, 0x00000100,
1103 0x3c2c0, 0xffffffff, 0x00000100,
1104 0x3c2c8, 0xffffffff, 0x00000100,
1105 0x3c2c4, 0xffffffff, 0x00000100,
1106 0x55e4, 0xffffffff, 0x00600100,
1107 0x3c280, 0xffffffff, 0x00000100,
1108 0x3c214, 0xffffffff, 0x06000100,
1109 0x3c220, 0xffffffff, 0x00000100,
1110 0x3c218, 0xffffffff, 0x06000100,
1111 0x3c204, 0xffffffff, 0x00000100,
1112 0x3c2e0, 0xffffffff, 0x00000100,
1113 0x3c224, 0xffffffff, 0x00000100,
1114 0x3c200, 0xffffffff, 0x00000100,
1115 0x3c230, 0xffffffff, 0x00000100,
1116 0x3c234, 0xffffffff, 0x00000100,
1117 0x3c250, 0xffffffff, 0x00000100,
1118 0x3c254, 0xffffffff, 0x00000100,
1119 0x3c258, 0xffffffff, 0x00000100,
1120 0x3c25c, 0xffffffff, 0x00000100,
1121 0x3c260, 0xffffffff, 0x00000100,
1122 0x3c27c, 0xffffffff, 0x00000100,
1123 0x3c278, 0xffffffff, 0x00000100,
1124 0x3c210, 0xffffffff, 0x06000100,
1125 0x3c290, 0xffffffff, 0x00000100,
1126 0x3c274, 0xffffffff, 0x00000100,
1127 0x3c2b4, 0xffffffff, 0x00000100,
1128 0x3c2b0, 0xffffffff, 0x00000100,
1129 0x3c270, 0xffffffff, 0x00000100,
1130 0x30800, 0xffffffff, 0xe0000000,
1131 0x3c020, 0xffffffff, 0x00010000,
1132 0x3c024, 0xffffffff, 0x00030002,
1133 0x3c028, 0xffffffff, 0x00040007,
1134 0x3c02c, 0xffffffff, 0x00060005,
1135 0x3c030, 0xffffffff, 0x00090008,
1136 0x3c034, 0xffffffff, 0x00010000,
1137 0x3c038, 0xffffffff, 0x00030002,
1138 0x3c03c, 0xffffffff, 0x00040007,
1139 0x3c040, 0xffffffff, 0x00060005,
1140 0x3c044, 0xffffffff, 0x00090008,
1141 0x3c048, 0xffffffff, 0x00010000,
1142 0x3c04c, 0xffffffff, 0x00030002,
1143 0x3c050, 0xffffffff, 0x00040007,
1144 0x3c054, 0xffffffff, 0x00060005,
1145 0x3c058, 0xffffffff, 0x00090008,
1146 0x3c05c, 0xffffffff, 0x00010000,
1147 0x3c060, 0xffffffff, 0x00030002,
1148 0x3c064, 0xffffffff, 0x00040007,
1149 0x3c068, 0xffffffff, 0x00060005,
1150 0x3c06c, 0xffffffff, 0x00090008,
1151 0x3c070, 0xffffffff, 0x00010000,
1152 0x3c074, 0xffffffff, 0x00030002,
1153 0x3c078, 0xffffffff, 0x00040007,
1154 0x3c07c, 0xffffffff, 0x00060005,
1155 0x3c080, 0xffffffff, 0x00090008,
1156 0x3c084, 0xffffffff, 0x00010000,
1157 0x3c088, 0xffffffff, 0x00030002,
1158 0x3c08c, 0xffffffff, 0x00040007,
1159 0x3c090, 0xffffffff, 0x00060005,
1160 0x3c094, 0xffffffff, 0x00090008,
1161 0x3c098, 0xffffffff, 0x00010000,
1162 0x3c09c, 0xffffffff, 0x00030002,
1163 0x3c0a0, 0xffffffff, 0x00040007,
1164 0x3c0a4, 0xffffffff, 0x00060005,
1165 0x3c0a8, 0xffffffff, 0x00090008,
1166 0x3c0ac, 0xffffffff, 0x00010000,
1167 0x3c0b0, 0xffffffff, 0x00030002,
1168 0x3c0b4, 0xffffffff, 0x00040007,
1169 0x3c0b8, 0xffffffff, 0x00060005,
1170 0x3c0bc, 0xffffffff, 0x00090008,
1171 0x3c000, 0xffffffff, 0x96e00200,
1172 0x8708, 0xffffffff, 0x00900100,
1173 0xc424, 0xffffffff, 0x0020003f,
1174 0x38, 0xffffffff, 0x0140001c,
1175 0x3c, 0x000f0000, 0x000f0000,
1176 0x220, 0xffffffff, 0xC060000C,
1177 0x224, 0xc0000fff, 0x00000100,
1178 0xf90, 0xffffffff, 0x00000100,
1179 0xf98, 0x00000101, 0x00000000,
1180 0x20a8, 0xffffffff, 0x00000104,
1181 0x55e4, 0xff000fff, 0x00000100,
1182 0x30cc, 0xc0000fff, 0x00000104,
1183 0xc1e4, 0x00000001, 0x00000001,
1184 0xd00c, 0xff000ff0, 0x00000100,
1185 0xd80c, 0xff000ff0, 0x00000100
1186};
1187
1188static const u32 kalindi_golden_spm_registers[] =
1189{
1190 0x30800, 0xe0ffffff, 0xe0000000
1191};
1192
1193static const u32 kalindi_golden_common_registers[] =
1194{
1195 0xc770, 0xffffffff, 0x00000800,
1196 0xc774, 0xffffffff, 0x00000800,
1197 0xc798, 0xffffffff, 0x00007fbf,
1198 0xc79c, 0xffffffff, 0x00007faf
1199};
1200
1201static const u32 kalindi_golden_registers[] =
1202{
1203 0x3c000, 0xffffdfff, 0x6e944040,
1204 0x55e4, 0xff607fff, 0xfc000100,
1205 0x3c220, 0xff000fff, 0x00000100,
1206 0x3c224, 0xff000fff, 0x00000100,
1207 0x3c200, 0xfffc0fff, 0x00000100,
1208 0x6ed8, 0x00010101, 0x00010000,
1209 0x9830, 0xffffffff, 0x00000000,
1210 0x9834, 0xf00fffff, 0x00000400,
1211 0x5bb0, 0x000000f0, 0x00000070,
1212 0x5bc0, 0xf0311fff, 0x80300000,
1213 0x98f8, 0x73773777, 0x12010001,
1214 0x98fc, 0xffffffff, 0x00000010,
1215 0x9b7c, 0x00ff0000, 0x00fc0000,
1216 0x8030, 0x00001f0f, 0x0000100a,
1217 0x2f48, 0x73773777, 0x12010001,
1218 0x2408, 0x000fffff, 0x000c007f,
1219 0x8a14, 0xf000003f, 0x00000007,
1220 0x8b24, 0x3fff3fff, 0x00ffcfff,
1221 0x30a04, 0x0000ff0f, 0x00000000,
1222 0x28a4c, 0x07ffffff, 0x06000000,
1223 0x4d8, 0x00000fff, 0x00000100,
1224 0x3e78, 0x00000001, 0x00000002,
1225 0xc768, 0x00000008, 0x00000008,
1226 0x8c00, 0x000000ff, 0x00000003,
1227 0x214f8, 0x01ff01ff, 0x00000002,
1228 0x21498, 0x007ff800, 0x00200000,
1229 0x2015c, 0xffffffff, 0x00000f40,
1230 0x88c4, 0x001f3ae3, 0x00000082,
1231 0x88d4, 0x0000001f, 0x00000010,
1232 0x30934, 0xffffffff, 0x00000000
1233};
1234
1235static const u32 kalindi_mgcg_cgcg_init[] =
1236{
1237 0xc420, 0xffffffff, 0xfffffffc,
1238 0x30800, 0xffffffff, 0xe0000000,
1239 0x3c2a0, 0xffffffff, 0x00000100,
1240 0x3c208, 0xffffffff, 0x00000100,
1241 0x3c2c0, 0xffffffff, 0x00000100,
1242 0x3c2c8, 0xffffffff, 0x00000100,
1243 0x3c2c4, 0xffffffff, 0x00000100,
1244 0x55e4, 0xffffffff, 0x00600100,
1245 0x3c280, 0xffffffff, 0x00000100,
1246 0x3c214, 0xffffffff, 0x06000100,
1247 0x3c220, 0xffffffff, 0x00000100,
1248 0x3c218, 0xffffffff, 0x06000100,
1249 0x3c204, 0xffffffff, 0x00000100,
1250 0x3c2e0, 0xffffffff, 0x00000100,
1251 0x3c224, 0xffffffff, 0x00000100,
1252 0x3c200, 0xffffffff, 0x00000100,
1253 0x3c230, 0xffffffff, 0x00000100,
1254 0x3c234, 0xffffffff, 0x00000100,
1255 0x3c250, 0xffffffff, 0x00000100,
1256 0x3c254, 0xffffffff, 0x00000100,
1257 0x3c258, 0xffffffff, 0x00000100,
1258 0x3c25c, 0xffffffff, 0x00000100,
1259 0x3c260, 0xffffffff, 0x00000100,
1260 0x3c27c, 0xffffffff, 0x00000100,
1261 0x3c278, 0xffffffff, 0x00000100,
1262 0x3c210, 0xffffffff, 0x06000100,
1263 0x3c290, 0xffffffff, 0x00000100,
1264 0x3c274, 0xffffffff, 0x00000100,
1265 0x3c2b4, 0xffffffff, 0x00000100,
1266 0x3c2b0, 0xffffffff, 0x00000100,
1267 0x3c270, 0xffffffff, 0x00000100,
1268 0x30800, 0xffffffff, 0xe0000000,
1269 0x3c020, 0xffffffff, 0x00010000,
1270 0x3c024, 0xffffffff, 0x00030002,
1271 0x3c028, 0xffffffff, 0x00040007,
1272 0x3c02c, 0xffffffff, 0x00060005,
1273 0x3c030, 0xffffffff, 0x00090008,
1274 0x3c034, 0xffffffff, 0x00010000,
1275 0x3c038, 0xffffffff, 0x00030002,
1276 0x3c03c, 0xffffffff, 0x00040007,
1277 0x3c040, 0xffffffff, 0x00060005,
1278 0x3c044, 0xffffffff, 0x00090008,
1279 0x3c000, 0xffffffff, 0x96e00200,
1280 0x8708, 0xffffffff, 0x00900100,
1281 0xc424, 0xffffffff, 0x0020003f,
1282 0x38, 0xffffffff, 0x0140001c,
1283 0x3c, 0x000f0000, 0x000f0000,
1284 0x220, 0xffffffff, 0xC060000C,
1285 0x224, 0xc0000fff, 0x00000100,
1286 0x20a8, 0xffffffff, 0x00000104,
1287 0x55e4, 0xff000fff, 0x00000100,
1288 0x30cc, 0xc0000fff, 0x00000104,
1289 0xc1e4, 0x00000001, 0x00000001,
1290 0xd00c, 0xff000ff0, 0x00000100,
1291 0xd80c, 0xff000ff0, 0x00000100
1292};
1293
1294static void cik_init_golden_registers(struct radeon_device *rdev)
1295{
1296 switch (rdev->family) {
1297 case CHIP_BONAIRE:
1298 radeon_program_register_sequence(rdev,
1299 bonaire_mgcg_cgcg_init,
1300 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1301 radeon_program_register_sequence(rdev,
1302 bonaire_golden_registers,
1303 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1304 radeon_program_register_sequence(rdev,
1305 bonaire_golden_common_registers,
1306 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1307 radeon_program_register_sequence(rdev,
1308 bonaire_golden_spm_registers,
1309 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1310 break;
1311 case CHIP_KABINI:
1312 radeon_program_register_sequence(rdev,
1313 kalindi_mgcg_cgcg_init,
1314 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1315 radeon_program_register_sequence(rdev,
1316 kalindi_golden_registers,
1317 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1318 radeon_program_register_sequence(rdev,
1319 kalindi_golden_common_registers,
1320 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1321 radeon_program_register_sequence(rdev,
1322 kalindi_golden_spm_registers,
1323 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1324 break;
1325 case CHIP_KAVERI:
1326 radeon_program_register_sequence(rdev,
1327 spectre_mgcg_cgcg_init,
1328 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1329 radeon_program_register_sequence(rdev,
1330 spectre_golden_registers,
1331 (const u32)ARRAY_SIZE(spectre_golden_registers));
1332 radeon_program_register_sequence(rdev,
1333 spectre_golden_common_registers,
1334 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1335 radeon_program_register_sequence(rdev,
1336 spectre_golden_spm_registers,
1337 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1338 break;
1339 default:
1340 break;
1341 }
1342}
1343
2c67912c
AD		 1344/**
1345 * cik_get_xclk - get the xclk
1346 *
1347 * @rdev: radeon_device pointer
1348 *
1349 * Returns the reference clock used by the gfx engine
1350 * (CIK).
1351 */
1352u32 cik_get_xclk(struct radeon_device *rdev)
1353{
1354 u32 reference_clock = rdev->clock.spll.reference_freq;
1355
1356 if (rdev->flags & RADEON_IS_IGP) {
1357 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1358 return reference_clock / 2;
1359 } else {
1360 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1361 return reference_clock / 4;
1362 }
1363 return reference_clock;
1364}
1365
75efdee1
AD		 1366/**
1367 * cik_mm_rdoorbell - read a doorbell dword
1368 *
1369 * @rdev: radeon_device pointer
1370 * @offset: byte offset into the aperture
1371 *
1372 * Returns the value in the doorbell aperture at the
1373 * requested offset (CIK).
1374 */
1375u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1376{
1377 if (offset < rdev->doorbell.size) {
1378 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1379 } else {
1380 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1381 return 0;
1382 }
1383}
1384
1385/**
1386 * cik_mm_wdoorbell - write a doorbell dword
1387 *
1388 * @rdev: radeon_device pointer
1389 * @offset: byte offset into the aperture
1390 * @v: value to write
1391 *
1392 * Writes @v to the doorbell aperture at the
1393 * requested offset (CIK).
1394 */
1395void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1396{
1397 if (offset < rdev->doorbell.size) {
1398 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1399 } else {
1400 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1401 }
1402}
1403
bc8273fe
AD		 1404#define BONAIRE_IO_MC_REGS_SIZE 36
1405
1406static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1407{
1408 {0x00000070, 0x04400000},
1409 {0x00000071, 0x80c01803},
1410 {0x00000072, 0x00004004},
1411 {0x00000073, 0x00000100},
1412 {0x00000074, 0x00ff0000},
1413 {0x00000075, 0x34000000},
1414 {0x00000076, 0x08000014},
1415 {0x00000077, 0x00cc08ec},
1416 {0x00000078, 0x00000400},
1417 {0x00000079, 0x00000000},
1418 {0x0000007a, 0x04090000},
1419 {0x0000007c, 0x00000000},
1420 {0x0000007e, 0x4408a8e8},
1421 {0x0000007f, 0x00000304},
1422 {0x00000080, 0x00000000},
1423 {0x00000082, 0x00000001},
1424 {0x00000083, 0x00000002},
1425 {0x00000084, 0xf3e4f400},
1426 {0x00000085, 0x052024e3},
1427 {0x00000087, 0x00000000},
1428 {0x00000088, 0x01000000},
1429 {0x0000008a, 0x1c0a0000},
1430 {0x0000008b, 0xff010000},
1431 {0x0000008d, 0xffffefff},
1432 {0x0000008e, 0xfff3efff},
1433 {0x0000008f, 0xfff3efbf},
1434 {0x00000092, 0xf7ffffff},
1435 {0x00000093, 0xffffff7f},
1436 {0x00000095, 0x00101101},
1437 {0x00000096, 0x00000fff},
1438 {0x00000097, 0x00116fff},
1439 {0x00000098, 0x60010000},
1440 {0x00000099, 0x10010000},
1441 {0x0000009a, 0x00006000},
1442 {0x0000009b, 0x00001000},
1443 {0x0000009f, 0x00b48000}
1444};
1445
b556b12e
AD		 1446/**
1447 * cik_srbm_select - select specific register instances
1448 *
1449 * @rdev: radeon_device pointer
1450 * @me: selected ME (micro engine)
1451 * @pipe: pipe
1452 * @queue: queue
1453 * @vmid: VMID
1454 *
1455 * Switches the currently active registers instances. Some
1456 * registers are instanced per VMID, others are instanced per
1457 * me/pipe/queue combination.
1458 */
1459static void cik_srbm_select(struct radeon_device *rdev,
1460 u32 me, u32 pipe, u32 queue, u32 vmid)
1461{
1462 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1463 MEID(me & 0x3) |
1464 VMID(vmid & 0xf) |
1465 QUEUEID(queue & 0x7));
1466 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1467}
1468
bc8273fe
AD		 1469/* ucode loading */
1470/**
1471 * ci_mc_load_microcode - load MC ucode into the hw
1472 *
1473 * @rdev: radeon_device pointer
1474 *
1475 * Load the GDDR MC ucode into the hw (CIK).
1476 * Returns 0 on success, error on failure.
1477 */
1478static int ci_mc_load_microcode(struct radeon_device *rdev)
1479{
1480 const __be32 *fw_data;
1481 u32 running, blackout = 0;
1482 u32 *io_mc_regs;
1483 int i, ucode_size, regs_size;
1484
1485 if (!rdev->mc_fw)
1486 return -EINVAL;
1487
1488 switch (rdev->family) {
1489 case CHIP_BONAIRE:
1490 default:
1491 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1492 ucode_size = CIK_MC_UCODE_SIZE;
1493 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1494 break;
1495 }
1496
1497 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1498
1499 if (running == 0) {
1500 if (running) {
1501 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1502 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1503 }
1504
1505 /* reset the engine and set to writable */
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1507 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508
1509 /* load mc io regs */
1510 for (i = 0; i < regs_size; i++) {
1511 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1512 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513 }
1514 /* load the MC ucode */
1515 fw_data = (const __be32 *)rdev->mc_fw->data;
1516 for (i = 0; i < ucode_size; i++)
1517 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518
1519 /* put the engine back into the active state */
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1522 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523
1524 /* wait for training to complete */
1525 for (i = 0; i < rdev->usec_timeout; i++) {
1526 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1527 break;
1528 udelay(1);
1529 }
1530 for (i = 0; i < rdev->usec_timeout; i++) {
1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1532 break;
1533 udelay(1);
1534 }
1535
1536 if (running)
1537 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1538 }
1539
1540 return 0;
1541}
1542
02c81327
AD		 1543/**
1544 * cik_init_microcode - load ucode images from disk
1545 *
1546 * @rdev: radeon_device pointer
1547 *
1548 * Use the firmware interface to load the ucode images into
1549 * the driver (not loaded into hw).
1550 * Returns 0 on success, error on failure.
1551 */
1552static int cik_init_microcode(struct radeon_device *rdev)
1553{
02c81327
AD		 1554 const char *chip_name;
1555 size_t pfp_req_size, me_req_size, ce_req_size,
21a93e13 1556 mec_req_size, rlc_req_size, mc_req_size,
cc8dbbb4 1557 sdma_req_size, smc_req_size;
02c81327
AD		 1558 char fw_name[30];
1559 int err;
1560
1561 DRM_DEBUG("\n");
1562
02c81327
AD		 1563 switch (rdev->family) {
1564 case CHIP_BONAIRE:
1565 chip_name = "BONAIRE";
1566 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1567 me_req_size = CIK_ME_UCODE_SIZE * 4;
1568 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1569 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1570 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1571 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1572 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1573 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327
AD		 1574 break;
1575 case CHIP_KAVERI:
1576 chip_name = "KAVERI";
1577 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1578 me_req_size = CIK_ME_UCODE_SIZE * 4;
1579 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1580 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1581 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1582 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1583 break;
1584 case CHIP_KABINI:
1585 chip_name = "KABINI";
1586 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1587 me_req_size = CIK_ME_UCODE_SIZE * 4;
1588 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1589 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1590 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1591 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1592 break;
1593 default: BUG();
1594 }
1595
1596 DRM_INFO("Loading %s Microcode\n", chip_name);
1597
1598 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1599 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1600 if (err)
1601 goto out;
1602 if (rdev->pfp_fw->size != pfp_req_size) {
1603 printk(KERN_ERR
1604 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1605 rdev->pfp_fw->size, fw_name);
1606 err = -EINVAL;
1607 goto out;
1608 }
1609
1610 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1611 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1612 if (err)
1613 goto out;
1614 if (rdev->me_fw->size != me_req_size) {
1615 printk(KERN_ERR
1616 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1617 rdev->me_fw->size, fw_name);
1618 err = -EINVAL;
1619 }
1620
1621 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1622 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1623 if (err)
1624 goto out;
1625 if (rdev->ce_fw->size != ce_req_size) {
1626 printk(KERN_ERR
1627 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1628 rdev->ce_fw->size, fw_name);
1629 err = -EINVAL;
1630 }
1631
1632 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1633 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1634 if (err)
1635 goto out;
1636 if (rdev->mec_fw->size != mec_req_size) {
1637 printk(KERN_ERR
1638 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1639 rdev->mec_fw->size, fw_name);
1640 err = -EINVAL;
1641 }
1642
1643 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1644 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1645 if (err)
1646 goto out;
1647 if (rdev->rlc_fw->size != rlc_req_size) {
1648 printk(KERN_ERR
1649 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1650 rdev->rlc_fw->size, fw_name);
1651 err = -EINVAL;
1652 }
1653
21a93e13 1654 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1655 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1656 if (err)
1657 goto out;
1658 if (rdev->sdma_fw->size != sdma_req_size) {
1659 printk(KERN_ERR
1660 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1661 rdev->sdma_fw->size, fw_name);
1662 err = -EINVAL;
1663 }
1664
cc8dbbb4 1665 /* No SMC, MC ucode on APUs */
02c81327
AD		 1666 if (!(rdev->flags & RADEON_IS_IGP)) {
1667 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1668 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1669 if (err)
1670 goto out;
1671 if (rdev->mc_fw->size != mc_req_size) {
1672 printk(KERN_ERR
1673 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1674 rdev->mc_fw->size, fw_name);
1675 err = -EINVAL;
1676 }
cc8dbbb4
AD		 1677
1678 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1679 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1680 if (err) {
1681 printk(KERN_ERR
1682 "smc: error loading firmware \"%s\"\n",
1683 fw_name);
1684 release_firmware(rdev->smc_fw);
1685 rdev->smc_fw = NULL;
1686 } else if (rdev->smc_fw->size != smc_req_size) {
1687 printk(KERN_ERR
1688 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1689 rdev->smc_fw->size, fw_name);
1690 err = -EINVAL;
1691 }
02c81327
AD		 1692 }
1693
1694out:
02c81327
AD		 1695 if (err) {
1696 if (err != -EINVAL)
1697 printk(KERN_ERR
1698 "cik_cp: Failed to load firmware \"%s\"\n",
1699 fw_name);
1700 release_firmware(rdev->pfp_fw);
1701 rdev->pfp_fw = NULL;
1702 release_firmware(rdev->me_fw);
1703 rdev->me_fw = NULL;
1704 release_firmware(rdev->ce_fw);
1705 rdev->ce_fw = NULL;
1706 release_firmware(rdev->rlc_fw);
1707 rdev->rlc_fw = NULL;
1708 release_firmware(rdev->mc_fw);
1709 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1710 release_firmware(rdev->smc_fw);
1711 rdev->smc_fw = NULL;
02c81327
AD		 1712 }
1713 return err;
1714}
1715
8cc1a532
AD		 1716/*
1717 * Core functions
1718 */
1719/**
1720 * cik_tiling_mode_table_init - init the hw tiling table
1721 *
1722 * @rdev: radeon_device pointer
1723 *
1724 * Starting with SI, the tiling setup is done globally in a
1725 * set of 32 tiling modes. Rather than selecting each set of
1726 * parameters per surface as on older asics, we just select
1727 * which index in the tiling table we want to use, and the
1728 * surface uses those parameters (CIK).
1729 */
1730static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1731{
1732 const u32 num_tile_mode_states = 32;
1733 const u32 num_secondary_tile_mode_states = 16;
1734 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1735 u32 num_pipe_configs;
1736 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1737 rdev->config.cik.max_shader_engines;
1738
1739 switch (rdev->config.cik.mem_row_size_in_kb) {
1740 case 1:
1741 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1742 break;
1743 case 2:
1744 default:
1745 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1746 break;
1747 case 4:
1748 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1749 break;
1750 }
1751
1752 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1753 if (num_pipe_configs > 8)
1754 num_pipe_configs = 8; /* ??? */
1755
1756 if (num_pipe_configs == 8) {
1757 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1758 switch (reg_offset) {
1759 case 0:
1760 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1761 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1762 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1763 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1764 break;
1765 case 1:
1766 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1767 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1768 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1769 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1770 break;
1771 case 2:
1772 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1773 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1774 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1775 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1776 break;
1777 case 3:
1778 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1779 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1780 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1781 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1782 break;
1783 case 4:
1784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1786 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1787 TILE_SPLIT(split_equal_to_row_size));
1788 break;
1789 case 5:
1790 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1792 break;
1793 case 6:
1794 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1796 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1798 break;
1799 case 7:
1800 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1801 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1802 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1803 TILE_SPLIT(split_equal_to_row_size));
1804 break;
1805 case 8:
1806 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1807 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1808 break;
1809 case 9:
1810 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1811 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1812 break;
1813 case 10:
1814 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1815 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1816 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1817 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1818 break;
1819 case 11:
1820 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1821 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1822 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1823 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1824 break;
1825 case 12:
1826 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1827 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1828 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1829 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1830 break;
1831 case 13:
1832 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1833 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1834 break;
1835 case 14:
1836 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1837 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1838 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1839 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1840 break;
1841 case 16:
1842 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1844 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1846 break;
1847 case 17:
1848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1850 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1852 break;
1853 case 27:
1854 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1855 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1856 break;
1857 case 28:
1858 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1859 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1860 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1861 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1862 break;
1863 case 29:
1864 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1865 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1866 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1867 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1868 break;
1869 case 30:
1870 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1871 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1872 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1873 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1874 break;
1875 default:
1876 gb_tile_moden = 0;
1877 break;
1878 }
39aee490 1879 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 1880 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1881 }
1882 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1883 switch (reg_offset) {
1884 case 0:
1885 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1886 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1887 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1888 NUM_BANKS(ADDR_SURF_16_BANK));
1889 break;
1890 case 1:
1891 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1892 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1893 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1894 NUM_BANKS(ADDR_SURF_16_BANK));
1895 break;
1896 case 2:
1897 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1898 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1899 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1900 NUM_BANKS(ADDR_SURF_16_BANK));
1901 break;
1902 case 3:
1903 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1904 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1905 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1906 NUM_BANKS(ADDR_SURF_16_BANK));
1907 break;
1908 case 4:
1909 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1910 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1911 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1912 NUM_BANKS(ADDR_SURF_8_BANK));
1913 break;
1914 case 5:
1915 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1916 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1917 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1918 NUM_BANKS(ADDR_SURF_4_BANK));
1919 break;
1920 case 6:
1921 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1922 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1923 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1924 NUM_BANKS(ADDR_SURF_2_BANK));
1925 break;
1926 case 8:
1927 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1928 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1929 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1930 NUM_BANKS(ADDR_SURF_16_BANK));
1931 break;
1932 case 9:
1933 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1934 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1935 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1936 NUM_BANKS(ADDR_SURF_16_BANK));
1937 break;
1938 case 10:
1939 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1940 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1941 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1942 NUM_BANKS(ADDR_SURF_16_BANK));
1943 break;
1944 case 11:
1945 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1946 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1947 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1948 NUM_BANKS(ADDR_SURF_16_BANK));
1949 break;
1950 case 12:
1951 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1952 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1953 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1954 NUM_BANKS(ADDR_SURF_8_BANK));
1955 break;
1956 case 13:
1957 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1958 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1959 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1960 NUM_BANKS(ADDR_SURF_4_BANK));
1961 break;
1962 case 14:
1963 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1966 NUM_BANKS(ADDR_SURF_2_BANK));
1967 break;
1968 default:
1969 gb_tile_moden = 0;
1970 break;
1971 }
1972 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1973 }
1974 } else if (num_pipe_configs == 4) {
1975 if (num_rbs == 4) {
1976 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1977 switch (reg_offset) {
1978 case 0:
1979 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1981 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1982 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1983 break;
1984 case 1:
1985 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1986 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1987 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1988 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1989 break;
1990 case 2:
1991 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1992 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1993 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1994 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1995 break;
1996 case 3:
1997 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1998 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1999 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2000 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2001 break;
2002 case 4:
2003 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2004 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2005 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2006 TILE_SPLIT(split_equal_to_row_size));
2007 break;
2008 case 5:
2009 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2010 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2011 break;
2012 case 6:
2013 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2015 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2016 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2017 break;
2018 case 7:
2019 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2021 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2022 TILE_SPLIT(split_equal_to_row_size));
2023 break;
2024 case 8:
2025 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2026 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2027 break;
2028 case 9:
2029 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2030 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2031 break;
2032 case 10:
2033 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2034 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2035 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2036 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2037 break;
2038 case 11:
2039 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2040 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2041 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2042 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2043 break;
2044 case 12:
2045 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2046 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2047 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2048 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2049 break;
2050 case 13:
2051 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2052 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2053 break;
2054 case 14:
2055 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2056 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2057 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2058 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2059 break;
2060 case 16:
2061 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2062 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2063 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2064 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2065 break;
2066 case 17:
2067 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2068 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2069 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071 break;
2072 case 27:
2073 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2074 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2075 break;
2076 case 28:
2077 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2078 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2079 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2080 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2081 break;
2082 case 29:
2083 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2084 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2085 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2087 break;
2088 case 30:
2089 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2090 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2091 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2092 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2093 break;
2094 default:
2095 gb_tile_moden = 0;
2096 break;
2097 }
39aee490 2098 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2099 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2100 }
2101 } else if (num_rbs < 4) {
2102 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2103 switch (reg_offset) {
2104 case 0:
2105 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2106 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2107 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2108 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2109 break;
2110 case 1:
2111 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2112 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2113 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2114 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2115 break;
2116 case 2:
2117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2119 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2121 break;
2122 case 3:
2123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2125 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2127 break;
2128 case 4:
2129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2131 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2132 TILE_SPLIT(split_equal_to_row_size));
2133 break;
2134 case 5:
2135 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2137 break;
2138 case 6:
2139 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2140 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2141 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2142 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2143 break;
2144 case 7:
2145 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2146 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2147 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2148 TILE_SPLIT(split_equal_to_row_size));
2149 break;
2150 case 8:
2151 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2152 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2153 break;
2154 case 9:
2155 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2156 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2157 break;
2158 case 10:
2159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2161 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2163 break;
2164 case 11:
2165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2167 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2169 break;
2170 case 12:
2171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2173 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2175 break;
2176 case 13:
2177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2178 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2179 break;
2180 case 14:
2181 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2182 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2183 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2184 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2185 break;
2186 case 16:
2187 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2188 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2189 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2190 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2191 break;
2192 case 17:
2193 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2194 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2195 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2196 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2197 break;
2198 case 27:
2199 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2200 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2201 break;
2202 case 28:
2203 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2204 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2205 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2206 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2207 break;
2208 case 29:
2209 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2210 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2211 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2212 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2213 break;
2214 case 30:
2215 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2216 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2217 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2218 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2219 break;
2220 default:
2221 gb_tile_moden = 0;
2222 break;
2223 }
39aee490 2224 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2225 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2226 }
2227 }
2228 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2229 switch (reg_offset) {
2230 case 0:
2231 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2232 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2233 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2234 NUM_BANKS(ADDR_SURF_16_BANK));
2235 break;
2236 case 1:
2237 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2239 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2240 NUM_BANKS(ADDR_SURF_16_BANK));
2241 break;
2242 case 2:
2243 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2246 NUM_BANKS(ADDR_SURF_16_BANK));
2247 break;
2248 case 3:
2249 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2251 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2252 NUM_BANKS(ADDR_SURF_16_BANK));
2253 break;
2254 case 4:
2255 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2257 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2258 NUM_BANKS(ADDR_SURF_16_BANK));
2259 break;
2260 case 5:
2261 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2264 NUM_BANKS(ADDR_SURF_8_BANK));
2265 break;
2266 case 6:
2267 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2268 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2269 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2270 NUM_BANKS(ADDR_SURF_4_BANK));
2271 break;
2272 case 8:
2273 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2276 NUM_BANKS(ADDR_SURF_16_BANK));
2277 break;
2278 case 9:
2279 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2282 NUM_BANKS(ADDR_SURF_16_BANK));
2283 break;
2284 case 10:
2285 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2288 NUM_BANKS(ADDR_SURF_16_BANK));
2289 break;
2290 case 11:
2291 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2292 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2293 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2294 NUM_BANKS(ADDR_SURF_16_BANK));
2295 break;
2296 case 12:
2297 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2298 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2299 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2300 NUM_BANKS(ADDR_SURF_16_BANK));
2301 break;
2302 case 13:
2303 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2306 NUM_BANKS(ADDR_SURF_8_BANK));
2307 break;
2308 case 14:
2309 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2310 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2311 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2312 NUM_BANKS(ADDR_SURF_4_BANK));
2313 break;
2314 default:
2315 gb_tile_moden = 0;
2316 break;
2317 }
2318 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2319 }
2320 } else if (num_pipe_configs == 2) {
2321 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2322 switch (reg_offset) {
2323 case 0:
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P2) |
2327 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2328 break;
2329 case 1:
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2332 PIPE_CONFIG(ADDR_SURF_P2) |
2333 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2334 break;
2335 case 2:
2336 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2337 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2338 PIPE_CONFIG(ADDR_SURF_P2) |
2339 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2340 break;
2341 case 3:
2342 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2343 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2344 PIPE_CONFIG(ADDR_SURF_P2) |
2345 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2346 break;
2347 case 4:
2348 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2349 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2350 PIPE_CONFIG(ADDR_SURF_P2) |
2351 TILE_SPLIT(split_equal_to_row_size));
2352 break;
2353 case 5:
2354 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2356 break;
2357 case 6:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P2) |
2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2362 break;
2363 case 7:
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P2) |
2367 TILE_SPLIT(split_equal_to_row_size));
2368 break;
2369 case 8:
2370 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2371 break;
2372 case 9:
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2375 break;
2376 case 10:
2377 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2379 PIPE_CONFIG(ADDR_SURF_P2) |
2380 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2381 break;
2382 case 11:
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2384 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P2) |
2386 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2387 break;
2388 case 12:
2389 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P2) |
2392 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2393 break;
2394 case 13:
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2397 break;
2398 case 14:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2401 PIPE_CONFIG(ADDR_SURF_P2) |
2402 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2403 break;
2404 case 16:
2405 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P2) |
2408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409 break;
2410 case 17:
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P2) |
2414 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2415 break;
2416 case 27:
2417 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2419 break;
2420 case 28:
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2422 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P2) |
2424 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425 break;
2426 case 29:
2427 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2428 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2429 PIPE_CONFIG(ADDR_SURF_P2) |
2430 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2431 break;
2432 case 30:
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P2) |
2436 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2437 break;
2438 default:
2439 gb_tile_moden = 0;
2440 break;
2441 }
39aee490 2442 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2443 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2444 }
2445 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2446 switch (reg_offset) {
2447 case 0:
2448 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2451 NUM_BANKS(ADDR_SURF_16_BANK));
2452 break;
2453 case 1:
2454 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2457 NUM_BANKS(ADDR_SURF_16_BANK));
2458 break;
2459 case 2:
2460 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2463 NUM_BANKS(ADDR_SURF_16_BANK));
2464 break;
2465 case 3:
2466 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2469 NUM_BANKS(ADDR_SURF_16_BANK));
2470 break;
2471 case 4:
2472 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2475 NUM_BANKS(ADDR_SURF_16_BANK));
2476 break;
2477 case 5:
2478 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2480 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2481 NUM_BANKS(ADDR_SURF_16_BANK));
2482 break;
2483 case 6:
2484 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2487 NUM_BANKS(ADDR_SURF_8_BANK));
2488 break;
2489 case 8:
2490 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2493 NUM_BANKS(ADDR_SURF_16_BANK));
2494 break;
2495 case 9:
2496 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2499 NUM_BANKS(ADDR_SURF_16_BANK));
2500 break;
2501 case 10:
2502 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2503 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2504 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2505 NUM_BANKS(ADDR_SURF_16_BANK));
2506 break;
2507 case 11:
2508 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2511 NUM_BANKS(ADDR_SURF_16_BANK));
2512 break;
2513 case 12:
2514 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2517 NUM_BANKS(ADDR_SURF_16_BANK));
2518 break;
2519 case 13:
2520 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2523 NUM_BANKS(ADDR_SURF_16_BANK));
2524 break;
2525 case 14:
2526 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2528 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2529 NUM_BANKS(ADDR_SURF_8_BANK));
2530 break;
2531 default:
2532 gb_tile_moden = 0;
2533 break;
2534 }
2535 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2536 }
2537 } else
2538 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2539}
2540
2541/**
2542 * cik_select_se_sh - select which SE, SH to address
2543 *
2544 * @rdev: radeon_device pointer
2545 * @se_num: shader engine to address
2546 * @sh_num: sh block to address
2547 *
2548 * Select which SE, SH combinations to address. Certain
2549 * registers are instanced per SE or SH. 0xffffffff means
2550 * broadcast to all SEs or SHs (CIK).
2551 */
2552static void cik_select_se_sh(struct radeon_device *rdev,
2553 u32 se_num, u32 sh_num)
2554{
2555 u32 data = INSTANCE_BROADCAST_WRITES;
2556
2557 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 2558 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 2559 else if (se_num == 0xffffffff)
2560 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2561 else if (sh_num == 0xffffffff)
2562 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2563 else
2564 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2565 WREG32(GRBM_GFX_INDEX, data);
2566}
2567
2568/**
2569 * cik_create_bitmask - create a bitmask
2570 *
2571 * @bit_width: length of the mask
2572 *
2573 * create a variable length bit mask (CIK).
2574 * Returns the bitmask.
2575 */
2576static u32 cik_create_bitmask(u32 bit_width)
2577{
2578 u32 i, mask = 0;
2579
2580 for (i = 0; i < bit_width; i++) {
2581 mask <<= 1;
2582 mask |= 1;
2583 }
2584 return mask;
2585}
2586
2587/**
2588 * cik_select_se_sh - select which SE, SH to address
2589 *
2590 * @rdev: radeon_device pointer
2591 * @max_rb_num: max RBs (render backends) for the asic
2592 * @se_num: number of SEs (shader engines) for the asic
2593 * @sh_per_se: number of SH blocks per SE for the asic
2594 *
2595 * Calculates the bitmask of disabled RBs (CIK).
2596 * Returns the disabled RB bitmask.
2597 */
2598static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2599 u32 max_rb_num, u32 se_num,
2600 u32 sh_per_se)
2601{
2602 u32 data, mask;
2603
2604 data = RREG32(CC_RB_BACKEND_DISABLE);
2605 if (data & 1)
2606 data &= BACKEND_DISABLE_MASK;
2607 else
2608 data = 0;
2609 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2610
2611 data >>= BACKEND_DISABLE_SHIFT;
2612
2613 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2614
2615 return data & mask;
2616}
2617
2618/**
2619 * cik_setup_rb - setup the RBs on the asic
2620 *
2621 * @rdev: radeon_device pointer
2622 * @se_num: number of SEs (shader engines) for the asic
2623 * @sh_per_se: number of SH blocks per SE for the asic
2624 * @max_rb_num: max RBs (render backends) for the asic
2625 *
2626 * Configures per-SE/SH RB registers (CIK).
2627 */
2628static void cik_setup_rb(struct radeon_device *rdev,
2629 u32 se_num, u32 sh_per_se,
2630 u32 max_rb_num)
2631{
2632 int i, j;
2633 u32 data, mask;
2634 u32 disabled_rbs = 0;
2635 u32 enabled_rbs = 0;
2636
2637 for (i = 0; i < se_num; i++) {
2638 for (j = 0; j < sh_per_se; j++) {
2639 cik_select_se_sh(rdev, i, j);
2640 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2641 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2642 }
2643 }
2644 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2645
2646 mask = 1;
2647 for (i = 0; i < max_rb_num; i++) {
2648 if (!(disabled_rbs & mask))
2649 enabled_rbs |= mask;
2650 mask <<= 1;
2651 }
2652
2653 for (i = 0; i < se_num; i++) {
2654 cik_select_se_sh(rdev, i, 0xffffffff);
2655 data = 0;
2656 for (j = 0; j < sh_per_se; j++) {
2657 switch (enabled_rbs & 3) {
2658 case 1:
2659 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2660 break;
2661 case 2:
2662 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2663 break;
2664 case 3:
2665 default:
2666 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2667 break;
2668 }
2669 enabled_rbs >>= 2;
2670 }
2671 WREG32(PA_SC_RASTER_CONFIG, data);
2672 }
2673 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2674}
2675
2676/**
2677 * cik_gpu_init - setup the 3D engine
2678 *
2679 * @rdev: radeon_device pointer
2680 *
2681 * Configures the 3D engine and tiling configuration
2682 * registers so that the 3D engine is usable.
2683 */
2684static void cik_gpu_init(struct radeon_device *rdev)
2685{
2686 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2687 u32 mc_shared_chmap, mc_arb_ramcfg;
2688 u32 hdp_host_path_cntl;
2689 u32 tmp;
2690 int i, j;
2691
2692 switch (rdev->family) {
2693 case CHIP_BONAIRE:
2694 rdev->config.cik.max_shader_engines = 2;
2695 rdev->config.cik.max_tile_pipes = 4;
2696 rdev->config.cik.max_cu_per_sh = 7;
2697 rdev->config.cik.max_sh_per_se = 1;
2698 rdev->config.cik.max_backends_per_se = 2;
2699 rdev->config.cik.max_texture_channel_caches = 4;
2700 rdev->config.cik.max_gprs = 256;
2701 rdev->config.cik.max_gs_threads = 32;
2702 rdev->config.cik.max_hw_contexts = 8;
2703
2704 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2705 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2706 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2707 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2708 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2709 break;
2710 case CHIP_KAVERI:
b2e4c70a
AD		 2711 rdev->config.cik.max_shader_engines = 1;
2712 rdev->config.cik.max_tile_pipes = 4;
2713 if ((rdev->pdev->device == 0x1304) ||
2714 (rdev->pdev->device == 0x1305) ||
2715 (rdev->pdev->device == 0x130C) ||
2716 (rdev->pdev->device == 0x130F) ||
2717 (rdev->pdev->device == 0x1310) ||
2718 (rdev->pdev->device == 0x1311) ||
2719 (rdev->pdev->device == 0x131C)) {
2720 rdev->config.cik.max_cu_per_sh = 8;
2721 rdev->config.cik.max_backends_per_se = 2;
2722 } else if ((rdev->pdev->device == 0x1309) ||
2723 (rdev->pdev->device == 0x130A) ||
2724 (rdev->pdev->device == 0x130D) ||
2725 (rdev->pdev->device == 0x1313)) {
2726 rdev->config.cik.max_cu_per_sh = 6;
2727 rdev->config.cik.max_backends_per_se = 2;
2728 } else if ((rdev->pdev->device == 0x1306) ||
2729 (rdev->pdev->device == 0x1307) ||
2730 (rdev->pdev->device == 0x130B) ||
2731 (rdev->pdev->device == 0x130E) ||
2732 (rdev->pdev->device == 0x1315) ||
2733 (rdev->pdev->device == 0x131B)) {
2734 rdev->config.cik.max_cu_per_sh = 4;
2735 rdev->config.cik.max_backends_per_se = 1;
2736 } else {
2737 rdev->config.cik.max_cu_per_sh = 3;
2738 rdev->config.cik.max_backends_per_se = 1;
2739 }
2740 rdev->config.cik.max_sh_per_se = 1;
2741 rdev->config.cik.max_texture_channel_caches = 4;
2742 rdev->config.cik.max_gprs = 256;
2743 rdev->config.cik.max_gs_threads = 16;
2744 rdev->config.cik.max_hw_contexts = 8;
2745
2746 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2747 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2748 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2749 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2750 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 2751 break;
2752 case CHIP_KABINI:
2753 default:
2754 rdev->config.cik.max_shader_engines = 1;
2755 rdev->config.cik.max_tile_pipes = 2;
2756 rdev->config.cik.max_cu_per_sh = 2;
2757 rdev->config.cik.max_sh_per_se = 1;
2758 rdev->config.cik.max_backends_per_se = 1;
2759 rdev->config.cik.max_texture_channel_caches = 2;
2760 rdev->config.cik.max_gprs = 256;
2761 rdev->config.cik.max_gs_threads = 16;
2762 rdev->config.cik.max_hw_contexts = 8;
2763
2764 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2765 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2766 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2767 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2768 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2769 break;
2770 }
2771
2772 /* Initialize HDP */
2773 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2774 WREG32((0x2c14 + j), 0x00000000);
2775 WREG32((0x2c18 + j), 0x00000000);
2776 WREG32((0x2c1c + j), 0x00000000);
2777 WREG32((0x2c20 + j), 0x00000000);
2778 WREG32((0x2c24 + j), 0x00000000);
2779 }
2780
2781 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2782
2783 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2784
2785 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2786 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2787
2788 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2789 rdev->config.cik.mem_max_burst_length_bytes = 256;
2790 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2791 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2792 if (rdev->config.cik.mem_row_size_in_kb > 4)
2793 rdev->config.cik.mem_row_size_in_kb = 4;
2794 /* XXX use MC settings? */
2795 rdev->config.cik.shader_engine_tile_size = 32;
2796 rdev->config.cik.num_gpus = 1;
2797 rdev->config.cik.multi_gpu_tile_size = 64;
2798
2799 /* fix up row size */
2800 gb_addr_config &= ~ROW_SIZE_MASK;
2801 switch (rdev->config.cik.mem_row_size_in_kb) {
2802 case 1:
2803 default:
2804 gb_addr_config |= ROW_SIZE(0);
2805 break;
2806 case 2:
2807 gb_addr_config |= ROW_SIZE(1);
2808 break;
2809 case 4:
2810 gb_addr_config |= ROW_SIZE(2);
2811 break;
2812 }
2813
2814 /* setup tiling info dword. gb_addr_config is not adequate since it does
2815 * not have bank info, so create a custom tiling dword.
2816 * bits 3:0 num_pipes
2817 * bits 7:4 num_banks
2818 * bits 11:8 group_size
2819 * bits 15:12 row_size
2820 */
2821 rdev->config.cik.tile_config = 0;
2822 switch (rdev->config.cik.num_tile_pipes) {
2823 case 1:
2824 rdev->config.cik.tile_config |= (0 << 0);
2825 break;
2826 case 2:
2827 rdev->config.cik.tile_config |= (1 << 0);
2828 break;
2829 case 4:
2830 rdev->config.cik.tile_config |= (2 << 0);
2831 break;
2832 case 8:
2833 default:
2834 /* XXX what about 12? */
2835 rdev->config.cik.tile_config |= (3 << 0);
2836 break;
2837 }
2838 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
2839 rdev->config.cik.tile_config |= 1 << 4;
2840 else
2841 rdev->config.cik.tile_config |= 0 << 4;
2842 rdev->config.cik.tile_config |=
2843 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2844 rdev->config.cik.tile_config |=
2845 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2846
2847 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2848 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2849 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 2850 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2851 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 2852 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2853 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2854 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 2855
2856 cik_tiling_mode_table_init(rdev);
2857
2858 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2859 rdev->config.cik.max_sh_per_se,
2860 rdev->config.cik.max_backends_per_se);
2861
2862 /* set HW defaults for 3D engine */
2863 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2864
2865 WREG32(SX_DEBUG_1, 0x20);
2866
2867 WREG32(TA_CNTL_AUX, 0x00010000);
2868
2869 tmp = RREG32(SPI_CONFIG_CNTL);
2870 tmp |= 0x03000000;
2871 WREG32(SPI_CONFIG_CNTL, tmp);
2872
2873 WREG32(SQ_CONFIG, 1);
2874
2875 WREG32(DB_DEBUG, 0);
2876
2877 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2878 tmp |= 0x00000400;
2879 WREG32(DB_DEBUG2, tmp);
2880
2881 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2882 tmp |= 0x00020200;
2883 WREG32(DB_DEBUG3, tmp);
2884
2885 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2886 tmp |= 0x00018208;
2887 WREG32(CB_HW_CONTROL, tmp);
2888
2889 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2890
2891 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2892 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2893 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2894 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2895
2896 WREG32(VGT_NUM_INSTANCES, 1);
2897
2898 WREG32(CP_PERFMON_CNTL, 0);
2899
2900 WREG32(SQ_CONFIG, 0);
2901
2902 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2903 FORCE_EOV_MAX_REZ_CNT(255)));
2904
2905 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2906 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2907
2908 WREG32(VGT_GS_VERTEX_REUSE, 16);
2909 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2910
2911 tmp = RREG32(HDP_MISC_CNTL);
2912 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2913 WREG32(HDP_MISC_CNTL, tmp);
2914
2915 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2916 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2917
2918 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2919 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2920
2921 udelay(50);
2922}
2923
2cae3bc3
AD		 2924/*
2925 * GPU scratch registers helpers function.
2926 */
2927/**
2928 * cik_scratch_init - setup driver info for CP scratch regs
2929 *
2930 * @rdev: radeon_device pointer
2931 *
2932 * Set up the number and offset of the CP scratch registers.
2933 * NOTE: use of CP scratch registers is a legacy inferface and
2934 * is not used by default on newer asics (r6xx+). On newer asics,
2935 * memory buffers are used for fences rather than scratch regs.
2936 */
2937static void cik_scratch_init(struct radeon_device *rdev)
2938{
2939 int i;
2940
2941 rdev->scratch.num_reg = 7;
2942 rdev->scratch.reg_base = SCRATCH_REG0;
2943 for (i = 0; i < rdev->scratch.num_reg; i++) {
2944 rdev->scratch.free[i] = true;
2945 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2946 }
2947}
2948
fbc832c7
AD		 2949/**
2950 * cik_ring_test - basic gfx ring test
2951 *
2952 * @rdev: radeon_device pointer
2953 * @ring: radeon_ring structure holding ring information
2954 *
2955 * Allocate a scratch register and write to it using the gfx ring (CIK).
2956 * Provides a basic gfx ring test to verify that the ring is working.
2957 * Used by cik_cp_gfx_resume();
2958 * Returns 0 on success, error on failure.
2959 */
2960int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2961{
2962 uint32_t scratch;
2963 uint32_t tmp = 0;
2964 unsigned i;
2965 int r;
2966
2967 r = radeon_scratch_get(rdev, &scratch);
2968 if (r) {
2969 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2970 return r;
2971 }
2972 WREG32(scratch, 0xCAFEDEAD);
2973 r = radeon_ring_lock(rdev, ring, 3);
2974 if (r) {
2975 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2976 radeon_scratch_free(rdev, scratch);
2977 return r;
2978 }
2979 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2980 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2981 radeon_ring_write(ring, 0xDEADBEEF);
2982 radeon_ring_unlock_commit(rdev, ring);
963e81f9 2983
fbc832c7
AD		 2984 for (i = 0; i < rdev->usec_timeout; i++) {
2985 tmp = RREG32(scratch);
2986 if (tmp == 0xDEADBEEF)
2987 break;
2988 DRM_UDELAY(1);
2989 }
2990 if (i < rdev->usec_timeout) {
2991 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2992 } else {
2993 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2994 ring->idx, scratch, tmp);
2995 r = -EINVAL;
2996 }
2997 radeon_scratch_free(rdev, scratch);
2998 return r;
2999}
3000
2cae3bc3 3001/**
b07fdd38 3002 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3003 *
3004 * @rdev: radeon_device pointer
3005 * @fence: radeon fence object
3006 *
3007 * Emits a fence sequnce number on the gfx ring and flushes
3008 * GPU caches.
3009 */
b07fdd38
AD		 3010void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3011 struct radeon_fence *fence)
2cae3bc3
AD		 3012{
3013 struct radeon_ring *ring = &rdev->ring[fence->ring];
3014 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3015
3016 /* EVENT_WRITE_EOP - flush caches, send int */
3017 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3018 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3019 EOP_TC_ACTION_EN |
3020 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3021 EVENT_INDEX(5)));
3022 radeon_ring_write(ring, addr & 0xfffffffc);
3023 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3024 radeon_ring_write(ring, fence->seq);
3025 radeon_ring_write(ring, 0);
3026 /* HDP flush */
3027 /* We should be using the new WAIT_REG_MEM special op packet here
3028 * but it causes the CP to hang
3029 */
3030 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3031 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3032 WRITE_DATA_DST_SEL(0)));
3033 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3034 radeon_ring_write(ring, 0);
3035 radeon_ring_write(ring, 0);
3036}
3037
b07fdd38
AD		 3038/**
3039 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3040 *
3041 * @rdev: radeon_device pointer
3042 * @fence: radeon fence object
3043 *
3044 * Emits a fence sequnce number on the compute ring and flushes
3045 * GPU caches.
3046 */
3047void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3048 struct radeon_fence *fence)
3049{
3050 struct radeon_ring *ring = &rdev->ring[fence->ring];
3051 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3052
3053 /* RELEASE_MEM - flush caches, send int */
3054 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3055 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3056 EOP_TC_ACTION_EN |
3057 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3058 EVENT_INDEX(5)));
3059 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3060 radeon_ring_write(ring, addr & 0xfffffffc);
3061 radeon_ring_write(ring, upper_32_bits(addr));
3062 radeon_ring_write(ring, fence->seq);
3063 radeon_ring_write(ring, 0);
3064 /* HDP flush */
3065 /* We should be using the new WAIT_REG_MEM special op packet here
3066 * but it causes the CP to hang
3067 */
3068 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3069 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3070 WRITE_DATA_DST_SEL(0)));
3071 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3072 radeon_ring_write(ring, 0);
3073 radeon_ring_write(ring, 0);
3074}
3075
2cae3bc3
AD		 3076void cik_semaphore_ring_emit(struct radeon_device *rdev,
3077 struct radeon_ring *ring,
3078 struct radeon_semaphore *semaphore,
3079 bool emit_wait)
3080{
3081 uint64_t addr = semaphore->gpu_addr;
3082 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3083
3084 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3085 radeon_ring_write(ring, addr & 0xffffffff);
3086 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3087}
3088
3089/*
3090 * IB stuff
3091 */
3092/**
3093 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3094 *
3095 * @rdev: radeon_device pointer
3096 * @ib: radeon indirect buffer object
3097 *
3098 * Emits an DE (drawing engine) or CE (constant engine) IB
3099 * on the gfx ring. IBs are usually generated by userspace
3100 * acceleration drivers and submitted to the kernel for
3101 * sheduling on the ring. This function schedules the IB
3102 * on the gfx ring for execution by the GPU.
3103 */
3104void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3105{
3106 struct radeon_ring *ring = &rdev->ring[ib->ring];
3107 u32 header, control = INDIRECT_BUFFER_VALID;
3108
3109 if (ib->is_const_ib) {
3110 /* set switch buffer packet before const IB */
3111 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3112 radeon_ring_write(ring, 0);
3113
3114 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3115 } else {
3116 u32 next_rptr;
3117 if (ring->rptr_save_reg) {
3118 next_rptr = ring->wptr + 3 + 4;
3119 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3120 radeon_ring_write(ring, ((ring->rptr_save_reg -
3121 PACKET3_SET_UCONFIG_REG_START) >> 2));
3122 radeon_ring_write(ring, next_rptr);
3123 } else if (rdev->wb.enabled) {
3124 next_rptr = ring->wptr + 5 + 4;
3125 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3126 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3127 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3128 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3129 radeon_ring_write(ring, next_rptr);
3130 }
3131
3132 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3133 }
3134
3135 control |= ib->length_dw |
3136 (ib->vm ? (ib->vm->id << 24) : 0);
3137
3138 radeon_ring_write(ring, header);
3139 radeon_ring_write(ring,
3140#ifdef __BIG_ENDIAN
3141 (2 << 0) |
3142#endif
3143 (ib->gpu_addr & 0xFFFFFFFC));
3144 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3145 radeon_ring_write(ring, control);
3146}
3147
fbc832c7
AD		 3148/**
3149 * cik_ib_test - basic gfx ring IB test
3150 *
3151 * @rdev: radeon_device pointer
3152 * @ring: radeon_ring structure holding ring information
3153 *
3154 * Allocate an IB and execute it on the gfx ring (CIK).
3155 * Provides a basic gfx ring test to verify that IBs are working.
3156 * Returns 0 on success, error on failure.
3157 */
3158int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3159{
3160 struct radeon_ib ib;
3161 uint32_t scratch;
3162 uint32_t tmp = 0;
3163 unsigned i;
3164 int r;
3165
3166 r = radeon_scratch_get(rdev, &scratch);
3167 if (r) {
3168 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3169 return r;
3170 }
3171 WREG32(scratch, 0xCAFEDEAD);
3172 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3173 if (r) {
3174 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3175 return r;
3176 }
3177 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3178 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3179 ib.ptr[2] = 0xDEADBEEF;
3180 ib.length_dw = 3;
3181 r = radeon_ib_schedule(rdev, &ib, NULL);
3182 if (r) {
3183 radeon_scratch_free(rdev, scratch);
3184 radeon_ib_free(rdev, &ib);
3185 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3186 return r;
3187 }
3188 r = radeon_fence_wait(ib.fence, false);
3189 if (r) {
3190 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3191 return r;
3192 }
3193 for (i = 0; i < rdev->usec_timeout; i++) {
3194 tmp = RREG32(scratch);
3195 if (tmp == 0xDEADBEEF)
3196 break;
3197 DRM_UDELAY(1);
3198 }
3199 if (i < rdev->usec_timeout) {
3200 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3201 } else {
3202 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3203 scratch, tmp);
3204 r = -EINVAL;
3205 }
3206 radeon_scratch_free(rdev, scratch);
3207 radeon_ib_free(rdev, &ib);
3208 return r;
3209}
3210
841cf442
AD		 3211/*
3212 * CP.
3213 * On CIK, gfx and compute now have independant command processors.
3214 *
3215 * GFX
3216 * Gfx consists of a single ring and can process both gfx jobs and
3217 * compute jobs. The gfx CP consists of three microengines (ME):
3218 * PFP - Pre-Fetch Parser
3219 * ME - Micro Engine
3220 * CE - Constant Engine
3221 * The PFP and ME make up what is considered the Drawing Engine (DE).
3222 * The CE is an asynchronous engine used for updating buffer desciptors
3223 * used by the DE so that they can be loaded into cache in parallel
3224 * while the DE is processing state update packets.
3225 *
3226 * Compute
3227 * The compute CP consists of two microengines (ME):
3228 * MEC1 - Compute MicroEngine 1
3229 * MEC2 - Compute MicroEngine 2
3230 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3231 * The queues are exposed to userspace and are programmed directly
3232 * by the compute runtime.
3233 */
3234/**
3235 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3236 *
3237 * @rdev: radeon_device pointer
3238 * @enable: enable or disable the MEs
3239 *
3240 * Halts or unhalts the gfx MEs.
3241 */
3242static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3243{
3244 if (enable)
3245 WREG32(CP_ME_CNTL, 0);
3246 else {
3247 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3248 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3249 }
3250 udelay(50);
3251}
3252
3253/**
3254 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3255 *
3256 * @rdev: radeon_device pointer
3257 *
3258 * Loads the gfx PFP, ME, and CE ucode.
3259 * Returns 0 for success, -EINVAL if the ucode is not available.
3260 */
3261static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3262{
3263 const __be32 *fw_data;
3264 int i;
3265
3266 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3267 return -EINVAL;
3268
3269 cik_cp_gfx_enable(rdev, false);
3270
3271 /* PFP */
3272 fw_data = (const __be32 *)rdev->pfp_fw->data;
3273 WREG32(CP_PFP_UCODE_ADDR, 0);
3274 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3275 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3276 WREG32(CP_PFP_UCODE_ADDR, 0);
3277
3278 /* CE */
3279 fw_data = (const __be32 *)rdev->ce_fw->data;
3280 WREG32(CP_CE_UCODE_ADDR, 0);
3281 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3282 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3283 WREG32(CP_CE_UCODE_ADDR, 0);
3284
3285 /* ME */
3286 fw_data = (const __be32 *)rdev->me_fw->data;
3287 WREG32(CP_ME_RAM_WADDR, 0);
3288 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3289 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3290 WREG32(CP_ME_RAM_WADDR, 0);
3291
3292 WREG32(CP_PFP_UCODE_ADDR, 0);
3293 WREG32(CP_CE_UCODE_ADDR, 0);
3294 WREG32(CP_ME_RAM_WADDR, 0);
3295 WREG32(CP_ME_RAM_RADDR, 0);
3296 return 0;
3297}
3298
3299/**
3300 * cik_cp_gfx_start - start the gfx ring
3301 *
3302 * @rdev: radeon_device pointer
3303 *
3304 * Enables the ring and loads the clear state context and other
3305 * packets required to init the ring.
3306 * Returns 0 for success, error for failure.
3307 */
3308static int cik_cp_gfx_start(struct radeon_device *rdev)
3309{
3310 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3311 int r, i;
3312
3313 /* init the CP */
3314 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3315 WREG32(CP_ENDIAN_SWAP, 0);
3316 WREG32(CP_DEVICE_ID, 1);
3317
3318 cik_cp_gfx_enable(rdev, true);
3319
3320 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3321 if (r) {
3322 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3323 return r;
3324 }
3325
3326 /* init the CE partitions. CE only used for gfx on CIK */
3327 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3328 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3329 radeon_ring_write(ring, 0xc000);
3330 radeon_ring_write(ring, 0xc000);
3331
3332 /* setup clear context state */
3333 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3334 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3335
3336 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3337 radeon_ring_write(ring, 0x80000000);
3338 radeon_ring_write(ring, 0x80000000);
3339
3340 for (i = 0; i < cik_default_size; i++)
3341 radeon_ring_write(ring, cik_default_state[i]);
3342
3343 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3344 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3345
3346 /* set clear context state */
3347 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3348 radeon_ring_write(ring, 0);
3349
3350 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3351 radeon_ring_write(ring, 0x00000316);
3352 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3353 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3354
3355 radeon_ring_unlock_commit(rdev, ring);
3356
3357 return 0;
3358}
3359
3360/**
3361 * cik_cp_gfx_fini - stop the gfx ring
3362 *
3363 * @rdev: radeon_device pointer
3364 *
3365 * Stop the gfx ring and tear down the driver ring
3366 * info.
3367 */
3368static void cik_cp_gfx_fini(struct radeon_device *rdev)
3369{
3370 cik_cp_gfx_enable(rdev, false);
3371 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3372}
3373
3374/**
3375 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3376 *
3377 * @rdev: radeon_device pointer
3378 *
3379 * Program the location and size of the gfx ring buffer
3380 * and test it to make sure it's working.
3381 * Returns 0 for success, error for failure.
3382 */
3383static int cik_cp_gfx_resume(struct radeon_device *rdev)
3384{
3385 struct radeon_ring *ring;
3386 u32 tmp;
3387 u32 rb_bufsz;
3388 u64 rb_addr;
3389 int r;
3390
3391 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3392 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3393
3394 /* Set the write pointer delay */
3395 WREG32(CP_RB_WPTR_DELAY, 0);
3396
3397 /* set the RB to use vmid 0 */
3398 WREG32(CP_RB_VMID, 0);
3399
3400 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3401
3402 /* ring 0 - compute and gfx */
3403 /* Set ring buffer size */
3404 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3405 rb_bufsz = drm_order(ring->ring_size / 8);
3406 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3407#ifdef __BIG_ENDIAN
3408 tmp |= BUF_SWAP_32BIT;
3409#endif
3410 WREG32(CP_RB0_CNTL, tmp);
3411
3412 /* Initialize the ring buffer's read and write pointers */
3413 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3414 ring->wptr = 0;
3415 WREG32(CP_RB0_WPTR, ring->wptr);
3416
3417 /* set the wb address wether it's enabled or not */
3418 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3419 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3420
3421 /* scratch register shadowing is no longer supported */
3422 WREG32(SCRATCH_UMSK, 0);
3423
3424 if (!rdev->wb.enabled)
3425 tmp |= RB_NO_UPDATE;
3426
3427 mdelay(1);
3428 WREG32(CP_RB0_CNTL, tmp);
3429
3430 rb_addr = ring->gpu_addr >> 8;
3431 WREG32(CP_RB0_BASE, rb_addr);
3432 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3433
3434 ring->rptr = RREG32(CP_RB0_RPTR);
3435
3436 /* start the ring */
3437 cik_cp_gfx_start(rdev);
3438 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3439 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3440 if (r) {
3441 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3442 return r;
3443 }
3444 return 0;
3445}
3446
963e81f9
AD		 3447u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3448 struct radeon_ring *ring)
3449{
3450 u32 rptr;
3451
3452
3453
3454 if (rdev->wb.enabled) {
3455 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3456 } else {
f61d5b46 3457 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3458 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3459 rptr = RREG32(CP_HQD_PQ_RPTR);
3460 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3461 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3462 }
963e81f9
AD		 3463
3464 return rptr;
3465}
3466
3467u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3468 struct radeon_ring *ring)
3469{
3470 u32 wptr;
3471
3472 if (rdev->wb.enabled) {
3473 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3474 } else {
f61d5b46 3475 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3476 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3477 wptr = RREG32(CP_HQD_PQ_WPTR);
3478 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3479 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3480 }
963e81f9
AD		 3481
3482 return wptr;
3483}
3484
3485void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3486 struct radeon_ring *ring)
3487{
2e1e6dad
CK		 3488 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3489 WDOORBELL32(ring->doorbell_offset, ring->wptr);
963e81f9
AD		 3490}
3491
841cf442
AD		 3492/**
3493 * cik_cp_compute_enable - enable/disable the compute CP MEs
3494 *
3495 * @rdev: radeon_device pointer
3496 * @enable: enable or disable the MEs
3497 *
3498 * Halts or unhalts the compute MEs.
3499 */
3500static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3501{
3502 if (enable)
3503 WREG32(CP_MEC_CNTL, 0);
3504 else
3505 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3506 udelay(50);
3507}
3508
3509/**
3510 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3511 *
3512 * @rdev: radeon_device pointer
3513 *
3514 * Loads the compute MEC1&2 ucode.
3515 * Returns 0 for success, -EINVAL if the ucode is not available.
3516 */
3517static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3518{
3519 const __be32 *fw_data;
3520 int i;
3521
3522 if (!rdev->mec_fw)
3523 return -EINVAL;
3524
3525 cik_cp_compute_enable(rdev, false);
3526
3527 /* MEC1 */
3528 fw_data = (const __be32 *)rdev->mec_fw->data;
3529 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3530 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3531 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3532 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3533
3534 if (rdev->family == CHIP_KAVERI) {
3535 /* MEC2 */
3536 fw_data = (const __be32 *)rdev->mec_fw->data;
3537 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3538 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3539 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3540 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3541 }
3542
3543 return 0;
3544}
3545
3546/**
3547 * cik_cp_compute_start - start the compute queues
3548 *
3549 * @rdev: radeon_device pointer
3550 *
3551 * Enable the compute queues.
3552 * Returns 0 for success, error for failure.
3553 */
3554static int cik_cp_compute_start(struct radeon_device *rdev)
3555{
963e81f9
AD		 3556 cik_cp_compute_enable(rdev, true);
3557
841cf442
AD		 3558 return 0;
3559}
3560
3561/**
3562 * cik_cp_compute_fini - stop the compute queues
3563 *
3564 * @rdev: radeon_device pointer
3565 *
3566 * Stop the compute queues and tear down the driver queue
3567 * info.
3568 */
3569static void cik_cp_compute_fini(struct radeon_device *rdev)
3570{
963e81f9
AD		 3571 int i, idx, r;
3572
841cf442 3573 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 3574
3575 for (i = 0; i < 2; i++) {
3576 if (i == 0)
3577 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3578 else
3579 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3580
3581 if (rdev->ring[idx].mqd_obj) {
3582 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3583 if (unlikely(r != 0))
3584 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3585
3586 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3587 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3588
3589 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3590 rdev->ring[idx].mqd_obj = NULL;
3591 }
3592 }
841cf442
AD		 3593}
3594
963e81f9
AD		 3595static void cik_mec_fini(struct radeon_device *rdev)
3596{
3597 int r;
3598
3599 if (rdev->mec.hpd_eop_obj) {
3600 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3601 if (unlikely(r != 0))
3602 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3603 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3604 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3605
3606 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3607 rdev->mec.hpd_eop_obj = NULL;
3608 }
3609}
3610
3611#define MEC_HPD_SIZE 2048
3612
3613static int cik_mec_init(struct radeon_device *rdev)
3614{
3615 int r;
3616 u32 *hpd;
3617
3618 /*
3619 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3620 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3621 */
3622 if (rdev->family == CHIP_KAVERI)
3623 rdev->mec.num_mec = 2;
3624 else
3625 rdev->mec.num_mec = 1;
3626 rdev->mec.num_pipe = 4;
3627 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3628
3629 if (rdev->mec.hpd_eop_obj == NULL) {
3630 r = radeon_bo_create(rdev,
3631 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3632 PAGE_SIZE, true,
3633 RADEON_GEM_DOMAIN_GTT, NULL,
3634 &rdev->mec.hpd_eop_obj);
3635 if (r) {
3636 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3637 return r;
3638 }
3639 }
3640
3641 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3642 if (unlikely(r != 0)) {
3643 cik_mec_fini(rdev);
3644 return r;
3645 }
3646 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3647 &rdev->mec.hpd_eop_gpu_addr);
3648 if (r) {
3649 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3650 cik_mec_fini(rdev);
3651 return r;
3652 }
3653 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3654 if (r) {
3655 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3656 cik_mec_fini(rdev);
3657 return r;
3658 }
3659
3660 /* clear memory. Not sure if this is required or not */
3661 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3662
3663 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3664 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3665
3666 return 0;
3667}
3668
3669struct hqd_registers
3670{
3671 u32 cp_mqd_base_addr;
3672 u32 cp_mqd_base_addr_hi;
3673 u32 cp_hqd_active;
3674 u32 cp_hqd_vmid;
3675 u32 cp_hqd_persistent_state;
3676 u32 cp_hqd_pipe_priority;
3677 u32 cp_hqd_queue_priority;
3678 u32 cp_hqd_quantum;
3679 u32 cp_hqd_pq_base;
3680 u32 cp_hqd_pq_base_hi;
3681 u32 cp_hqd_pq_rptr;
3682 u32 cp_hqd_pq_rptr_report_addr;
3683 u32 cp_hqd_pq_rptr_report_addr_hi;
3684 u32 cp_hqd_pq_wptr_poll_addr;
3685 u32 cp_hqd_pq_wptr_poll_addr_hi;
3686 u32 cp_hqd_pq_doorbell_control;
3687 u32 cp_hqd_pq_wptr;
3688 u32 cp_hqd_pq_control;
3689 u32 cp_hqd_ib_base_addr;
3690 u32 cp_hqd_ib_base_addr_hi;
3691 u32 cp_hqd_ib_rptr;
3692 u32 cp_hqd_ib_control;
3693 u32 cp_hqd_iq_timer;
3694 u32 cp_hqd_iq_rptr;
3695 u32 cp_hqd_dequeue_request;
3696 u32 cp_hqd_dma_offload;
3697 u32 cp_hqd_sema_cmd;
3698 u32 cp_hqd_msg_type;
3699 u32 cp_hqd_atomic0_preop_lo;
3700 u32 cp_hqd_atomic0_preop_hi;
3701 u32 cp_hqd_atomic1_preop_lo;
3702 u32 cp_hqd_atomic1_preop_hi;
3703 u32 cp_hqd_hq_scheduler0;
3704 u32 cp_hqd_hq_scheduler1;
3705 u32 cp_mqd_control;
3706};
3707
3708struct bonaire_mqd
3709{
3710 u32 header;
3711 u32 dispatch_initiator;
3712 u32 dimensions[3];
3713 u32 start_idx[3];
3714 u32 num_threads[3];
3715 u32 pipeline_stat_enable;
3716 u32 perf_counter_enable;
3717 u32 pgm[2];
3718 u32 tba[2];
3719 u32 tma[2];
3720 u32 pgm_rsrc[2];
3721 u32 vmid;
3722 u32 resource_limits;
3723 u32 static_thread_mgmt01[2];
3724 u32 tmp_ring_size;
3725 u32 static_thread_mgmt23[2];
3726 u32 restart[3];
3727 u32 thread_trace_enable;
3728 u32 reserved1;
3729 u32 user_data[16];
3730 u32 vgtcs_invoke_count[2];
3731 struct hqd_registers queue_state;
3732 u32 dequeue_cntr;
3733 u32 interrupt_queue[64];
3734};
3735
841cf442
AD		 3736/**
3737 * cik_cp_compute_resume - setup the compute queue registers
3738 *
3739 * @rdev: radeon_device pointer
3740 *
3741 * Program the compute queues and test them to make sure they
3742 * are working.
3743 * Returns 0 for success, error for failure.
3744 */
3745static int cik_cp_compute_resume(struct radeon_device *rdev)
3746{
963e81f9
AD		 3747 int r, i, idx;
3748 u32 tmp;
3749 bool use_doorbell = true;
3750 u64 hqd_gpu_addr;
3751 u64 mqd_gpu_addr;
3752 u64 eop_gpu_addr;
3753 u64 wb_gpu_addr;
3754 u32 *buf;
3755 struct bonaire_mqd *mqd;
841cf442 3756
841cf442
AD		 3757 r = cik_cp_compute_start(rdev);
3758 if (r)
3759 return r;
963e81f9
AD		 3760
3761 /* fix up chicken bits */
3762 tmp = RREG32(CP_CPF_DEBUG);
3763 tmp |= (1 << 23);
3764 WREG32(CP_CPF_DEBUG, tmp);
3765
3766 /* init the pipes */
f61d5b46 3767 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3768 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3769 int me = (i < 4) ? 1 : 2;
3770 int pipe = (i < 4) ? i : (i - 4);
3771
3772 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3773
3774 cik_srbm_select(rdev, me, pipe, 0, 0);
3775
3776 /* write the EOP addr */
3777 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3778 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3779
3780 /* set the VMID assigned */
3781 WREG32(CP_HPD_EOP_VMID, 0);
3782
3783 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3784 tmp = RREG32(CP_HPD_EOP_CONTROL);
3785 tmp &= ~EOP_SIZE_MASK;
3786 tmp |= drm_order(MEC_HPD_SIZE / 8);
3787 WREG32(CP_HPD_EOP_CONTROL, tmp);
3788 }
3789 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3790 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 3791
3792 /* init the queues. Just two for now. */
3793 for (i = 0; i < 2; i++) {
3794 if (i == 0)
3795 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3796 else
3797 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3798
3799 if (rdev->ring[idx].mqd_obj == NULL) {
3800 r = radeon_bo_create(rdev,
3801 sizeof(struct bonaire_mqd),
3802 PAGE_SIZE, true,
3803 RADEON_GEM_DOMAIN_GTT, NULL,
3804 &rdev->ring[idx].mqd_obj);
3805 if (r) {
3806 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3807 return r;
3808 }
3809 }
3810
3811 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3812 if (unlikely(r != 0)) {
3813 cik_cp_compute_fini(rdev);
3814 return r;
3815 }
3816 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3817 &mqd_gpu_addr);
3818 if (r) {
3819 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3820 cik_cp_compute_fini(rdev);
3821 return r;
3822 }
3823 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3824 if (r) {
3825 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3826 cik_cp_compute_fini(rdev);
3827 return r;
3828 }
3829
3830 /* doorbell offset */
3831 rdev->ring[idx].doorbell_offset =
3832 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3833
3834 /* init the mqd struct */
3835 memset(buf, 0, sizeof(struct bonaire_mqd));
3836
3837 mqd = (struct bonaire_mqd *)buf;
3838 mqd->header = 0xC0310800;
3839 mqd->static_thread_mgmt01[0] = 0xffffffff;
3840 mqd->static_thread_mgmt01[1] = 0xffffffff;
3841 mqd->static_thread_mgmt23[0] = 0xffffffff;
3842 mqd->static_thread_mgmt23[1] = 0xffffffff;
3843
f61d5b46 3844 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3845 cik_srbm_select(rdev, rdev->ring[idx].me,
3846 rdev->ring[idx].pipe,
3847 rdev->ring[idx].queue, 0);
3848
3849 /* disable wptr polling */
3850 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3851 tmp &= ~WPTR_POLL_EN;
3852 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3853
3854 /* enable doorbell? */
3855 mqd->queue_state.cp_hqd_pq_doorbell_control =
3856 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3857 if (use_doorbell)
3858 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3859 else
3860 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3861 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3862 mqd->queue_state.cp_hqd_pq_doorbell_control);
3863
3864 /* disable the queue if it's active */
3865 mqd->queue_state.cp_hqd_dequeue_request = 0;
3866 mqd->queue_state.cp_hqd_pq_rptr = 0;
3867 mqd->queue_state.cp_hqd_pq_wptr= 0;
3868 if (RREG32(CP_HQD_ACTIVE) & 1) {
3869 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3870 for (i = 0; i < rdev->usec_timeout; i++) {
3871 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3872 break;
3873 udelay(1);
3874 }
3875 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3876 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3877 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3878 }
3879
3880 /* set the pointer to the MQD */
3881 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3882 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3883 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3884 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3885 /* set MQD vmid to 0 */
3886 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3887 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3888 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3889
3890 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3891 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3892 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3893 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3894 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3895 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3896
3897 /* set up the HQD, this is similar to CP_RB0_CNTL */
3898 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3899 mqd->queue_state.cp_hqd_pq_control &=
3900 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3901
3902 mqd->queue_state.cp_hqd_pq_control |=
3903 drm_order(rdev->ring[idx].ring_size / 8);
3904 mqd->queue_state.cp_hqd_pq_control |=
3905 (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8);
3906#ifdef __BIG_ENDIAN
3907 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3908#endif
3909 mqd->queue_state.cp_hqd_pq_control &=
3910 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3911 mqd->queue_state.cp_hqd_pq_control |=
3912 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3913 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3914
3915 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3916 if (i == 0)
3917 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3918 else
3919 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3920 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3921 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3922 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3923 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3924 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3925
3926 /* set the wb address wether it's enabled or not */
3927 if (i == 0)
3928 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3929 else
3930 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3931 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3932 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3933 upper_32_bits(wb_gpu_addr) & 0xffff;
3934 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3935 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3936 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3937 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3938
3939 /* enable the doorbell if requested */
3940 if (use_doorbell) {
3941 mqd->queue_state.cp_hqd_pq_doorbell_control =
3942 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3943 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3944 mqd->queue_state.cp_hqd_pq_doorbell_control |=
3945 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3946 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3947 mqd->queue_state.cp_hqd_pq_doorbell_control &=
3948 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3949
3950 } else {
3951 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3952 }
3953 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3954 mqd->queue_state.cp_hqd_pq_doorbell_control);
3955
3956 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3957 rdev->ring[idx].wptr = 0;
3958 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3959 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3960 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3961 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3962
3963 /* set the vmid for the queue */
3964 mqd->queue_state.cp_hqd_vmid = 0;
3965 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3966
3967 /* activate the queue */
3968 mqd->queue_state.cp_hqd_active = 1;
3969 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3970
3971 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3972 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 3973
3974 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3975 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3976
3977 rdev->ring[idx].ready = true;
3978 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3979 if (r)
3980 rdev->ring[idx].ready = false;
3981 }
3982
841cf442
AD		 3983 return 0;
3984}
3985
841cf442
AD		 3986static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3987{
3988 cik_cp_gfx_enable(rdev, enable);
3989 cik_cp_compute_enable(rdev, enable);
3990}
3991
841cf442
AD		 3992static int cik_cp_load_microcode(struct radeon_device *rdev)
3993{
3994 int r;
3995
3996 r = cik_cp_gfx_load_microcode(rdev);
3997 if (r)
3998 return r;
3999 r = cik_cp_compute_load_microcode(rdev);
4000 if (r)
4001 return r;
4002
4003 return 0;
4004}
4005
841cf442
AD		 4006static void cik_cp_fini(struct radeon_device *rdev)
4007{
4008 cik_cp_gfx_fini(rdev);
4009 cik_cp_compute_fini(rdev);
4010}
4011
841cf442
AD		 4012static int cik_cp_resume(struct radeon_device *rdev)
4013{
4014 int r;
4015
841cf442
AD		 4016 r = cik_cp_load_microcode(rdev);
4017 if (r)
4018 return r;
4019
4020 r = cik_cp_gfx_resume(rdev);
4021 if (r)
4022 return r;
4023 r = cik_cp_compute_resume(rdev);
4024 if (r)
4025 return r;
4026
4027 return 0;
4028}
4029
cc066715 4030static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4031{
6f2043ce
AD		 4032 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4033 RREG32(GRBM_STATUS));
4034 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4035 RREG32(GRBM_STATUS2));
4036 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4037 RREG32(GRBM_STATUS_SE0));
4038 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4039 RREG32(GRBM_STATUS_SE1));
4040 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4041 RREG32(GRBM_STATUS_SE2));
4042 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4043 RREG32(GRBM_STATUS_SE3));
4044 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4045 RREG32(SRBM_STATUS));
4046 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4047 RREG32(SRBM_STATUS2));
cc066715
AD		 4048 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4049 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4050 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4051 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4052 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4053 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4054 RREG32(CP_STALLED_STAT1));
4055 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4056 RREG32(CP_STALLED_STAT2));
4057 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4058 RREG32(CP_STALLED_STAT3));
4059 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4060 RREG32(CP_CPF_BUSY_STAT));
4061 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4062 RREG32(CP_CPF_STALLED_STAT1));
4063 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4064 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4065 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4066 RREG32(CP_CPC_STALLED_STAT1));
4067 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4068}
6f2043ce 4069
cc066715
AD		 4070/**
4071 * cik_gpu_check_soft_reset - check which blocks are busy
4072 *
4073 * @rdev: radeon_device pointer
4074 *
4075 * Check which blocks are busy and return the relevant reset
4076 * mask to be used by cik_gpu_soft_reset().
4077 * Returns a mask of the blocks to be reset.
4078 */
2483b4ea 4079u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
cc066715
AD		 4080{
4081 u32 reset_mask = 0;
4082 u32 tmp;
6f2043ce 4083
cc066715
AD		 4084 /* GRBM_STATUS */
4085 tmp = RREG32(GRBM_STATUS);
4086 if (tmp & (PA_BUSY | SC_BUSY |
4087 BCI_BUSY | SX_BUSY |
4088 TA_BUSY | VGT_BUSY |
4089 DB_BUSY | CB_BUSY |
4090 GDS_BUSY | SPI_BUSY |
4091 IA_BUSY | IA_BUSY_NO_DMA))
4092 reset_mask |= RADEON_RESET_GFX;
4093
4094 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4095 reset_mask |= RADEON_RESET_CP;
4096
4097 /* GRBM_STATUS2 */
4098 tmp = RREG32(GRBM_STATUS2);
4099 if (tmp & RLC_BUSY)
4100 reset_mask |= RADEON_RESET_RLC;
4101
4102 /* SDMA0_STATUS_REG */
4103 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4104 if (!(tmp & SDMA_IDLE))
4105 reset_mask |= RADEON_RESET_DMA;
4106
4107 /* SDMA1_STATUS_REG */
4108 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4109 if (!(tmp & SDMA_IDLE))
4110 reset_mask |= RADEON_RESET_DMA1;
4111
4112 /* SRBM_STATUS2 */
4113 tmp = RREG32(SRBM_STATUS2);
4114 if (tmp & SDMA_BUSY)
4115 reset_mask |= RADEON_RESET_DMA;
4116
4117 if (tmp & SDMA1_BUSY)
4118 reset_mask |= RADEON_RESET_DMA1;
4119
4120 /* SRBM_STATUS */
4121 tmp = RREG32(SRBM_STATUS);
4122
4123 if (tmp & IH_BUSY)
4124 reset_mask |= RADEON_RESET_IH;
4125
4126 if (tmp & SEM_BUSY)
4127 reset_mask |= RADEON_RESET_SEM;
4128
4129 if (tmp & GRBM_RQ_PENDING)
4130 reset_mask |= RADEON_RESET_GRBM;
4131
4132 if (tmp & VMC_BUSY)
4133 reset_mask |= RADEON_RESET_VMC;
4134
4135 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4136 MCC_BUSY | MCD_BUSY))
4137 reset_mask |= RADEON_RESET_MC;
4138
4139 if (evergreen_is_display_hung(rdev))
4140 reset_mask |= RADEON_RESET_DISPLAY;
4141
4142 /* Skip MC reset as it's mostly likely not hung, just busy */
4143 if (reset_mask & RADEON_RESET_MC) {
4144 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4145 reset_mask &= ~RADEON_RESET_MC;
4146 }
4147
4148 return reset_mask;
6f2043ce
AD		 4149}
4150
4151/**
cc066715 4152 * cik_gpu_soft_reset - soft reset GPU
6f2043ce
AD		 4153 *
4154 * @rdev: radeon_device pointer
cc066715 4155 * @reset_mask: mask of which blocks to reset
6f2043ce 4156 *
cc066715 4157 * Soft reset the blocks specified in @reset_mask.
6f2043ce 4158 */
cc066715 4159static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
6f2043ce
AD		 4160{
4161 struct evergreen_mc_save save;
cc066715
AD		 4162 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4163 u32 tmp;
4164
4165 if (reset_mask == 0)
4166 return;
4167
4168 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4169
4170 cik_print_gpu_status_regs(rdev);
4171 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4172 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4173 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4174 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4175
4176 /* stop the rlc */
4177 cik_rlc_stop(rdev);
4178
4179 /* Disable GFX parsing/prefetching */
4180 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4181
4182 /* Disable MEC parsing/prefetching */
4183 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4184
4185 if (reset_mask & RADEON_RESET_DMA) {
4186 /* sdma0 */
4187 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4188 tmp |= SDMA_HALT;
4189 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4190 }
4191 if (reset_mask & RADEON_RESET_DMA1) {
4192 /* sdma1 */
4193 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4194 tmp |= SDMA_HALT;
4195 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4196 }
6f2043ce 4197
6f2043ce 4198 evergreen_mc_stop(rdev, &save);
cc066715 4199 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4200 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4201 }
6f2043ce 4202
cc066715
AD		 4203 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4204 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4205
4206 if (reset_mask & RADEON_RESET_CP) {
4207 grbm_soft_reset |= SOFT_RESET_CP;
4208
4209 srbm_soft_reset |= SOFT_RESET_GRBM;
4210 }
4211
4212 if (reset_mask & RADEON_RESET_DMA)
4213 srbm_soft_reset |= SOFT_RESET_SDMA;
4214
4215 if (reset_mask & RADEON_RESET_DMA1)
4216 srbm_soft_reset |= SOFT_RESET_SDMA1;
4217
4218 if (reset_mask & RADEON_RESET_DISPLAY)
4219 srbm_soft_reset |= SOFT_RESET_DC;
4220
4221 if (reset_mask & RADEON_RESET_RLC)
4222 grbm_soft_reset |= SOFT_RESET_RLC;
4223
4224 if (reset_mask & RADEON_RESET_SEM)
4225 srbm_soft_reset |= SOFT_RESET_SEM;
4226
4227 if (reset_mask & RADEON_RESET_IH)
4228 srbm_soft_reset |= SOFT_RESET_IH;
4229
4230 if (reset_mask & RADEON_RESET_GRBM)
4231 srbm_soft_reset |= SOFT_RESET_GRBM;
4232
4233 if (reset_mask & RADEON_RESET_VMC)
4234 srbm_soft_reset |= SOFT_RESET_VMC;
4235
4236 if (!(rdev->flags & RADEON_IS_IGP)) {
4237 if (reset_mask & RADEON_RESET_MC)
4238 srbm_soft_reset |= SOFT_RESET_MC;
4239 }
4240
4241 if (grbm_soft_reset) {
4242 tmp = RREG32(GRBM_SOFT_RESET);
4243 tmp |= grbm_soft_reset;
4244 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4245 WREG32(GRBM_SOFT_RESET, tmp);
4246 tmp = RREG32(GRBM_SOFT_RESET);
4247
4248 udelay(50);
4249
4250 tmp &= ~grbm_soft_reset;
4251 WREG32(GRBM_SOFT_RESET, tmp);
4252 tmp = RREG32(GRBM_SOFT_RESET);
4253 }
4254
4255 if (srbm_soft_reset) {
4256 tmp = RREG32(SRBM_SOFT_RESET);
4257 tmp |= srbm_soft_reset;
4258 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4259 WREG32(SRBM_SOFT_RESET, tmp);
4260 tmp = RREG32(SRBM_SOFT_RESET);
4261
4262 udelay(50);
4263
4264 tmp &= ~srbm_soft_reset;
4265 WREG32(SRBM_SOFT_RESET, tmp);
4266 tmp = RREG32(SRBM_SOFT_RESET);
4267 }
6f2043ce 4268
6f2043ce
AD		 4269 /* Wait a little for things to settle down */
4270 udelay(50);
cc066715 4271
6f2043ce 4272 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4273 udelay(50);
4274
4275 cik_print_gpu_status_regs(rdev);
6f2043ce
AD		 4276}
4277
4278/**
cc066715 4279 * cik_asic_reset - soft reset GPU
6f2043ce
AD		 4280 *
4281 * @rdev: radeon_device pointer
4282 *
cc066715
AD		 4283 * Look up which blocks are hung and attempt
4284 * to reset them.
6f2043ce
AD		 4285 * Returns 0 for success.
4286 */
4287int cik_asic_reset(struct radeon_device *rdev)
4288{
cc066715 4289 u32 reset_mask;
6f2043ce 4290
cc066715
AD		 4291 reset_mask = cik_gpu_check_soft_reset(rdev);
4292
4293 if (reset_mask)
4294 r600_set_bios_scratch_engine_hung(rdev, true);
4295
4296 cik_gpu_soft_reset(rdev, reset_mask);
6f2043ce 4297
cc066715
AD		 4298 reset_mask = cik_gpu_check_soft_reset(rdev);
4299
4300 if (!reset_mask)
4301 r600_set_bios_scratch_engine_hung(rdev, false);
4302
4303 return 0;
4304}
4305
4306/**
4307 * cik_gfx_is_lockup - check if the 3D engine is locked up
4308 *
4309 * @rdev: radeon_device pointer
4310 * @ring: radeon_ring structure holding ring information
4311 *
4312 * Check if the 3D engine is locked up (CIK).
4313 * Returns true if the engine is locked, false if not.
4314 */
4315bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4316{
4317 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4318
4319 if (!(reset_mask & (RADEON_RESET_GFX |
4320 RADEON_RESET_COMPUTE |
4321 RADEON_RESET_CP))) {
4322 radeon_ring_lockup_update(ring);
4323 return false;
4324 }
4325 /* force CP activities */
4326 radeon_ring_force_activity(rdev, ring);
4327 return radeon_ring_test_lockup(rdev, ring);
6f2043ce 4328}
1c49165d
AD		 4329
4330/* MC */
4331/**
4332 * cik_mc_program - program the GPU memory controller
4333 *
4334 * @rdev: radeon_device pointer
4335 *
4336 * Set the location of vram, gart, and AGP in the GPU's
4337 * physical address space (CIK).
4338 */
4339static void cik_mc_program(struct radeon_device *rdev)
4340{
4341 struct evergreen_mc_save save;
4342 u32 tmp;
4343 int i, j;
4344
4345 /* Initialize HDP */
4346 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4347 WREG32((0x2c14 + j), 0x00000000);
4348 WREG32((0x2c18 + j), 0x00000000);
4349 WREG32((0x2c1c + j), 0x00000000);
4350 WREG32((0x2c20 + j), 0x00000000);
4351 WREG32((0x2c24 + j), 0x00000000);
4352 }
4353 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4354
4355 evergreen_mc_stop(rdev, &save);
4356 if (radeon_mc_wait_for_idle(rdev)) {
4357 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4358 }
4359 /* Lockout access through VGA aperture*/
4360 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4361 /* Update configuration */
4362 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4363 rdev->mc.vram_start >> 12);
4364 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4365 rdev->mc.vram_end >> 12);
4366 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4367 rdev->vram_scratch.gpu_addr >> 12);
4368 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4369 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4370 WREG32(MC_VM_FB_LOCATION, tmp);
4371 /* XXX double check these! */
4372 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4373 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4374 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4375 WREG32(MC_VM_AGP_BASE, 0);
4376 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4377 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4378 if (radeon_mc_wait_for_idle(rdev)) {
4379 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4380 }
4381 evergreen_mc_resume(rdev, &save);
4382 /* we need to own VRAM, so turn off the VGA renderer here
4383 * to stop it overwriting our objects */
4384 rv515_vga_render_disable(rdev);
4385}
4386
4387/**
4388 * cik_mc_init - initialize the memory controller driver params
4389 *
4390 * @rdev: radeon_device pointer
4391 *
4392 * Look up the amount of vram, vram width, and decide how to place
4393 * vram and gart within the GPU's physical address space (CIK).
4394 * Returns 0 for success.
4395 */
4396static int cik_mc_init(struct radeon_device *rdev)
4397{
4398 u32 tmp;
4399 int chansize, numchan;
4400
4401 /* Get VRAM informations */
4402 rdev->mc.vram_is_ddr = true;
4403 tmp = RREG32(MC_ARB_RAMCFG);
4404 if (tmp & CHANSIZE_MASK) {
4405 chansize = 64;
4406 } else {
4407 chansize = 32;
4408 }
4409 tmp = RREG32(MC_SHARED_CHMAP);
4410 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4411 case 0:
4412 default:
4413 numchan = 1;
4414 break;
4415 case 1:
4416 numchan = 2;
4417 break;
4418 case 2:
4419 numchan = 4;
4420 break;
4421 case 3:
4422 numchan = 8;
4423 break;
4424 case 4:
4425 numchan = 3;
4426 break;
4427 case 5:
4428 numchan = 6;
4429 break;
4430 case 6:
4431 numchan = 10;
4432 break;
4433 case 7:
4434 numchan = 12;
4435 break;
4436 case 8:
4437 numchan = 16;
4438 break;
4439 }
4440 rdev->mc.vram_width = numchan * chansize;
4441 /* Could aper size report 0 ? */
4442 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4443 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4444 /* size in MB on si */
4445 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4446 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4447 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4448 si_vram_gtt_location(rdev, &rdev->mc);
4449 radeon_update_bandwidth_info(rdev);
4450
4451 return 0;
4452}
4453
4454/*
4455 * GART
4456 * VMID 0 is the physical GPU addresses as used by the kernel.
4457 * VMIDs 1-15 are used for userspace clients and are handled
4458 * by the radeon vm/hsa code.
4459 */
4460/**
4461 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4462 *
4463 * @rdev: radeon_device pointer
4464 *
4465 * Flush the TLB for the VMID 0 page table (CIK).
4466 */
4467void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4468{
4469 /* flush hdp cache */
4470 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4471
4472 /* bits 0-15 are the VM contexts0-15 */
4473 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4474}
4475
4476/**
4477 * cik_pcie_gart_enable - gart enable
4478 *
4479 * @rdev: radeon_device pointer
4480 *
4481 * This sets up the TLBs, programs the page tables for VMID0,
4482 * sets up the hw for VMIDs 1-15 which are allocated on
4483 * demand, and sets up the global locations for the LDS, GDS,
4484 * and GPUVM for FSA64 clients (CIK).
4485 * Returns 0 for success, errors for failure.
4486 */
4487static int cik_pcie_gart_enable(struct radeon_device *rdev)
4488{
4489 int r, i;
4490
4491 if (rdev->gart.robj == NULL) {
4492 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4493 return -EINVAL;
4494 }
4495 r = radeon_gart_table_vram_pin(rdev);
4496 if (r)
4497 return r;
4498 radeon_gart_restore(rdev);
4499 /* Setup TLB control */
4500 WREG32(MC_VM_MX_L1_TLB_CNTL,
4501 (0xA << 7) |
4502 ENABLE_L1_TLB |
4503 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4504 ENABLE_ADVANCED_DRIVER_MODEL |
4505 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4506 /* Setup L2 cache */
4507 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4508 ENABLE_L2_FRAGMENT_PROCESSING |
4509 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4510 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4511 EFFECTIVE_L2_QUEUE_SIZE(7) |
4512 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4513 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4514 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4515 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4516 /* setup context0 */
4517 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4518 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4519 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4520 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4521 (u32)(rdev->dummy_page.addr >> 12));
4522 WREG32(VM_CONTEXT0_CNTL2, 0);
4523 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4524 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4525
4526 WREG32(0x15D4, 0);
4527 WREG32(0x15D8, 0);
4528 WREG32(0x15DC, 0);
4529
4530 /* empty context1-15 */
4531 /* FIXME start with 4G, once using 2 level pt switch to full
4532 * vm size space
4533 */
4534 /* set vm size, must be a multiple of 4 */
4535 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4536 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4537 for (i = 1; i < 16; i++) {
4538 if (i < 8)
4539 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4540 rdev->gart.table_addr >> 12);
4541 else
4542 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4543 rdev->gart.table_addr >> 12);
4544 }
4545
4546 /* enable context1-15 */
4547 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4548 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 4549 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 4550 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 4551 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4552 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4553 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4554 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4555 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4556 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4557 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4558 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4559 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4560 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4561 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4562 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 4563
4564 /* TC cache setup ??? */
4565 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4566 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4567 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4568
4569 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4570 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4571 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4572 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4573 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4574
4575 WREG32(TC_CFG_L1_VOLATILE, 0);
4576 WREG32(TC_CFG_L2_VOLATILE, 0);
4577
4578 if (rdev->family == CHIP_KAVERI) {
4579 u32 tmp = RREG32(CHUB_CONTROL);
4580 tmp &= ~BYPASS_VM;
4581 WREG32(CHUB_CONTROL, tmp);
4582 }
4583
4584 /* XXX SH_MEM regs */
4585 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 4586 mutex_lock(&rdev->srbm_mutex);
1c49165d 4587 for (i = 0; i < 16; i++) {
b556b12e 4588 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 4589 /* CP and shaders */
1c49165d
AD		 4590 WREG32(SH_MEM_CONFIG, 0);
4591 WREG32(SH_MEM_APE1_BASE, 1);
4592 WREG32(SH_MEM_APE1_LIMIT, 0);
4593 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 4594 /* SDMA GFX */
4595 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4596 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4597 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4598 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4599 /* XXX SDMA RLC - todo */
1c49165d 4600 }
b556b12e 4601 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4602 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 4603
4604 cik_pcie_gart_tlb_flush(rdev);
4605 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4606 (unsigned)(rdev->mc.gtt_size >> 20),
4607 (unsigned long long)rdev->gart.table_addr);
4608 rdev->gart.ready = true;
4609 return 0;
4610}
4611
4612/**
4613 * cik_pcie_gart_disable - gart disable
4614 *
4615 * @rdev: radeon_device pointer
4616 *
4617 * This disables all VM page table (CIK).
4618 */
4619static void cik_pcie_gart_disable(struct radeon_device *rdev)
4620{
4621 /* Disable all tables */
4622 WREG32(VM_CONTEXT0_CNTL, 0);
4623 WREG32(VM_CONTEXT1_CNTL, 0);
4624 /* Setup TLB control */
4625 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4626 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4627 /* Setup L2 cache */
4628 WREG32(VM_L2_CNTL,
4629 ENABLE_L2_FRAGMENT_PROCESSING |
4630 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4631 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4632 EFFECTIVE_L2_QUEUE_SIZE(7) |
4633 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4634 WREG32(VM_L2_CNTL2, 0);
4635 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4636 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4637 radeon_gart_table_vram_unpin(rdev);
4638}
4639
4640/**
4641 * cik_pcie_gart_fini - vm fini callback
4642 *
4643 * @rdev: radeon_device pointer
4644 *
4645 * Tears down the driver GART/VM setup (CIK).
4646 */
4647static void cik_pcie_gart_fini(struct radeon_device *rdev)
4648{
4649 cik_pcie_gart_disable(rdev);
4650 radeon_gart_table_vram_free(rdev);
4651 radeon_gart_fini(rdev);
4652}
4653
4654/* vm parser */
4655/**
4656 * cik_ib_parse - vm ib_parse callback
4657 *
4658 * @rdev: radeon_device pointer
4659 * @ib: indirect buffer pointer
4660 *
4661 * CIK uses hw IB checking so this is a nop (CIK).
4662 */
4663int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4664{
4665 return 0;
4666}
4667
4668/*
4669 * vm
4670 * VMID 0 is the physical GPU addresses as used by the kernel.
4671 * VMIDs 1-15 are used for userspace clients and are handled
4672 * by the radeon vm/hsa code.
4673 */
4674/**
4675 * cik_vm_init - cik vm init callback
4676 *
4677 * @rdev: radeon_device pointer
4678 *
4679 * Inits cik specific vm parameters (number of VMs, base of vram for
4680 * VMIDs 1-15) (CIK).
4681 * Returns 0 for success.
4682 */
4683int cik_vm_init(struct radeon_device *rdev)
4684{
4685 /* number of VMs */
4686 rdev->vm_manager.nvm = 16;
4687 /* base offset of vram pages */
4688 if (rdev->flags & RADEON_IS_IGP) {
4689 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4690 tmp <<= 22;
4691 rdev->vm_manager.vram_base_offset = tmp;
4692 } else
4693 rdev->vm_manager.vram_base_offset = 0;
4694
4695 return 0;
4696}
4697
4698/**
4699 * cik_vm_fini - cik vm fini callback
4700 *
4701 * @rdev: radeon_device pointer
4702 *
4703 * Tear down any asic specific VM setup (CIK).
4704 */
4705void cik_vm_fini(struct radeon_device *rdev)
4706{
4707}
4708
3ec7d11b
AD		 4709/**
4710 * cik_vm_decode_fault - print human readable fault info
4711 *
4712 * @rdev: radeon_device pointer
4713 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4714 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4715 *
4716 * Print human readable fault information (CIK).
4717 */
4718static void cik_vm_decode_fault(struct radeon_device *rdev,
4719 u32 status, u32 addr, u32 mc_client)
4720{
4721 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4722 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4723 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4724 char *block = (char *)&mc_client;
4725
4726 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4727 protections, vmid, addr,
4728 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4729 block, mc_id);
4730}
4731
f96ab484
AD		 4732/**
4733 * cik_vm_flush - cik vm flush using the CP
4734 *
4735 * @rdev: radeon_device pointer
4736 *
4737 * Update the page table base and flush the VM TLB
4738 * using the CP (CIK).
4739 */
4740void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4741{
4742 struct radeon_ring *ring = &rdev->ring[ridx];
4743
4744 if (vm == NULL)
4745 return;
4746
4747 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4748 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4749 WRITE_DATA_DST_SEL(0)));
4750 if (vm->id < 8) {
4751 radeon_ring_write(ring,
4752 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4753 } else {
4754 radeon_ring_write(ring,
4755 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4756 }
4757 radeon_ring_write(ring, 0);
4758 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4759
4760 /* update SH_MEM_* regs */
4761 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4762 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4763 WRITE_DATA_DST_SEL(0)));
4764 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4765 radeon_ring_write(ring, 0);
4766 radeon_ring_write(ring, VMID(vm->id));
4767
4768 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4769 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4770 WRITE_DATA_DST_SEL(0)));
4771 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4772 radeon_ring_write(ring, 0);
4773
4774 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4775 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4776 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4777 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4778
4779 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4780 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4781 WRITE_DATA_DST_SEL(0)));
4782 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4783 radeon_ring_write(ring, 0);
4784 radeon_ring_write(ring, VMID(0));
4785
4786 /* HDP flush */
4787 /* We should be using the WAIT_REG_MEM packet here like in
4788 * cik_fence_ring_emit(), but it causes the CP to hang in this
4789 * context...
4790 */
4791 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4792 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4793 WRITE_DATA_DST_SEL(0)));
4794 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4795 radeon_ring_write(ring, 0);
4796 radeon_ring_write(ring, 0);
4797
4798 /* bits 0-15 are the VM contexts0-15 */
4799 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4800 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4801 WRITE_DATA_DST_SEL(0)));
4802 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4803 radeon_ring_write(ring, 0);
4804 radeon_ring_write(ring, 1 << vm->id);
4805
b07fdd38
AD		 4806 /* compute doesn't have PFP */
4807 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4808 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4809 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4810 radeon_ring_write(ring, 0x0);
4811 }
f96ab484
AD		 4812}
4813
d0e092d9
AD		 4814/**
4815 * cik_vm_set_page - update the page tables using sDMA
4816 *
4817 * @rdev: radeon_device pointer
4818 * @ib: indirect buffer to fill with commands
4819 * @pe: addr of the page entry
4820 * @addr: dst addr to write into pe
4821 * @count: number of page entries to update
4822 * @incr: increase next addr by incr bytes
4823 * @flags: access flags
4824 *
4825 * Update the page tables using CP or sDMA (CIK).
4826 */
4827void cik_vm_set_page(struct radeon_device *rdev,
4828 struct radeon_ib *ib,
4829 uint64_t pe,
4830 uint64_t addr, unsigned count,
4831 uint32_t incr, uint32_t flags)
4832{
4833 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4834 uint64_t value;
4835 unsigned ndw;
4836
4837 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4838 /* CP */
4839 while (count) {
4840 ndw = 2 + count * 2;
4841 if (ndw > 0x3FFE)
4842 ndw = 0x3FFE;
4843
4844 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4845 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4846 WRITE_DATA_DST_SEL(1));
4847 ib->ptr[ib->length_dw++] = pe;
4848 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4849 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4850 if (flags & RADEON_VM_PAGE_SYSTEM) {
4851 value = radeon_vm_map_gart(rdev, addr);
4852 value &= 0xFFFFFFFFFFFFF000ULL;
4853 } else if (flags & RADEON_VM_PAGE_VALID) {
4854 value = addr;
4855 } else {
4856 value = 0;
4857 }
4858 addr += incr;
4859 value |= r600_flags;
4860 ib->ptr[ib->length_dw++] = value;
4861 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4862 }
4863 }
4864 } else {
4865 /* DMA */
2483b4ea 4866 cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
605de6b9 4867 }
605de6b9
AD		 4868}
4869
f6796cae
AD		 4870/*
4871 * RLC
4872 * The RLC is a multi-purpose microengine that handles a
4873 * variety of functions, the most important of which is
4874 * the interrupt controller.
4875 */
866d83de
AD		 4876static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4877 bool enable)
f6796cae 4878{
866d83de 4879 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 4880
866d83de
AD		 4881 if (enable)
4882 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4883 else
4884 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 4885 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 4886}
f6796cae 4887
866d83de
AD		 4888static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
4889{
4890 u32 tmp;
f6796cae 4891
866d83de
AD		 4892 tmp = RREG32(RLC_LB_CNTL);
4893 if (enable)
4894 tmp |= LOAD_BALANCE_ENABLE;
4895 else
4896 tmp &= ~LOAD_BALANCE_ENABLE;
4897 WREG32(RLC_LB_CNTL, tmp);
4898}
f6796cae 4899
866d83de
AD		 4900static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4901{
4902 u32 i, j, k;
4903 u32 mask;
f6796cae
AD		 4904
4905 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4906 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4907 cik_select_se_sh(rdev, i, j);
4908 for (k = 0; k < rdev->usec_timeout; k++) {
4909 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4910 break;
4911 udelay(1);
4912 }
4913 }
4914 }
4915 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4916
4917 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4918 for (k = 0; k < rdev->usec_timeout; k++) {
4919 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4920 break;
4921 udelay(1);
4922 }
4923}
4924
22c775ce
AD		 4925static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
4926{
4927 u32 tmp;
4928
4929 tmp = RREG32(RLC_CNTL);
4930 if (tmp != rlc)
4931 WREG32(RLC_CNTL, rlc);
4932}
4933
4934static u32 cik_halt_rlc(struct radeon_device *rdev)
4935{
4936 u32 data, orig;
4937
4938 orig = data = RREG32(RLC_CNTL);
4939
4940 if (data & RLC_ENABLE) {
4941 u32 i;
4942
4943 data &= ~RLC_ENABLE;
4944 WREG32(RLC_CNTL, data);
4945
4946 for (i = 0; i < rdev->usec_timeout; i++) {
4947 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
4948 break;
4949 udelay(1);
4950 }
4951
4952 cik_wait_for_rlc_serdes(rdev);
4953 }
4954
4955 return orig;
4956}
4957
a412fce0
AD		 4958void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
4959{
4960 u32 tmp, i, mask;
4961
4962 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
4963 WREG32(RLC_GPR_REG2, tmp);
4964
4965 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
4966 for (i = 0; i < rdev->usec_timeout; i++) {
4967 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
4968 break;
4969 udelay(1);
4970 }
4971
4972 for (i = 0; i < rdev->usec_timeout; i++) {
4973 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
4974 break;
4975 udelay(1);
4976 }
4977}
4978
4979void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
4980{
4981 u32 tmp;
4982
4983 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
4984 WREG32(RLC_GPR_REG2, tmp);
4985}
4986
866d83de
AD		 4987/**
4988 * cik_rlc_stop - stop the RLC ME
4989 *
4990 * @rdev: radeon_device pointer
4991 *
4992 * Halt the RLC ME (MicroEngine) (CIK).
4993 */
4994static void cik_rlc_stop(struct radeon_device *rdev)
4995{
22c775ce 4996 WREG32(RLC_CNTL, 0);
866d83de
AD		 4997
4998 cik_enable_gui_idle_interrupt(rdev, false);
4999
866d83de
AD		 5000 cik_wait_for_rlc_serdes(rdev);
5001}
5002
f6796cae
AD		 5003/**
5004 * cik_rlc_start - start the RLC ME
5005 *
5006 * @rdev: radeon_device pointer
5007 *
5008 * Unhalt the RLC ME (MicroEngine) (CIK).
5009 */
5010static void cik_rlc_start(struct radeon_device *rdev)
5011{
f6796cae
AD		 5012 WREG32(RLC_CNTL, RLC_ENABLE);
5013
866d83de 5014 cik_enable_gui_idle_interrupt(rdev, true);
f6796cae
AD		 5015
5016 udelay(50);
5017}
5018
5019/**
5020 * cik_rlc_resume - setup the RLC hw
5021 *
5022 * @rdev: radeon_device pointer
5023 *
5024 * Initialize the RLC registers, load the ucode,
5025 * and start the RLC (CIK).
5026 * Returns 0 for success, -EINVAL if the ucode is not available.
5027 */
5028static int cik_rlc_resume(struct radeon_device *rdev)
5029{
22c775ce 5030 u32 i, size, tmp;
f6796cae
AD		 5031 const __be32 *fw_data;
5032
5033 if (!rdev->rlc_fw)
5034 return -EINVAL;
5035
5036 switch (rdev->family) {
5037 case CHIP_BONAIRE:
5038 default:
5039 size = BONAIRE_RLC_UCODE_SIZE;
5040 break;
5041 case CHIP_KAVERI:
5042 size = KV_RLC_UCODE_SIZE;
5043 break;
5044 case CHIP_KABINI:
5045 size = KB_RLC_UCODE_SIZE;
5046 break;
5047 }
5048
5049 cik_rlc_stop(rdev);
5050
22c775ce
AD		 5051 /* disable CG */
5052 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5053 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5054
866d83de 5055 si_rlc_reset(rdev);
f6796cae 5056
22c775ce
AD		 5057 cik_init_pg(rdev);
5058
5059 cik_init_cg(rdev);
5060
f6796cae
AD		 5061 WREG32(RLC_LB_CNTR_INIT, 0);
5062 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5063
5064 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5065 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5066 WREG32(RLC_LB_PARAMS, 0x00600408);
5067 WREG32(RLC_LB_CNTL, 0x80000004);
5068
5069 WREG32(RLC_MC_CNTL, 0);
5070 WREG32(RLC_UCODE_CNTL, 0);
5071
5072 fw_data = (const __be32 *)rdev->rlc_fw->data;
5073 WREG32(RLC_GPM_UCODE_ADDR, 0);
5074 for (i = 0; i < size; i++)
5075 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5076 WREG32(RLC_GPM_UCODE_ADDR, 0);
5077
866d83de
AD		 5078 /* XXX - find out what chips support lbpw */
5079 cik_enable_lbpw(rdev, false);
5080
22c775ce
AD		 5081 if (rdev->family == CHIP_BONAIRE)
5082 WREG32(RLC_DRIVER_DMA_STATUS, 0);
f6796cae
AD		 5083
5084 cik_rlc_start(rdev);
5085
5086 return 0;
5087}
a59781bb 5088
22c775ce
AD		 5089static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5090{
5091 u32 data, orig, tmp, tmp2;
5092
5093 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5094
473359bc 5095 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6
AD		 5096 cik_enable_gui_idle_interrupt(rdev, true);
5097
22c775ce
AD		 5098 tmp = cik_halt_rlc(rdev);
5099
5100 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5101 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5102 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5103 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5104 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5105
5106 cik_update_rlc(rdev, tmp);
5107
5108 data |= CGCG_EN | CGLS_EN;
5109 } else {
ddc76ff6
AD		 5110 cik_enable_gui_idle_interrupt(rdev, false);
5111
22c775ce
AD		 5112 RREG32(CB_CGTT_SCLK_CTRL);
5113 RREG32(CB_CGTT_SCLK_CTRL);
5114 RREG32(CB_CGTT_SCLK_CTRL);
5115 RREG32(CB_CGTT_SCLK_CTRL);
5116
5117 data &= ~(CGCG_EN | CGLS_EN);
5118 }
5119
5120 if (orig != data)
5121 WREG32(RLC_CGCG_CGLS_CTRL, data);
5122
5123}
5124
5125static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5126{
5127 u32 data, orig, tmp = 0;
5128
473359bc
AD		 5129 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5130 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5131 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5132 orig = data = RREG32(CP_MEM_SLP_CNTL);
5133 data |= CP_MEM_LS_EN;
5134 if (orig != data)
5135 WREG32(CP_MEM_SLP_CNTL, data);
5136 }
5137 }
22c775ce
AD		 5138
5139 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5140 data &= 0xfffffffd;
5141 if (orig != data)
5142 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5143
5144 tmp = cik_halt_rlc(rdev);
5145
5146 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5147 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5148 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5149 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5150 WREG32(RLC_SERDES_WR_CTRL, data);
5151
5152 cik_update_rlc(rdev, tmp);
5153
473359bc
AD		 5154 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5155 orig = data = RREG32(CGTS_SM_CTRL_REG);
5156 data &= ~SM_MODE_MASK;
5157 data |= SM_MODE(0x2);
5158 data |= SM_MODE_ENABLE;
5159 data &= ~CGTS_OVERRIDE;
5160 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5161 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5162 data &= ~CGTS_LS_OVERRIDE;
5163 data &= ~ON_MONITOR_ADD_MASK;
5164 data |= ON_MONITOR_ADD_EN;
5165 data |= ON_MONITOR_ADD(0x96);
5166 if (orig != data)
5167 WREG32(CGTS_SM_CTRL_REG, data);
5168 }
22c775ce
AD		 5169 } else {
5170 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5171 data |= 0x00000002;
5172 if (orig != data)
5173 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5174
5175 data = RREG32(RLC_MEM_SLP_CNTL);
5176 if (data & RLC_MEM_LS_EN) {
5177 data &= ~RLC_MEM_LS_EN;
5178 WREG32(RLC_MEM_SLP_CNTL, data);
5179 }
5180
5181 data = RREG32(CP_MEM_SLP_CNTL);
5182 if (data & CP_MEM_LS_EN) {
5183 data &= ~CP_MEM_LS_EN;
5184 WREG32(CP_MEM_SLP_CNTL, data);
5185 }
5186
5187 orig = data = RREG32(CGTS_SM_CTRL_REG);
5188 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5189 if (orig != data)
5190 WREG32(CGTS_SM_CTRL_REG, data);
5191
5192 tmp = cik_halt_rlc(rdev);
5193
5194 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5195 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5196 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5197 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5198 WREG32(RLC_SERDES_WR_CTRL, data);
5199
5200 cik_update_rlc(rdev, tmp);
5201 }
5202}
5203
5204static const u32 mc_cg_registers[] =
5205{
5206 MC_HUB_MISC_HUB_CG,
5207 MC_HUB_MISC_SIP_CG,
5208 MC_HUB_MISC_VM_CG,
5209 MC_XPB_CLK_GAT,
5210 ATC_MISC_CG,
5211 MC_CITF_MISC_WR_CG,
5212 MC_CITF_MISC_RD_CG,
5213 MC_CITF_MISC_VM_CG,
5214 VM_L2_CG,
5215};
5216
5217static void cik_enable_mc_ls(struct radeon_device *rdev,
5218 bool enable)
5219{
5220 int i;
5221 u32 orig, data;
5222
5223 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5224 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5225 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5226 data |= MC_LS_ENABLE;
5227 else
5228 data &= ~MC_LS_ENABLE;
5229 if (data != orig)
5230 WREG32(mc_cg_registers[i], data);
5231 }
5232}
5233
5234static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5235 bool enable)
5236{
5237 int i;
5238 u32 orig, data;
5239
5240 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5241 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5242 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5243 data |= MC_CG_ENABLE;
5244 else
5245 data &= ~MC_CG_ENABLE;
5246 if (data != orig)
5247 WREG32(mc_cg_registers[i], data);
5248 }
5249}
5250
5251static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5252 bool enable)
5253{
5254 u32 orig, data;
5255
473359bc 5256 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5257 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5258 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5259 } else {
5260 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5261 data |= 0xff000000;
5262 if (data != orig)
5263 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5264
5265 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5266 data |= 0xff000000;
5267 if (data != orig)
5268 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5269 }
5270}
5271
5272static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5273 bool enable)
5274{
5275 u32 orig, data;
5276
473359bc 5277 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5278 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5279 data |= 0x100;
5280 if (orig != data)
5281 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5282
5283 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5284 data |= 0x100;
5285 if (orig != data)
5286 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5287 } else {
5288 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5289 data &= ~0x100;
5290 if (orig != data)
5291 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5292
5293 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5294 data &= ~0x100;
5295 if (orig != data)
5296 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5297 }
5298}
5299
5300static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5301 bool enable)
5302{
5303 u32 orig, data;
5304
473359bc 5305 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 5306 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5307 data = 0xfff;
5308 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5309
5310 orig = data = RREG32(UVD_CGC_CTRL);
5311 data |= DCM;
5312 if (orig != data)
5313 WREG32(UVD_CGC_CTRL, data);
5314 } else {
5315 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5316 data &= ~0xfff;
5317 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5318
5319 orig = data = RREG32(UVD_CGC_CTRL);
5320 data &= ~DCM;
5321 if (orig != data)
5322 WREG32(UVD_CGC_CTRL, data);
5323 }
5324}
5325
473359bc
AD		 5326static void cik_enable_bif_mgls(struct radeon_device *rdev,
5327 bool enable)
5328{
5329 u32 orig, data;
5330
5331 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5332
5333 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5334 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5335 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5336 else
5337 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5338 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5339
5340 if (orig != data)
5341 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5342}
5343
22c775ce
AD		 5344static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5345 bool enable)
5346{
5347 u32 orig, data;
5348
5349 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5350
473359bc 5351 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 5352 data &= ~CLOCK_GATING_DIS;
5353 else
5354 data |= CLOCK_GATING_DIS;
5355
5356 if (orig != data)
5357 WREG32(HDP_HOST_PATH_CNTL, data);
5358}
5359
5360static void cik_enable_hdp_ls(struct radeon_device *rdev,
5361 bool enable)
5362{
5363 u32 orig, data;
5364
5365 orig = data = RREG32(HDP_MEM_POWER_LS);
5366
473359bc 5367 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 5368 data |= HDP_LS_ENABLE;
5369 else
5370 data &= ~HDP_LS_ENABLE;
5371
5372 if (orig != data)
5373 WREG32(HDP_MEM_POWER_LS, data);
5374}
5375
5376void cik_update_cg(struct radeon_device *rdev,
5377 u32 block, bool enable)
5378{
5379 if (block & RADEON_CG_BLOCK_GFX) {
5380 /* order matters! */
5381 if (enable) {
5382 cik_enable_mgcg(rdev, true);
5383 cik_enable_cgcg(rdev, true);
5384 } else {
5385 cik_enable_cgcg(rdev, false);
5386 cik_enable_mgcg(rdev, false);
5387 }
5388 }
5389
5390 if (block & RADEON_CG_BLOCK_MC) {
5391 if (!(rdev->flags & RADEON_IS_IGP)) {
5392 cik_enable_mc_mgcg(rdev, enable);
5393 cik_enable_mc_ls(rdev, enable);
5394 }
5395 }
5396
5397 if (block & RADEON_CG_BLOCK_SDMA) {
5398 cik_enable_sdma_mgcg(rdev, enable);
5399 cik_enable_sdma_mgls(rdev, enable);
5400 }
5401
473359bc
AD		 5402 if (block & RADEON_CG_BLOCK_BIF) {
5403 cik_enable_bif_mgls(rdev, enable);
5404 }
5405
22c775ce
AD		 5406 if (block & RADEON_CG_BLOCK_UVD) {
5407 if (rdev->has_uvd)
5408 cik_enable_uvd_mgcg(rdev, enable);
5409 }
5410
5411 if (block & RADEON_CG_BLOCK_HDP) {
5412 cik_enable_hdp_mgcg(rdev, enable);
5413 cik_enable_hdp_ls(rdev, enable);
5414 }
5415}
5416
5417static void cik_init_cg(struct radeon_device *rdev)
5418{
5419
ddc76ff6 5420 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 5421
5422 if (rdev->has_uvd)
5423 si_init_uvd_internal_cg(rdev);
5424
5425 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5426 RADEON_CG_BLOCK_SDMA |
473359bc 5427 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 5428 RADEON_CG_BLOCK_UVD |
5429 RADEON_CG_BLOCK_HDP), true);
5430}
5431
473359bc
AD		 5432static void cik_fini_cg(struct radeon_device *rdev)
5433{
5434 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5435 RADEON_CG_BLOCK_SDMA |
5436 RADEON_CG_BLOCK_BIF |
5437 RADEON_CG_BLOCK_UVD |
5438 RADEON_CG_BLOCK_HDP), false);
5439
5440 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5441}
5442
22c775ce
AD		 5443static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5444 bool enable)
5445{
5446 u32 data, orig;
5447
5448 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5449 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5450 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5451 else
5452 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5453 if (orig != data)
5454 WREG32(RLC_PG_CNTL, data);
5455}
5456
5457static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5458 bool enable)
5459{
5460 u32 data, orig;
5461
5462 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5463 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5464 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5465 else
5466 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5467 if (orig != data)
5468 WREG32(RLC_PG_CNTL, data);
5469}
5470
5471static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5472{
5473 u32 data, orig;
5474
5475 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5476 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 5477 data &= ~DISABLE_CP_PG;
5478 else
5479 data |= DISABLE_CP_PG;
5480 if (orig != data)
5481 WREG32(RLC_PG_CNTL, data);
5482}
5483
5484static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5485{
5486 u32 data, orig;
5487
5488 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5489 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 5490 data &= ~DISABLE_GDS_PG;
5491 else
5492 data |= DISABLE_GDS_PG;
5493 if (orig != data)
5494 WREG32(RLC_PG_CNTL, data);
5495}
5496
5497#define CP_ME_TABLE_SIZE 96
5498#define CP_ME_TABLE_OFFSET 2048
5499#define CP_MEC_TABLE_OFFSET 4096
5500
5501void cik_init_cp_pg_table(struct radeon_device *rdev)
5502{
5503 const __be32 *fw_data;
5504 volatile u32 *dst_ptr;
5505 int me, i, max_me = 4;
5506 u32 bo_offset = 0;
5507 u32 table_offset;
5508
5509 if (rdev->family == CHIP_KAVERI)
5510 max_me = 5;
5511
5512 if (rdev->rlc.cp_table_ptr == NULL)
5513 return;
5514
5515 /* write the cp table buffer */
5516 dst_ptr = rdev->rlc.cp_table_ptr;
5517 for (me = 0; me < max_me; me++) {
5518 if (me == 0) {
5519 fw_data = (const __be32 *)rdev->ce_fw->data;
5520 table_offset = CP_ME_TABLE_OFFSET;
5521 } else if (me == 1) {
5522 fw_data = (const __be32 *)rdev->pfp_fw->data;
5523 table_offset = CP_ME_TABLE_OFFSET;
5524 } else if (me == 2) {
5525 fw_data = (const __be32 *)rdev->me_fw->data;
5526 table_offset = CP_ME_TABLE_OFFSET;
5527 } else {
5528 fw_data = (const __be32 *)rdev->mec_fw->data;
5529 table_offset = CP_MEC_TABLE_OFFSET;
5530 }
5531
5532 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
5533 dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
5534 }
5535 bo_offset += CP_ME_TABLE_SIZE;
5536 }
5537}
5538
5539static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5540 bool enable)
5541{
5542 u32 data, orig;
5543
473359bc 5544 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) {
22c775ce
AD		 5545 orig = data = RREG32(RLC_PG_CNTL);
5546 data |= GFX_PG_ENABLE;
5547 if (orig != data)
5548 WREG32(RLC_PG_CNTL, data);
5549
5550 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5551 data |= AUTO_PG_EN;
5552 if (orig != data)
5553 WREG32(RLC_AUTO_PG_CTRL, data);
5554 } else {
5555 orig = data = RREG32(RLC_PG_CNTL);
5556 data &= ~GFX_PG_ENABLE;
5557 if (orig != data)
5558 WREG32(RLC_PG_CNTL, data);
5559
5560 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5561 data &= ~AUTO_PG_EN;
5562 if (orig != data)
5563 WREG32(RLC_AUTO_PG_CTRL, data);
5564
5565 data = RREG32(DB_RENDER_CONTROL);
5566 }
5567}
5568
5569static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5570{
5571 u32 mask = 0, tmp, tmp1;
5572 int i;
5573
5574 cik_select_se_sh(rdev, se, sh);
5575 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5576 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5577 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5578
5579 tmp &= 0xffff0000;
5580
5581 tmp |= tmp1;
5582 tmp >>= 16;
5583
5584 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5585 mask <<= 1;
5586 mask |= 1;
5587 }
5588
5589 return (~tmp) & mask;
5590}
5591
5592static void cik_init_ao_cu_mask(struct radeon_device *rdev)
5593{
5594 u32 i, j, k, active_cu_number = 0;
5595 u32 mask, counter, cu_bitmap;
5596 u32 tmp = 0;
5597
5598 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5599 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5600 mask = 1;
5601 cu_bitmap = 0;
5602 counter = 0;
5603 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5604 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5605 if (counter < 2)
5606 cu_bitmap |= mask;
5607 counter ++;
5608 }
5609 mask <<= 1;
5610 }
5611
5612 active_cu_number += counter;
5613 tmp |= (cu_bitmap << (i * 16 + j * 8));
5614 }
5615 }
5616
5617 WREG32(RLC_PG_AO_CU_MASK, tmp);
5618
5619 tmp = RREG32(RLC_MAX_PG_CU);
5620 tmp &= ~MAX_PU_CU_MASK;
5621 tmp |= MAX_PU_CU(active_cu_number);
5622 WREG32(RLC_MAX_PG_CU, tmp);
5623}
5624
5625static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5626 bool enable)
5627{
5628 u32 data, orig;
5629
5630 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5631 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 5632 data |= STATIC_PER_CU_PG_ENABLE;
5633 else
5634 data &= ~STATIC_PER_CU_PG_ENABLE;
5635 if (orig != data)
5636 WREG32(RLC_PG_CNTL, data);
5637}
5638
5639static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5640 bool enable)
5641{
5642 u32 data, orig;
5643
5644 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5645 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce
AD		 5646 data |= DYN_PER_CU_PG_ENABLE;
5647 else
5648 data &= ~DYN_PER_CU_PG_ENABLE;
5649 if (orig != data)
5650 WREG32(RLC_PG_CNTL, data);
5651}
5652
5653#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5654#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
5655
5656static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5657{
5658 u32 data, orig;
5659 u32 i;
5660
5661 if (rdev->rlc.cs_data) {
5662 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5663 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 5664 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce
AD		 5665 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
5666 } else {
5667 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5668 for (i = 0; i < 3; i++)
5669 WREG32(RLC_GPM_SCRATCH_DATA, 0);
5670 }
5671 if (rdev->rlc.reg_list) {
5672 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5673 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5674 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
5675 }
5676
5677 orig = data = RREG32(RLC_PG_CNTL);
5678 data |= GFX_PG_SRC;
5679 if (orig != data)
5680 WREG32(RLC_PG_CNTL, data);
5681
5682 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5683 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
5684
5685 data = RREG32(CP_RB_WPTR_POLL_CNTL);
5686 data &= ~IDLE_POLL_COUNT_MASK;
5687 data |= IDLE_POLL_COUNT(0x60);
5688 WREG32(CP_RB_WPTR_POLL_CNTL, data);
5689
5690 data = 0x10101010;
5691 WREG32(RLC_PG_DELAY, data);
5692
5693 data = RREG32(RLC_PG_DELAY_2);
5694 data &= ~0xff;
5695 data |= 0x3;
5696 WREG32(RLC_PG_DELAY_2, data);
5697
5698 data = RREG32(RLC_AUTO_PG_CTRL);
5699 data &= ~GRBM_REG_SGIT_MASK;
5700 data |= GRBM_REG_SGIT(0x700);
5701 WREG32(RLC_AUTO_PG_CTRL, data);
5702
5703}
5704
5705static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
5706{
473359bc
AD		 5707 cik_enable_gfx_cgpg(rdev, enable);
5708 cik_enable_gfx_static_mgpg(rdev, enable);
5709 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce
AD		 5710}
5711
a0f38609
AD		 5712u32 cik_get_csb_size(struct radeon_device *rdev)
5713{
5714 u32 count = 0;
5715 const struct cs_section_def *sect = NULL;
5716 const struct cs_extent_def *ext = NULL;
5717
5718 if (rdev->rlc.cs_data == NULL)
5719 return 0;
5720
5721 /* begin clear state */
5722 count += 2;
5723 /* context control state */
5724 count += 3;
5725
5726 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5727 for (ext = sect->section; ext->extent != NULL; ++ext) {
5728 if (sect->id == SECT_CONTEXT)
5729 count += 2 + ext->reg_count;
5730 else
5731 return 0;
5732 }
5733 }
5734 /* pa_sc_raster_config/pa_sc_raster_config1 */
5735 count += 4;
5736 /* end clear state */
5737 count += 2;
5738 /* clear state */
5739 count += 2;
5740
5741 return count;
5742}
5743
5744void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5745{
5746 u32 count = 0, i;
5747 const struct cs_section_def *sect = NULL;
5748 const struct cs_extent_def *ext = NULL;
5749
5750 if (rdev->rlc.cs_data == NULL)
5751 return;
5752 if (buffer == NULL)
5753 return;
5754
5755 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5756 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5757
5758 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5759 buffer[count++] = 0x80000000;
5760 buffer[count++] = 0x80000000;
5761
5762 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5763 for (ext = sect->section; ext->extent != NULL; ++ext) {
5764 if (sect->id == SECT_CONTEXT) {
5765 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5766 buffer[count++] = ext->reg_index - 0xa000;
5767 for (i = 0; i < ext->reg_count; i++)
5768 buffer[count++] = ext->extent[i];
5769 } else {
5770 return;
5771 }
5772 }
5773 }
5774
5775 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
5776 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5777 switch (rdev->family) {
5778 case CHIP_BONAIRE:
5779 buffer[count++] = 0x16000012;
5780 buffer[count++] = 0x00000000;
5781 break;
5782 case CHIP_KAVERI:
5783 buffer[count++] = 0x00000000; /* XXX */
5784 buffer[count++] = 0x00000000;
5785 break;
5786 case CHIP_KABINI:
5787 buffer[count++] = 0x00000000; /* XXX */
5788 buffer[count++] = 0x00000000;
5789 break;
5790 default:
5791 buffer[count++] = 0x00000000;
5792 buffer[count++] = 0x00000000;
5793 break;
5794 }
5795
5796 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5797 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5798
5799 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5800 buffer[count++] = 0;
5801}
5802
473359bc 5803static void cik_init_pg(struct radeon_device *rdev)
22c775ce 5804{
473359bc 5805 if (rdev->pg_flags) {
22c775ce
AD		 5806 cik_enable_sck_slowdown_on_pu(rdev, true);
5807 cik_enable_sck_slowdown_on_pd(rdev, true);
473359bc
AD		 5808 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5809 cik_init_gfx_cgpg(rdev);
5810 cik_enable_cp_pg(rdev, true);
5811 cik_enable_gds_pg(rdev, true);
5812 }
22c775ce
AD		 5813 cik_init_ao_cu_mask(rdev);
5814 cik_update_gfx_pg(rdev, true);
5815 }
5816}
5817
473359bc
AD		 5818static void cik_fini_pg(struct radeon_device *rdev)
5819{
5820 if (rdev->pg_flags) {
5821 cik_update_gfx_pg(rdev, false);
5822 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5823 cik_enable_cp_pg(rdev, false);
5824 cik_enable_gds_pg(rdev, false);
5825 }
5826 }
5827}
5828
a59781bb
AD		 5829/*
5830 * Interrupts
5831 * Starting with r6xx, interrupts are handled via a ring buffer.
5832 * Ring buffers are areas of GPU accessible memory that the GPU
5833 * writes interrupt vectors into and the host reads vectors out of.
5834 * There is a rptr (read pointer) that determines where the
5835 * host is currently reading, and a wptr (write pointer)
5836 * which determines where the GPU has written. When the
5837 * pointers are equal, the ring is idle. When the GPU
5838 * writes vectors to the ring buffer, it increments the
5839 * wptr. When there is an interrupt, the host then starts
5840 * fetching commands and processing them until the pointers are
5841 * equal again at which point it updates the rptr.
5842 */
5843
5844/**
5845 * cik_enable_interrupts - Enable the interrupt ring buffer
5846 *
5847 * @rdev: radeon_device pointer
5848 *
5849 * Enable the interrupt ring buffer (CIK).
5850 */
5851static void cik_enable_interrupts(struct radeon_device *rdev)
5852{
5853 u32 ih_cntl = RREG32(IH_CNTL);
5854 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5855
5856 ih_cntl |= ENABLE_INTR;
5857 ih_rb_cntl |= IH_RB_ENABLE;
5858 WREG32(IH_CNTL, ih_cntl);
5859 WREG32(IH_RB_CNTL, ih_rb_cntl);
5860 rdev->ih.enabled = true;
5861}
5862
5863/**
5864 * cik_disable_interrupts - Disable the interrupt ring buffer
5865 *
5866 * @rdev: radeon_device pointer
5867 *
5868 * Disable the interrupt ring buffer (CIK).
5869 */
5870static void cik_disable_interrupts(struct radeon_device *rdev)
5871{
5872 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5873 u32 ih_cntl = RREG32(IH_CNTL);
5874
5875 ih_rb_cntl &= ~IH_RB_ENABLE;
5876 ih_cntl &= ~ENABLE_INTR;
5877 WREG32(IH_RB_CNTL, ih_rb_cntl);
5878 WREG32(IH_CNTL, ih_cntl);
5879 /* set rptr, wptr to 0 */
5880 WREG32(IH_RB_RPTR, 0);
5881 WREG32(IH_RB_WPTR, 0);
5882 rdev->ih.enabled = false;
5883 rdev->ih.rptr = 0;
5884}
5885
5886/**
5887 * cik_disable_interrupt_state - Disable all interrupt sources
5888 *
5889 * @rdev: radeon_device pointer
5890 *
5891 * Clear all interrupt enable bits used by the driver (CIK).
5892 */
5893static void cik_disable_interrupt_state(struct radeon_device *rdev)
5894{
5895 u32 tmp;
5896
5897 /* gfx ring */
5898 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
21a93e13
AD		 5899 /* sdma */
5900 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5901 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5902 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5903 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 5904 /* compute queues */
5905 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
5906 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
5907 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
5908 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
5909 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
5910 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
5911 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
5912 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
5913 /* grbm */
5914 WREG32(GRBM_INT_CNTL, 0);
5915 /* vline/vblank, etc. */
5916 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5917 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5918 if (rdev->num_crtc >= 4) {
5919 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5920 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5921 }
5922 if (rdev->num_crtc >= 6) {
5923 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5924 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5925 }
5926
5927 /* dac hotplug */
5928 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5929
5930 /* digital hotplug */
5931 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5932 WREG32(DC_HPD1_INT_CONTROL, tmp);
5933 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5934 WREG32(DC_HPD2_INT_CONTROL, tmp);
5935 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5936 WREG32(DC_HPD3_INT_CONTROL, tmp);
5937 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5938 WREG32(DC_HPD4_INT_CONTROL, tmp);
5939 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5940 WREG32(DC_HPD5_INT_CONTROL, tmp);
5941 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5942 WREG32(DC_HPD6_INT_CONTROL, tmp);
5943
5944}
5945
5946/**
5947 * cik_irq_init - init and enable the interrupt ring
5948 *
5949 * @rdev: radeon_device pointer
5950 *
5951 * Allocate a ring buffer for the interrupt controller,
5952 * enable the RLC, disable interrupts, enable the IH
5953 * ring buffer and enable it (CIK).
5954 * Called at device load and reume.
5955 * Returns 0 for success, errors for failure.
5956 */
5957static int cik_irq_init(struct radeon_device *rdev)
5958{
5959 int ret = 0;
5960 int rb_bufsz;
5961 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5962
5963 /* allocate ring */
5964 ret = r600_ih_ring_alloc(rdev);
5965 if (ret)
5966 return ret;
5967
5968 /* disable irqs */
5969 cik_disable_interrupts(rdev);
5970
5971 /* init rlc */
5972 ret = cik_rlc_resume(rdev);
5973 if (ret) {
5974 r600_ih_ring_fini(rdev);
5975 return ret;
5976 }
5977
5978 /* setup interrupt control */
5979 /* XXX this should actually be a bus address, not an MC address. same on older asics */
5980 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5981 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5982 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5983 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5984 */
5985 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5986 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5987 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5988 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5989
5990 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5991 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5992
5993 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5994 IH_WPTR_OVERFLOW_CLEAR |
5995 (rb_bufsz << 1));
5996
5997 if (rdev->wb.enabled)
5998 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5999
6000 /* set the writeback address whether it's enabled or not */
6001 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6002 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6003
6004 WREG32(IH_RB_CNTL, ih_rb_cntl);
6005
6006 /* set rptr, wptr to 0 */
6007 WREG32(IH_RB_RPTR, 0);
6008 WREG32(IH_RB_WPTR, 0);
6009
6010 /* Default settings for IH_CNTL (disabled at first) */
6011 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6012 /* RPTR_REARM only works if msi's are enabled */
6013 if (rdev->msi_enabled)
6014 ih_cntl |= RPTR_REARM;
6015 WREG32(IH_CNTL, ih_cntl);
6016
6017 /* force the active interrupt state to all disabled */
6018 cik_disable_interrupt_state(rdev);
6019
6020 pci_set_master(rdev->pdev);
6021
6022 /* enable irqs */
6023 cik_enable_interrupts(rdev);
6024
6025 return ret;
6026}
6027
6028/**
6029 * cik_irq_set - enable/disable interrupt sources
6030 *
6031 * @rdev: radeon_device pointer
6032 *
6033 * Enable interrupt sources on the GPU (vblanks, hpd,
6034 * etc.) (CIK).
6035 * Returns 0 for success, errors for failure.
6036 */
6037int cik_irq_set(struct radeon_device *rdev)
6038{
6039 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
6040 PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
2b0781a6
AD		 6041 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6042 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6043 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6044 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6045 u32 grbm_int_cntl = 0;
21a93e13 6046 u32 dma_cntl, dma_cntl1;
41a524ab 6047 u32 thermal_int;
a59781bb
AD		 6048
6049 if (!rdev->irq.installed) {
6050 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6051 return -EINVAL;
6052 }
6053 /* don't enable anything if the ih is disabled */
6054 if (!rdev->ih.enabled) {
6055 cik_disable_interrupts(rdev);
6056 /* force the active interrupt state to all disabled */
6057 cik_disable_interrupt_state(rdev);
6058 return 0;
6059 }
6060
6061 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6062 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6063 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6064 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6065 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6066 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6067
21a93e13
AD		 6068 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6069 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6070
2b0781a6
AD		 6071 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6072 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6073 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6074 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6075 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6076 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6077 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6078 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6079
cc8dbbb4
AD		 6080 if (rdev->flags & RADEON_IS_IGP)
6081 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6082 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6083 else
6084 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6085 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6086
a59781bb
AD		 6087 /* enable CP interrupts on all rings */
6088 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6089 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6090 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6091 }
2b0781a6
AD		 6092 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6093 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6094 DRM_DEBUG("si_irq_set: sw int cp1\n");
6095 if (ring->me == 1) {
6096 switch (ring->pipe) {
6097 case 0:
6098 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6099 break;
6100 case 1:
6101 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6102 break;
6103 case 2:
6104 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6105 break;
6106 case 3:
6107 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6108 break;
6109 default:
6110 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6111 break;
6112 }
6113 } else if (ring->me == 2) {
6114 switch (ring->pipe) {
6115 case 0:
6116 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6117 break;
6118 case 1:
6119 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6120 break;
6121 case 2:
6122 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6123 break;
6124 case 3:
6125 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6126 break;
6127 default:
6128 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6129 break;
6130 }
6131 } else {
6132 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6133 }
6134 }
6135 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6136 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6137 DRM_DEBUG("si_irq_set: sw int cp2\n");
6138 if (ring->me == 1) {
6139 switch (ring->pipe) {
6140 case 0:
6141 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6142 break;
6143 case 1:
6144 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6145 break;
6146 case 2:
6147 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6148 break;
6149 case 3:
6150 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6151 break;
6152 default:
6153 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6154 break;
6155 }
6156 } else if (ring->me == 2) {
6157 switch (ring->pipe) {
6158 case 0:
6159 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6160 break;
6161 case 1:
6162 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6163 break;
6164 case 2:
6165 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6166 break;
6167 case 3:
6168 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6169 break;
6170 default:
6171 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6172 break;
6173 }
6174 } else {
6175 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6176 }
6177 }
a59781bb 6178
21a93e13
AD		 6179 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6180 DRM_DEBUG("cik_irq_set: sw int dma\n");
6181 dma_cntl |= TRAP_ENABLE;
6182 }
6183
6184 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6185 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6186 dma_cntl1 |= TRAP_ENABLE;
6187 }
6188
a59781bb
AD		 6189 if (rdev->irq.crtc_vblank_int[0] ||
6190 atomic_read(&rdev->irq.pflip[0])) {
6191 DRM_DEBUG("cik_irq_set: vblank 0\n");
6192 crtc1 |= VBLANK_INTERRUPT_MASK;
6193 }
6194 if (rdev->irq.crtc_vblank_int[1] ||
6195 atomic_read(&rdev->irq.pflip[1])) {
6196 DRM_DEBUG("cik_irq_set: vblank 1\n");
6197 crtc2 |= VBLANK_INTERRUPT_MASK;
6198 }
6199 if (rdev->irq.crtc_vblank_int[2] ||
6200 atomic_read(&rdev->irq.pflip[2])) {
6201 DRM_DEBUG("cik_irq_set: vblank 2\n");
6202 crtc3 |= VBLANK_INTERRUPT_MASK;
6203 }
6204 if (rdev->irq.crtc_vblank_int[3] ||
6205 atomic_read(&rdev->irq.pflip[3])) {
6206 DRM_DEBUG("cik_irq_set: vblank 3\n");
6207 crtc4 |= VBLANK_INTERRUPT_MASK;
6208 }
6209 if (rdev->irq.crtc_vblank_int[4] ||
6210 atomic_read(&rdev->irq.pflip[4])) {
6211 DRM_DEBUG("cik_irq_set: vblank 4\n");
6212 crtc5 |= VBLANK_INTERRUPT_MASK;
6213 }
6214 if (rdev->irq.crtc_vblank_int[5] ||
6215 atomic_read(&rdev->irq.pflip[5])) {
6216 DRM_DEBUG("cik_irq_set: vblank 5\n");
6217 crtc6 |= VBLANK_INTERRUPT_MASK;
6218 }
6219 if (rdev->irq.hpd[0]) {
6220 DRM_DEBUG("cik_irq_set: hpd 1\n");
6221 hpd1 |= DC_HPDx_INT_EN;
6222 }
6223 if (rdev->irq.hpd[1]) {
6224 DRM_DEBUG("cik_irq_set: hpd 2\n");
6225 hpd2 |= DC_HPDx_INT_EN;
6226 }
6227 if (rdev->irq.hpd[2]) {
6228 DRM_DEBUG("cik_irq_set: hpd 3\n");
6229 hpd3 |= DC_HPDx_INT_EN;
6230 }
6231 if (rdev->irq.hpd[3]) {
6232 DRM_DEBUG("cik_irq_set: hpd 4\n");
6233 hpd4 |= DC_HPDx_INT_EN;
6234 }
6235 if (rdev->irq.hpd[4]) {
6236 DRM_DEBUG("cik_irq_set: hpd 5\n");
6237 hpd5 |= DC_HPDx_INT_EN;
6238 }
6239 if (rdev->irq.hpd[5]) {
6240 DRM_DEBUG("cik_irq_set: hpd 6\n");
6241 hpd6 |= DC_HPDx_INT_EN;
6242 }
6243
41a524ab
AD		 6244 if (rdev->irq.dpm_thermal) {
6245 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6246 if (rdev->flags & RADEON_IS_IGP)
6247 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6248 else
6249 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6250 }
6251
a59781bb
AD		 6252 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6253
21a93e13
AD		 6254 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6255 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6256
2b0781a6
AD		 6257 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6258 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6259 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6260 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6261 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6262 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6263 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6264 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6265
a59781bb
AD		 6266 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6267
6268 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6269 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6270 if (rdev->num_crtc >= 4) {
6271 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6272 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6273 }
6274 if (rdev->num_crtc >= 6) {
6275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6276 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6277 }
6278
6279 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6280 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6281 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6282 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6283 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6284 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6285
cc8dbbb4
AD		 6286 if (rdev->flags & RADEON_IS_IGP)
6287 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6288 else
6289 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 6290
a59781bb
AD		 6291 return 0;
6292}
6293
6294/**
6295 * cik_irq_ack - ack interrupt sources
6296 *
6297 * @rdev: radeon_device pointer
6298 *
6299 * Ack interrupt sources on the GPU (vblanks, hpd,
6300 * etc.) (CIK). Certain interrupts sources are sw
6301 * generated and do not require an explicit ack.
6302 */
6303static inline void cik_irq_ack(struct radeon_device *rdev)
6304{
6305 u32 tmp;
6306
6307 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6308 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6309 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6310 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6311 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6312 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6313 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6314
6315 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6316 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6317 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6318 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6319 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6320 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6321 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6322 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6323
6324 if (rdev->num_crtc >= 4) {
6325 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6326 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6327 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6328 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6329 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6330 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6331 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6332 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6333 }
6334
6335 if (rdev->num_crtc >= 6) {
6336 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6337 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6338 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6339 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6340 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6341 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6342 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6343 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6344 }
6345
6346 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6347 tmp = RREG32(DC_HPD1_INT_CONTROL);
6348 tmp |= DC_HPDx_INT_ACK;
6349 WREG32(DC_HPD1_INT_CONTROL, tmp);
6350 }
6351 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6352 tmp = RREG32(DC_HPD2_INT_CONTROL);
6353 tmp |= DC_HPDx_INT_ACK;
6354 WREG32(DC_HPD2_INT_CONTROL, tmp);
6355 }
6356 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6357 tmp = RREG32(DC_HPD3_INT_CONTROL);
6358 tmp |= DC_HPDx_INT_ACK;
6359 WREG32(DC_HPD3_INT_CONTROL, tmp);
6360 }
6361 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6362 tmp = RREG32(DC_HPD4_INT_CONTROL);
6363 tmp |= DC_HPDx_INT_ACK;
6364 WREG32(DC_HPD4_INT_CONTROL, tmp);
6365 }
6366 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6367 tmp = RREG32(DC_HPD5_INT_CONTROL);
6368 tmp |= DC_HPDx_INT_ACK;
6369 WREG32(DC_HPD5_INT_CONTROL, tmp);
6370 }
6371 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6372 tmp = RREG32(DC_HPD5_INT_CONTROL);
6373 tmp |= DC_HPDx_INT_ACK;
6374 WREG32(DC_HPD6_INT_CONTROL, tmp);
6375 }
6376}
6377
6378/**
6379 * cik_irq_disable - disable interrupts
6380 *
6381 * @rdev: radeon_device pointer
6382 *
6383 * Disable interrupts on the hw (CIK).
6384 */
6385static void cik_irq_disable(struct radeon_device *rdev)
6386{
6387 cik_disable_interrupts(rdev);
6388 /* Wait and acknowledge irq */
6389 mdelay(1);
6390 cik_irq_ack(rdev);
6391 cik_disable_interrupt_state(rdev);
6392}
6393
6394/**
6395 * cik_irq_disable - disable interrupts for suspend
6396 *
6397 * @rdev: radeon_device pointer
6398 *
6399 * Disable interrupts and stop the RLC (CIK).
6400 * Used for suspend.
6401 */
6402static void cik_irq_suspend(struct radeon_device *rdev)
6403{
6404 cik_irq_disable(rdev);
6405 cik_rlc_stop(rdev);
6406}
6407
6408/**
6409 * cik_irq_fini - tear down interrupt support
6410 *
6411 * @rdev: radeon_device pointer
6412 *
6413 * Disable interrupts on the hw and free the IH ring
6414 * buffer (CIK).
6415 * Used for driver unload.
6416 */
6417static void cik_irq_fini(struct radeon_device *rdev)
6418{
6419 cik_irq_suspend(rdev);
6420 r600_ih_ring_fini(rdev);
6421}
6422
6423/**
6424 * cik_get_ih_wptr - get the IH ring buffer wptr
6425 *
6426 * @rdev: radeon_device pointer
6427 *
6428 * Get the IH ring buffer wptr from either the register
6429 * or the writeback memory buffer (CIK). Also check for
6430 * ring buffer overflow and deal with it.
6431 * Used by cik_irq_process().
6432 * Returns the value of the wptr.
6433 */
6434static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6435{
6436 u32 wptr, tmp;
6437
6438 if (rdev->wb.enabled)
6439 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6440 else
6441 wptr = RREG32(IH_RB_WPTR);
6442
6443 if (wptr & RB_OVERFLOW) {
6444 /* When a ring buffer overflow happen start parsing interrupt
6445 * from the last not overwritten vector (wptr + 16). Hopefully
6446 * this should allow us to catchup.
6447 */
6448 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6449 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6450 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6451 tmp = RREG32(IH_RB_CNTL);
6452 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6453 WREG32(IH_RB_CNTL, tmp);
6454 }
6455 return (wptr & rdev->ih.ptr_mask);
6456}
6457
6458/* CIK IV Ring
6459 * Each IV ring entry is 128 bits:
6460 * [7:0] - interrupt source id
6461 * [31:8] - reserved
6462 * [59:32] - interrupt source data
6463 * [63:60] - reserved
21a93e13
AD		 6464 * [71:64] - RINGID
6465 * CP:
6466 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 6467 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6468 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6469 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6470 * PIPE_ID - ME0 0=3D
6471 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 6472 * SDMA:
6473 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6474 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6475 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 6476 * [79:72] - VMID
6477 * [95:80] - PASID
6478 * [127:96] - reserved
6479 */
6480/**
6481 * cik_irq_process - interrupt handler
6482 *
6483 * @rdev: radeon_device pointer
6484 *
6485 * Interrupt hander (CIK). Walk the IH ring,
6486 * ack interrupts and schedule work to handle
6487 * interrupt events.
6488 * Returns irq process return code.
6489 */
6490int cik_irq_process(struct radeon_device *rdev)
6491{
2b0781a6
AD		 6492 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6493 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 6494 u32 wptr;
6495 u32 rptr;
6496 u32 src_id, src_data, ring_id;
6497 u8 me_id, pipe_id, queue_id;
6498 u32 ring_index;
6499 bool queue_hotplug = false;
6500 bool queue_reset = false;
3ec7d11b 6501 u32 addr, status, mc_client;
41a524ab 6502 bool queue_thermal = false;
a59781bb
AD		 6503
6504 if (!rdev->ih.enabled || rdev->shutdown)
6505 return IRQ_NONE;
6506
6507 wptr = cik_get_ih_wptr(rdev);
6508
6509restart_ih:
6510 /* is somebody else already processing irqs? */
6511 if (atomic_xchg(&rdev->ih.lock, 1))
6512 return IRQ_NONE;
6513
6514 rptr = rdev->ih.rptr;
6515 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6516
6517 /* Order reading of wptr vs. reading of IH ring data */
6518 rmb();
6519
6520 /* display interrupts */
6521 cik_irq_ack(rdev);
6522
6523 while (rptr != wptr) {
6524 /* wptr/rptr are in bytes! */
6525 ring_index = rptr / 4;
6526 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6527 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6528 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 6529
6530 switch (src_id) {
6531 case 1: /* D1 vblank/vline */
6532 switch (src_data) {
6533 case 0: /* D1 vblank */
6534 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6535 if (rdev->irq.crtc_vblank_int[0]) {
6536 drm_handle_vblank(rdev->ddev, 0);
6537 rdev->pm.vblank_sync = true;
6538 wake_up(&rdev->irq.vblank_queue);
6539 }
6540 if (atomic_read(&rdev->irq.pflip[0]))
6541 radeon_crtc_handle_flip(rdev, 0);
6542 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6543 DRM_DEBUG("IH: D1 vblank\n");
6544 }
6545 break;
6546 case 1: /* D1 vline */
6547 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6548 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6549 DRM_DEBUG("IH: D1 vline\n");
6550 }
6551 break;
6552 default:
6553 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6554 break;
6555 }
6556 break;
6557 case 2: /* D2 vblank/vline */
6558 switch (src_data) {
6559 case 0: /* D2 vblank */
6560 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6561 if (rdev->irq.crtc_vblank_int[1]) {
6562 drm_handle_vblank(rdev->ddev, 1);
6563 rdev->pm.vblank_sync = true;
6564 wake_up(&rdev->irq.vblank_queue);
6565 }
6566 if (atomic_read(&rdev->irq.pflip[1]))
6567 radeon_crtc_handle_flip(rdev, 1);
6568 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6569 DRM_DEBUG("IH: D2 vblank\n");
6570 }
6571 break;
6572 case 1: /* D2 vline */
6573 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6574 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6575 DRM_DEBUG("IH: D2 vline\n");
6576 }
6577 break;
6578 default:
6579 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6580 break;
6581 }
6582 break;
6583 case 3: /* D3 vblank/vline */
6584 switch (src_data) {
6585 case 0: /* D3 vblank */
6586 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6587 if (rdev->irq.crtc_vblank_int[2]) {
6588 drm_handle_vblank(rdev->ddev, 2);
6589 rdev->pm.vblank_sync = true;
6590 wake_up(&rdev->irq.vblank_queue);
6591 }
6592 if (atomic_read(&rdev->irq.pflip[2]))
6593 radeon_crtc_handle_flip(rdev, 2);
6594 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6595 DRM_DEBUG("IH: D3 vblank\n");
6596 }
6597 break;
6598 case 1: /* D3 vline */
6599 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6600 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6601 DRM_DEBUG("IH: D3 vline\n");
6602 }
6603 break;
6604 default:
6605 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6606 break;
6607 }
6608 break;
6609 case 4: /* D4 vblank/vline */
6610 switch (src_data) {
6611 case 0: /* D4 vblank */
6612 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6613 if (rdev->irq.crtc_vblank_int[3]) {
6614 drm_handle_vblank(rdev->ddev, 3);
6615 rdev->pm.vblank_sync = true;
6616 wake_up(&rdev->irq.vblank_queue);
6617 }
6618 if (atomic_read(&rdev->irq.pflip[3]))
6619 radeon_crtc_handle_flip(rdev, 3);
6620 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6621 DRM_DEBUG("IH: D4 vblank\n");
6622 }
6623 break;
6624 case 1: /* D4 vline */
6625 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6626 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6627 DRM_DEBUG("IH: D4 vline\n");
6628 }
6629 break;
6630 default:
6631 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6632 break;
6633 }
6634 break;
6635 case 5: /* D5 vblank/vline */
6636 switch (src_data) {
6637 case 0: /* D5 vblank */
6638 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6639 if (rdev->irq.crtc_vblank_int[4]) {
6640 drm_handle_vblank(rdev->ddev, 4);
6641 rdev->pm.vblank_sync = true;
6642 wake_up(&rdev->irq.vblank_queue);
6643 }
6644 if (atomic_read(&rdev->irq.pflip[4]))
6645 radeon_crtc_handle_flip(rdev, 4);
6646 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6647 DRM_DEBUG("IH: D5 vblank\n");
6648 }
6649 break;
6650 case 1: /* D5 vline */
6651 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6652 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6653 DRM_DEBUG("IH: D5 vline\n");
6654 }
6655 break;
6656 default:
6657 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6658 break;
6659 }
6660 break;
6661 case 6: /* D6 vblank/vline */
6662 switch (src_data) {
6663 case 0: /* D6 vblank */
6664 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6665 if (rdev->irq.crtc_vblank_int[5]) {
6666 drm_handle_vblank(rdev->ddev, 5);
6667 rdev->pm.vblank_sync = true;
6668 wake_up(&rdev->irq.vblank_queue);
6669 }
6670 if (atomic_read(&rdev->irq.pflip[5]))
6671 radeon_crtc_handle_flip(rdev, 5);
6672 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6673 DRM_DEBUG("IH: D6 vblank\n");
6674 }
6675 break;
6676 case 1: /* D6 vline */
6677 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6678 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6679 DRM_DEBUG("IH: D6 vline\n");
6680 }
6681 break;
6682 default:
6683 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6684 break;
6685 }
6686 break;
6687 case 42: /* HPD hotplug */
6688 switch (src_data) {
6689 case 0:
6690 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6691 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6692 queue_hotplug = true;
6693 DRM_DEBUG("IH: HPD1\n");
6694 }
6695 break;
6696 case 1:
6697 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6698 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6699 queue_hotplug = true;
6700 DRM_DEBUG("IH: HPD2\n");
6701 }
6702 break;
6703 case 2:
6704 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6705 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6706 queue_hotplug = true;
6707 DRM_DEBUG("IH: HPD3\n");
6708 }
6709 break;
6710 case 3:
6711 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6712 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6713 queue_hotplug = true;
6714 DRM_DEBUG("IH: HPD4\n");
6715 }
6716 break;
6717 case 4:
6718 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6719 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6720 queue_hotplug = true;
6721 DRM_DEBUG("IH: HPD5\n");
6722 }
6723 break;
6724 case 5:
6725 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6726 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6727 queue_hotplug = true;
6728 DRM_DEBUG("IH: HPD6\n");
6729 }
6730 break;
6731 default:
6732 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6733 break;
6734 }
6735 break;
9d97c99b
AD		 6736 case 146:
6737 case 147:
3ec7d11b
AD		 6738 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6739 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6740 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 6741 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6742 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 6743 addr);
9d97c99b 6744 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 6745 status);
6746 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 6747 /* reset addr and status */
6748 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6749 break;
a59781bb
AD		 6750 case 176: /* GFX RB CP_INT */
6751 case 177: /* GFX IB CP_INT */
6752 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6753 break;
6754 case 181: /* CP EOP event */
6755 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 6756 /* XXX check the bitfield order! */
6757 me_id = (ring_id & 0x60) >> 5;
6758 pipe_id = (ring_id & 0x18) >> 3;
6759 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6760 switch (me_id) {
6761 case 0:
6762 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6763 break;
6764 case 1:
a59781bb 6765 case 2:
2b0781a6
AD		 6766 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6767 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6768 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6769 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 6770 break;
6771 }
6772 break;
6773 case 184: /* CP Privileged reg access */
6774 DRM_ERROR("Illegal register access in command stream\n");
6775 /* XXX check the bitfield order! */
6776 me_id = (ring_id & 0x60) >> 5;
6777 pipe_id = (ring_id & 0x18) >> 3;
6778 queue_id = (ring_id & 0x7) >> 0;
6779 switch (me_id) {
6780 case 0:
6781 /* This results in a full GPU reset, but all we need to do is soft
6782 * reset the CP for gfx
6783 */
6784 queue_reset = true;
6785 break;
6786 case 1:
6787 /* XXX compute */
2b0781a6 6788 queue_reset = true;
a59781bb
AD		 6789 break;
6790 case 2:
6791 /* XXX compute */
2b0781a6 6792 queue_reset = true;
a59781bb
AD		 6793 break;
6794 }
6795 break;
6796 case 185: /* CP Privileged inst */
6797 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 6798 /* XXX check the bitfield order! */
6799 me_id = (ring_id & 0x60) >> 5;
6800 pipe_id = (ring_id & 0x18) >> 3;
6801 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6802 switch (me_id) {
6803 case 0:
6804 /* This results in a full GPU reset, but all we need to do is soft
6805 * reset the CP for gfx
6806 */
6807 queue_reset = true;
6808 break;
6809 case 1:
6810 /* XXX compute */
2b0781a6 6811 queue_reset = true;
a59781bb
AD		 6812 break;
6813 case 2:
6814 /* XXX compute */
2b0781a6 6815 queue_reset = true;
a59781bb
AD		 6816 break;
6817 }
6818 break;
21a93e13
AD		 6819 case 224: /* SDMA trap event */
6820 /* XXX check the bitfield order! */
6821 me_id = (ring_id & 0x3) >> 0;
6822 queue_id = (ring_id & 0xc) >> 2;
6823 DRM_DEBUG("IH: SDMA trap\n");
6824 switch (me_id) {
6825 case 0:
6826 switch (queue_id) {
6827 case 0:
6828 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6829 break;
6830 case 1:
6831 /* XXX compute */
6832 break;
6833 case 2:
6834 /* XXX compute */
6835 break;
6836 }
6837 break;
6838 case 1:
6839 switch (queue_id) {
6840 case 0:
6841 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6842 break;
6843 case 1:
6844 /* XXX compute */
6845 break;
6846 case 2:
6847 /* XXX compute */
6848 break;
6849 }
6850 break;
6851 }
6852 break;
41a524ab
AD		 6853 case 230: /* thermal low to high */
6854 DRM_DEBUG("IH: thermal low to high\n");
6855 rdev->pm.dpm.thermal.high_to_low = false;
6856 queue_thermal = true;
6857 break;
6858 case 231: /* thermal high to low */
6859 DRM_DEBUG("IH: thermal high to low\n");
6860 rdev->pm.dpm.thermal.high_to_low = true;
6861 queue_thermal = true;
6862 break;
6863 case 233: /* GUI IDLE */
6864 DRM_DEBUG("IH: GUI idle\n");
6865 break;
21a93e13
AD		 6866 case 241: /* SDMA Privileged inst */
6867 case 247: /* SDMA Privileged inst */
6868 DRM_ERROR("Illegal instruction in SDMA command stream\n");
6869 /* XXX check the bitfield order! */
6870 me_id = (ring_id & 0x3) >> 0;
6871 queue_id = (ring_id & 0xc) >> 2;
6872 switch (me_id) {
6873 case 0:
6874 switch (queue_id) {
6875 case 0:
6876 queue_reset = true;
6877 break;
6878 case 1:
6879 /* XXX compute */
6880 queue_reset = true;
6881 break;
6882 case 2:
6883 /* XXX compute */
6884 queue_reset = true;
6885 break;
6886 }
6887 break;
6888 case 1:
6889 switch (queue_id) {
6890 case 0:
6891 queue_reset = true;
6892 break;
6893 case 1:
6894 /* XXX compute */
6895 queue_reset = true;
6896 break;
6897 case 2:
6898 /* XXX compute */
6899 queue_reset = true;
6900 break;
6901 }
6902 break;
6903 }
6904 break;
a59781bb
AD		 6905 default:
6906 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6907 break;
6908 }
6909
6910 /* wptr/rptr are in bytes! */
6911 rptr += 16;
6912 rptr &= rdev->ih.ptr_mask;
6913 }
6914 if (queue_hotplug)
6915 schedule_work(&rdev->hotplug_work);
6916 if (queue_reset)
6917 schedule_work(&rdev->reset_work);
41a524ab
AD		 6918 if (queue_thermal)
6919 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 6920 rdev->ih.rptr = rptr;
6921 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6922 atomic_set(&rdev->ih.lock, 0);
6923
6924 /* make sure wptr hasn't changed while processing */
6925 wptr = cik_get_ih_wptr(rdev);
6926 if (wptr != rptr)
6927 goto restart_ih;
6928
6929 return IRQ_HANDLED;
6930}
7bf94a2c
AD		 6931
6932/*
6933 * startup/shutdown callbacks
6934 */
6935/**
6936 * cik_startup - program the asic to a functional state
6937 *
6938 * @rdev: radeon_device pointer
6939 *
6940 * Programs the asic to a functional state (CIK).
6941 * Called by cik_init() and cik_resume().
6942 * Returns 0 for success, error for failure.
6943 */
6944static int cik_startup(struct radeon_device *rdev)
6945{
6946 struct radeon_ring *ring;
6947 int r;
6948
8a7cd276
AD		 6949 /* enable pcie gen2/3 link */
6950 cik_pcie_gen3_enable(rdev);
7235711a
AD		 6951 /* enable aspm */
6952 cik_program_aspm(rdev);
8a7cd276 6953
e5903d39
AD		 6954 /* scratch needs to be initialized before MC */
6955 r = r600_vram_scratch_init(rdev);
6956 if (r)
6957 return r;
6958
6fab3feb
AD		 6959 cik_mc_program(rdev);
6960
7bf94a2c
AD		 6961 if (rdev->flags & RADEON_IS_IGP) {
6962 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6963 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
6964 r = cik_init_microcode(rdev);
6965 if (r) {
6966 DRM_ERROR("Failed to load firmware!\n");
6967 return r;
6968 }
6969 }
6970 } else {
6971 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6972 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
6973 !rdev->mc_fw) {
6974 r = cik_init_microcode(rdev);
6975 if (r) {
6976 DRM_ERROR("Failed to load firmware!\n");
6977 return r;
6978 }
6979 }
6980
6981 r = ci_mc_load_microcode(rdev);
6982 if (r) {
6983 DRM_ERROR("Failed to load MC firmware!\n");
6984 return r;
6985 }
6986 }
6987
7bf94a2c
AD		 6988 r = cik_pcie_gart_enable(rdev);
6989 if (r)
6990 return r;
6991 cik_gpu_init(rdev);
6992
6993 /* allocate rlc buffers */
22c775ce
AD		 6994 if (rdev->flags & RADEON_IS_IGP) {
6995 if (rdev->family == CHIP_KAVERI) {
6996 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
6997 rdev->rlc.reg_list_size =
6998 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
6999 } else {
7000 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7001 rdev->rlc.reg_list_size =
7002 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7003 }
7004 }
7005 rdev->rlc.cs_data = ci_cs_data;
7006 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7007 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7008 if (r) {
7009 DRM_ERROR("Failed to init rlc BOs!\n");
7010 return r;
7011 }
7012
7013 /* allocate wb buffer */
7014 r = radeon_wb_init(rdev);
7015 if (r)
7016 return r;
7017
963e81f9
AD		 7018 /* allocate mec buffers */
7019 r = cik_mec_init(rdev);
7020 if (r) {
7021 DRM_ERROR("Failed to init MEC BOs!\n");
7022 return r;
7023 }
7024
7bf94a2c
AD		 7025 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7026 if (r) {
7027 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7028 return r;
7029 }
7030
963e81f9
AD		 7031 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7032 if (r) {
7033 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7034 return r;
7035 }
7036
7037 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7038 if (r) {
7039 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7040 return r;
7041 }
7042
7bf94a2c
AD		 7043 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7044 if (r) {
7045 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7046 return r;
7047 }
7048
7049 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7050 if (r) {
7051 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7052 return r;
7053 }
7054
2ce529da 7055 r = radeon_uvd_resume(rdev);
87167bb1 7056 if (!r) {
2ce529da
AD		 7057 r = uvd_v4_2_resume(rdev);
7058 if (!r) {
7059 r = radeon_fence_driver_start_ring(rdev,
7060 R600_RING_TYPE_UVD_INDEX);
7061 if (r)
7062 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7063 }
87167bb1
CK		 7064 }
7065 if (r)
7066 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7067
7bf94a2c
AD		 7068 /* Enable IRQ */
7069 if (!rdev->irq.installed) {
7070 r = radeon_irq_kms_init(rdev);
7071 if (r)
7072 return r;
7073 }
7074
7075 r = cik_irq_init(rdev);
7076 if (r) {
7077 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7078 radeon_irq_kms_fini(rdev);
7079 return r;
7080 }
7081 cik_irq_set(rdev);
7082
7083 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7084 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7085 CP_RB0_RPTR, CP_RB0_WPTR,
2e1e6dad 7086 RADEON_CP_PACKET2);
7bf94a2c
AD		 7087 if (r)
7088 return r;
7089
963e81f9 7090 /* set up the compute queues */
2615b53a 7091 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7092 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7093 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7094 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7095 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7096 if (r)
7097 return r;
7098 ring->me = 1; /* first MEC */
7099 ring->pipe = 0; /* first pipe */
7100 ring->queue = 0; /* first queue */
7101 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7102
2615b53a 7103 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7104 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7105 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7106 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7107 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7108 if (r)
7109 return r;
7110 /* dGPU only have 1 MEC */
7111 ring->me = 1; /* first MEC */
7112 ring->pipe = 0; /* first pipe */
7113 ring->queue = 1; /* second queue */
7114 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7115
7bf94a2c
AD		 7116 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7117 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7118 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7119 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
2e1e6dad 7120 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7121 if (r)
7122 return r;
7123
7124 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7125 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7126 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7127 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
2e1e6dad 7128 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7129 if (r)
7130 return r;
7131
7132 r = cik_cp_resume(rdev);
7133 if (r)
7134 return r;
7135
7136 r = cik_sdma_resume(rdev);
7137 if (r)
7138 return r;
7139
87167bb1
CK		 7140 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7141 if (ring->ring_size) {
02c9f7fa 7142 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
87167bb1 7143 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 7144 RADEON_CP_PACKET2);
87167bb1 7145 if (!r)
e409b128 7146 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7147 if (r)
7148 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7149 }
7150
7bf94a2c
AD		 7151 r = radeon_ib_pool_init(rdev);
7152 if (r) {
7153 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7154 return r;
7155 }
7156
7157 r = radeon_vm_manager_init(rdev);
7158 if (r) {
7159 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7160 return r;
7161 }
7162
b530602f
AD		 7163 r = dce6_audio_init(rdev);
7164 if (r)
7165 return r;
7166
7bf94a2c
AD		 7167 return 0;
7168}
7169
7170/**
7171 * cik_resume - resume the asic to a functional state
7172 *
7173 * @rdev: radeon_device pointer
7174 *
7175 * Programs the asic to a functional state (CIK).
7176 * Called at resume.
7177 * Returns 0 for success, error for failure.
7178 */
7179int cik_resume(struct radeon_device *rdev)
7180{
7181 int r;
7182
7183 /* post card */
7184 atom_asic_init(rdev->mode_info.atom_context);
7185
0aafd313
AD		 7186 /* init golden registers */
7187 cik_init_golden_registers(rdev);
7188
7bf94a2c
AD		 7189 rdev->accel_working = true;
7190 r = cik_startup(rdev);
7191 if (r) {
7192 DRM_ERROR("cik startup failed on resume\n");
7193 rdev->accel_working = false;
7194 return r;
7195 }
7196
7197 return r;
7198
7199}
7200
7201/**
7202 * cik_suspend - suspend the asic
7203 *
7204 * @rdev: radeon_device pointer
7205 *
7206 * Bring the chip into a state suitable for suspend (CIK).
7207 * Called at suspend.
7208 * Returns 0 for success.
7209 */
7210int cik_suspend(struct radeon_device *rdev)
7211{
b530602f 7212 dce6_audio_fini(rdev);
7bf94a2c
AD		 7213 radeon_vm_manager_fini(rdev);
7214 cik_cp_enable(rdev, false);
7215 cik_sdma_enable(rdev, false);
e409b128 7216 uvd_v1_0_fini(rdev);
87167bb1 7217 radeon_uvd_suspend(rdev);
473359bc
AD		 7218 cik_fini_pg(rdev);
7219 cik_fini_cg(rdev);
7bf94a2c
AD		 7220 cik_irq_suspend(rdev);
7221 radeon_wb_disable(rdev);
7222 cik_pcie_gart_disable(rdev);
7223 return 0;
7224}
7225
7226/* Plan is to move initialization in that function and use
7227 * helper function so that radeon_device_init pretty much
7228 * do nothing more than calling asic specific function. This
7229 * should also allow to remove a bunch of callback function
7230 * like vram_info.
7231 */
7232/**
7233 * cik_init - asic specific driver and hw init
7234 *
7235 * @rdev: radeon_device pointer
7236 *
7237 * Setup asic specific driver variables and program the hw
7238 * to a functional state (CIK).
7239 * Called at driver startup.
7240 * Returns 0 for success, errors for failure.
7241 */
7242int cik_init(struct radeon_device *rdev)
7243{
7244 struct radeon_ring *ring;
7245 int r;
7246
7247 /* Read BIOS */
7248 if (!radeon_get_bios(rdev)) {
7249 if (ASIC_IS_AVIVO(rdev))
7250 return -EINVAL;
7251 }
7252 /* Must be an ATOMBIOS */
7253 if (!rdev->is_atom_bios) {
7254 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7255 return -EINVAL;
7256 }
7257 r = radeon_atombios_init(rdev);
7258 if (r)
7259 return r;
7260
7261 /* Post card if necessary */
7262 if (!radeon_card_posted(rdev)) {
7263 if (!rdev->bios) {
7264 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7265 return -EINVAL;
7266 }
7267 DRM_INFO("GPU not posted. posting now...\n");
7268 atom_asic_init(rdev->mode_info.atom_context);
7269 }
0aafd313
AD		 7270 /* init golden registers */
7271 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7272 /* Initialize scratch registers */
7273 cik_scratch_init(rdev);
7274 /* Initialize surface registers */
7275 radeon_surface_init(rdev);
7276 /* Initialize clocks */
7277 radeon_get_clock_info(rdev->ddev);
7278
7279 /* Fence driver */
7280 r = radeon_fence_driver_init(rdev);
7281 if (r)
7282 return r;
7283
7284 /* initialize memory controller */
7285 r = cik_mc_init(rdev);
7286 if (r)
7287 return r;
7288 /* Memory manager */
7289 r = radeon_bo_init(rdev);
7290 if (r)
7291 return r;
7292
7293 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7294 ring->ring_obj = NULL;
7295 r600_ring_init(rdev, ring, 1024 * 1024);
7296
963e81f9
AD		 7297 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7298 ring->ring_obj = NULL;
7299 r600_ring_init(rdev, ring, 1024 * 1024);
7300 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7301 if (r)
7302 return r;
7303
7304 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7305 ring->ring_obj = NULL;
7306 r600_ring_init(rdev, ring, 1024 * 1024);
7307 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7308 if (r)
7309 return r;
7310
7bf94a2c
AD		 7311 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7312 ring->ring_obj = NULL;
7313 r600_ring_init(rdev, ring, 256 * 1024);
7314
7315 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7316 ring->ring_obj = NULL;
7317 r600_ring_init(rdev, ring, 256 * 1024);
7318
87167bb1
CK		 7319 r = radeon_uvd_init(rdev);
7320 if (!r) {
7321 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7322 ring->ring_obj = NULL;
7323 r600_ring_init(rdev, ring, 4096);
7324 }
7325
7bf94a2c
AD		 7326 rdev->ih.ring_obj = NULL;
7327 r600_ih_ring_init(rdev, 64 * 1024);
7328
7329 r = r600_pcie_gart_init(rdev);
7330 if (r)
7331 return r;
7332
7333 rdev->accel_working = true;
7334 r = cik_startup(rdev);
7335 if (r) {
7336 dev_err(rdev->dev, "disabling GPU acceleration\n");
7337 cik_cp_fini(rdev);
7338 cik_sdma_fini(rdev);
7339 cik_irq_fini(rdev);
1fd11777 7340 sumo_rlc_fini(rdev);
963e81f9 7341 cik_mec_fini(rdev);
7bf94a2c
AD		 7342 radeon_wb_fini(rdev);
7343 radeon_ib_pool_fini(rdev);
7344 radeon_vm_manager_fini(rdev);
7345 radeon_irq_kms_fini(rdev);
7346 cik_pcie_gart_fini(rdev);
7347 rdev->accel_working = false;
7348 }
7349
7350 /* Don't start up if the MC ucode is missing.
7351 * The default clocks and voltages before the MC ucode
7352 * is loaded are not suffient for advanced operations.
7353 */
7354 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7355 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7356 return -EINVAL;
7357 }
7358
7359 return 0;
7360}
7361
7362/**
7363 * cik_fini - asic specific driver and hw fini
7364 *
7365 * @rdev: radeon_device pointer
7366 *
7367 * Tear down the asic specific driver variables and program the hw
7368 * to an idle state (CIK).
7369 * Called at driver unload.
7370 */
7371void cik_fini(struct radeon_device *rdev)
7372{
7373 cik_cp_fini(rdev);
7374 cik_sdma_fini(rdev);
473359bc
AD		 7375 cik_fini_pg(rdev);
7376 cik_fini_cg(rdev);
7bf94a2c 7377 cik_irq_fini(rdev);
1fd11777 7378 sumo_rlc_fini(rdev);
963e81f9 7379 cik_mec_fini(rdev);
7bf94a2c
AD		 7380 radeon_wb_fini(rdev);
7381 radeon_vm_manager_fini(rdev);
7382 radeon_ib_pool_fini(rdev);
7383 radeon_irq_kms_fini(rdev);
e409b128 7384 uvd_v1_0_fini(rdev);
87167bb1 7385 radeon_uvd_fini(rdev);
7bf94a2c
AD		 7386 cik_pcie_gart_fini(rdev);
7387 r600_vram_scratch_fini(rdev);
7388 radeon_gem_fini(rdev);
7389 radeon_fence_driver_fini(rdev);
7390 radeon_bo_fini(rdev);
7391 radeon_atombios_fini(rdev);
7392 kfree(rdev->bios);
7393 rdev->bios = NULL;
7394}
cd84a27d
AD		 7395
7396/* display watermark setup */
7397/**
7398 * dce8_line_buffer_adjust - Set up the line buffer
7399 *
7400 * @rdev: radeon_device pointer
7401 * @radeon_crtc: the selected display controller
7402 * @mode: the current display mode on the selected display
7403 * controller
7404 *
7405 * Setup up the line buffer allocation for
7406 * the selected display controller (CIK).
7407 * Returns the line buffer size in pixels.
7408 */
7409static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7410 struct radeon_crtc *radeon_crtc,
7411 struct drm_display_mode *mode)
7412{
bc01a8c7
AD		 7413 u32 tmp, buffer_alloc, i;
7414 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 7415 /*
7416 * Line Buffer Setup
7417 * There are 6 line buffers, one for each display controllers.
7418 * There are 3 partitions per LB. Select the number of partitions
7419 * to enable based on the display width. For display widths larger
7420 * than 4096, you need use to use 2 display controllers and combine
7421 * them using the stereo blender.
7422 */
7423 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 7424 if (mode->crtc_hdisplay < 1920) {
cd84a27d 7425 tmp = 1;
bc01a8c7
AD		 7426 buffer_alloc = 2;
7427 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 7428 tmp = 2;
bc01a8c7
AD		 7429 buffer_alloc = 2;
7430 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 7431 tmp = 0;
bc01a8c7
AD		 7432 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7433 } else {
cd84a27d
AD		 7434 DRM_DEBUG_KMS("Mode too big for LB!\n");
7435 tmp = 0;
bc01a8c7 7436 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 7437 }
bc01a8c7 7438 } else {
cd84a27d 7439 tmp = 1;
bc01a8c7
AD		 7440 buffer_alloc = 0;
7441 }
cd84a27d
AD		 7442
7443 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7444 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7445
bc01a8c7
AD		 7446 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7447 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7448 for (i = 0; i < rdev->usec_timeout; i++) {
7449 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7450 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7451 break;
7452 udelay(1);
7453 }
7454
cd84a27d
AD		 7455 if (radeon_crtc->base.enabled && mode) {
7456 switch (tmp) {
7457 case 0:
7458 default:
7459 return 4096 * 2;
7460 case 1:
7461 return 1920 * 2;
7462 case 2:
7463 return 2560 * 2;
7464 }
7465 }
7466
7467 /* controller not enabled, so no lb used */
7468 return 0;
7469}
7470
7471/**
7472 * cik_get_number_of_dram_channels - get the number of dram channels
7473 *
7474 * @rdev: radeon_device pointer
7475 *
7476 * Look up the number of video ram channels (CIK).
7477 * Used for display watermark bandwidth calculations
7478 * Returns the number of dram channels
7479 */
7480static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7481{
7482 u32 tmp = RREG32(MC_SHARED_CHMAP);
7483
7484 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7485 case 0:
7486 default:
7487 return 1;
7488 case 1:
7489 return 2;
7490 case 2:
7491 return 4;
7492 case 3:
7493 return 8;
7494 case 4:
7495 return 3;
7496 case 5:
7497 return 6;
7498 case 6:
7499 return 10;
7500 case 7:
7501 return 12;
7502 case 8:
7503 return 16;
7504 }
7505}
7506
7507struct dce8_wm_params {
7508 u32 dram_channels; /* number of dram channels */
7509 u32 yclk; /* bandwidth per dram data pin in kHz */
7510 u32 sclk; /* engine clock in kHz */
7511 u32 disp_clk; /* display clock in kHz */
7512 u32 src_width; /* viewport width */
7513 u32 active_time; /* active display time in ns */
7514 u32 blank_time; /* blank time in ns */
7515 bool interlaced; /* mode is interlaced */
7516 fixed20_12 vsc; /* vertical scale ratio */
7517 u32 num_heads; /* number of active crtcs */
7518 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7519 u32 lb_size; /* line buffer allocated to pipe */
7520 u32 vtaps; /* vertical scaler taps */
7521};
7522
7523/**
7524 * dce8_dram_bandwidth - get the dram bandwidth
7525 *
7526 * @wm: watermark calculation data
7527 *
7528 * Calculate the raw dram bandwidth (CIK).
7529 * Used for display watermark bandwidth calculations
7530 * Returns the dram bandwidth in MBytes/s
7531 */
7532static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7533{
7534 /* Calculate raw DRAM Bandwidth */
7535 fixed20_12 dram_efficiency; /* 0.7 */
7536 fixed20_12 yclk, dram_channels, bandwidth;
7537 fixed20_12 a;
7538
7539 a.full = dfixed_const(1000);
7540 yclk.full = dfixed_const(wm->yclk);
7541 yclk.full = dfixed_div(yclk, a);
7542 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7543 a.full = dfixed_const(10);
7544 dram_efficiency.full = dfixed_const(7);
7545 dram_efficiency.full = dfixed_div(dram_efficiency, a);
7546 bandwidth.full = dfixed_mul(dram_channels, yclk);
7547 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7548
7549 return dfixed_trunc(bandwidth);
7550}
7551
7552/**
7553 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7554 *
7555 * @wm: watermark calculation data
7556 *
7557 * Calculate the dram bandwidth used for display (CIK).
7558 * Used for display watermark bandwidth calculations
7559 * Returns the dram bandwidth for display in MBytes/s
7560 */
7561static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7562{
7563 /* Calculate DRAM Bandwidth and the part allocated to display. */
7564 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7565 fixed20_12 yclk, dram_channels, bandwidth;
7566 fixed20_12 a;
7567
7568 a.full = dfixed_const(1000);
7569 yclk.full = dfixed_const(wm->yclk);
7570 yclk.full = dfixed_div(yclk, a);
7571 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7572 a.full = dfixed_const(10);
7573 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7574 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7575 bandwidth.full = dfixed_mul(dram_channels, yclk);
7576 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7577
7578 return dfixed_trunc(bandwidth);
7579}
7580
7581/**
7582 * dce8_data_return_bandwidth - get the data return bandwidth
7583 *
7584 * @wm: watermark calculation data
7585 *
7586 * Calculate the data return bandwidth used for display (CIK).
7587 * Used for display watermark bandwidth calculations
7588 * Returns the data return bandwidth in MBytes/s
7589 */
7590static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7591{
7592 /* Calculate the display Data return Bandwidth */
7593 fixed20_12 return_efficiency; /* 0.8 */
7594 fixed20_12 sclk, bandwidth;
7595 fixed20_12 a;
7596
7597 a.full = dfixed_const(1000);
7598 sclk.full = dfixed_const(wm->sclk);
7599 sclk.full = dfixed_div(sclk, a);
7600 a.full = dfixed_const(10);
7601 return_efficiency.full = dfixed_const(8);
7602 return_efficiency.full = dfixed_div(return_efficiency, a);
7603 a.full = dfixed_const(32);
7604 bandwidth.full = dfixed_mul(a, sclk);
7605 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7606
7607 return dfixed_trunc(bandwidth);
7608}
7609
7610/**
7611 * dce8_dmif_request_bandwidth - get the dmif bandwidth
7612 *
7613 * @wm: watermark calculation data
7614 *
7615 * Calculate the dmif bandwidth used for display (CIK).
7616 * Used for display watermark bandwidth calculations
7617 * Returns the dmif bandwidth in MBytes/s
7618 */
7619static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7620{
7621 /* Calculate the DMIF Request Bandwidth */
7622 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7623 fixed20_12 disp_clk, bandwidth;
7624 fixed20_12 a, b;
7625
7626 a.full = dfixed_const(1000);
7627 disp_clk.full = dfixed_const(wm->disp_clk);
7628 disp_clk.full = dfixed_div(disp_clk, a);
7629 a.full = dfixed_const(32);
7630 b.full = dfixed_mul(a, disp_clk);
7631
7632 a.full = dfixed_const(10);
7633 disp_clk_request_efficiency.full = dfixed_const(8);
7634 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7635
7636 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7637
7638 return dfixed_trunc(bandwidth);
7639}
7640
7641/**
7642 * dce8_available_bandwidth - get the min available bandwidth
7643 *
7644 * @wm: watermark calculation data
7645 *
7646 * Calculate the min available bandwidth used for display (CIK).
7647 * Used for display watermark bandwidth calculations
7648 * Returns the min available bandwidth in MBytes/s
7649 */
7650static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7651{
7652 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7653 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7654 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7655 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7656
7657 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7658}
7659
7660/**
7661 * dce8_average_bandwidth - get the average available bandwidth
7662 *
7663 * @wm: watermark calculation data
7664 *
7665 * Calculate the average available bandwidth used for display (CIK).
7666 * Used for display watermark bandwidth calculations
7667 * Returns the average available bandwidth in MBytes/s
7668 */
7669static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7670{
7671 /* Calculate the display mode Average Bandwidth
7672 * DisplayMode should contain the source and destination dimensions,
7673 * timing, etc.
7674 */
7675 fixed20_12 bpp;
7676 fixed20_12 line_time;
7677 fixed20_12 src_width;
7678 fixed20_12 bandwidth;
7679 fixed20_12 a;
7680
7681 a.full = dfixed_const(1000);
7682 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7683 line_time.full = dfixed_div(line_time, a);
7684 bpp.full = dfixed_const(wm->bytes_per_pixel);
7685 src_width.full = dfixed_const(wm->src_width);
7686 bandwidth.full = dfixed_mul(src_width, bpp);
7687 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7688 bandwidth.full = dfixed_div(bandwidth, line_time);
7689
7690 return dfixed_trunc(bandwidth);
7691}
7692
7693/**
7694 * dce8_latency_watermark - get the latency watermark
7695 *
7696 * @wm: watermark calculation data
7697 *
7698 * Calculate the latency watermark (CIK).
7699 * Used for display watermark bandwidth calculations
7700 * Returns the latency watermark in ns
7701 */
7702static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7703{
7704 /* First calculate the latency in ns */
7705 u32 mc_latency = 2000; /* 2000 ns. */
7706 u32 available_bandwidth = dce8_available_bandwidth(wm);
7707 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7708 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7709 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7710 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7711 (wm->num_heads * cursor_line_pair_return_time);
7712 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7713 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7714 u32 tmp, dmif_size = 12288;
7715 fixed20_12 a, b, c;
7716
7717 if (wm->num_heads == 0)
7718 return 0;
7719
7720 a.full = dfixed_const(2);
7721 b.full = dfixed_const(1);
7722 if ((wm->vsc.full > a.full) ||
7723 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7724 (wm->vtaps >= 5) ||
7725 ((wm->vsc.full >= a.full) && wm->interlaced))
7726 max_src_lines_per_dst_line = 4;
7727 else
7728 max_src_lines_per_dst_line = 2;
7729
7730 a.full = dfixed_const(available_bandwidth);
7731 b.full = dfixed_const(wm->num_heads);
7732 a.full = dfixed_div(a, b);
7733
7734 b.full = dfixed_const(mc_latency + 512);
7735 c.full = dfixed_const(wm->disp_clk);
7736 b.full = dfixed_div(b, c);
7737
7738 c.full = dfixed_const(dmif_size);
7739 b.full = dfixed_div(c, b);
7740
7741 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7742
7743 b.full = dfixed_const(1000);
7744 c.full = dfixed_const(wm->disp_clk);
7745 b.full = dfixed_div(c, b);
7746 c.full = dfixed_const(wm->bytes_per_pixel);
7747 b.full = dfixed_mul(b, c);
7748
7749 lb_fill_bw = min(tmp, dfixed_trunc(b));
7750
7751 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7752 b.full = dfixed_const(1000);
7753 c.full = dfixed_const(lb_fill_bw);
7754 b.full = dfixed_div(c, b);
7755 a.full = dfixed_div(a, b);
7756 line_fill_time = dfixed_trunc(a);
7757
7758 if (line_fill_time < wm->active_time)
7759 return latency;
7760 else
7761 return latency + (line_fill_time - wm->active_time);
7762
7763}
7764
7765/**
7766 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7767 * average and available dram bandwidth
7768 *
7769 * @wm: watermark calculation data
7770 *
7771 * Check if the display average bandwidth fits in the display
7772 * dram bandwidth (CIK).
7773 * Used for display watermark bandwidth calculations
7774 * Returns true if the display fits, false if not.
7775 */
7776static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7777{
7778 if (dce8_average_bandwidth(wm) <=
7779 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7780 return true;
7781 else
7782 return false;
7783}
7784
7785/**
7786 * dce8_average_bandwidth_vs_available_bandwidth - check
7787 * average and available bandwidth
7788 *
7789 * @wm: watermark calculation data
7790 *
7791 * Check if the display average bandwidth fits in the display
7792 * available bandwidth (CIK).
7793 * Used for display watermark bandwidth calculations
7794 * Returns true if the display fits, false if not.
7795 */
7796static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7797{
7798 if (dce8_average_bandwidth(wm) <=
7799 (dce8_available_bandwidth(wm) / wm->num_heads))
7800 return true;
7801 else
7802 return false;
7803}
7804
7805/**
7806 * dce8_check_latency_hiding - check latency hiding
7807 *
7808 * @wm: watermark calculation data
7809 *
7810 * Check latency hiding (CIK).
7811 * Used for display watermark bandwidth calculations
7812 * Returns true if the display fits, false if not.
7813 */
7814static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7815{
7816 u32 lb_partitions = wm->lb_size / wm->src_width;
7817 u32 line_time = wm->active_time + wm->blank_time;
7818 u32 latency_tolerant_lines;
7819 u32 latency_hiding;
7820 fixed20_12 a;
7821
7822 a.full = dfixed_const(1);
7823 if (wm->vsc.full > a.full)
7824 latency_tolerant_lines = 1;
7825 else {
7826 if (lb_partitions <= (wm->vtaps + 1))
7827 latency_tolerant_lines = 1;
7828 else
7829 latency_tolerant_lines = 2;
7830 }
7831
7832 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
7833
7834 if (dce8_latency_watermark(wm) <= latency_hiding)
7835 return true;
7836 else
7837 return false;
7838}
7839
7840/**
7841 * dce8_program_watermarks - program display watermarks
7842 *
7843 * @rdev: radeon_device pointer
7844 * @radeon_crtc: the selected display controller
7845 * @lb_size: line buffer size
7846 * @num_heads: number of display controllers in use
7847 *
7848 * Calculate and program the display watermarks for the
7849 * selected display controller (CIK).
7850 */
7851static void dce8_program_watermarks(struct radeon_device *rdev,
7852 struct radeon_crtc *radeon_crtc,
7853 u32 lb_size, u32 num_heads)
7854{
7855 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 7856 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 7857 u32 pixel_period;
7858 u32 line_time = 0;
7859 u32 latency_watermark_a = 0, latency_watermark_b = 0;
7860 u32 tmp, wm_mask;
7861
7862 if (radeon_crtc->base.enabled && num_heads && mode) {
7863 pixel_period = 1000000 / (u32)mode->clock;
7864 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
7865
58ea2dea
AD		 7866 /* watermark for high clocks */
7867 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7868 rdev->pm.dpm_enabled) {
7869 wm_high.yclk =
7870 radeon_dpm_get_mclk(rdev, false) * 10;
7871 wm_high.sclk =
7872 radeon_dpm_get_sclk(rdev, false) * 10;
7873 } else {
7874 wm_high.yclk = rdev->pm.current_mclk * 10;
7875 wm_high.sclk = rdev->pm.current_sclk * 10;
7876 }
7877
7878 wm_high.disp_clk = mode->clock;
7879 wm_high.src_width = mode->crtc_hdisplay;
7880 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
7881 wm_high.blank_time = line_time - wm_high.active_time;
7882 wm_high.interlaced = false;
cd84a27d 7883 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 7884 wm_high.interlaced = true;
7885 wm_high.vsc = radeon_crtc->vsc;
7886 wm_high.vtaps = 1;
cd84a27d 7887 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 7888 wm_high.vtaps = 2;
7889 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
7890 wm_high.lb_size = lb_size;
7891 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
7892 wm_high.num_heads = num_heads;
cd84a27d
AD		 7893
7894 /* set for high clocks */
58ea2dea
AD		 7895 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
7896
7897 /* possibly force display priority to high */
7898 /* should really do this at mode validation time... */
7899 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
7900 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
7901 !dce8_check_latency_hiding(&wm_high) ||
7902 (rdev->disp_priority == 2)) {
7903 DRM_DEBUG_KMS("force priority to high\n");
7904 }
7905
7906 /* watermark for low clocks */
7907 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7908 rdev->pm.dpm_enabled) {
7909 wm_low.yclk =
7910 radeon_dpm_get_mclk(rdev, true) * 10;
7911 wm_low.sclk =
7912 radeon_dpm_get_sclk(rdev, true) * 10;
7913 } else {
7914 wm_low.yclk = rdev->pm.current_mclk * 10;
7915 wm_low.sclk = rdev->pm.current_sclk * 10;
7916 }
7917
7918 wm_low.disp_clk = mode->clock;
7919 wm_low.src_width = mode->crtc_hdisplay;
7920 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
7921 wm_low.blank_time = line_time - wm_low.active_time;
7922 wm_low.interlaced = false;
7923 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7924 wm_low.interlaced = true;
7925 wm_low.vsc = radeon_crtc->vsc;
7926 wm_low.vtaps = 1;
7927 if (radeon_crtc->rmx_type != RMX_OFF)
7928 wm_low.vtaps = 2;
7929 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
7930 wm_low.lb_size = lb_size;
7931 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
7932 wm_low.num_heads = num_heads;
7933
cd84a27d 7934 /* set for low clocks */
58ea2dea 7935 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 7936
7937 /* possibly force display priority to high */
7938 /* should really do this at mode validation time... */
58ea2dea
AD		 7939 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
7940 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
7941 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 7942 (rdev->disp_priority == 2)) {
7943 DRM_DEBUG_KMS("force priority to high\n");
7944 }
7945 }
7946
7947 /* select wm A */
7948 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7949 tmp = wm_mask;
7950 tmp &= ~LATENCY_WATERMARK_MASK(3);
7951 tmp |= LATENCY_WATERMARK_MASK(1);
7952 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7953 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7954 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
7955 LATENCY_HIGH_WATERMARK(line_time)));
7956 /* select wm B */
7957 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7958 tmp &= ~LATENCY_WATERMARK_MASK(3);
7959 tmp |= LATENCY_WATERMARK_MASK(2);
7960 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7961 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7962 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
7963 LATENCY_HIGH_WATERMARK(line_time)));
7964 /* restore original selection */
7965 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 7966
7967 /* save values for DPM */
7968 radeon_crtc->line_time = line_time;
7969 radeon_crtc->wm_high = latency_watermark_a;
7970 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 7971}
7972
7973/**
7974 * dce8_bandwidth_update - program display watermarks
7975 *
7976 * @rdev: radeon_device pointer
7977 *
7978 * Calculate and program the display watermarks and line
7979 * buffer allocation (CIK).
7980 */
7981void dce8_bandwidth_update(struct radeon_device *rdev)
7982{
7983 struct drm_display_mode *mode = NULL;
7984 u32 num_heads = 0, lb_size;
7985 int i;
7986
7987 radeon_update_display_priority(rdev);
7988
7989 for (i = 0; i < rdev->num_crtc; i++) {
7990 if (rdev->mode_info.crtcs[i]->base.enabled)
7991 num_heads++;
7992 }
7993 for (i = 0; i < rdev->num_crtc; i++) {
7994 mode = &rdev->mode_info.crtcs[i]->base.mode;
7995 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
7996 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
7997 }
7998}
44fa346f
AD		 7999
8000/**
8001 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8002 *
8003 * @rdev: radeon_device pointer
8004 *
8005 * Fetches a GPU clock counter snapshot (SI).
8006 * Returns the 64 bit clock counter snapshot.
8007 */
8008uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8009{
8010 uint64_t clock;
8011
8012 mutex_lock(&rdev->gpu_clock_mutex);
8013 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8014 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8015 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8016 mutex_unlock(&rdev->gpu_clock_mutex);
8017 return clock;
8018}
8019
87167bb1
CK		 8020static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8021 u32 cntl_reg, u32 status_reg)
8022{
8023 int r, i;
8024 struct atom_clock_dividers dividers;
8025 uint32_t tmp;
8026
8027 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8028 clock, false, &dividers);
8029 if (r)
8030 return r;
8031
8032 tmp = RREG32_SMC(cntl_reg);
8033 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8034 tmp |= dividers.post_divider;
8035 WREG32_SMC(cntl_reg, tmp);
8036
8037 for (i = 0; i < 100; i++) {
8038 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8039 break;
8040 mdelay(10);
8041 }
8042 if (i == 100)
8043 return -ETIMEDOUT;
8044
8045 return 0;
8046}
8047
8048int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8049{
8050 int r = 0;
8051
8052 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8053 if (r)
8054 return r;
8055
8056 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8057 return r;
8058}
8059
8a7cd276
AD		 8060static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8061{
8062 struct pci_dev *root = rdev->pdev->bus->self;
8063 int bridge_pos, gpu_pos;
8064 u32 speed_cntl, mask, current_data_rate;
8065 int ret, i;
8066 u16 tmp16;
8067
8068 if (radeon_pcie_gen2 == 0)
8069 return;
8070
8071 if (rdev->flags & RADEON_IS_IGP)
8072 return;
8073
8074 if (!(rdev->flags & RADEON_IS_PCIE))
8075 return;
8076
8077 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8078 if (ret != 0)
8079 return;
8080
8081 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8082 return;
8083
8084 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8085 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8086 LC_CURRENT_DATA_RATE_SHIFT;
8087 if (mask & DRM_PCIE_SPEED_80) {
8088 if (current_data_rate == 2) {
8089 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8090 return;
8091 }
8092 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8093 } else if (mask & DRM_PCIE_SPEED_50) {
8094 if (current_data_rate == 1) {
8095 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8096 return;
8097 }
8098 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8099 }
8100
8101 bridge_pos = pci_pcie_cap(root);
8102 if (!bridge_pos)
8103 return;
8104
8105 gpu_pos = pci_pcie_cap(rdev->pdev);
8106 if (!gpu_pos)
8107 return;
8108
8109 if (mask & DRM_PCIE_SPEED_80) {
8110 /* re-try equalization if gen3 is not already enabled */
8111 if (current_data_rate != 2) {
8112 u16 bridge_cfg, gpu_cfg;
8113 u16 bridge_cfg2, gpu_cfg2;
8114 u32 max_lw, current_lw, tmp;
8115
8116 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8117 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8118
8119 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8120 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8121
8122 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8123 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8124
8125 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8126 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8127 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8128
8129 if (current_lw < max_lw) {
8130 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8131 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8132 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8133 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8134 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8135 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8136 }
8137 }
8138
8139 for (i = 0; i < 10; i++) {
8140 /* check status */
8141 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8142 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8143 break;
8144
8145 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8146 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8147
8148 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8149 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8150
8151 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8152 tmp |= LC_SET_QUIESCE;
8153 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8154
8155 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8156 tmp |= LC_REDO_EQ;
8157 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8158
8159 mdelay(100);
8160
8161 /* linkctl */
8162 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8163 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8164 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8165 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8166
8167 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8168 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8169 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8170 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8171
8172 /* linkctl2 */
8173 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8174 tmp16 &= ~((1 << 4) | (7 << 9));
8175 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8176 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8177
8178 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8179 tmp16 &= ~((1 << 4) | (7 << 9));
8180 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8181 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8182
8183 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8184 tmp &= ~LC_SET_QUIESCE;
8185 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8186 }
8187 }
8188 }
8189
8190 /* set the link speed */
8191 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8192 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8193 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8194
8195 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8196 tmp16 &= ~0xf;
8197 if (mask & DRM_PCIE_SPEED_80)
8198 tmp16 |= 3; /* gen3 */
8199 else if (mask & DRM_PCIE_SPEED_50)
8200 tmp16 |= 2; /* gen2 */
8201 else
8202 tmp16 |= 1; /* gen1 */
8203 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8204
8205 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8206 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8207 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8208
8209 for (i = 0; i < rdev->usec_timeout; i++) {
8210 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8211 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8212 break;
8213 udelay(1);
8214 }
8215}
7235711a
AD		 8216
8217static void cik_program_aspm(struct radeon_device *rdev)
8218{
8219 u32 data, orig;
8220 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8221 bool disable_clkreq = false;
8222
8223 if (radeon_aspm == 0)
8224 return;
8225
8226 /* XXX double check IGPs */
8227 if (rdev->flags & RADEON_IS_IGP)
8228 return;
8229
8230 if (!(rdev->flags & RADEON_IS_PCIE))
8231 return;
8232
8233 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8234 data &= ~LC_XMIT_N_FTS_MASK;
8235 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8236 if (orig != data)
8237 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8238
8239 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8240 data |= LC_GO_TO_RECOVERY;
8241 if (orig != data)
8242 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8243
8244 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8245 data |= P_IGNORE_EDB_ERR;
8246 if (orig != data)
8247 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8248
8249 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8250 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8251 data |= LC_PMI_TO_L1_DIS;
8252 if (!disable_l0s)
8253 data |= LC_L0S_INACTIVITY(7);
8254
8255 if (!disable_l1) {
8256 data |= LC_L1_INACTIVITY(7);
8257 data &= ~LC_PMI_TO_L1_DIS;
8258 if (orig != data)
8259 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8260
8261 if (!disable_plloff_in_l1) {
8262 bool clk_req_support;
8263
8264 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8265 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8266 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8267 if (orig != data)
8268 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8269
8270 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8271 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8272 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8273 if (orig != data)
8274 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8275
8276 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8277 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8278 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8279 if (orig != data)
8280 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8281
8282 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8283 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8284 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8285 if (orig != data)
8286 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8287
8288 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8289 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8290 data |= LC_DYN_LANES_PWR_STATE(3);
8291 if (orig != data)
8292 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8293
8294 if (!disable_clkreq) {
8295 struct pci_dev *root = rdev->pdev->bus->self;
8296 u32 lnkcap;
8297
8298 clk_req_support = false;
8299 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8300 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8301 clk_req_support = true;
8302 } else {
8303 clk_req_support = false;
8304 }
8305
8306 if (clk_req_support) {
8307 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8308 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8309 if (orig != data)
8310 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8311
8312 orig = data = RREG32_SMC(THM_CLK_CNTL);
8313 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8314 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8315 if (orig != data)
8316 WREG32_SMC(THM_CLK_CNTL, data);
8317
8318 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8319 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8320 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8321 if (orig != data)
8322 WREG32_SMC(MISC_CLK_CTRL, data);
8323
8324 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8325 data &= ~BCLK_AS_XCLK;
8326 if (orig != data)
8327 WREG32_SMC(CG_CLKPIN_CNTL, data);
8328
8329 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8330 data &= ~FORCE_BIF_REFCLK_EN;
8331 if (orig != data)
8332 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8333
8334 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8335 data &= ~MPLL_CLKOUT_SEL_MASK;
8336 data |= MPLL_CLKOUT_SEL(4);
8337 if (orig != data)
8338 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8339 }
8340 }
8341 } else {
8342 if (orig != data)
8343 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8344 }
8345
8346 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8347 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8348 if (orig != data)
8349 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8350
8351 if (!disable_l0s) {
8352 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8353 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8354 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8355 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8356 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8357 data &= ~LC_L0S_INACTIVITY_MASK;
8358 if (orig != data)
8359 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8360 }
8361 }
8362 }
8363}
Linux kernel - Deliverables
This page took 0.40107 seconds and 5 git commands to generate.