projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-4.1' of git://git.kernel.org/pub/scm/linux/kernel/git/kishon/linux...
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
e28740ec 36#include "radeon_kfd.h"
02c81327
AD		 37
38MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 43MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 44MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 45MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 46MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 47
48MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49MODULE_FIRMWARE("radeon/bonaire_me.bin");
50MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55MODULE_FIRMWARE("radeon/bonaire_smc.bin");
56
d4775655
AD		 57MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 62MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 63MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 66
67MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68MODULE_FIRMWARE("radeon/hawaii_me.bin");
69MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74MODULE_FIRMWARE("radeon/hawaii_smc.bin");
75
02c81327
AD		 76MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 81MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 82
83MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84MODULE_FIRMWARE("radeon/kaveri_me.bin");
85MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
90
02c81327
AD		 91MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92MODULE_FIRMWARE("radeon/KABINI_me.bin");
93MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 96MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 97
98MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99MODULE_FIRMWARE("radeon/kabini_me.bin");
100MODULE_FIRMWARE("radeon/kabini_ce.bin");
101MODULE_FIRMWARE("radeon/kabini_mec.bin");
102MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103MODULE_FIRMWARE("radeon/kabini_sdma.bin");
104
f73a9e83
SL		 105MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 111
f2c6b0f4
AD		 112MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113MODULE_FIRMWARE("radeon/mullins_me.bin");
114MODULE_FIRMWARE("radeon/mullins_ce.bin");
115MODULE_FIRMWARE("radeon/mullins_mec.bin");
116MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117MODULE_FIRMWARE("radeon/mullins_sdma.bin");
118
a59781bb
AD		 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 123extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 124extern void sumo_rlc_fini(struct radeon_device *rdev);
125extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 126extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 127extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 128extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 129static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 130extern int cik_sdma_resume(struct radeon_device *rdev);
131extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 133extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 134static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 135static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 136static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 137static void cik_init_pg(struct radeon_device *rdev);
138static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 139static void cik_fini_pg(struct radeon_device *rdev);
140static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 141static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 bool enable);
6f2043ce 143
286d9cc6
AD		 144/* get temperature in millidegrees */
145int ci_get_temp(struct radeon_device *rdev)
146{
147 u32 temp;
148 int actual_temp = 0;
149
150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
151 CTF_TEMP_SHIFT;
152
153 if (temp & 0x200)
154 actual_temp = 255;
155 else
156 actual_temp = temp & 0x1ff;
157
158 actual_temp = actual_temp * 1000;
159
160 return actual_temp;
161}
162
163/* get temperature in millidegrees */
164int kv_get_temp(struct radeon_device *rdev)
165{
166 u32 temp;
167 int actual_temp = 0;
168
169 temp = RREG32_SMC(0xC0300E0C);
170
171 if (temp)
172 actual_temp = (temp / 8) - 49;
173 else
174 actual_temp = 0;
175
176 actual_temp = actual_temp * 1000;
177
178 return actual_temp;
179}
6f2043ce 180
6e2c3c0a
AD		 181/*
182 * Indirect registers accessor
183 */
184u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
185{
0a5b7b0b 186 unsigned long flags;
6e2c3c0a
AD		 187 u32 r;
188
0a5b7b0b 189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 190 WREG32(PCIE_INDEX, reg);
191 (void)RREG32(PCIE_INDEX);
192 r = RREG32(PCIE_DATA);
0a5b7b0b 193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 194 return r;
195}
196
197void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
198{
0a5b7b0b
AD		 199 unsigned long flags;
200
201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 202 WREG32(PCIE_INDEX, reg);
203 (void)RREG32(PCIE_INDEX);
204 WREG32(PCIE_DATA, v);
205 (void)RREG32(PCIE_DATA);
0a5b7b0b 206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 207}
208
22c775ce
AD		 209static const u32 spectre_rlc_save_restore_register_list[] =
210{
211 (0x0e00 << 16) | (0xc12c >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc140 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc150 >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc15c >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc168 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc170 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc178 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc204 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b4 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2b8 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2bc >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc2c0 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x8228 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x829c >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0x869c >> 2),
240 0x00000000,
241 (0x0600 << 16) | (0x98f4 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x98f8 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0x9900 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc260 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x90e8 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c000 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x3c00c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x8c1c >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0x9700 >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x4e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x5e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x6e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x7e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x8e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0x9e00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xae00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0xbe00 << 16) | (0xcd20 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x89bc >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x8900 >> 2),
280 0x00000000,
281 0x3,
282 (0x0e00 << 16) | (0xc130 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc134 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc1fc >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc208 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc264 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc268 >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc26c >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc270 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc274 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc278 >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc27c >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc280 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc284 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc288 >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc28c >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc290 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc294 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc298 >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc29c >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a0 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a4 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2a8 >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2ac >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0xc2b0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x301d0 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30238 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30250 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30254 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x30258 >> 2),
339 0x00000000,
340 (0x0e00 << 16) | (0x3025c >> 2),
341 0x00000000,
342 (0x4e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x5e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x6e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x7e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x8e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0x9e00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xae00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0xbe00 << 16) | (0xc900 >> 2),
357 0x00000000,
358 (0x4e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x5e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x6e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x7e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x8e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0x9e00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xae00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0xbe00 << 16) | (0xc904 >> 2),
373 0x00000000,
374 (0x4e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x5e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x6e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x7e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x8e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0x9e00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xae00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0xbe00 << 16) | (0xc908 >> 2),
389 0x00000000,
390 (0x4e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x5e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x6e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x7e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x8e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0x9e00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xae00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0xbe00 << 16) | (0xc90c >> 2),
405 0x00000000,
406 (0x4e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x5e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x6e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x7e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x8e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0x9e00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xae00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0xbe00 << 16) | (0xc910 >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0xc99c >> 2),
423 0x00000000,
424 (0x0e00 << 16) | (0x9834 >> 2),
425 0x00000000,
426 (0x0000 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0001 << 16) | (0x30f00 >> 2),
429 0x00000000,
430 (0x0000 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0001 << 16) | (0x30f04 >> 2),
433 0x00000000,
434 (0x0000 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0001 << 16) | (0x30f08 >> 2),
437 0x00000000,
438 (0x0000 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0001 << 16) | (0x30f0c >> 2),
441 0x00000000,
442 (0x0600 << 16) | (0x9b7c >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a14 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x8a18 >> 2),
447 0x00000000,
448 (0x0600 << 16) | (0x30a00 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bf0 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8bcc >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x8b24 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x30a04 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a10 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a14 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a18 >> 2),
463 0x00000000,
464 (0x0600 << 16) | (0x30a2c >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc700 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc704 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc708 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0xc768 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc770 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc774 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc778 >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc77c >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc780 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc784 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc788 >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc78c >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc798 >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc79c >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a0 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a4 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7a8 >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7ac >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b0 >> 2),
503 0x00000000,
504 (0x0400 << 16) | (0xc7b4 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x9100 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x3c010 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92a8 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92ac >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b4 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92b8 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92bc >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c0 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c4 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92c8 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92cc >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x92d0 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c00 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c04 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c20 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c38 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0x8c3c >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0xae00 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0x9604 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac08 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac0c >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac10 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac14 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac58 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac68 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac6c >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac70 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac74 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac78 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac7c >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac80 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac84 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac88 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0xac8c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x970c >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9714 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x9718 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x971c >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x4e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x5e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x6e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x7e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x8e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0x9e00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xae00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0xbe00 << 16) | (0x31068 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd10 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xcd14 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b0 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b4 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88b8 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0x88bc >> 2),
611 0x00000000,
612 (0x0400 << 16) | (0x89c0 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c4 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88c8 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d0 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d4 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x88d8 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8980 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x30938 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x3093c >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x30940 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x89a0 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x30904 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x89b4 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c210 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c214 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x3c218 >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x8904 >> 2),
647 0x00000000,
648 0x5,
649 (0x0e00 << 16) | (0x8c28 >> 2),
650 (0x0e00 << 16) | (0x8c2c >> 2),
651 (0x0e00 << 16) | (0x8c30 >> 2),
652 (0x0e00 << 16) | (0x8c34 >> 2),
653 (0x0e00 << 16) | (0x9600 >> 2),
654};
655
656static const u32 kalindi_rlc_save_restore_register_list[] =
657{
658 (0x0e00 << 16) | (0xc12c >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc140 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc150 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc15c >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc168 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc170 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc204 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b4 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2b8 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2bc >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc2c0 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x8228 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x829c >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x869c >> 2),
685 0x00000000,
686 (0x0600 << 16) | (0x98f4 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x98f8 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x9900 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0xc260 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x90e8 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c000 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x3c00c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x8c1c >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x9700 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x4e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x5e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x6e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x7e00 << 16) | (0xcd20 >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x89bc >> 2),
715 0x00000000,
716 (0x0e00 << 16) | (0x8900 >> 2),
717 0x00000000,
718 0x3,
719 (0x0e00 << 16) | (0xc130 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc134 >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc1fc >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc208 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc264 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc268 >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc26c >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc270 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc274 >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc28c >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc290 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc294 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc298 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a0 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a4 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2a8 >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0xc2ac >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x301d0 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30238 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30250 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30254 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x30258 >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x3025c >> 2),
764 0x00000000,
765 (0x4e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x5e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x6e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x7e00 << 16) | (0xc900 >> 2),
772 0x00000000,
773 (0x4e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x5e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x6e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x7e00 << 16) | (0xc904 >> 2),
780 0x00000000,
781 (0x4e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x5e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x6e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x7e00 << 16) | (0xc908 >> 2),
788 0x00000000,
789 (0x4e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x5e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x6e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x7e00 << 16) | (0xc90c >> 2),
796 0x00000000,
797 (0x4e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x5e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x6e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x7e00 << 16) | (0xc910 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc99c >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x9834 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f00 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f04 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f08 >> 2),
814 0x00000000,
815 (0x0000 << 16) | (0x30f0c >> 2),
816 0x00000000,
817 (0x0600 << 16) | (0x9b7c >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a14 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x8a18 >> 2),
822 0x00000000,
823 (0x0600 << 16) | (0x30a00 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bf0 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8bcc >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x8b24 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0x30a04 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a10 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a14 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a18 >> 2),
838 0x00000000,
839 (0x0600 << 16) | (0x30a2c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc700 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc704 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc708 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xc768 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc770 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc774 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc798 >> 2),
854 0x00000000,
855 (0x0400 << 16) | (0xc79c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x9100 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x3c010 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c00 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c04 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c20 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c38 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x8c3c >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0xae00 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x9604 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac08 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac0c >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac10 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac14 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac58 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac68 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac6c >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac70 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac74 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac78 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac7c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac80 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac84 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac88 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xac8c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x970c >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9714 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x9718 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x971c >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x4e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x5e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x6e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x7e00 << 16) | (0x31068 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd10 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0xcd14 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b0 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b4 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88b8 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0x88bc >> 2),
934 0x00000000,
935 (0x0400 << 16) | (0x89c0 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c4 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88c8 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d0 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d4 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x88d8 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x8980 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x30938 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x3093c >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x30940 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x89a0 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30900 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x30904 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x89b4 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3e1fc >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c210 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c214 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x3c218 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x8904 >> 2),
972 0x00000000,
973 0x5,
974 (0x0e00 << 16) | (0x8c28 >> 2),
975 (0x0e00 << 16) | (0x8c2c >> 2),
976 (0x0e00 << 16) | (0x8c30 >> 2),
977 (0x0e00 << 16) | (0x8c34 >> 2),
978 (0x0e00 << 16) | (0x9600 >> 2),
979};
980
0aafd313
AD		 981static const u32 bonaire_golden_spm_registers[] =
982{
983 0x30800, 0xe0ffffff, 0xe0000000
984};
985
986static const u32 bonaire_golden_common_registers[] =
987{
988 0xc770, 0xffffffff, 0x00000800,
989 0xc774, 0xffffffff, 0x00000800,
990 0xc798, 0xffffffff, 0x00007fbf,
991 0xc79c, 0xffffffff, 0x00007faf
992};
993
994static const u32 bonaire_golden_registers[] =
995{
996 0x3354, 0x00000333, 0x00000333,
997 0x3350, 0x000c0fc0, 0x00040200,
998 0x9a10, 0x00010000, 0x00058208,
999 0x3c000, 0xffff1fff, 0x00140000,
1000 0x3c200, 0xfdfc0fff, 0x00000100,
1001 0x3c234, 0x40000000, 0x40000200,
1002 0x9830, 0xffffffff, 0x00000000,
1003 0x9834, 0xf00fffff, 0x00000400,
1004 0x9838, 0x0002021c, 0x00020200,
1005 0xc78, 0x00000080, 0x00000000,
1006 0x5bb0, 0x000000f0, 0x00000070,
1007 0x5bc0, 0xf0311fff, 0x80300000,
1008 0x98f8, 0x73773777, 0x12010001,
1009 0x350c, 0x00810000, 0x408af000,
1010 0x7030, 0x31000111, 0x00000011,
1011 0x2f48, 0x73773777, 0x12010001,
1012 0x220c, 0x00007fb6, 0x0021a1b1,
1013 0x2210, 0x00007fb6, 0x002021b1,
1014 0x2180, 0x00007fb6, 0x00002191,
1015 0x2218, 0x00007fb6, 0x002121b1,
1016 0x221c, 0x00007fb6, 0x002021b1,
1017 0x21dc, 0x00007fb6, 0x00002191,
1018 0x21e0, 0x00007fb6, 0x00002191,
1019 0x3628, 0x0000003f, 0x0000000a,
1020 0x362c, 0x0000003f, 0x0000000a,
1021 0x2ae4, 0x00073ffe, 0x000022a2,
1022 0x240c, 0x000007ff, 0x00000000,
1023 0x8a14, 0xf000003f, 0x00000007,
1024 0x8bf0, 0x00002001, 0x00000001,
1025 0x8b24, 0xffffffff, 0x00ffffff,
1026 0x30a04, 0x0000ff0f, 0x00000000,
1027 0x28a4c, 0x07ffffff, 0x06000000,
1028 0x4d8, 0x00000fff, 0x00000100,
1029 0x3e78, 0x00000001, 0x00000002,
1030 0x9100, 0x03000000, 0x0362c688,
1031 0x8c00, 0x000000ff, 0x00000001,
1032 0xe40, 0x00001fff, 0x00001fff,
1033 0x9060, 0x0000007f, 0x00000020,
1034 0x9508, 0x00010000, 0x00010000,
1035 0xac14, 0x000003ff, 0x000000f3,
1036 0xac0c, 0xffffffff, 0x00001032
1037};
1038
1039static const u32 bonaire_mgcg_cgcg_init[] =
1040{
1041 0xc420, 0xffffffff, 0xfffffffc,
1042 0x30800, 0xffffffff, 0xe0000000,
1043 0x3c2a0, 0xffffffff, 0x00000100,
1044 0x3c208, 0xffffffff, 0x00000100,
1045 0x3c2c0, 0xffffffff, 0xc0000100,
1046 0x3c2c8, 0xffffffff, 0xc0000100,
1047 0x3c2c4, 0xffffffff, 0xc0000100,
1048 0x55e4, 0xffffffff, 0x00600100,
1049 0x3c280, 0xffffffff, 0x00000100,
1050 0x3c214, 0xffffffff, 0x06000100,
1051 0x3c220, 0xffffffff, 0x00000100,
1052 0x3c218, 0xffffffff, 0x06000100,
1053 0x3c204, 0xffffffff, 0x00000100,
1054 0x3c2e0, 0xffffffff, 0x00000100,
1055 0x3c224, 0xffffffff, 0x00000100,
1056 0x3c200, 0xffffffff, 0x00000100,
1057 0x3c230, 0xffffffff, 0x00000100,
1058 0x3c234, 0xffffffff, 0x00000100,
1059 0x3c250, 0xffffffff, 0x00000100,
1060 0x3c254, 0xffffffff, 0x00000100,
1061 0x3c258, 0xffffffff, 0x00000100,
1062 0x3c25c, 0xffffffff, 0x00000100,
1063 0x3c260, 0xffffffff, 0x00000100,
1064 0x3c27c, 0xffffffff, 0x00000100,
1065 0x3c278, 0xffffffff, 0x00000100,
1066 0x3c210, 0xffffffff, 0x06000100,
1067 0x3c290, 0xffffffff, 0x00000100,
1068 0x3c274, 0xffffffff, 0x00000100,
1069 0x3c2b4, 0xffffffff, 0x00000100,
1070 0x3c2b0, 0xffffffff, 0x00000100,
1071 0x3c270, 0xffffffff, 0x00000100,
1072 0x30800, 0xffffffff, 0xe0000000,
1073 0x3c020, 0xffffffff, 0x00010000,
1074 0x3c024, 0xffffffff, 0x00030002,
1075 0x3c028, 0xffffffff, 0x00040007,
1076 0x3c02c, 0xffffffff, 0x00060005,
1077 0x3c030, 0xffffffff, 0x00090008,
1078 0x3c034, 0xffffffff, 0x00010000,
1079 0x3c038, 0xffffffff, 0x00030002,
1080 0x3c03c, 0xffffffff, 0x00040007,
1081 0x3c040, 0xffffffff, 0x00060005,
1082 0x3c044, 0xffffffff, 0x00090008,
1083 0x3c048, 0xffffffff, 0x00010000,
1084 0x3c04c, 0xffffffff, 0x00030002,
1085 0x3c050, 0xffffffff, 0x00040007,
1086 0x3c054, 0xffffffff, 0x00060005,
1087 0x3c058, 0xffffffff, 0x00090008,
1088 0x3c05c, 0xffffffff, 0x00010000,
1089 0x3c060, 0xffffffff, 0x00030002,
1090 0x3c064, 0xffffffff, 0x00040007,
1091 0x3c068, 0xffffffff, 0x00060005,
1092 0x3c06c, 0xffffffff, 0x00090008,
1093 0x3c070, 0xffffffff, 0x00010000,
1094 0x3c074, 0xffffffff, 0x00030002,
1095 0x3c078, 0xffffffff, 0x00040007,
1096 0x3c07c, 0xffffffff, 0x00060005,
1097 0x3c080, 0xffffffff, 0x00090008,
1098 0x3c084, 0xffffffff, 0x00010000,
1099 0x3c088, 0xffffffff, 0x00030002,
1100 0x3c08c, 0xffffffff, 0x00040007,
1101 0x3c090, 0xffffffff, 0x00060005,
1102 0x3c094, 0xffffffff, 0x00090008,
1103 0x3c098, 0xffffffff, 0x00010000,
1104 0x3c09c, 0xffffffff, 0x00030002,
1105 0x3c0a0, 0xffffffff, 0x00040007,
1106 0x3c0a4, 0xffffffff, 0x00060005,
1107 0x3c0a8, 0xffffffff, 0x00090008,
1108 0x3c000, 0xffffffff, 0x96e00200,
1109 0x8708, 0xffffffff, 0x00900100,
1110 0xc424, 0xffffffff, 0x0020003f,
1111 0x38, 0xffffffff, 0x0140001c,
1112 0x3c, 0x000f0000, 0x000f0000,
1113 0x220, 0xffffffff, 0xC060000C,
1114 0x224, 0xc0000fff, 0x00000100,
1115 0xf90, 0xffffffff, 0x00000100,
1116 0xf98, 0x00000101, 0x00000000,
1117 0x20a8, 0xffffffff, 0x00000104,
1118 0x55e4, 0xff000fff, 0x00000100,
1119 0x30cc, 0xc0000fff, 0x00000104,
1120 0xc1e4, 0x00000001, 0x00000001,
1121 0xd00c, 0xff000ff0, 0x00000100,
1122 0xd80c, 0xff000ff0, 0x00000100
1123};
1124
1125static const u32 spectre_golden_spm_registers[] =
1126{
1127 0x30800, 0xe0ffffff, 0xe0000000
1128};
1129
1130static const u32 spectre_golden_common_registers[] =
1131{
1132 0xc770, 0xffffffff, 0x00000800,
1133 0xc774, 0xffffffff, 0x00000800,
1134 0xc798, 0xffffffff, 0x00007fbf,
1135 0xc79c, 0xffffffff, 0x00007faf
1136};
1137
1138static const u32 spectre_golden_registers[] =
1139{
1140 0x3c000, 0xffff1fff, 0x96940200,
1141 0x3c00c, 0xffff0001, 0xff000000,
1142 0x3c200, 0xfffc0fff, 0x00000100,
1143 0x6ed8, 0x00010101, 0x00010000,
1144 0x9834, 0xf00fffff, 0x00000400,
1145 0x9838, 0xfffffffc, 0x00020200,
1146 0x5bb0, 0x000000f0, 0x00000070,
1147 0x5bc0, 0xf0311fff, 0x80300000,
1148 0x98f8, 0x73773777, 0x12010001,
1149 0x9b7c, 0x00ff0000, 0x00fc0000,
1150 0x2f48, 0x73773777, 0x12010001,
1151 0x8a14, 0xf000003f, 0x00000007,
1152 0x8b24, 0xffffffff, 0x00ffffff,
1153 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1154 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1155 0x3e78, 0x00000001, 0x00000002,
1156 0x913c, 0xffff03df, 0x00000004,
1157 0xc768, 0x00000008, 0x00000008,
1158 0x8c00, 0x000008ff, 0x00000800,
1159 0x9508, 0x00010000, 0x00010000,
1160 0xac0c, 0xffffffff, 0x54763210,
1161 0x214f8, 0x01ff01ff, 0x00000002,
1162 0x21498, 0x007ff800, 0x00200000,
1163 0x2015c, 0xffffffff, 0x00000f40,
1164 0x30934, 0xffffffff, 0x00000001
1165};
1166
1167static const u32 spectre_mgcg_cgcg_init[] =
1168{
1169 0xc420, 0xffffffff, 0xfffffffc,
1170 0x30800, 0xffffffff, 0xe0000000,
1171 0x3c2a0, 0xffffffff, 0x00000100,
1172 0x3c208, 0xffffffff, 0x00000100,
1173 0x3c2c0, 0xffffffff, 0x00000100,
1174 0x3c2c8, 0xffffffff, 0x00000100,
1175 0x3c2c4, 0xffffffff, 0x00000100,
1176 0x55e4, 0xffffffff, 0x00600100,
1177 0x3c280, 0xffffffff, 0x00000100,
1178 0x3c214, 0xffffffff, 0x06000100,
1179 0x3c220, 0xffffffff, 0x00000100,
1180 0x3c218, 0xffffffff, 0x06000100,
1181 0x3c204, 0xffffffff, 0x00000100,
1182 0x3c2e0, 0xffffffff, 0x00000100,
1183 0x3c224, 0xffffffff, 0x00000100,
1184 0x3c200, 0xffffffff, 0x00000100,
1185 0x3c230, 0xffffffff, 0x00000100,
1186 0x3c234, 0xffffffff, 0x00000100,
1187 0x3c250, 0xffffffff, 0x00000100,
1188 0x3c254, 0xffffffff, 0x00000100,
1189 0x3c258, 0xffffffff, 0x00000100,
1190 0x3c25c, 0xffffffff, 0x00000100,
1191 0x3c260, 0xffffffff, 0x00000100,
1192 0x3c27c, 0xffffffff, 0x00000100,
1193 0x3c278, 0xffffffff, 0x00000100,
1194 0x3c210, 0xffffffff, 0x06000100,
1195 0x3c290, 0xffffffff, 0x00000100,
1196 0x3c274, 0xffffffff, 0x00000100,
1197 0x3c2b4, 0xffffffff, 0x00000100,
1198 0x3c2b0, 0xffffffff, 0x00000100,
1199 0x3c270, 0xffffffff, 0x00000100,
1200 0x30800, 0xffffffff, 0xe0000000,
1201 0x3c020, 0xffffffff, 0x00010000,
1202 0x3c024, 0xffffffff, 0x00030002,
1203 0x3c028, 0xffffffff, 0x00040007,
1204 0x3c02c, 0xffffffff, 0x00060005,
1205 0x3c030, 0xffffffff, 0x00090008,
1206 0x3c034, 0xffffffff, 0x00010000,
1207 0x3c038, 0xffffffff, 0x00030002,
1208 0x3c03c, 0xffffffff, 0x00040007,
1209 0x3c040, 0xffffffff, 0x00060005,
1210 0x3c044, 0xffffffff, 0x00090008,
1211 0x3c048, 0xffffffff, 0x00010000,
1212 0x3c04c, 0xffffffff, 0x00030002,
1213 0x3c050, 0xffffffff, 0x00040007,
1214 0x3c054, 0xffffffff, 0x00060005,
1215 0x3c058, 0xffffffff, 0x00090008,
1216 0x3c05c, 0xffffffff, 0x00010000,
1217 0x3c060, 0xffffffff, 0x00030002,
1218 0x3c064, 0xffffffff, 0x00040007,
1219 0x3c068, 0xffffffff, 0x00060005,
1220 0x3c06c, 0xffffffff, 0x00090008,
1221 0x3c070, 0xffffffff, 0x00010000,
1222 0x3c074, 0xffffffff, 0x00030002,
1223 0x3c078, 0xffffffff, 0x00040007,
1224 0x3c07c, 0xffffffff, 0x00060005,
1225 0x3c080, 0xffffffff, 0x00090008,
1226 0x3c084, 0xffffffff, 0x00010000,
1227 0x3c088, 0xffffffff, 0x00030002,
1228 0x3c08c, 0xffffffff, 0x00040007,
1229 0x3c090, 0xffffffff, 0x00060005,
1230 0x3c094, 0xffffffff, 0x00090008,
1231 0x3c098, 0xffffffff, 0x00010000,
1232 0x3c09c, 0xffffffff, 0x00030002,
1233 0x3c0a0, 0xffffffff, 0x00040007,
1234 0x3c0a4, 0xffffffff, 0x00060005,
1235 0x3c0a8, 0xffffffff, 0x00090008,
1236 0x3c0ac, 0xffffffff, 0x00010000,
1237 0x3c0b0, 0xffffffff, 0x00030002,
1238 0x3c0b4, 0xffffffff, 0x00040007,
1239 0x3c0b8, 0xffffffff, 0x00060005,
1240 0x3c0bc, 0xffffffff, 0x00090008,
1241 0x3c000, 0xffffffff, 0x96e00200,
1242 0x8708, 0xffffffff, 0x00900100,
1243 0xc424, 0xffffffff, 0x0020003f,
1244 0x38, 0xffffffff, 0x0140001c,
1245 0x3c, 0x000f0000, 0x000f0000,
1246 0x220, 0xffffffff, 0xC060000C,
1247 0x224, 0xc0000fff, 0x00000100,
1248 0xf90, 0xffffffff, 0x00000100,
1249 0xf98, 0x00000101, 0x00000000,
1250 0x20a8, 0xffffffff, 0x00000104,
1251 0x55e4, 0xff000fff, 0x00000100,
1252 0x30cc, 0xc0000fff, 0x00000104,
1253 0xc1e4, 0x00000001, 0x00000001,
1254 0xd00c, 0xff000ff0, 0x00000100,
1255 0xd80c, 0xff000ff0, 0x00000100
1256};
1257
1258static const u32 kalindi_golden_spm_registers[] =
1259{
1260 0x30800, 0xe0ffffff, 0xe0000000
1261};
1262
1263static const u32 kalindi_golden_common_registers[] =
1264{
1265 0xc770, 0xffffffff, 0x00000800,
1266 0xc774, 0xffffffff, 0x00000800,
1267 0xc798, 0xffffffff, 0x00007fbf,
1268 0xc79c, 0xffffffff, 0x00007faf
1269};
1270
1271static const u32 kalindi_golden_registers[] =
1272{
1273 0x3c000, 0xffffdfff, 0x6e944040,
1274 0x55e4, 0xff607fff, 0xfc000100,
1275 0x3c220, 0xff000fff, 0x00000100,
1276 0x3c224, 0xff000fff, 0x00000100,
1277 0x3c200, 0xfffc0fff, 0x00000100,
1278 0x6ed8, 0x00010101, 0x00010000,
1279 0x9830, 0xffffffff, 0x00000000,
1280 0x9834, 0xf00fffff, 0x00000400,
1281 0x5bb0, 0x000000f0, 0x00000070,
1282 0x5bc0, 0xf0311fff, 0x80300000,
1283 0x98f8, 0x73773777, 0x12010001,
1284 0x98fc, 0xffffffff, 0x00000010,
1285 0x9b7c, 0x00ff0000, 0x00fc0000,
1286 0x8030, 0x00001f0f, 0x0000100a,
1287 0x2f48, 0x73773777, 0x12010001,
1288 0x2408, 0x000fffff, 0x000c007f,
1289 0x8a14, 0xf000003f, 0x00000007,
1290 0x8b24, 0x3fff3fff, 0x00ffcfff,
1291 0x30a04, 0x0000ff0f, 0x00000000,
1292 0x28a4c, 0x07ffffff, 0x06000000,
1293 0x4d8, 0x00000fff, 0x00000100,
1294 0x3e78, 0x00000001, 0x00000002,
1295 0xc768, 0x00000008, 0x00000008,
1296 0x8c00, 0x000000ff, 0x00000003,
1297 0x214f8, 0x01ff01ff, 0x00000002,
1298 0x21498, 0x007ff800, 0x00200000,
1299 0x2015c, 0xffffffff, 0x00000f40,
1300 0x88c4, 0x001f3ae3, 0x00000082,
1301 0x88d4, 0x0000001f, 0x00000010,
1302 0x30934, 0xffffffff, 0x00000000
1303};
1304
1305static const u32 kalindi_mgcg_cgcg_init[] =
1306{
1307 0xc420, 0xffffffff, 0xfffffffc,
1308 0x30800, 0xffffffff, 0xe0000000,
1309 0x3c2a0, 0xffffffff, 0x00000100,
1310 0x3c208, 0xffffffff, 0x00000100,
1311 0x3c2c0, 0xffffffff, 0x00000100,
1312 0x3c2c8, 0xffffffff, 0x00000100,
1313 0x3c2c4, 0xffffffff, 0x00000100,
1314 0x55e4, 0xffffffff, 0x00600100,
1315 0x3c280, 0xffffffff, 0x00000100,
1316 0x3c214, 0xffffffff, 0x06000100,
1317 0x3c220, 0xffffffff, 0x00000100,
1318 0x3c218, 0xffffffff, 0x06000100,
1319 0x3c204, 0xffffffff, 0x00000100,
1320 0x3c2e0, 0xffffffff, 0x00000100,
1321 0x3c224, 0xffffffff, 0x00000100,
1322 0x3c200, 0xffffffff, 0x00000100,
1323 0x3c230, 0xffffffff, 0x00000100,
1324 0x3c234, 0xffffffff, 0x00000100,
1325 0x3c250, 0xffffffff, 0x00000100,
1326 0x3c254, 0xffffffff, 0x00000100,
1327 0x3c258, 0xffffffff, 0x00000100,
1328 0x3c25c, 0xffffffff, 0x00000100,
1329 0x3c260, 0xffffffff, 0x00000100,
1330 0x3c27c, 0xffffffff, 0x00000100,
1331 0x3c278, 0xffffffff, 0x00000100,
1332 0x3c210, 0xffffffff, 0x06000100,
1333 0x3c290, 0xffffffff, 0x00000100,
1334 0x3c274, 0xffffffff, 0x00000100,
1335 0x3c2b4, 0xffffffff, 0x00000100,
1336 0x3c2b0, 0xffffffff, 0x00000100,
1337 0x3c270, 0xffffffff, 0x00000100,
1338 0x30800, 0xffffffff, 0xe0000000,
1339 0x3c020, 0xffffffff, 0x00010000,
1340 0x3c024, 0xffffffff, 0x00030002,
1341 0x3c028, 0xffffffff, 0x00040007,
1342 0x3c02c, 0xffffffff, 0x00060005,
1343 0x3c030, 0xffffffff, 0x00090008,
1344 0x3c034, 0xffffffff, 0x00010000,
1345 0x3c038, 0xffffffff, 0x00030002,
1346 0x3c03c, 0xffffffff, 0x00040007,
1347 0x3c040, 0xffffffff, 0x00060005,
1348 0x3c044, 0xffffffff, 0x00090008,
1349 0x3c000, 0xffffffff, 0x96e00200,
1350 0x8708, 0xffffffff, 0x00900100,
1351 0xc424, 0xffffffff, 0x0020003f,
1352 0x38, 0xffffffff, 0x0140001c,
1353 0x3c, 0x000f0000, 0x000f0000,
1354 0x220, 0xffffffff, 0xC060000C,
1355 0x224, 0xc0000fff, 0x00000100,
1356 0x20a8, 0xffffffff, 0x00000104,
1357 0x55e4, 0xff000fff, 0x00000100,
1358 0x30cc, 0xc0000fff, 0x00000104,
1359 0xc1e4, 0x00000001, 0x00000001,
1360 0xd00c, 0xff000ff0, 0x00000100,
1361 0xd80c, 0xff000ff0, 0x00000100
1362};
1363
8efff337
AD		 1364static const u32 hawaii_golden_spm_registers[] =
1365{
1366 0x30800, 0xe0ffffff, 0xe0000000
1367};
1368
1369static const u32 hawaii_golden_common_registers[] =
1370{
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x28350, 0xffffffff, 0x3a00161a,
1373 0x28354, 0xffffffff, 0x0000002e,
1374 0x9a10, 0xffffffff, 0x00018208,
1375 0x98f8, 0xffffffff, 0x12011003
1376};
1377
1378static const u32 hawaii_golden_registers[] =
1379{
1380 0x3354, 0x00000333, 0x00000333,
1381 0x9a10, 0x00010000, 0x00058208,
1382 0x9830, 0xffffffff, 0x00000000,
1383 0x9834, 0xf00fffff, 0x00000400,
1384 0x9838, 0x0002021c, 0x00020200,
1385 0xc78, 0x00000080, 0x00000000,
1386 0x5bb0, 0x000000f0, 0x00000070,
1387 0x5bc0, 0xf0311fff, 0x80300000,
1388 0x350c, 0x00810000, 0x408af000,
1389 0x7030, 0x31000111, 0x00000011,
1390 0x2f48, 0x73773777, 0x12010001,
1391 0x2120, 0x0000007f, 0x0000001b,
1392 0x21dc, 0x00007fb6, 0x00002191,
1393 0x3628, 0x0000003f, 0x0000000a,
1394 0x362c, 0x0000003f, 0x0000000a,
1395 0x2ae4, 0x00073ffe, 0x000022a2,
1396 0x240c, 0x000007ff, 0x00000000,
1397 0x8bf0, 0x00002001, 0x00000001,
1398 0x8b24, 0xffffffff, 0x00ffffff,
1399 0x30a04, 0x0000ff0f, 0x00000000,
1400 0x28a4c, 0x07ffffff, 0x06000000,
1401 0x3e78, 0x00000001, 0x00000002,
1402 0xc768, 0x00000008, 0x00000008,
1403 0xc770, 0x00000f00, 0x00000800,
1404 0xc774, 0x00000f00, 0x00000800,
1405 0xc798, 0x00ffffff, 0x00ff7fbf,
1406 0xc79c, 0x00ffffff, 0x00ff7faf,
1407 0x8c00, 0x000000ff, 0x00000800,
1408 0xe40, 0x00001fff, 0x00001fff,
1409 0x9060, 0x0000007f, 0x00000020,
1410 0x9508, 0x00010000, 0x00010000,
1411 0xae00, 0x00100000, 0x000ff07c,
1412 0xac14, 0x000003ff, 0x0000000f,
1413 0xac10, 0xffffffff, 0x7564fdec,
1414 0xac0c, 0xffffffff, 0x3120b9a8,
1415 0xac08, 0x20000000, 0x0f9c0000
1416};
1417
1418static const u32 hawaii_mgcg_cgcg_init[] =
1419{
1420 0xc420, 0xffffffff, 0xfffffffd,
1421 0x30800, 0xffffffff, 0xe0000000,
1422 0x3c2a0, 0xffffffff, 0x00000100,
1423 0x3c208, 0xffffffff, 0x00000100,
1424 0x3c2c0, 0xffffffff, 0x00000100,
1425 0x3c2c8, 0xffffffff, 0x00000100,
1426 0x3c2c4, 0xffffffff, 0x00000100,
1427 0x55e4, 0xffffffff, 0x00200100,
1428 0x3c280, 0xffffffff, 0x00000100,
1429 0x3c214, 0xffffffff, 0x06000100,
1430 0x3c220, 0xffffffff, 0x00000100,
1431 0x3c218, 0xffffffff, 0x06000100,
1432 0x3c204, 0xffffffff, 0x00000100,
1433 0x3c2e0, 0xffffffff, 0x00000100,
1434 0x3c224, 0xffffffff, 0x00000100,
1435 0x3c200, 0xffffffff, 0x00000100,
1436 0x3c230, 0xffffffff, 0x00000100,
1437 0x3c234, 0xffffffff, 0x00000100,
1438 0x3c250, 0xffffffff, 0x00000100,
1439 0x3c254, 0xffffffff, 0x00000100,
1440 0x3c258, 0xffffffff, 0x00000100,
1441 0x3c25c, 0xffffffff, 0x00000100,
1442 0x3c260, 0xffffffff, 0x00000100,
1443 0x3c27c, 0xffffffff, 0x00000100,
1444 0x3c278, 0xffffffff, 0x00000100,
1445 0x3c210, 0xffffffff, 0x06000100,
1446 0x3c290, 0xffffffff, 0x00000100,
1447 0x3c274, 0xffffffff, 0x00000100,
1448 0x3c2b4, 0xffffffff, 0x00000100,
1449 0x3c2b0, 0xffffffff, 0x00000100,
1450 0x3c270, 0xffffffff, 0x00000100,
1451 0x30800, 0xffffffff, 0xe0000000,
1452 0x3c020, 0xffffffff, 0x00010000,
1453 0x3c024, 0xffffffff, 0x00030002,
1454 0x3c028, 0xffffffff, 0x00040007,
1455 0x3c02c, 0xffffffff, 0x00060005,
1456 0x3c030, 0xffffffff, 0x00090008,
1457 0x3c034, 0xffffffff, 0x00010000,
1458 0x3c038, 0xffffffff, 0x00030002,
1459 0x3c03c, 0xffffffff, 0x00040007,
1460 0x3c040, 0xffffffff, 0x00060005,
1461 0x3c044, 0xffffffff, 0x00090008,
1462 0x3c048, 0xffffffff, 0x00010000,
1463 0x3c04c, 0xffffffff, 0x00030002,
1464 0x3c050, 0xffffffff, 0x00040007,
1465 0x3c054, 0xffffffff, 0x00060005,
1466 0x3c058, 0xffffffff, 0x00090008,
1467 0x3c05c, 0xffffffff, 0x00010000,
1468 0x3c060, 0xffffffff, 0x00030002,
1469 0x3c064, 0xffffffff, 0x00040007,
1470 0x3c068, 0xffffffff, 0x00060005,
1471 0x3c06c, 0xffffffff, 0x00090008,
1472 0x3c070, 0xffffffff, 0x00010000,
1473 0x3c074, 0xffffffff, 0x00030002,
1474 0x3c078, 0xffffffff, 0x00040007,
1475 0x3c07c, 0xffffffff, 0x00060005,
1476 0x3c080, 0xffffffff, 0x00090008,
1477 0x3c084, 0xffffffff, 0x00010000,
1478 0x3c088, 0xffffffff, 0x00030002,
1479 0x3c08c, 0xffffffff, 0x00040007,
1480 0x3c090, 0xffffffff, 0x00060005,
1481 0x3c094, 0xffffffff, 0x00090008,
1482 0x3c098, 0xffffffff, 0x00010000,
1483 0x3c09c, 0xffffffff, 0x00030002,
1484 0x3c0a0, 0xffffffff, 0x00040007,
1485 0x3c0a4, 0xffffffff, 0x00060005,
1486 0x3c0a8, 0xffffffff, 0x00090008,
1487 0x3c0ac, 0xffffffff, 0x00010000,
1488 0x3c0b0, 0xffffffff, 0x00030002,
1489 0x3c0b4, 0xffffffff, 0x00040007,
1490 0x3c0b8, 0xffffffff, 0x00060005,
1491 0x3c0bc, 0xffffffff, 0x00090008,
1492 0x3c0c0, 0xffffffff, 0x00010000,
1493 0x3c0c4, 0xffffffff, 0x00030002,
1494 0x3c0c8, 0xffffffff, 0x00040007,
1495 0x3c0cc, 0xffffffff, 0x00060005,
1496 0x3c0d0, 0xffffffff, 0x00090008,
1497 0x3c0d4, 0xffffffff, 0x00010000,
1498 0x3c0d8, 0xffffffff, 0x00030002,
1499 0x3c0dc, 0xffffffff, 0x00040007,
1500 0x3c0e0, 0xffffffff, 0x00060005,
1501 0x3c0e4, 0xffffffff, 0x00090008,
1502 0x3c0e8, 0xffffffff, 0x00010000,
1503 0x3c0ec, 0xffffffff, 0x00030002,
1504 0x3c0f0, 0xffffffff, 0x00040007,
1505 0x3c0f4, 0xffffffff, 0x00060005,
1506 0x3c0f8, 0xffffffff, 0x00090008,
1507 0xc318, 0xffffffff, 0x00020200,
1508 0x3350, 0xffffffff, 0x00000200,
1509 0x15c0, 0xffffffff, 0x00000400,
1510 0x55e8, 0xffffffff, 0x00000000,
1511 0x2f50, 0xffffffff, 0x00000902,
1512 0x3c000, 0xffffffff, 0x96940200,
1513 0x8708, 0xffffffff, 0x00900100,
1514 0xc424, 0xffffffff, 0x0020003f,
1515 0x38, 0xffffffff, 0x0140001c,
1516 0x3c, 0x000f0000, 0x000f0000,
1517 0x220, 0xffffffff, 0xc060000c,
1518 0x224, 0xc0000fff, 0x00000100,
1519 0xf90, 0xffffffff, 0x00000100,
1520 0xf98, 0x00000101, 0x00000000,
1521 0x20a8, 0xffffffff, 0x00000104,
1522 0x55e4, 0xff000fff, 0x00000100,
1523 0x30cc, 0xc0000fff, 0x00000104,
1524 0xc1e4, 0x00000001, 0x00000001,
1525 0xd00c, 0xff000ff0, 0x00000100,
1526 0xd80c, 0xff000ff0, 0x00000100
1527};
1528
f73a9e83
SL		 1529static const u32 godavari_golden_registers[] =
1530{
1531 0x55e4, 0xff607fff, 0xfc000100,
1532 0x6ed8, 0x00010101, 0x00010000,
1533 0x9830, 0xffffffff, 0x00000000,
1534 0x98302, 0xf00fffff, 0x00000400,
1535 0x6130, 0xffffffff, 0x00010000,
1536 0x5bb0, 0x000000f0, 0x00000070,
1537 0x5bc0, 0xf0311fff, 0x80300000,
1538 0x98f8, 0x73773777, 0x12010001,
1539 0x98fc, 0xffffffff, 0x00000010,
1540 0x8030, 0x00001f0f, 0x0000100a,
1541 0x2f48, 0x73773777, 0x12010001,
1542 0x2408, 0x000fffff, 0x000c007f,
1543 0x8a14, 0xf000003f, 0x00000007,
1544 0x8b24, 0xffffffff, 0x00ff0fff,
1545 0x30a04, 0x0000ff0f, 0x00000000,
1546 0x28a4c, 0x07ffffff, 0x06000000,
1547 0x4d8, 0x00000fff, 0x00000100,
1548 0xd014, 0x00010000, 0x00810001,
1549 0xd814, 0x00010000, 0x00810001,
1550 0x3e78, 0x00000001, 0x00000002,
1551 0xc768, 0x00000008, 0x00000008,
1552 0xc770, 0x00000f00, 0x00000800,
1553 0xc774, 0x00000f00, 0x00000800,
1554 0xc798, 0x00ffffff, 0x00ff7fbf,
1555 0xc79c, 0x00ffffff, 0x00ff7faf,
1556 0x8c00, 0x000000ff, 0x00000001,
1557 0x214f8, 0x01ff01ff, 0x00000002,
1558 0x21498, 0x007ff800, 0x00200000,
1559 0x2015c, 0xffffffff, 0x00000f40,
1560 0x88c4, 0x001f3ae3, 0x00000082,
1561 0x88d4, 0x0000001f, 0x00000010,
1562 0x30934, 0xffffffff, 0x00000000
1563};
1564
1565
0aafd313
AD		 1566static void cik_init_golden_registers(struct radeon_device *rdev)
1567{
1c0a4625
OG		 1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1569 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1570 switch (rdev->family) {
1571 case CHIP_BONAIRE:
1572 radeon_program_register_sequence(rdev,
1573 bonaire_mgcg_cgcg_init,
1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1575 radeon_program_register_sequence(rdev,
1576 bonaire_golden_registers,
1577 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1578 radeon_program_register_sequence(rdev,
1579 bonaire_golden_common_registers,
1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1581 radeon_program_register_sequence(rdev,
1582 bonaire_golden_spm_registers,
1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1584 break;
1585 case CHIP_KABINI:
1586 radeon_program_register_sequence(rdev,
1587 kalindi_mgcg_cgcg_init,
1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1589 radeon_program_register_sequence(rdev,
1590 kalindi_golden_registers,
1591 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1592 radeon_program_register_sequence(rdev,
1593 kalindi_golden_common_registers,
1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1595 radeon_program_register_sequence(rdev,
1596 kalindi_golden_spm_registers,
1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1598 break;
f73a9e83
SL		 1599 case CHIP_MULLINS:
1600 radeon_program_register_sequence(rdev,
1601 kalindi_mgcg_cgcg_init,
1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1603 radeon_program_register_sequence(rdev,
1604 godavari_golden_registers,
1605 (const u32)ARRAY_SIZE(godavari_golden_registers));
1606 radeon_program_register_sequence(rdev,
1607 kalindi_golden_common_registers,
1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1609 radeon_program_register_sequence(rdev,
1610 kalindi_golden_spm_registers,
1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1612 break;
0aafd313
AD		 1613 case CHIP_KAVERI:
1614 radeon_program_register_sequence(rdev,
1615 spectre_mgcg_cgcg_init,
1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1617 radeon_program_register_sequence(rdev,
1618 spectre_golden_registers,
1619 (const u32)ARRAY_SIZE(spectre_golden_registers));
1620 radeon_program_register_sequence(rdev,
1621 spectre_golden_common_registers,
1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1623 radeon_program_register_sequence(rdev,
1624 spectre_golden_spm_registers,
1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1626 break;
8efff337
AD		 1627 case CHIP_HAWAII:
1628 radeon_program_register_sequence(rdev,
1629 hawaii_mgcg_cgcg_init,
1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1631 radeon_program_register_sequence(rdev,
1632 hawaii_golden_registers,
1633 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1634 radeon_program_register_sequence(rdev,
1635 hawaii_golden_common_registers,
1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1637 radeon_program_register_sequence(rdev,
1638 hawaii_golden_spm_registers,
1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1640 break;
0aafd313
AD		 1641 default:
1642 break;
1643 }
1c0a4625 1644 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1645}
1646
2c67912c
AD		 1647/**
1648 * cik_get_xclk - get the xclk
1649 *
1650 * @rdev: radeon_device pointer
1651 *
1652 * Returns the reference clock used by the gfx engine
1653 * (CIK).
1654 */
1655u32 cik_get_xclk(struct radeon_device *rdev)
1656{
1657 u32 reference_clock = rdev->clock.spll.reference_freq;
1658
1659 if (rdev->flags & RADEON_IS_IGP) {
1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1661 return reference_clock / 2;
1662 } else {
1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1664 return reference_clock / 4;
1665 }
1666 return reference_clock;
1667}
1668
75efdee1
AD		 1669/**
1670 * cik_mm_rdoorbell - read a doorbell dword
1671 *
1672 * @rdev: radeon_device pointer
d5754ab8 1673 * @index: doorbell index
75efdee1
AD		 1674 *
1675 * Returns the value in the doorbell aperture at the
d5754ab8 1676 * requested doorbell index (CIK).
75efdee1 1677 */
d5754ab8 1678u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1679{
d5754ab8
AL		 1680 if (index < rdev->doorbell.num_doorbells) {
1681 return readl(rdev->doorbell.ptr + index);
75efdee1 1682 } else {
d5754ab8 1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1684 return 0;
1685 }
1686}
1687
1688/**
1689 * cik_mm_wdoorbell - write a doorbell dword
1690 *
1691 * @rdev: radeon_device pointer
d5754ab8 1692 * @index: doorbell index
75efdee1
AD		 1693 * @v: value to write
1694 *
1695 * Writes @v to the doorbell aperture at the
d5754ab8 1696 * requested doorbell index (CIK).
75efdee1 1697 */
d5754ab8 1698void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1699{
d5754ab8
AL		 1700 if (index < rdev->doorbell.num_doorbells) {
1701 writel(v, rdev->doorbell.ptr + index);
75efdee1 1702 } else {
d5754ab8 1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1704 }
1705}
1706
bc8273fe
AD		 1707#define BONAIRE_IO_MC_REGS_SIZE 36
1708
1709static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1710{
1711 {0x00000070, 0x04400000},
1712 {0x00000071, 0x80c01803},
1713 {0x00000072, 0x00004004},
1714 {0x00000073, 0x00000100},
1715 {0x00000074, 0x00ff0000},
1716 {0x00000075, 0x34000000},
1717 {0x00000076, 0x08000014},
1718 {0x00000077, 0x00cc08ec},
1719 {0x00000078, 0x00000400},
1720 {0x00000079, 0x00000000},
1721 {0x0000007a, 0x04090000},
1722 {0x0000007c, 0x00000000},
1723 {0x0000007e, 0x4408a8e8},
1724 {0x0000007f, 0x00000304},
1725 {0x00000080, 0x00000000},
1726 {0x00000082, 0x00000001},
1727 {0x00000083, 0x00000002},
1728 {0x00000084, 0xf3e4f400},
1729 {0x00000085, 0x052024e3},
1730 {0x00000087, 0x00000000},
1731 {0x00000088, 0x01000000},
1732 {0x0000008a, 0x1c0a0000},
1733 {0x0000008b, 0xff010000},
1734 {0x0000008d, 0xffffefff},
1735 {0x0000008e, 0xfff3efff},
1736 {0x0000008f, 0xfff3efbf},
1737 {0x00000092, 0xf7ffffff},
1738 {0x00000093, 0xffffff7f},
1739 {0x00000095, 0x00101101},
1740 {0x00000096, 0x00000fff},
1741 {0x00000097, 0x00116fff},
1742 {0x00000098, 0x60010000},
1743 {0x00000099, 0x10010000},
1744 {0x0000009a, 0x00006000},
1745 {0x0000009b, 0x00001000},
1746 {0x0000009f, 0x00b48000}
1747};
1748
d4775655
AD		 1749#define HAWAII_IO_MC_REGS_SIZE 22
1750
1751static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1752{
1753 {0x0000007d, 0x40000000},
1754 {0x0000007e, 0x40180304},
1755 {0x0000007f, 0x0000ff00},
1756 {0x00000081, 0x00000000},
1757 {0x00000083, 0x00000800},
1758 {0x00000086, 0x00000000},
1759 {0x00000087, 0x00000100},
1760 {0x00000088, 0x00020100},
1761 {0x00000089, 0x00000000},
1762 {0x0000008b, 0x00040000},
1763 {0x0000008c, 0x00000100},
1764 {0x0000008e, 0xff010000},
1765 {0x00000090, 0xffffefff},
1766 {0x00000091, 0xfff3efff},
1767 {0x00000092, 0xfff3efbf},
1768 {0x00000093, 0xf7ffffff},
1769 {0x00000094, 0xffffff7f},
1770 {0x00000095, 0x00000fff},
1771 {0x00000096, 0x00116fff},
1772 {0x00000097, 0x60010000},
1773 {0x00000098, 0x10010000},
1774 {0x0000009f, 0x00c79000}
1775};
1776
1777
b556b12e
AD		 1778/**
1779 * cik_srbm_select - select specific register instances
1780 *
1781 * @rdev: radeon_device pointer
1782 * @me: selected ME (micro engine)
1783 * @pipe: pipe
1784 * @queue: queue
1785 * @vmid: VMID
1786 *
1787 * Switches the currently active registers instances. Some
1788 * registers are instanced per VMID, others are instanced per
1789 * me/pipe/queue combination.
1790 */
1791static void cik_srbm_select(struct radeon_device *rdev,
1792 u32 me, u32 pipe, u32 queue, u32 vmid)
1793{
1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1795 MEID(me & 0x3) |
1796 VMID(vmid & 0xf) |
1797 QUEUEID(queue & 0x7));
1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1799}
1800
bc8273fe
AD		 1801/* ucode loading */
1802/**
1803 * ci_mc_load_microcode - load MC ucode into the hw
1804 *
1805 * @rdev: radeon_device pointer
1806 *
1807 * Load the GDDR MC ucode into the hw (CIK).
1808 * Returns 0 on success, error on failure.
1809 */
6c7bccea 1810int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1811{
f2c6b0f4
AD		 1812 const __be32 *fw_data = NULL;
1813 const __le32 *new_fw_data = NULL;
9feb3dda 1814 u32 running, blackout = 0, tmp;
f2c6b0f4
AD		 1815 u32 *io_mc_regs = NULL;
1816 const __le32 *new_io_mc_regs = NULL;
bcddee29 1817 int i, regs_size, ucode_size;
bc8273fe
AD		 1818
1819 if (!rdev->mc_fw)
1820 return -EINVAL;
1821
f2c6b0f4
AD		 1822 if (rdev->new_fw) {
1823 const struct mc_firmware_header_v1_0 *hdr =
1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1825
f2c6b0f4
AD		 1826 radeon_ucode_print_mc_hdr(&hdr->header);
1827
1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1829 new_io_mc_regs = (const __le32 *)
1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1832 new_fw_data = (const __le32 *)
1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1834 } else {
1835 ucode_size = rdev->mc_fw->size / 4;
1836
1837 switch (rdev->family) {
1838 case CHIP_BONAIRE:
1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1840 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1841 break;
1842 case CHIP_HAWAII:
1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1844 regs_size = HAWAII_IO_MC_REGS_SIZE;
1845 break;
1846 default:
1847 return -EINVAL;
1848 }
1849 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1850 }
1851
1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1853
1854 if (running == 0) {
1855 if (running) {
1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1858 }
1859
1860 /* reset the engine and set to writable */
1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1863
1864 /* load mc io regs */
1865 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1866 if (rdev->new_fw) {
1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1869 } else {
1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1872 }
bc8273fe 1873 }
9feb3dda
AD		 1874
1875 tmp = RREG32(MC_SEQ_MISC0);
1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1881 }
1882
bc8273fe 1883 /* load the MC ucode */
f2c6b0f4
AD		 1884 for (i = 0; i < ucode_size; i++) {
1885 if (rdev->new_fw)
1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1887 else
1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1889 }
bc8273fe
AD		 1890
1891 /* put the engine back into the active state */
1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1895
1896 /* wait for training to complete */
1897 for (i = 0; i < rdev->usec_timeout; i++) {
1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1899 break;
1900 udelay(1);
1901 }
1902 for (i = 0; i < rdev->usec_timeout; i++) {
1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1904 break;
1905 udelay(1);
1906 }
1907
1908 if (running)
1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1910 }
1911
1912 return 0;
1913}
1914
02c81327
AD		 1915/**
1916 * cik_init_microcode - load ucode images from disk
1917 *
1918 * @rdev: radeon_device pointer
1919 *
1920 * Use the firmware interface to load the ucode images into
1921 * the driver (not loaded into hw).
1922 * Returns 0 on success, error on failure.
1923 */
1924static int cik_init_microcode(struct radeon_device *rdev)
1925{
02c81327 1926 const char *chip_name;
f2c6b0f4 1927 const char *new_chip_name;
02c81327 1928 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1929 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1931 char fw_name[30];
f2c6b0f4 1932 int new_fw = 0;
02c81327 1933 int err;
f2c6b0f4 1934 int num_fw;
02c81327
AD		 1935
1936 DRM_DEBUG("\n");
1937
02c81327
AD		 1938 switch (rdev->family) {
1939 case CHIP_BONAIRE:
1940 chip_name = "BONAIRE";
f2c6b0f4 1941 new_chip_name = "bonaire";
02c81327
AD		 1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 1951 num_fw = 8;
02c81327 1952 break;
d4775655
AD		 1953 case CHIP_HAWAII:
1954 chip_name = "HAWAII";
f2c6b0f4 1955 new_chip_name = "hawaii";
d4775655
AD		 1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 1965 num_fw = 8;
d4775655 1966 break;
02c81327
AD		 1967 case CHIP_KAVERI:
1968 chip_name = "KAVERI";
f2c6b0f4 1969 new_chip_name = "kaveri";
02c81327
AD		 1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1971 me_req_size = CIK_ME_UCODE_SIZE * 4;
1972 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1976 num_fw = 7;
02c81327
AD		 1977 break;
1978 case CHIP_KABINI:
1979 chip_name = "KABINI";
f2c6b0f4 1980 new_chip_name = "kabini";
02c81327
AD		 1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1982 me_req_size = CIK_ME_UCODE_SIZE * 4;
1983 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1987 num_fw = 6;
02c81327 1988 break;
f73a9e83
SL		 1989 case CHIP_MULLINS:
1990 chip_name = "MULLINS";
f2c6b0f4 1991 new_chip_name = "mullins";
f73a9e83
SL		 1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1993 me_req_size = CIK_ME_UCODE_SIZE * 4;
1994 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1998 num_fw = 6;
f73a9e83 1999 break;
02c81327
AD		 2000 default: BUG();
2001 }
2002
f2c6b0f4 2003 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2004
f2c6b0f4 2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2007 if (err) {
2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2010 if (err)
2011 goto out;
2012 if (rdev->pfp_fw->size != pfp_req_size) {
2013 printk(KERN_ERR
2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2015 rdev->pfp_fw->size, fw_name);
2016 err = -EINVAL;
2017 goto out;
2018 }
2019 } else {
2020 err = radeon_ucode_validate(rdev->pfp_fw);
2021 if (err) {
2022 printk(KERN_ERR
2023 "cik_fw: validation failed for firmware \"%s\"\n",
2024 fw_name);
2025 goto out;
2026 } else {
2027 new_fw++;
2028 }
02c81327
AD		 2029 }
2030
f2c6b0f4 2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2033 if (err) {
2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2036 if (err)
2037 goto out;
2038 if (rdev->me_fw->size != me_req_size) {
2039 printk(KERN_ERR
2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2041 rdev->me_fw->size, fw_name);
2042 err = -EINVAL;
2043 }
2044 } else {
2045 err = radeon_ucode_validate(rdev->me_fw);
2046 if (err) {
2047 printk(KERN_ERR
2048 "cik_fw: validation failed for firmware \"%s\"\n",
2049 fw_name);
2050 goto out;
2051 } else {
2052 new_fw++;
2053 }
02c81327
AD		 2054 }
2055
f2c6b0f4 2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2058 if (err) {
2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2061 if (err)
2062 goto out;
2063 if (rdev->ce_fw->size != ce_req_size) {
2064 printk(KERN_ERR
2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2066 rdev->ce_fw->size, fw_name);
2067 err = -EINVAL;
2068 }
2069 } else {
2070 err = radeon_ucode_validate(rdev->ce_fw);
2071 if (err) {
2072 printk(KERN_ERR
2073 "cik_fw: validation failed for firmware \"%s\"\n",
2074 fw_name);
2075 goto out;
2076 } else {
2077 new_fw++;
2078 }
02c81327
AD		 2079 }
2080
f2c6b0f4 2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2083 if (err) {
2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2086 if (err)
2087 goto out;
2088 if (rdev->mec_fw->size != mec_req_size) {
2089 printk(KERN_ERR
2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2091 rdev->mec_fw->size, fw_name);
2092 err = -EINVAL;
2093 }
2094 } else {
2095 err = radeon_ucode_validate(rdev->mec_fw);
2096 if (err) {
2097 printk(KERN_ERR
2098 "cik_fw: validation failed for firmware \"%s\"\n",
2099 fw_name);
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
2104 }
2105
2106 if (rdev->family == CHIP_KAVERI) {
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2109 if (err) {
2110 goto out;
2111 } else {
2112 err = radeon_ucode_validate(rdev->mec2_fw);
2113 if (err) {
2114 goto out;
2115 } else {
2116 new_fw++;
2117 }
2118 }
02c81327
AD		 2119 }
2120
f2c6b0f4 2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2123 if (err) {
2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2126 if (err)
2127 goto out;
2128 if (rdev->rlc_fw->size != rlc_req_size) {
2129 printk(KERN_ERR
2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2131 rdev->rlc_fw->size, fw_name);
2132 err = -EINVAL;
2133 }
2134 } else {
2135 err = radeon_ucode_validate(rdev->rlc_fw);
2136 if (err) {
2137 printk(KERN_ERR
2138 "cik_fw: validation failed for firmware \"%s\"\n",
2139 fw_name);
2140 goto out;
2141 } else {
2142 new_fw++;
2143 }
02c81327
AD		 2144 }
2145
f2c6b0f4 2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2148 if (err) {
2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2151 if (err)
2152 goto out;
2153 if (rdev->sdma_fw->size != sdma_req_size) {
2154 printk(KERN_ERR
2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2156 rdev->sdma_fw->size, fw_name);
2157 err = -EINVAL;
2158 }
2159 } else {
2160 err = radeon_ucode_validate(rdev->sdma_fw);
2161 if (err) {
2162 printk(KERN_ERR
2163 "cik_fw: validation failed for firmware \"%s\"\n",
2164 fw_name);
2165 goto out;
2166 } else {
2167 new_fw++;
2168 }
21a93e13
AD		 2169 }
2170
cc8dbbb4 2171 /* No SMC, MC ucode on APUs */
02c81327 2172 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2175 if (err) {
f2c6b0f4 2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2178 if (err) {
2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2181 if (err)
2182 goto out;
2183 }
2184 if ((rdev->mc_fw->size != mc_req_size) &&
2185 (rdev->mc_fw->size != mc2_req_size)){
2186 printk(KERN_ERR
2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2188 rdev->mc_fw->size, fw_name);
2189 err = -EINVAL;
2190 }
2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2192 } else {
2193 err = radeon_ucode_validate(rdev->mc_fw);
2194 if (err) {
2195 printk(KERN_ERR
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2197 fw_name);
277babc3 2198 goto out;
f2c6b0f4
AD		 2199 } else {
2200 new_fw++;
2201 }
277babc3 2202 }
cc8dbbb4 2203
f2c6b0f4 2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2206 if (err) {
f2c6b0f4
AD		 2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2209 if (err) {
2210 printk(KERN_ERR
2211 "smc: error loading firmware \"%s\"\n",
2212 fw_name);
2213 release_firmware(rdev->smc_fw);
2214 rdev->smc_fw = NULL;
2215 err = 0;
2216 } else if (rdev->smc_fw->size != smc_req_size) {
2217 printk(KERN_ERR
2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2219 rdev->smc_fw->size, fw_name);
2220 err = -EINVAL;
2221 }
2222 } else {
2223 err = radeon_ucode_validate(rdev->smc_fw);
2224 if (err) {
2225 printk(KERN_ERR
2226 "cik_fw: validation failed for firmware \"%s\"\n",
2227 fw_name);
2228 goto out;
2229 } else {
2230 new_fw++;
2231 }
cc8dbbb4 2232 }
02c81327
AD		 2233 }
2234
f2c6b0f4
AD		 2235 if (new_fw == 0) {
2236 rdev->new_fw = false;
2237 } else if (new_fw < num_fw) {
2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2239 err = -EINVAL;
2240 } else {
2241 rdev->new_fw = true;
2242 }
2243
02c81327 2244out:
02c81327
AD		 2245 if (err) {
2246 if (err != -EINVAL)
2247 printk(KERN_ERR
2248 "cik_cp: Failed to load firmware \"%s\"\n",
2249 fw_name);
2250 release_firmware(rdev->pfp_fw);
2251 rdev->pfp_fw = NULL;
2252 release_firmware(rdev->me_fw);
2253 rdev->me_fw = NULL;
2254 release_firmware(rdev->ce_fw);
2255 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2256 release_firmware(rdev->mec_fw);
2257 rdev->mec_fw = NULL;
2258 release_firmware(rdev->mec2_fw);
2259 rdev->mec2_fw = NULL;
02c81327
AD		 2260 release_firmware(rdev->rlc_fw);
2261 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2262 release_firmware(rdev->sdma_fw);
2263 rdev->sdma_fw = NULL;
02c81327
AD		 2264 release_firmware(rdev->mc_fw);
2265 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2266 release_firmware(rdev->smc_fw);
2267 rdev->smc_fw = NULL;
02c81327
AD		 2268 }
2269 return err;
2270}
2271
8cc1a532
AD		 2272/*
2273 * Core functions
2274 */
2275/**
2276 * cik_tiling_mode_table_init - init the hw tiling table
2277 *
2278 * @rdev: radeon_device pointer
2279 *
2280 * Starting with SI, the tiling setup is done globally in a
2281 * set of 32 tiling modes. Rather than selecting each set of
2282 * parameters per surface as on older asics, we just select
2283 * which index in the tiling table we want to use, and the
2284 * surface uses those parameters (CIK).
2285 */
2286static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2287{
2288 const u32 num_tile_mode_states = 32;
2289 const u32 num_secondary_tile_mode_states = 16;
2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2291 u32 num_pipe_configs;
2292 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2293 rdev->config.cik.max_shader_engines;
2294
2295 switch (rdev->config.cik.mem_row_size_in_kb) {
2296 case 1:
2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2298 break;
2299 case 2:
2300 default:
2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2302 break;
2303 case 4:
2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2305 break;
2306 }
2307
2308 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2309 if (num_pipe_configs > 8)
21e438af 2310 num_pipe_configs = 16;
8cc1a532 2311
21e438af
AD		 2312 if (num_pipe_configs == 16) {
2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2314 switch (reg_offset) {
2315 case 0:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2320 break;
2321 case 1:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2326 break;
2327 case 2:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2332 break;
2333 case 3:
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2338 break;
2339 case 4:
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343 TILE_SPLIT(split_equal_to_row_size));
2344 break;
2345 case 5:
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2349 break;
2350 case 6:
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2355 break;
2356 case 7:
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360 TILE_SPLIT(split_equal_to_row_size));
2361 break;
2362 case 8:
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2365 break;
2366 case 9:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2370 break;
2371 case 10:
2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2376 break;
2377 case 11:
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2382 break;
2383 case 12:
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2388 break;
2389 case 13:
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2393 break;
2394 case 14:
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2399 break;
2400 case 16:
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405 break;
2406 case 17:
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2411 break;
2412 case 27:
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2416 break;
2417 case 28:
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2422 break;
2423 case 29:
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2428 break;
2429 case 30:
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434 break;
2435 default:
2436 gb_tile_moden = 0;
2437 break;
2438 }
2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2441 }
2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2443 switch (reg_offset) {
2444 case 0:
2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2449 break;
2450 case 1:
2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2454 NUM_BANKS(ADDR_SURF_16_BANK));
2455 break;
2456 case 2:
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2461 break;
2462 case 3:
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2467 break;
2468 case 4:
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472 NUM_BANKS(ADDR_SURF_8_BANK));
2473 break;
2474 case 5:
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2478 NUM_BANKS(ADDR_SURF_4_BANK));
2479 break;
2480 case 6:
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_2_BANK));
2485 break;
2486 case 8:
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2491 break;
2492 case 9:
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_16_BANK));
2497 break;
2498 case 10:
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2503 break;
2504 case 11:
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2508 NUM_BANKS(ADDR_SURF_8_BANK));
2509 break;
2510 case 12:
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2514 NUM_BANKS(ADDR_SURF_4_BANK));
2515 break;
2516 case 13:
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2520 NUM_BANKS(ADDR_SURF_2_BANK));
2521 break;
2522 case 14:
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2526 NUM_BANKS(ADDR_SURF_2_BANK));
2527 break;
2528 default:
2529 gb_tile_moden = 0;
2530 break;
2531 }
1b2c4869 2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2534 }
2535 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2537 switch (reg_offset) {
2538 case 0:
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2543 break;
2544 case 1:
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2549 break;
2550 case 2:
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2555 break;
2556 case 3:
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2561 break;
2562 case 4:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2567 break;
2568 case 5:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2572 break;
2573 case 6:
2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2578 break;
2579 case 7:
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 TILE_SPLIT(split_equal_to_row_size));
2584 break;
2585 case 8:
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2588 break;
2589 case 9:
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2593 break;
2594 case 10:
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 break;
2600 case 11:
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2605 break;
2606 case 12:
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2611 break;
2612 case 13:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2616 break;
2617 case 14:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 16:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 17:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 27:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2639 break;
2640 case 28:
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2645 break;
2646 case 29:
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2651 break;
2652 case 30:
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 break;
2658 default:
2659 gb_tile_moden = 0;
2660 break;
2661 }
39aee490 2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2664 }
2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2666 switch (reg_offset) {
2667 case 0:
2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2671 NUM_BANKS(ADDR_SURF_16_BANK));
2672 break;
2673 case 1:
2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2677 NUM_BANKS(ADDR_SURF_16_BANK));
2678 break;
2679 case 2:
2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2683 NUM_BANKS(ADDR_SURF_16_BANK));
2684 break;
2685 case 3:
2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2689 NUM_BANKS(ADDR_SURF_16_BANK));
2690 break;
2691 case 4:
2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2695 NUM_BANKS(ADDR_SURF_8_BANK));
2696 break;
2697 case 5:
2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2701 NUM_BANKS(ADDR_SURF_4_BANK));
2702 break;
2703 case 6:
2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2707 NUM_BANKS(ADDR_SURF_2_BANK));
2708 break;
2709 case 8:
2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2713 NUM_BANKS(ADDR_SURF_16_BANK));
2714 break;
2715 case 9:
2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2719 NUM_BANKS(ADDR_SURF_16_BANK));
2720 break;
2721 case 10:
2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2725 NUM_BANKS(ADDR_SURF_16_BANK));
2726 break;
2727 case 11:
2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2731 NUM_BANKS(ADDR_SURF_16_BANK));
2732 break;
2733 case 12:
2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2737 NUM_BANKS(ADDR_SURF_8_BANK));
2738 break;
2739 case 13:
2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2743 NUM_BANKS(ADDR_SURF_4_BANK));
2744 break;
2745 case 14:
2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2749 NUM_BANKS(ADDR_SURF_2_BANK));
2750 break;
2751 default:
2752 gb_tile_moden = 0;
2753 break;
2754 }
32f79a8a 2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2757 }
2758 } else if (num_pipe_configs == 4) {
2759 if (num_rbs == 4) {
2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2761 switch (reg_offset) {
2762 case 0:
2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2767 break;
2768 case 1:
2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2773 break;
2774 case 2:
2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2779 break;
2780 case 3:
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2785 break;
2786 case 4:
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2791 break;
2792 case 5:
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2796 break;
2797 case 6:
2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2802 break;
2803 case 7:
2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2807 TILE_SPLIT(split_equal_to_row_size));
2808 break;
2809 case 8:
2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2811 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2812 break;
2813 case 9:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2817 break;
2818 case 10:
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 break;
2824 case 11:
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2829 break;
2830 case 12:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2835 break;
2836 case 13:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2840 break;
2841 case 14:
2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2846 break;
2847 case 16:
2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2852 break;
2853 case 17:
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2858 break;
2859 case 27:
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2863 break;
2864 case 28:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 29:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 30:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2881 break;
2882 default:
2883 gb_tile_moden = 0;
2884 break;
2885 }
39aee490 2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2888 }
2889 } else if (num_rbs < 4) {
2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2891 switch (reg_offset) {
2892 case 0:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2897 break;
2898 case 1:
2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2903 break;
2904 case 2:
2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2909 break;
2910 case 3:
2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2915 break;
2916 case 4:
2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2920 TILE_SPLIT(split_equal_to_row_size));
2921 break;
2922 case 5:
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2926 break;
2927 case 6:
2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2932 break;
2933 case 7:
2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2937 TILE_SPLIT(split_equal_to_row_size));
2938 break;
2939 case 8:
2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2941 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2942 break;
2943 case 9:
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2947 break;
2948 case 10:
2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2953 break;
2954 case 11:
2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959 break;
2960 case 12:
2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2965 break;
2966 case 13:
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2970 break;
2971 case 14:
2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2976 break;
2977 case 16:
2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2982 break;
2983 case 17:
2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2988 break;
2989 case 27:
2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2993 break;
2994 case 28:
2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2999 break;
3000 case 29:
3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3005 break;
3006 case 30:
3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3011 break;
3012 default:
3013 gb_tile_moden = 0;
3014 break;
3015 }
39aee490 3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3018 }
3019 }
3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3021 switch (reg_offset) {
3022 case 0:
3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3027 break;
3028 case 1:
3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3032 NUM_BANKS(ADDR_SURF_16_BANK));
3033 break;
3034 case 2:
3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_16_BANK));
3039 break;
3040 case 3:
3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3044 NUM_BANKS(ADDR_SURF_16_BANK));
3045 break;
3046 case 4:
3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3050 NUM_BANKS(ADDR_SURF_16_BANK));
3051 break;
3052 case 5:
3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3056 NUM_BANKS(ADDR_SURF_8_BANK));
3057 break;
3058 case 6:
3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3062 NUM_BANKS(ADDR_SURF_4_BANK));
3063 break;
3064 case 8:
3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3068 NUM_BANKS(ADDR_SURF_16_BANK));
3069 break;
3070 case 9:
3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3074 NUM_BANKS(ADDR_SURF_16_BANK));
3075 break;
3076 case 10:
3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3080 NUM_BANKS(ADDR_SURF_16_BANK));
3081 break;
3082 case 11:
3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3086 NUM_BANKS(ADDR_SURF_16_BANK));
3087 break;
3088 case 12:
3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3092 NUM_BANKS(ADDR_SURF_16_BANK));
3093 break;
3094 case 13:
3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3098 NUM_BANKS(ADDR_SURF_8_BANK));
3099 break;
3100 case 14:
3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3104 NUM_BANKS(ADDR_SURF_4_BANK));
3105 break;
3106 default:
3107 gb_tile_moden = 0;
3108 break;
3109 }
32f79a8a 3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3112 }
3113 } else if (num_pipe_configs == 2) {
3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3115 switch (reg_offset) {
3116 case 0:
3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3119 PIPE_CONFIG(ADDR_SURF_P2) |
3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3121 break;
3122 case 1:
3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3125 PIPE_CONFIG(ADDR_SURF_P2) |
3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3127 break;
3128 case 2:
3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3131 PIPE_CONFIG(ADDR_SURF_P2) |
3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3133 break;
3134 case 3:
3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3137 PIPE_CONFIG(ADDR_SURF_P2) |
3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3139 break;
3140 case 4:
3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3143 PIPE_CONFIG(ADDR_SURF_P2) |
3144 TILE_SPLIT(split_equal_to_row_size));
3145 break;
3146 case 5:
3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3148 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3150 break;
3151 case 6:
3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3154 PIPE_CONFIG(ADDR_SURF_P2) |
3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3156 break;
3157 case 7:
3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3160 PIPE_CONFIG(ADDR_SURF_P2) |
3161 TILE_SPLIT(split_equal_to_row_size));
3162 break;
3163 case 8:
020ff546
MO		 3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3165 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3166 break;
3167 case 9:
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3171 break;
3172 case 10:
3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3175 PIPE_CONFIG(ADDR_SURF_P2) |
3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3177 break;
3178 case 11:
3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3181 PIPE_CONFIG(ADDR_SURF_P2) |
3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3183 break;
3184 case 12:
3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3187 PIPE_CONFIG(ADDR_SURF_P2) |
3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3189 break;
3190 case 13:
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3192 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3194 break;
3195 case 14:
3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3198 PIPE_CONFIG(ADDR_SURF_P2) |
3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3200 break;
3201 case 16:
3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3204 PIPE_CONFIG(ADDR_SURF_P2) |
3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3206 break;
3207 case 17:
3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3210 PIPE_CONFIG(ADDR_SURF_P2) |
3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3212 break;
3213 case 27:
3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3216 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3217 break;
3218 case 28:
3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3221 PIPE_CONFIG(ADDR_SURF_P2) |
3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3223 break;
3224 case 29:
3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3227 PIPE_CONFIG(ADDR_SURF_P2) |
3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3229 break;
3230 case 30:
3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3233 PIPE_CONFIG(ADDR_SURF_P2) |
3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3235 break;
3236 default:
3237 gb_tile_moden = 0;
3238 break;
3239 }
39aee490 3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3242 }
3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3244 switch (reg_offset) {
3245 case 0:
3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3249 NUM_BANKS(ADDR_SURF_16_BANK));
3250 break;
3251 case 1:
3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3255 NUM_BANKS(ADDR_SURF_16_BANK));
3256 break;
3257 case 2:
3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3261 NUM_BANKS(ADDR_SURF_16_BANK));
3262 break;
3263 case 3:
3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3267 NUM_BANKS(ADDR_SURF_16_BANK));
3268 break;
3269 case 4:
3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3273 NUM_BANKS(ADDR_SURF_16_BANK));
3274 break;
3275 case 5:
3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3279 NUM_BANKS(ADDR_SURF_16_BANK));
3280 break;
3281 case 6:
3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3285 NUM_BANKS(ADDR_SURF_8_BANK));
3286 break;
3287 case 8:
3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3291 NUM_BANKS(ADDR_SURF_16_BANK));
3292 break;
3293 case 9:
3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3297 NUM_BANKS(ADDR_SURF_16_BANK));
3298 break;
3299 case 10:
3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3303 NUM_BANKS(ADDR_SURF_16_BANK));
3304 break;
3305 case 11:
3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3309 NUM_BANKS(ADDR_SURF_16_BANK));
3310 break;
3311 case 12:
3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3315 NUM_BANKS(ADDR_SURF_16_BANK));
3316 break;
3317 case 13:
3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3321 NUM_BANKS(ADDR_SURF_16_BANK));
3322 break;
3323 case 14:
3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3327 NUM_BANKS(ADDR_SURF_8_BANK));
3328 break;
3329 default:
3330 gb_tile_moden = 0;
3331 break;
3332 }
32f79a8a 3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3335 }
3336 } else
3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3338}
3339
3340/**
3341 * cik_select_se_sh - select which SE, SH to address
3342 *
3343 * @rdev: radeon_device pointer
3344 * @se_num: shader engine to address
3345 * @sh_num: sh block to address
3346 *
3347 * Select which SE, SH combinations to address. Certain
3348 * registers are instanced per SE or SH. 0xffffffff means
3349 * broadcast to all SEs or SHs (CIK).
3350 */
3351static void cik_select_se_sh(struct radeon_device *rdev,
3352 u32 se_num, u32 sh_num)
3353{
3354 u32 data = INSTANCE_BROADCAST_WRITES;
3355
3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3358 else if (se_num == 0xffffffff)
3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3360 else if (sh_num == 0xffffffff)
3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3362 else
3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3364 WREG32(GRBM_GFX_INDEX, data);
3365}
3366
3367/**
3368 * cik_create_bitmask - create a bitmask
3369 *
3370 * @bit_width: length of the mask
3371 *
3372 * create a variable length bit mask (CIK).
3373 * Returns the bitmask.
3374 */
3375static u32 cik_create_bitmask(u32 bit_width)
3376{
3377 u32 i, mask = 0;
3378
3379 for (i = 0; i < bit_width; i++) {
3380 mask <<= 1;
3381 mask |= 1;
3382 }
3383 return mask;
3384}
3385
3386/**
972c5ddb 3387 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3388 *
3389 * @rdev: radeon_device pointer
3390 * @max_rb_num: max RBs (render backends) for the asic
3391 * @se_num: number of SEs (shader engines) for the asic
3392 * @sh_per_se: number of SH blocks per SE for the asic
3393 *
3394 * Calculates the bitmask of disabled RBs (CIK).
3395 * Returns the disabled RB bitmask.
3396 */
3397static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3398 u32 max_rb_num_per_se,
8cc1a532
AD		 3399 u32 sh_per_se)
3400{
3401 u32 data, mask;
3402
3403 data = RREG32(CC_RB_BACKEND_DISABLE);
3404 if (data & 1)
3405 data &= BACKEND_DISABLE_MASK;
3406 else
3407 data = 0;
3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3409
3410 data >>= BACKEND_DISABLE_SHIFT;
3411
9fadb352 3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3413
3414 return data & mask;
3415}
3416
3417/**
3418 * cik_setup_rb - setup the RBs on the asic
3419 *
3420 * @rdev: radeon_device pointer
3421 * @se_num: number of SEs (shader engines) for the asic
3422 * @sh_per_se: number of SH blocks per SE for the asic
3423 * @max_rb_num: max RBs (render backends) for the asic
3424 *
3425 * Configures per-SE/SH RB registers (CIK).
3426 */
3427static void cik_setup_rb(struct radeon_device *rdev,
3428 u32 se_num, u32 sh_per_se,
9fadb352 3429 u32 max_rb_num_per_se)
8cc1a532
AD		 3430{
3431 int i, j;
3432 u32 data, mask;
3433 u32 disabled_rbs = 0;
3434 u32 enabled_rbs = 0;
3435
1c0a4625 3436 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3437 for (i = 0; i < se_num; i++) {
3438 for (j = 0; j < sh_per_se; j++) {
3439 cik_select_se_sh(rdev, i, j);
9fadb352 3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3441 if (rdev->family == CHIP_HAWAII)
3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3443 else
3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3445 }
3446 }
3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3448 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3449
3450 mask = 1;
9fadb352 3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3452 if (!(disabled_rbs & mask))
3453 enabled_rbs |= mask;
3454 mask <<= 1;
3455 }
3456
439a1cff
MO		 3457 rdev->config.cik.backend_enable_mask = enabled_rbs;
3458
1c0a4625 3459 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3460 for (i = 0; i < se_num; i++) {
3461 cik_select_se_sh(rdev, i, 0xffffffff);
3462 data = 0;
3463 for (j = 0; j < sh_per_se; j++) {
3464 switch (enabled_rbs & 3) {
fc821b70
AD		 3465 case 0:
3466 if (j == 0)
3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3468 else
3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3470 break;
8cc1a532
AD		 3471 case 1:
3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3473 break;
3474 case 2:
3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3476 break;
3477 case 3:
3478 default:
3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3480 break;
3481 }
3482 enabled_rbs >>= 2;
3483 }
3484 WREG32(PA_SC_RASTER_CONFIG, data);
3485 }
3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3487 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3488}
3489
3490/**
3491 * cik_gpu_init - setup the 3D engine
3492 *
3493 * @rdev: radeon_device pointer
3494 *
3495 * Configures the 3D engine and tiling configuration
3496 * registers so that the 3D engine is usable.
3497 */
3498static void cik_gpu_init(struct radeon_device *rdev)
3499{
3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3501 u32 mc_shared_chmap, mc_arb_ramcfg;
3502 u32 hdp_host_path_cntl;
3503 u32 tmp;
6101b3ae 3504 int i, j;
8cc1a532
AD		 3505
3506 switch (rdev->family) {
3507 case CHIP_BONAIRE:
3508 rdev->config.cik.max_shader_engines = 2;
3509 rdev->config.cik.max_tile_pipes = 4;
3510 rdev->config.cik.max_cu_per_sh = 7;
3511 rdev->config.cik.max_sh_per_se = 1;
3512 rdev->config.cik.max_backends_per_se = 2;
3513 rdev->config.cik.max_texture_channel_caches = 4;
3514 rdev->config.cik.max_gprs = 256;
3515 rdev->config.cik.max_gs_threads = 32;
3516 rdev->config.cik.max_hw_contexts = 8;
3517
3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3523 break;
b496038b
AD		 3524 case CHIP_HAWAII:
3525 rdev->config.cik.max_shader_engines = 4;
3526 rdev->config.cik.max_tile_pipes = 16;
3527 rdev->config.cik.max_cu_per_sh = 11;
3528 rdev->config.cik.max_sh_per_se = 1;
3529 rdev->config.cik.max_backends_per_se = 4;
3530 rdev->config.cik.max_texture_channel_caches = 16;
3531 rdev->config.cik.max_gprs = 256;
3532 rdev->config.cik.max_gs_threads = 32;
3533 rdev->config.cik.max_hw_contexts = 8;
3534
3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3540 break;
8cc1a532 3541 case CHIP_KAVERI:
b2e4c70a
AD		 3542 rdev->config.cik.max_shader_engines = 1;
3543 rdev->config.cik.max_tile_pipes = 4;
3544 if ((rdev->pdev->device == 0x1304) ||
3545 (rdev->pdev->device == 0x1305) ||
3546 (rdev->pdev->device == 0x130C) ||
3547 (rdev->pdev->device == 0x130F) ||
3548 (rdev->pdev->device == 0x1310) ||
3549 (rdev->pdev->device == 0x1311) ||
3550 (rdev->pdev->device == 0x131C)) {
3551 rdev->config.cik.max_cu_per_sh = 8;
3552 rdev->config.cik.max_backends_per_se = 2;
3553 } else if ((rdev->pdev->device == 0x1309) ||
3554 (rdev->pdev->device == 0x130A) ||
3555 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3556 (rdev->pdev->device == 0x1313) ||
3557 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3558 rdev->config.cik.max_cu_per_sh = 6;
3559 rdev->config.cik.max_backends_per_se = 2;
3560 } else if ((rdev->pdev->device == 0x1306) ||
3561 (rdev->pdev->device == 0x1307) ||
3562 (rdev->pdev->device == 0x130B) ||
3563 (rdev->pdev->device == 0x130E) ||
3564 (rdev->pdev->device == 0x1315) ||
6dc14baf 3565 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3566 (rdev->pdev->device == 0x131B)) {
3567 rdev->config.cik.max_cu_per_sh = 4;
3568 rdev->config.cik.max_backends_per_se = 1;
3569 } else {
3570 rdev->config.cik.max_cu_per_sh = 3;
3571 rdev->config.cik.max_backends_per_se = 1;
3572 }
3573 rdev->config.cik.max_sh_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 4;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3578
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3584 break;
3585 case CHIP_KABINI:
f73a9e83 3586 case CHIP_MULLINS:
8cc1a532
AD		 3587 default:
3588 rdev->config.cik.max_shader_engines = 1;
3589 rdev->config.cik.max_tile_pipes = 2;
3590 rdev->config.cik.max_cu_per_sh = 2;
3591 rdev->config.cik.max_sh_per_se = 1;
3592 rdev->config.cik.max_backends_per_se = 1;
3593 rdev->config.cik.max_texture_channel_caches = 2;
3594 rdev->config.cik.max_gprs = 256;
3595 rdev->config.cik.max_gs_threads = 16;
3596 rdev->config.cik.max_hw_contexts = 8;
3597
3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3603 break;
3604 }
3605
3606 /* Initialize HDP */
3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3608 WREG32((0x2c14 + j), 0x00000000);
3609 WREG32((0x2c18 + j), 0x00000000);
3610 WREG32((0x2c1c + j), 0x00000000);
3611 WREG32((0x2c20 + j), 0x00000000);
3612 WREG32((0x2c24 + j), 0x00000000);
3613 }
3614
3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3616 WREG32(SRBM_INT_CNTL, 0x1);
3617 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3618
3619 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3620
3621 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3622 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3623
3624 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3625 rdev->config.cik.mem_max_burst_length_bytes = 256;
3626 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3627 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3628 if (rdev->config.cik.mem_row_size_in_kb > 4)
3629 rdev->config.cik.mem_row_size_in_kb = 4;
3630 /* XXX use MC settings? */
3631 rdev->config.cik.shader_engine_tile_size = 32;
3632 rdev->config.cik.num_gpus = 1;
3633 rdev->config.cik.multi_gpu_tile_size = 64;
3634
3635 /* fix up row size */
3636 gb_addr_config &= ~ROW_SIZE_MASK;
3637 switch (rdev->config.cik.mem_row_size_in_kb) {
3638 case 1:
3639 default:
3640 gb_addr_config |= ROW_SIZE(0);
3641 break;
3642 case 2:
3643 gb_addr_config |= ROW_SIZE(1);
3644 break;
3645 case 4:
3646 gb_addr_config |= ROW_SIZE(2);
3647 break;
3648 }
3649
3650 /* setup tiling info dword. gb_addr_config is not adequate since it does
3651 * not have bank info, so create a custom tiling dword.
3652 * bits 3:0 num_pipes
3653 * bits 7:4 num_banks
3654 * bits 11:8 group_size
3655 * bits 15:12 row_size
3656 */
3657 rdev->config.cik.tile_config = 0;
3658 switch (rdev->config.cik.num_tile_pipes) {
3659 case 1:
3660 rdev->config.cik.tile_config |= (0 << 0);
3661 break;
3662 case 2:
3663 rdev->config.cik.tile_config |= (1 << 0);
3664 break;
3665 case 4:
3666 rdev->config.cik.tile_config |= (2 << 0);
3667 break;
3668 case 8:
3669 default:
3670 /* XXX what about 12? */
3671 rdev->config.cik.tile_config |= (3 << 0);
3672 break;
3673 }
a537314e
MD		 3674 rdev->config.cik.tile_config |=
3675 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3676 rdev->config.cik.tile_config |=
3677 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3678 rdev->config.cik.tile_config |=
3679 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3680
3681 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3682 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3683 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3684 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3685 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3686 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3687 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3688 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3689
3690 cik_tiling_mode_table_init(rdev);
3691
3692 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3693 rdev->config.cik.max_sh_per_se,
3694 rdev->config.cik.max_backends_per_se);
3695
52da51f0 3696 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3697 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3698 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3699 rdev->config.cik.active_cus +=
3700 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3701 }
3702 }
3703
8cc1a532
AD		 3704 /* set HW defaults for 3D engine */
3705 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3706
1c0a4625
OG		 3707 mutex_lock(&rdev->grbm_idx_mutex);
3708 /*
3709 * making sure that the following register writes will be broadcasted
3710 * to all the shaders
3711 */
3712 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3713 WREG32(SX_DEBUG_1, 0x20);
3714
3715 WREG32(TA_CNTL_AUX, 0x00010000);
3716
3717 tmp = RREG32(SPI_CONFIG_CNTL);
3718 tmp |= 0x03000000;
3719 WREG32(SPI_CONFIG_CNTL, tmp);
3720
3721 WREG32(SQ_CONFIG, 1);
3722
3723 WREG32(DB_DEBUG, 0);
3724
3725 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3726 tmp |= 0x00000400;
3727 WREG32(DB_DEBUG2, tmp);
3728
3729 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3730 tmp |= 0x00020200;
3731 WREG32(DB_DEBUG3, tmp);
3732
3733 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3734 tmp |= 0x00018208;
3735 WREG32(CB_HW_CONTROL, tmp);
3736
3737 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3738
3739 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3740 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3741 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3742 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3743
3744 WREG32(VGT_NUM_INSTANCES, 1);
3745
3746 WREG32(CP_PERFMON_CNTL, 0);
3747
3748 WREG32(SQ_CONFIG, 0);
3749
3750 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3751 FORCE_EOV_MAX_REZ_CNT(255)));
3752
3753 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3754 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3755
3756 WREG32(VGT_GS_VERTEX_REUSE, 16);
3757 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3758
3759 tmp = RREG32(HDP_MISC_CNTL);
3760 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3761 WREG32(HDP_MISC_CNTL, tmp);
3762
3763 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3764 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3765
3766 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3767 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3768 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3769
3770 udelay(50);
3771}
3772
2cae3bc3
AD		 3773/*
3774 * GPU scratch registers helpers function.
3775 */
3776/**
3777 * cik_scratch_init - setup driver info for CP scratch regs
3778 *
3779 * @rdev: radeon_device pointer
3780 *
3781 * Set up the number and offset of the CP scratch registers.
3782 * NOTE: use of CP scratch registers is a legacy inferface and
3783 * is not used by default on newer asics (r6xx+). On newer asics,
3784 * memory buffers are used for fences rather than scratch regs.
3785 */
3786static void cik_scratch_init(struct radeon_device *rdev)
3787{
3788 int i;
3789
3790 rdev->scratch.num_reg = 7;
3791 rdev->scratch.reg_base = SCRATCH_REG0;
3792 for (i = 0; i < rdev->scratch.num_reg; i++) {
3793 rdev->scratch.free[i] = true;
3794 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3795 }
3796}
3797
fbc832c7
AD		 3798/**
3799 * cik_ring_test - basic gfx ring test
3800 *
3801 * @rdev: radeon_device pointer
3802 * @ring: radeon_ring structure holding ring information
3803 *
3804 * Allocate a scratch register and write to it using the gfx ring (CIK).
3805 * Provides a basic gfx ring test to verify that the ring is working.
3806 * Used by cik_cp_gfx_resume();
3807 * Returns 0 on success, error on failure.
3808 */
3809int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3810{
3811 uint32_t scratch;
3812 uint32_t tmp = 0;
3813 unsigned i;
3814 int r;
3815
3816 r = radeon_scratch_get(rdev, &scratch);
3817 if (r) {
3818 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3819 return r;
3820 }
3821 WREG32(scratch, 0xCAFEDEAD);
3822 r = radeon_ring_lock(rdev, ring, 3);
3823 if (r) {
3824 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3825 radeon_scratch_free(rdev, scratch);
3826 return r;
3827 }
3828 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3829 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3830 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3831 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3832
fbc832c7
AD		 3833 for (i = 0; i < rdev->usec_timeout; i++) {
3834 tmp = RREG32(scratch);
3835 if (tmp == 0xDEADBEEF)
3836 break;
3837 DRM_UDELAY(1);
3838 }
3839 if (i < rdev->usec_timeout) {
3840 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3841 } else {
3842 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3843 ring->idx, scratch, tmp);
3844 r = -EINVAL;
3845 }
3846 radeon_scratch_free(rdev, scratch);
3847 return r;
3848}
3849
780f5ddd
AD		 3850/**
3851 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3852 *
3853 * @rdev: radeon_device pointer
3854 * @ridx: radeon ring index
3855 *
3856 * Emits an hdp flush on the cp.
3857 */
3858static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3859 int ridx)
3860{
3861 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3862 u32 ref_and_mask;
780f5ddd 3863
5d259067
AD		 3864 switch (ring->idx) {
3865 case CAYMAN_RING_TYPE_CP1_INDEX:
3866 case CAYMAN_RING_TYPE_CP2_INDEX:
3867 default:
3868 switch (ring->me) {
3869 case 0:
3870 ref_and_mask = CP2 << ring->pipe;
3871 break;
3872 case 1:
3873 ref_and_mask = CP6 << ring->pipe;
3874 break;
3875 default:
3876 return;
3877 }
3878 break;
3879 case RADEON_RING_TYPE_GFX_INDEX:
3880 ref_and_mask = CP0;
3881 break;
3882 }
3883
3884 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3885 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3886 WAIT_REG_MEM_FUNCTION(3) | /* == */
3887 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3888 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3889 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3890 radeon_ring_write(ring, ref_and_mask);
3891 radeon_ring_write(ring, ref_and_mask);
3892 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3893}
3894
2cae3bc3 3895/**
b07fdd38 3896 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3897 *
3898 * @rdev: radeon_device pointer
3899 * @fence: radeon fence object
3900 *
3901 * Emits a fence sequnce number on the gfx ring and flushes
3902 * GPU caches.
3903 */
b07fdd38
AD		 3904void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3905 struct radeon_fence *fence)
2cae3bc3
AD		 3906{
3907 struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3909
a9c73a0e
CK		 3910 /* Workaround for cache flush problems. First send a dummy EOP
3911 * event down the pipe with seq one below.
3912 */
3913 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3914 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3915 EOP_TC_ACTION_EN |
3916 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3917 EVENT_INDEX(5)));
3918 radeon_ring_write(ring, addr & 0xfffffffc);
3919 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3920 DATA_SEL(1) | INT_SEL(0));
3921 radeon_ring_write(ring, fence->seq - 1);
3922 radeon_ring_write(ring, 0);
3923
3924 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3925 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3926 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3927 EOP_TC_ACTION_EN |
3928 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3929 EVENT_INDEX(5)));
3930 radeon_ring_write(ring, addr & 0xfffffffc);
3931 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3932 radeon_ring_write(ring, fence->seq);
3933 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3934}
3935
b07fdd38
AD		 3936/**
3937 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3938 *
3939 * @rdev: radeon_device pointer
3940 * @fence: radeon fence object
3941 *
3942 * Emits a fence sequnce number on the compute ring and flushes
3943 * GPU caches.
3944 */
3945void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3946 struct radeon_fence *fence)
3947{
3948 struct radeon_ring *ring = &rdev->ring[fence->ring];
3949 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3950
3951 /* RELEASE_MEM - flush caches, send int */
3952 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3953 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3954 EOP_TC_ACTION_EN |
3955 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3956 EVENT_INDEX(5)));
3957 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3958 radeon_ring_write(ring, addr & 0xfffffffc);
3959 radeon_ring_write(ring, upper_32_bits(addr));
3960 radeon_ring_write(ring, fence->seq);
3961 radeon_ring_write(ring, 0);
b07fdd38
AD		 3962}
3963
86302eea
CK		 3964/**
3965 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3966 *
3967 * @rdev: radeon_device pointer
3968 * @ring: radeon ring buffer object
3969 * @semaphore: radeon semaphore object
3970 * @emit_wait: Is this a sempahore wait?
3971 *
3972 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3973 * from running ahead of semaphore waits.
3974 */
1654b817 3975bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3976 struct radeon_ring *ring,
3977 struct radeon_semaphore *semaphore,
3978 bool emit_wait)
3979{
3980 uint64_t addr = semaphore->gpu_addr;
3981 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3982
3983 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3984 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3985 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3986
86302eea
CK		 3987 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3988 /* Prevent the PFP from running ahead of the semaphore wait */
3989 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3990 radeon_ring_write(ring, 0x0);
3991 }
3992
1654b817 3993 return true;
2cae3bc3
AD		 3994}
3995
c9dbd705
AD		 3996/**
3997 * cik_copy_cpdma - copy pages using the CP DMA engine
3998 *
3999 * @rdev: radeon_device pointer
4000 * @src_offset: src GPU address
4001 * @dst_offset: dst GPU address
4002 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 4003 * @resv: reservation object to sync to
c9dbd705
AD		 4004 *
4005 * Copy GPU paging using the CP DMA engine (CIK+).
4006 * Used by the radeon ttm implementation to move pages if
4007 * registered as the asic copy callback.
4008 */
57d20a43
CK		 4009struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4010 uint64_t src_offset, uint64_t dst_offset,
4011 unsigned num_gpu_pages,
4012 struct reservation_object *resv)
c9dbd705 4013{
57d20a43 4014 struct radeon_fence *fence;
975700d2 4015 struct radeon_sync sync;
c9dbd705
AD		 4016 int ring_index = rdev->asic->copy.blit_ring_index;
4017 struct radeon_ring *ring = &rdev->ring[ring_index];
4018 u32 size_in_bytes, cur_size_in_bytes, control;
4019 int i, num_loops;
4020 int r = 0;
4021
975700d2 4022 radeon_sync_create(&sync);
c9dbd705
AD		 4023
4024 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4025 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4026 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4027 if (r) {
4028 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 4029 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4030 return ERR_PTR(r);
c9dbd705
AD		 4031 }
4032
975700d2
CK		 4033 radeon_sync_resv(rdev, &sync, resv, false);
4034 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 4035
4036 for (i = 0; i < num_loops; i++) {
4037 cur_size_in_bytes = size_in_bytes;
4038 if (cur_size_in_bytes > 0x1fffff)
4039 cur_size_in_bytes = 0x1fffff;
4040 size_in_bytes -= cur_size_in_bytes;
4041 control = 0;
4042 if (size_in_bytes == 0)
4043 control |= PACKET3_DMA_DATA_CP_SYNC;
4044 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4045 radeon_ring_write(ring, control);
4046 radeon_ring_write(ring, lower_32_bits(src_offset));
4047 radeon_ring_write(ring, upper_32_bits(src_offset));
4048 radeon_ring_write(ring, lower_32_bits(dst_offset));
4049 radeon_ring_write(ring, upper_32_bits(dst_offset));
4050 radeon_ring_write(ring, cur_size_in_bytes);
4051 src_offset += cur_size_in_bytes;
4052 dst_offset += cur_size_in_bytes;
4053 }
4054
57d20a43 4055 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 4056 if (r) {
4057 radeon_ring_unlock_undo(rdev, ring);
975700d2 4058 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4059 return ERR_PTR(r);
c9dbd705
AD		 4060 }
4061
1538a9e0 4062 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 4063 radeon_sync_free(rdev, &sync, fence);
c9dbd705 4064
57d20a43 4065 return fence;
c9dbd705
AD		 4066}
4067
2cae3bc3
AD		 4068/*
4069 * IB stuff
4070 */
4071/**
4072 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4073 *
4074 * @rdev: radeon_device pointer
4075 * @ib: radeon indirect buffer object
4076 *
4077 * Emits an DE (drawing engine) or CE (constant engine) IB
4078 * on the gfx ring. IBs are usually generated by userspace
4079 * acceleration drivers and submitted to the kernel for
4080 * sheduling on the ring. This function schedules the IB
4081 * on the gfx ring for execution by the GPU.
4082 */
4083void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4084{
4085 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 4086 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 4087 u32 header, control = INDIRECT_BUFFER_VALID;
4088
4089 if (ib->is_const_ib) {
4090 /* set switch buffer packet before const IB */
4091 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4092 radeon_ring_write(ring, 0);
4093
4094 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4095 } else {
4096 u32 next_rptr;
4097 if (ring->rptr_save_reg) {
4098 next_rptr = ring->wptr + 3 + 4;
4099 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4100 radeon_ring_write(ring, ((ring->rptr_save_reg -
4101 PACKET3_SET_UCONFIG_REG_START) >> 2));
4102 radeon_ring_write(ring, next_rptr);
4103 } else if (rdev->wb.enabled) {
4104 next_rptr = ring->wptr + 5 + 4;
4105 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4106 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4107 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4108 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4109 radeon_ring_write(ring, next_rptr);
4110 }
4111
4112 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4113 }
4114
7c42bc1a 4115 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 4116
4117 radeon_ring_write(ring, header);
4118 radeon_ring_write(ring,
4119#ifdef __BIG_ENDIAN
4120 (2 << 0) |
4121#endif
4122 (ib->gpu_addr & 0xFFFFFFFC));
4123 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4124 radeon_ring_write(ring, control);
4125}
4126
fbc832c7
AD		 4127/**
4128 * cik_ib_test - basic gfx ring IB test
4129 *
4130 * @rdev: radeon_device pointer
4131 * @ring: radeon_ring structure holding ring information
4132 *
4133 * Allocate an IB and execute it on the gfx ring (CIK).
4134 * Provides a basic gfx ring test to verify that IBs are working.
4135 * Returns 0 on success, error on failure.
4136 */
4137int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4138{
4139 struct radeon_ib ib;
4140 uint32_t scratch;
4141 uint32_t tmp = 0;
4142 unsigned i;
4143 int r;
4144
4145 r = radeon_scratch_get(rdev, &scratch);
4146 if (r) {
4147 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4148 return r;
4149 }
4150 WREG32(scratch, 0xCAFEDEAD);
4151 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4152 if (r) {
4153 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4154 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4155 return r;
4156 }
4157 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4158 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4159 ib.ptr[2] = 0xDEADBEEF;
4160 ib.length_dw = 3;
1538a9e0 4161 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 4162 if (r) {
4163 radeon_scratch_free(rdev, scratch);
4164 radeon_ib_free(rdev, &ib);
4165 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4166 return r;
4167 }
4168 r = radeon_fence_wait(ib.fence, false);
4169 if (r) {
4170 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4171 radeon_scratch_free(rdev, scratch);
4172 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4173 return r;
4174 }
4175 for (i = 0; i < rdev->usec_timeout; i++) {
4176 tmp = RREG32(scratch);
4177 if (tmp == 0xDEADBEEF)
4178 break;
4179 DRM_UDELAY(1);
4180 }
4181 if (i < rdev->usec_timeout) {
4182 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4183 } else {
4184 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4185 scratch, tmp);
4186 r = -EINVAL;
4187 }
4188 radeon_scratch_free(rdev, scratch);
4189 radeon_ib_free(rdev, &ib);
4190 return r;
4191}
4192
841cf442
AD		 4193/*
4194 * CP.
4195 * On CIK, gfx and compute now have independant command processors.
4196 *
4197 * GFX
4198 * Gfx consists of a single ring and can process both gfx jobs and
4199 * compute jobs. The gfx CP consists of three microengines (ME):
4200 * PFP - Pre-Fetch Parser
4201 * ME - Micro Engine
4202 * CE - Constant Engine
4203 * The PFP and ME make up what is considered the Drawing Engine (DE).
4204 * The CE is an asynchronous engine used for updating buffer desciptors
4205 * used by the DE so that they can be loaded into cache in parallel
4206 * while the DE is processing state update packets.
4207 *
4208 * Compute
4209 * The compute CP consists of two microengines (ME):
4210 * MEC1 - Compute MicroEngine 1
4211 * MEC2 - Compute MicroEngine 2
4212 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4213 * The queues are exposed to userspace and are programmed directly
4214 * by the compute runtime.
4215 */
4216/**
4217 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4218 *
4219 * @rdev: radeon_device pointer
4220 * @enable: enable or disable the MEs
4221 *
4222 * Halts or unhalts the gfx MEs.
4223 */
4224static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4225{
4226 if (enable)
4227 WREG32(CP_ME_CNTL, 0);
4228 else {
50efa51a
AD		 4229 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4230 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4231 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4232 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4233 }
4234 udelay(50);
4235}
4236
4237/**
4238 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4239 *
4240 * @rdev: radeon_device pointer
4241 *
4242 * Loads the gfx PFP, ME, and CE ucode.
4243 * Returns 0 for success, -EINVAL if the ucode is not available.
4244 */
4245static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4246{
841cf442
AD		 4247 int i;
4248
4249 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4250 return -EINVAL;
4251
4252 cik_cp_gfx_enable(rdev, false);
4253
f2c6b0f4
AD		 4254 if (rdev->new_fw) {
4255 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4256 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4257 const struct gfx_firmware_header_v1_0 *ce_hdr =
4258 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4259 const struct gfx_firmware_header_v1_0 *me_hdr =
4260 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4261 const __le32 *fw_data;
4262 u32 fw_size;
4263
4264 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4265 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4266 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4267
4268 /* PFP */
4269 fw_data = (const __le32 *)
4270 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4271 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4272 WREG32(CP_PFP_UCODE_ADDR, 0);
4273 for (i = 0; i < fw_size; i++)
4274 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4275 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 4276
4277 /* CE */
4278 fw_data = (const __le32 *)
4279 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4280 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4281 WREG32(CP_CE_UCODE_ADDR, 0);
4282 for (i = 0; i < fw_size; i++)
4283 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4284 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 4285
4286 /* ME */
4287 fw_data = (const __be32 *)
4288 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4289 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4290 WREG32(CP_ME_RAM_WADDR, 0);
4291 for (i = 0; i < fw_size; i++)
4292 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4293 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4294 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4295 } else {
4296 const __be32 *fw_data;
4297
4298 /* PFP */
4299 fw_data = (const __be32 *)rdev->pfp_fw->data;
4300 WREG32(CP_PFP_UCODE_ADDR, 0);
4301 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4302 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4303 WREG32(CP_PFP_UCODE_ADDR, 0);
4304
4305 /* CE */
4306 fw_data = (const __be32 *)rdev->ce_fw->data;
4307 WREG32(CP_CE_UCODE_ADDR, 0);
4308 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4309 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4310 WREG32(CP_CE_UCODE_ADDR, 0);
4311
4312 /* ME */
4313 fw_data = (const __be32 *)rdev->me_fw->data;
4314 WREG32(CP_ME_RAM_WADDR, 0);
4315 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4316 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4317 WREG32(CP_ME_RAM_WADDR, 0);
4318 }
841cf442 4319
841cf442
AD		 4320 return 0;
4321}
4322
4323/**
4324 * cik_cp_gfx_start - start the gfx ring
4325 *
4326 * @rdev: radeon_device pointer
4327 *
4328 * Enables the ring and loads the clear state context and other
4329 * packets required to init the ring.
4330 * Returns 0 for success, error for failure.
4331 */
4332static int cik_cp_gfx_start(struct radeon_device *rdev)
4333{
4334 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4335 int r, i;
4336
4337 /* init the CP */
4338 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4339 WREG32(CP_ENDIAN_SWAP, 0);
4340 WREG32(CP_DEVICE_ID, 1);
4341
4342 cik_cp_gfx_enable(rdev, true);
4343
4344 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4345 if (r) {
4346 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4347 return r;
4348 }
4349
4350 /* init the CE partitions. CE only used for gfx on CIK */
4351 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4352 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4353 radeon_ring_write(ring, 0x8000);
4354 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4355
4356 /* setup clear context state */
4357 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4358 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4359
4360 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4361 radeon_ring_write(ring, 0x80000000);
4362 radeon_ring_write(ring, 0x80000000);
4363
4364 for (i = 0; i < cik_default_size; i++)
4365 radeon_ring_write(ring, cik_default_state[i]);
4366
4367 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4368 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4369
4370 /* set clear context state */
4371 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4372 radeon_ring_write(ring, 0);
4373
4374 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4375 radeon_ring_write(ring, 0x00000316);
4376 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4377 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4378
1538a9e0 4379 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4380
4381 return 0;
4382}
4383
4384/**
4385 * cik_cp_gfx_fini - stop the gfx ring
4386 *
4387 * @rdev: radeon_device pointer
4388 *
4389 * Stop the gfx ring and tear down the driver ring
4390 * info.
4391 */
4392static void cik_cp_gfx_fini(struct radeon_device *rdev)
4393{
4394 cik_cp_gfx_enable(rdev, false);
4395 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4396}
4397
4398/**
4399 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4400 *
4401 * @rdev: radeon_device pointer
4402 *
4403 * Program the location and size of the gfx ring buffer
4404 * and test it to make sure it's working.
4405 * Returns 0 for success, error for failure.
4406 */
4407static int cik_cp_gfx_resume(struct radeon_device *rdev)
4408{
4409 struct radeon_ring *ring;
4410 u32 tmp;
4411 u32 rb_bufsz;
4412 u64 rb_addr;
4413 int r;
4414
4415 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4416 if (rdev->family != CHIP_HAWAII)
4417 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4418
4419 /* Set the write pointer delay */
4420 WREG32(CP_RB_WPTR_DELAY, 0);
4421
4422 /* set the RB to use vmid 0 */
4423 WREG32(CP_RB_VMID, 0);
4424
4425 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4426
4427 /* ring 0 - compute and gfx */
4428 /* Set ring buffer size */
4429 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4430 rb_bufsz = order_base_2(ring->ring_size / 8);
4431 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4432#ifdef __BIG_ENDIAN
4433 tmp |= BUF_SWAP_32BIT;
4434#endif
4435 WREG32(CP_RB0_CNTL, tmp);
4436
4437 /* Initialize the ring buffer's read and write pointers */
4438 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4439 ring->wptr = 0;
4440 WREG32(CP_RB0_WPTR, ring->wptr);
4441
4442 /* set the wb address wether it's enabled or not */
4443 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4444 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4445
4446 /* scratch register shadowing is no longer supported */
4447 WREG32(SCRATCH_UMSK, 0);
4448
4449 if (!rdev->wb.enabled)
4450 tmp |= RB_NO_UPDATE;
4451
4452 mdelay(1);
4453 WREG32(CP_RB0_CNTL, tmp);
4454
4455 rb_addr = ring->gpu_addr >> 8;
4456 WREG32(CP_RB0_BASE, rb_addr);
4457 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4458
841cf442
AD		 4459 /* start the ring */
4460 cik_cp_gfx_start(rdev);
4461 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4462 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4463 if (r) {
4464 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4465 return r;
4466 }
50efa51a
AD		 4467
4468 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4469 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4470
841cf442
AD		 4471 return 0;
4472}
4473
ea31bf69
AD		 4474u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4475 struct radeon_ring *ring)
963e81f9
AD		 4476{
4477 u32 rptr;
4478
ea31bf69
AD		 4479 if (rdev->wb.enabled)
4480 rptr = rdev->wb.wb[ring->rptr_offs/4];
4481 else
4482 rptr = RREG32(CP_RB0_RPTR);
4483
4484 return rptr;
4485}
4486
4487u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4488 struct radeon_ring *ring)
4489{
4490 u32 wptr;
4491
4492 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4493
ea31bf69
AD		 4494 return wptr;
4495}
4496
4497void cik_gfx_set_wptr(struct radeon_device *rdev,
4498 struct radeon_ring *ring)
4499{
4500 WREG32(CP_RB0_WPTR, ring->wptr);
4501 (void)RREG32(CP_RB0_WPTR);
4502}
4503
4504u32 cik_compute_get_rptr(struct radeon_device *rdev,
4505 struct radeon_ring *ring)
4506{
4507 u32 rptr;
963e81f9
AD		 4508
4509 if (rdev->wb.enabled) {
ea31bf69 4510 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4511 } else {
f61d5b46 4512 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4513 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4514 rptr = RREG32(CP_HQD_PQ_RPTR);
4515 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4516 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4517 }
963e81f9
AD		 4518
4519 return rptr;
4520}
4521
ea31bf69
AD		 4522u32 cik_compute_get_wptr(struct radeon_device *rdev,
4523 struct radeon_ring *ring)
963e81f9
AD		 4524{
4525 u32 wptr;
4526
4527 if (rdev->wb.enabled) {
ea31bf69
AD		 4528 /* XXX check if swapping is necessary on BE */
4529 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4530 } else {
f61d5b46 4531 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4532 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4533 wptr = RREG32(CP_HQD_PQ_WPTR);
4534 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4535 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4536 }
963e81f9
AD		 4537
4538 return wptr;
4539}
4540
ea31bf69
AD		 4541void cik_compute_set_wptr(struct radeon_device *rdev,
4542 struct radeon_ring *ring)
963e81f9 4543{
ea31bf69
AD		 4544 /* XXX check if swapping is necessary on BE */
4545 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4546 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4547}
4548
841cf442
AD		 4549/**
4550 * cik_cp_compute_enable - enable/disable the compute CP MEs
4551 *
4552 * @rdev: radeon_device pointer
4553 * @enable: enable or disable the MEs
4554 *
4555 * Halts or unhalts the compute MEs.
4556 */
4557static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4558{
4559 if (enable)
4560 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4561 else {
841cf442 4562 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4563 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4564 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4565 }
841cf442
AD		 4566 udelay(50);
4567}
4568
4569/**
4570 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4571 *
4572 * @rdev: radeon_device pointer
4573 *
4574 * Loads the compute MEC1&2 ucode.
4575 * Returns 0 for success, -EINVAL if the ucode is not available.
4576 */
4577static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4578{
841cf442
AD		 4579 int i;
4580
4581 if (!rdev->mec_fw)
4582 return -EINVAL;
4583
4584 cik_cp_compute_enable(rdev, false);
4585
f2c6b0f4
AD		 4586 if (rdev->new_fw) {
4587 const struct gfx_firmware_header_v1_0 *mec_hdr =
4588 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4589 const __le32 *fw_data;
4590 u32 fw_size;
4591
4592 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4593
4594 /* MEC1 */
4595 fw_data = (const __le32 *)
4596 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4597 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4598 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4599 for (i = 0; i < fw_size; i++)
4600 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4601 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4602
841cf442 4603 /* MEC2 */
f2c6b0f4
AD		 4604 if (rdev->family == CHIP_KAVERI) {
4605 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4606 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4607
4608 fw_data = (const __le32 *)
4609 (rdev->mec2_fw->data +
4610 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4611 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4612 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4613 for (i = 0; i < fw_size; i++)
4614 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4615 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4616 }
4617 } else {
4618 const __be32 *fw_data;
4619
4620 /* MEC1 */
841cf442 4621 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4622 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4623 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4624 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4625 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4626
4627 if (rdev->family == CHIP_KAVERI) {
4628 /* MEC2 */
4629 fw_data = (const __be32 *)rdev->mec_fw->data;
4630 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4631 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4632 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4633 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4634 }
841cf442
AD		 4635 }
4636
4637 return 0;
4638}
4639
4640/**
4641 * cik_cp_compute_start - start the compute queues
4642 *
4643 * @rdev: radeon_device pointer
4644 *
4645 * Enable the compute queues.
4646 * Returns 0 for success, error for failure.
4647 */
4648static int cik_cp_compute_start(struct radeon_device *rdev)
4649{
963e81f9
AD		 4650 cik_cp_compute_enable(rdev, true);
4651
841cf442
AD		 4652 return 0;
4653}
4654
4655/**
4656 * cik_cp_compute_fini - stop the compute queues
4657 *
4658 * @rdev: radeon_device pointer
4659 *
4660 * Stop the compute queues and tear down the driver queue
4661 * info.
4662 */
4663static void cik_cp_compute_fini(struct radeon_device *rdev)
4664{
963e81f9
AD		 4665 int i, idx, r;
4666
841cf442 4667 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4668
4669 for (i = 0; i < 2; i++) {
4670 if (i == 0)
4671 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4672 else
4673 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4674
4675 if (rdev->ring[idx].mqd_obj) {
4676 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4677 if (unlikely(r != 0))
4678 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4679
4680 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4681 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4682
4683 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4684 rdev->ring[idx].mqd_obj = NULL;
4685 }
4686 }
841cf442
AD		 4687}
4688
963e81f9
AD		 4689static void cik_mec_fini(struct radeon_device *rdev)
4690{
4691 int r;
4692
4693 if (rdev->mec.hpd_eop_obj) {
4694 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4695 if (unlikely(r != 0))
4696 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4697 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4698 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4699
4700 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4701 rdev->mec.hpd_eop_obj = NULL;
4702 }
4703}
4704
4705#define MEC_HPD_SIZE 2048
4706
4707static int cik_mec_init(struct radeon_device *rdev)
4708{
4709 int r;
4710 u32 *hpd;
4711
4712 /*
4713 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4714 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4715 * Nonetheless, we assign only 1 pipe because all other pipes will
4716 * be handled by KFD
963e81f9 4717 */
62a7b7fb
OG		 4718 rdev->mec.num_mec = 1;
4719 rdev->mec.num_pipe = 1;
963e81f9
AD		 4720 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4721
4722 if (rdev->mec.hpd_eop_obj == NULL) {
4723 r = radeon_bo_create(rdev,
4724 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4725 PAGE_SIZE, true,
831b6966 4726 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4727 &rdev->mec.hpd_eop_obj);
4728 if (r) {
4729 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4730 return r;
4731 }
4732 }
4733
4734 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4735 if (unlikely(r != 0)) {
4736 cik_mec_fini(rdev);
4737 return r;
4738 }
4739 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4740 &rdev->mec.hpd_eop_gpu_addr);
4741 if (r) {
4742 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4743 cik_mec_fini(rdev);
4744 return r;
4745 }
4746 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4747 if (r) {
4748 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4749 cik_mec_fini(rdev);
4750 return r;
4751 }
4752
4753 /* clear memory. Not sure if this is required or not */
4754 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4755
4756 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4757 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4758
4759 return 0;
4760}
4761
4762struct hqd_registers
4763{
4764 u32 cp_mqd_base_addr;
4765 u32 cp_mqd_base_addr_hi;
4766 u32 cp_hqd_active;
4767 u32 cp_hqd_vmid;
4768 u32 cp_hqd_persistent_state;
4769 u32 cp_hqd_pipe_priority;
4770 u32 cp_hqd_queue_priority;
4771 u32 cp_hqd_quantum;
4772 u32 cp_hqd_pq_base;
4773 u32 cp_hqd_pq_base_hi;
4774 u32 cp_hqd_pq_rptr;
4775 u32 cp_hqd_pq_rptr_report_addr;
4776 u32 cp_hqd_pq_rptr_report_addr_hi;
4777 u32 cp_hqd_pq_wptr_poll_addr;
4778 u32 cp_hqd_pq_wptr_poll_addr_hi;
4779 u32 cp_hqd_pq_doorbell_control;
4780 u32 cp_hqd_pq_wptr;
4781 u32 cp_hqd_pq_control;
4782 u32 cp_hqd_ib_base_addr;
4783 u32 cp_hqd_ib_base_addr_hi;
4784 u32 cp_hqd_ib_rptr;
4785 u32 cp_hqd_ib_control;
4786 u32 cp_hqd_iq_timer;
4787 u32 cp_hqd_iq_rptr;
4788 u32 cp_hqd_dequeue_request;
4789 u32 cp_hqd_dma_offload;
4790 u32 cp_hqd_sema_cmd;
4791 u32 cp_hqd_msg_type;
4792 u32 cp_hqd_atomic0_preop_lo;
4793 u32 cp_hqd_atomic0_preop_hi;
4794 u32 cp_hqd_atomic1_preop_lo;
4795 u32 cp_hqd_atomic1_preop_hi;
4796 u32 cp_hqd_hq_scheduler0;
4797 u32 cp_hqd_hq_scheduler1;
4798 u32 cp_mqd_control;
4799};
4800
4801struct bonaire_mqd
4802{
4803 u32 header;
4804 u32 dispatch_initiator;
4805 u32 dimensions[3];
4806 u32 start_idx[3];
4807 u32 num_threads[3];
4808 u32 pipeline_stat_enable;
4809 u32 perf_counter_enable;
4810 u32 pgm[2];
4811 u32 tba[2];
4812 u32 tma[2];
4813 u32 pgm_rsrc[2];
4814 u32 vmid;
4815 u32 resource_limits;
4816 u32 static_thread_mgmt01[2];
4817 u32 tmp_ring_size;
4818 u32 static_thread_mgmt23[2];
4819 u32 restart[3];
4820 u32 thread_trace_enable;
4821 u32 reserved1;
4822 u32 user_data[16];
4823 u32 vgtcs_invoke_count[2];
4824 struct hqd_registers queue_state;
4825 u32 dequeue_cntr;
4826 u32 interrupt_queue[64];
4827};
4828
841cf442
AD		 4829/**
4830 * cik_cp_compute_resume - setup the compute queue registers
4831 *
4832 * @rdev: radeon_device pointer
4833 *
4834 * Program the compute queues and test them to make sure they
4835 * are working.
4836 * Returns 0 for success, error for failure.
4837 */
4838static int cik_cp_compute_resume(struct radeon_device *rdev)
4839{
370ce45b 4840 int r, i, j, idx;
963e81f9
AD		 4841 u32 tmp;
4842 bool use_doorbell = true;
4843 u64 hqd_gpu_addr;
4844 u64 mqd_gpu_addr;
4845 u64 eop_gpu_addr;
4846 u64 wb_gpu_addr;
4847 u32 *buf;
4848 struct bonaire_mqd *mqd;
841cf442 4849
841cf442
AD		 4850 r = cik_cp_compute_start(rdev);
4851 if (r)
4852 return r;
963e81f9
AD		 4853
4854 /* fix up chicken bits */
4855 tmp = RREG32(CP_CPF_DEBUG);
4856 tmp |= (1 << 23);
4857 WREG32(CP_CPF_DEBUG, tmp);
4858
4859 /* init the pipes */
f61d5b46 4860 mutex_lock(&rdev->srbm_mutex);
963e81f9 4861
62a7b7fb 4862 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4863
62a7b7fb 4864 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4865
62a7b7fb
OG		 4866 /* write the EOP addr */
4867 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4868 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4869
62a7b7fb
OG		 4870 /* set the VMID assigned */
4871 WREG32(CP_HPD_EOP_VMID, 0);
4872
4873 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4874 tmp = RREG32(CP_HPD_EOP_CONTROL);
4875 tmp &= ~EOP_SIZE_MASK;
4876 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4877 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4878
f61d5b46 4879 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4880
4881 /* init the queues. Just two for now. */
4882 for (i = 0; i < 2; i++) {
4883 if (i == 0)
4884 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4885 else
4886 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4887
4888 if (rdev->ring[idx].mqd_obj == NULL) {
4889 r = radeon_bo_create(rdev,
4890 sizeof(struct bonaire_mqd),
4891 PAGE_SIZE, true,
02376d82 4892 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4893 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4894 if (r) {
4895 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4896 return r;
4897 }
4898 }
4899
4900 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4901 if (unlikely(r != 0)) {
4902 cik_cp_compute_fini(rdev);
4903 return r;
4904 }
4905 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4906 &mqd_gpu_addr);
4907 if (r) {
4908 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4909 cik_cp_compute_fini(rdev);
4910 return r;
4911 }
4912 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4913 if (r) {
4914 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4915 cik_cp_compute_fini(rdev);
4916 return r;
4917 }
4918
963e81f9
AD		 4919 /* init the mqd struct */
4920 memset(buf, 0, sizeof(struct bonaire_mqd));
4921
4922 mqd = (struct bonaire_mqd *)buf;
4923 mqd->header = 0xC0310800;
4924 mqd->static_thread_mgmt01[0] = 0xffffffff;
4925 mqd->static_thread_mgmt01[1] = 0xffffffff;
4926 mqd->static_thread_mgmt23[0] = 0xffffffff;
4927 mqd->static_thread_mgmt23[1] = 0xffffffff;
4928
f61d5b46 4929 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4930 cik_srbm_select(rdev, rdev->ring[idx].me,
4931 rdev->ring[idx].pipe,
4932 rdev->ring[idx].queue, 0);
4933
4934 /* disable wptr polling */
4935 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4936 tmp &= ~WPTR_POLL_EN;
4937 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4938
4939 /* enable doorbell? */
4940 mqd->queue_state.cp_hqd_pq_doorbell_control =
4941 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4942 if (use_doorbell)
4943 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4944 else
4945 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4946 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4947 mqd->queue_state.cp_hqd_pq_doorbell_control);
4948
4949 /* disable the queue if it's active */
4950 mqd->queue_state.cp_hqd_dequeue_request = 0;
4951 mqd->queue_state.cp_hqd_pq_rptr = 0;
4952 mqd->queue_state.cp_hqd_pq_wptr= 0;
4953 if (RREG32(CP_HQD_ACTIVE) & 1) {
4954 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4955 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4956 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4957 break;
4958 udelay(1);
4959 }
4960 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4961 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4962 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4963 }
4964
4965 /* set the pointer to the MQD */
4966 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4967 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4968 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4969 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4970 /* set MQD vmid to 0 */
4971 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4972 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4973 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4974
4975 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4976 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4977 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4978 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4979 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4980 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4981
4982 /* set up the HQD, this is similar to CP_RB0_CNTL */
4983 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4984 mqd->queue_state.cp_hqd_pq_control &=
4985 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4986
4987 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4988 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4989 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4990 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4991#ifdef __BIG_ENDIAN
4992 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4993#endif
4994 mqd->queue_state.cp_hqd_pq_control &=
4995 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4996 mqd->queue_state.cp_hqd_pq_control |=
4997 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4998 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4999
5000 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5001 if (i == 0)
5002 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5003 else
5004 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5005 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5006 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5007 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5008 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5009 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5010
5011 /* set the wb address wether it's enabled or not */
5012 if (i == 0)
5013 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5014 else
5015 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5016 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5017 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5018 upper_32_bits(wb_gpu_addr) & 0xffff;
5019 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5020 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5021 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5022 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5023
5024 /* enable the doorbell if requested */
5025 if (use_doorbell) {
5026 mqd->queue_state.cp_hqd_pq_doorbell_control =
5027 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5028 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5029 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 5030 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 5031 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5032 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5033 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5034
5035 } else {
5036 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5037 }
5038 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5039 mqd->queue_state.cp_hqd_pq_doorbell_control);
5040
5041 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5042 rdev->ring[idx].wptr = 0;
5043 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5044 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5045 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5046
5047 /* set the vmid for the queue */
5048 mqd->queue_state.cp_hqd_vmid = 0;
5049 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5050
5051 /* activate the queue */
5052 mqd->queue_state.cp_hqd_active = 1;
5053 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5054
5055 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5056 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5057
5058 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5059 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5060
5061 rdev->ring[idx].ready = true;
5062 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5063 if (r)
5064 rdev->ring[idx].ready = false;
5065 }
5066
841cf442
AD		 5067 return 0;
5068}
5069
841cf442
AD		 5070static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5071{
5072 cik_cp_gfx_enable(rdev, enable);
5073 cik_cp_compute_enable(rdev, enable);
5074}
5075
841cf442
AD		 5076static int cik_cp_load_microcode(struct radeon_device *rdev)
5077{
5078 int r;
5079
5080 r = cik_cp_gfx_load_microcode(rdev);
5081 if (r)
5082 return r;
5083 r = cik_cp_compute_load_microcode(rdev);
5084 if (r)
5085 return r;
5086
5087 return 0;
5088}
5089
841cf442
AD		 5090static void cik_cp_fini(struct radeon_device *rdev)
5091{
5092 cik_cp_gfx_fini(rdev);
5093 cik_cp_compute_fini(rdev);
5094}
5095
841cf442
AD		 5096static int cik_cp_resume(struct radeon_device *rdev)
5097{
5098 int r;
5099
4214faf6
AD		 5100 cik_enable_gui_idle_interrupt(rdev, false);
5101
841cf442
AD		 5102 r = cik_cp_load_microcode(rdev);
5103 if (r)
5104 return r;
5105
5106 r = cik_cp_gfx_resume(rdev);
5107 if (r)
5108 return r;
5109 r = cik_cp_compute_resume(rdev);
5110 if (r)
5111 return r;
5112
4214faf6
AD		 5113 cik_enable_gui_idle_interrupt(rdev, true);
5114
841cf442
AD		 5115 return 0;
5116}
5117
cc066715 5118static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5119{
6f2043ce
AD		 5120 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5121 RREG32(GRBM_STATUS));
5122 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5123 RREG32(GRBM_STATUS2));
5124 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5125 RREG32(GRBM_STATUS_SE0));
5126 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5127 RREG32(GRBM_STATUS_SE1));
5128 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5129 RREG32(GRBM_STATUS_SE2));
5130 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5131 RREG32(GRBM_STATUS_SE3));
5132 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5133 RREG32(SRBM_STATUS));
5134 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5135 RREG32(SRBM_STATUS2));
cc066715
AD		 5136 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5137 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5138 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5139 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5140 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5141 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5142 RREG32(CP_STALLED_STAT1));
5143 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5144 RREG32(CP_STALLED_STAT2));
5145 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5146 RREG32(CP_STALLED_STAT3));
5147 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5148 RREG32(CP_CPF_BUSY_STAT));
5149 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5150 RREG32(CP_CPF_STALLED_STAT1));
5151 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5152 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5153 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5154 RREG32(CP_CPC_STALLED_STAT1));
5155 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5156}
6f2043ce 5157
21a93e13 5158/**
cc066715 5159 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5160 *
5161 * @rdev: radeon_device pointer
21a93e13 5162 *
cc066715
AD		 5163 * Check which blocks are busy and return the relevant reset
5164 * mask to be used by cik_gpu_soft_reset().
5165 * Returns a mask of the blocks to be reset.
21a93e13 5166 */
2483b4ea 5167u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5168{
cc066715
AD		 5169 u32 reset_mask = 0;
5170 u32 tmp;
21a93e13 5171
cc066715
AD		 5172 /* GRBM_STATUS */
5173 tmp = RREG32(GRBM_STATUS);
5174 if (tmp & (PA_BUSY | SC_BUSY |
5175 BCI_BUSY | SX_BUSY |
5176 TA_BUSY | VGT_BUSY |
5177 DB_BUSY | CB_BUSY |
5178 GDS_BUSY | SPI_BUSY |
5179 IA_BUSY | IA_BUSY_NO_DMA))
5180 reset_mask |= RADEON_RESET_GFX;
21a93e13 5181
cc066715
AD		 5182 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5183 reset_mask |= RADEON_RESET_CP;
21a93e13 5184
cc066715
AD		 5185 /* GRBM_STATUS2 */
5186 tmp = RREG32(GRBM_STATUS2);
5187 if (tmp & RLC_BUSY)
5188 reset_mask |= RADEON_RESET_RLC;
21a93e13 5189
cc066715
AD		 5190 /* SDMA0_STATUS_REG */
5191 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5192 if (!(tmp & SDMA_IDLE))
5193 reset_mask |= RADEON_RESET_DMA;
21a93e13 5194
cc066715
AD		 5195 /* SDMA1_STATUS_REG */
5196 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5197 if (!(tmp & SDMA_IDLE))
5198 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5199
cc066715
AD		 5200 /* SRBM_STATUS2 */
5201 tmp = RREG32(SRBM_STATUS2);
5202 if (tmp & SDMA_BUSY)
5203 reset_mask |= RADEON_RESET_DMA;
21a93e13 5204
cc066715
AD		 5205 if (tmp & SDMA1_BUSY)
5206 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5207
cc066715
AD		 5208 /* SRBM_STATUS */
5209 tmp = RREG32(SRBM_STATUS);
21a93e13 5210
cc066715
AD		 5211 if (tmp & IH_BUSY)
5212 reset_mask |= RADEON_RESET_IH;
21a93e13 5213
cc066715
AD		 5214 if (tmp & SEM_BUSY)
5215 reset_mask |= RADEON_RESET_SEM;
21a93e13 5216
cc066715
AD		 5217 if (tmp & GRBM_RQ_PENDING)
5218 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5219
cc066715
AD		 5220 if (tmp & VMC_BUSY)
5221 reset_mask |= RADEON_RESET_VMC;
21a93e13 5222
cc066715
AD		 5223 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5224 MCC_BUSY | MCD_BUSY))
5225 reset_mask |= RADEON_RESET_MC;
21a93e13 5226
cc066715
AD		 5227 if (evergreen_is_display_hung(rdev))
5228 reset_mask |= RADEON_RESET_DISPLAY;
5229
5230 /* Skip MC reset as it's mostly likely not hung, just busy */
5231 if (reset_mask & RADEON_RESET_MC) {
5232 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5233 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5234 }
cc066715
AD		 5235
5236 return reset_mask;
21a93e13
AD		 5237}
5238
5239/**
cc066715 5240 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5241 *
5242 * @rdev: radeon_device pointer
cc066715 5243 * @reset_mask: mask of which blocks to reset
21a93e13 5244 *
cc066715 5245 * Soft reset the blocks specified in @reset_mask.
21a93e13 5246 */
cc066715 5247static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5248{
6f2043ce 5249 struct evergreen_mc_save save;
cc066715
AD		 5250 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5251 u32 tmp;
21a93e13 5252
cc066715
AD		 5253 if (reset_mask == 0)
5254 return;
21a93e13 5255
cc066715 5256 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5257
cc066715
AD		 5258 cik_print_gpu_status_regs(rdev);
5259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5261 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5262 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5263
fb2c7f4d
AD		 5264 /* disable CG/PG */
5265 cik_fini_pg(rdev);
5266 cik_fini_cg(rdev);
5267
cc066715
AD		 5268 /* stop the rlc */
5269 cik_rlc_stop(rdev);
21a93e13 5270
cc066715
AD		 5271 /* Disable GFX parsing/prefetching */
5272 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5273
cc066715
AD		 5274 /* Disable MEC parsing/prefetching */
5275 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5276
cc066715
AD		 5277 if (reset_mask & RADEON_RESET_DMA) {
5278 /* sdma0 */
5279 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5280 tmp |= SDMA_HALT;
5281 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5282 }
5283 if (reset_mask & RADEON_RESET_DMA1) {
5284 /* sdma1 */
5285 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5286 tmp |= SDMA_HALT;
5287 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5288 }
21a93e13 5289
6f2043ce 5290 evergreen_mc_stop(rdev, &save);
cc066715 5291 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5292 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5293 }
21a93e13 5294
cc066715
AD		 5295 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5296 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5297
cc066715
AD		 5298 if (reset_mask & RADEON_RESET_CP) {
5299 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5300
cc066715
AD		 5301 srbm_soft_reset |= SOFT_RESET_GRBM;
5302 }
21a93e13 5303
cc066715
AD		 5304 if (reset_mask & RADEON_RESET_DMA)
5305 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5306
cc066715
AD		 5307 if (reset_mask & RADEON_RESET_DMA1)
5308 srbm_soft_reset |= SOFT_RESET_SDMA1;
5309
5310 if (reset_mask & RADEON_RESET_DISPLAY)
5311 srbm_soft_reset |= SOFT_RESET_DC;
5312
5313 if (reset_mask & RADEON_RESET_RLC)
5314 grbm_soft_reset |= SOFT_RESET_RLC;
5315
5316 if (reset_mask & RADEON_RESET_SEM)
5317 srbm_soft_reset |= SOFT_RESET_SEM;
5318
5319 if (reset_mask & RADEON_RESET_IH)
5320 srbm_soft_reset |= SOFT_RESET_IH;
5321
5322 if (reset_mask & RADEON_RESET_GRBM)
5323 srbm_soft_reset |= SOFT_RESET_GRBM;
5324
5325 if (reset_mask & RADEON_RESET_VMC)
5326 srbm_soft_reset |= SOFT_RESET_VMC;
5327
5328 if (!(rdev->flags & RADEON_IS_IGP)) {
5329 if (reset_mask & RADEON_RESET_MC)
5330 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5331 }
5332
cc066715
AD		 5333 if (grbm_soft_reset) {
5334 tmp = RREG32(GRBM_SOFT_RESET);
5335 tmp |= grbm_soft_reset;
5336 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5337 WREG32(GRBM_SOFT_RESET, tmp);
5338 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5339
cc066715 5340 udelay(50);
21a93e13 5341
cc066715
AD		 5342 tmp &= ~grbm_soft_reset;
5343 WREG32(GRBM_SOFT_RESET, tmp);
5344 tmp = RREG32(GRBM_SOFT_RESET);
5345 }
21a93e13 5346
cc066715
AD		 5347 if (srbm_soft_reset) {
5348 tmp = RREG32(SRBM_SOFT_RESET);
5349 tmp |= srbm_soft_reset;
5350 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5351 WREG32(SRBM_SOFT_RESET, tmp);
5352 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5353
cc066715 5354 udelay(50);
21a93e13 5355
cc066715
AD		 5356 tmp &= ~srbm_soft_reset;
5357 WREG32(SRBM_SOFT_RESET, tmp);
5358 tmp = RREG32(SRBM_SOFT_RESET);
5359 }
21a93e13 5360
6f2043ce
AD		 5361 /* Wait a little for things to settle down */
5362 udelay(50);
21a93e13 5363
6f2043ce 5364 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5365 udelay(50);
5366
5367 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5368}
5369
0279ed19
AD		 5370struct kv_reset_save_regs {
5371 u32 gmcon_reng_execute;
5372 u32 gmcon_misc;
5373 u32 gmcon_misc3;
5374};
5375
5376static void kv_save_regs_for_reset(struct radeon_device *rdev,
5377 struct kv_reset_save_regs *save)
5378{
5379 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5380 save->gmcon_misc = RREG32(GMCON_MISC);
5381 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5382
5383 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5384 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5385 STCTRL_STUTTER_EN));
5386}
5387
5388static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5389 struct kv_reset_save_regs *save)
5390{
5391 int i;
5392
5393 WREG32(GMCON_PGFSM_WRITE, 0);
5394 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5395
5396 for (i = 0; i < 5; i++)
5397 WREG32(GMCON_PGFSM_WRITE, 0);
5398
5399 WREG32(GMCON_PGFSM_WRITE, 0);
5400 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5401
5402 for (i = 0; i < 5; i++)
5403 WREG32(GMCON_PGFSM_WRITE, 0);
5404
5405 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5406 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5407
5408 for (i = 0; i < 5; i++)
5409 WREG32(GMCON_PGFSM_WRITE, 0);
5410
5411 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5412 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5413
5414 for (i = 0; i < 5; i++)
5415 WREG32(GMCON_PGFSM_WRITE, 0);
5416
5417 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5418 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5419
5420 for (i = 0; i < 5; i++)
5421 WREG32(GMCON_PGFSM_WRITE, 0);
5422
5423 WREG32(GMCON_PGFSM_WRITE, 0);
5424 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5425
5426 for (i = 0; i < 5; i++)
5427 WREG32(GMCON_PGFSM_WRITE, 0);
5428
5429 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5430 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5431
5432 for (i = 0; i < 5; i++)
5433 WREG32(GMCON_PGFSM_WRITE, 0);
5434
5435 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5436 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5437
5438 for (i = 0; i < 5; i++)
5439 WREG32(GMCON_PGFSM_WRITE, 0);
5440
5441 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5442 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5443
5444 for (i = 0; i < 5; i++)
5445 WREG32(GMCON_PGFSM_WRITE, 0);
5446
5447 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5448 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5449
5450 for (i = 0; i < 5; i++)
5451 WREG32(GMCON_PGFSM_WRITE, 0);
5452
5453 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5454 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5455
5456 WREG32(GMCON_MISC3, save->gmcon_misc3);
5457 WREG32(GMCON_MISC, save->gmcon_misc);
5458 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5459}
5460
5461static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5462{
5463 struct evergreen_mc_save save;
5464 struct kv_reset_save_regs kv_save = { 0 };
5465 u32 tmp, i;
5466
5467 dev_info(rdev->dev, "GPU pci config reset\n");
5468
5469 /* disable dpm? */
5470
5471 /* disable cg/pg */
5472 cik_fini_pg(rdev);
5473 cik_fini_cg(rdev);
5474
5475 /* Disable GFX parsing/prefetching */
5476 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5477
5478 /* Disable MEC parsing/prefetching */
5479 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5480
5481 /* sdma0 */
5482 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5483 tmp |= SDMA_HALT;
5484 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5485 /* sdma1 */
5486 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5487 tmp |= SDMA_HALT;
5488 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5489 /* XXX other engines? */
5490
5491 /* halt the rlc, disable cp internal ints */
5492 cik_rlc_stop(rdev);
5493
5494 udelay(50);
5495
5496 /* disable mem access */
5497 evergreen_mc_stop(rdev, &save);
5498 if (evergreen_mc_wait_for_idle(rdev)) {
5499 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5500 }
5501
5502 if (rdev->flags & RADEON_IS_IGP)
5503 kv_save_regs_for_reset(rdev, &kv_save);
5504
5505 /* disable BM */
5506 pci_clear_master(rdev->pdev);
5507 /* reset */
5508 radeon_pci_config_reset(rdev);
5509
5510 udelay(100);
5511
5512 /* wait for asic to come out of reset */
5513 for (i = 0; i < rdev->usec_timeout; i++) {
5514 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5515 break;
5516 udelay(1);
5517 }
5518
5519 /* does asic init need to be run first??? */
5520 if (rdev->flags & RADEON_IS_IGP)
5521 kv_restore_regs_for_reset(rdev, &kv_save);
5522}
5523
21a93e13 5524/**
cc066715 5525 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5526 *
5527 * @rdev: radeon_device pointer
5528 *
cc066715
AD		 5529 * Look up which blocks are hung and attempt
5530 * to reset them.
6f2043ce 5531 * Returns 0 for success.
21a93e13 5532 */
6f2043ce 5533int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5534{
cc066715 5535 u32 reset_mask;
21a93e13 5536
cc066715 5537 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5538
cc066715
AD		 5539 if (reset_mask)
5540 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5541
0279ed19 5542 /* try soft reset */
cc066715 5543 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5544
cc066715
AD		 5545 reset_mask = cik_gpu_check_soft_reset(rdev);
5546
0279ed19
AD		 5547 /* try pci config reset */
5548 if (reset_mask && radeon_hard_reset)
5549 cik_gpu_pci_config_reset(rdev);
5550
5551 reset_mask = cik_gpu_check_soft_reset(rdev);
5552
cc066715
AD		 5553 if (!reset_mask)
5554 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5555
5556 return 0;
5557}
5558
5559/**
cc066715 5560 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5561 *
5562 * @rdev: radeon_device pointer
cc066715 5563 * @ring: radeon_ring structure holding ring information
21a93e13 5564 *
cc066715
AD		 5565 * Check if the 3D engine is locked up (CIK).
5566 * Returns true if the engine is locked, false if not.
21a93e13 5567 */
cc066715 5568bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5569{
cc066715 5570 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5571
cc066715
AD		 5572 if (!(reset_mask & (RADEON_RESET_GFX |
5573 RADEON_RESET_COMPUTE |
5574 RADEON_RESET_CP))) {
ff212f25 5575 radeon_ring_lockup_update(rdev, ring);
cc066715 5576 return false;
21a93e13 5577 }
cc066715 5578 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5579}
5580
1c49165d 5581/* MC */
21a93e13 5582/**
1c49165d 5583 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5584 *
5585 * @rdev: radeon_device pointer
21a93e13 5586 *
1c49165d
AD		 5587 * Set the location of vram, gart, and AGP in the GPU's
5588 * physical address space (CIK).
21a93e13 5589 */
1c49165d 5590static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5591{
1c49165d 5592 struct evergreen_mc_save save;
21a93e13 5593 u32 tmp;
1c49165d 5594 int i, j;
21a93e13 5595
1c49165d
AD		 5596 /* Initialize HDP */
5597 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5598 WREG32((0x2c14 + j), 0x00000000);
5599 WREG32((0x2c18 + j), 0x00000000);
5600 WREG32((0x2c1c + j), 0x00000000);
5601 WREG32((0x2c20 + j), 0x00000000);
5602 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5603 }
1c49165d 5604 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5605
1c49165d
AD		 5606 evergreen_mc_stop(rdev, &save);
5607 if (radeon_mc_wait_for_idle(rdev)) {
5608 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5609 }
1c49165d
AD		 5610 /* Lockout access through VGA aperture*/
5611 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5612 /* Update configuration */
5613 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5614 rdev->mc.vram_start >> 12);
5615 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5616 rdev->mc.vram_end >> 12);
5617 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5618 rdev->vram_scratch.gpu_addr >> 12);
5619 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5620 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5621 WREG32(MC_VM_FB_LOCATION, tmp);
5622 /* XXX double check these! */
5623 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5624 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5625 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5626 WREG32(MC_VM_AGP_BASE, 0);
5627 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5628 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5629 if (radeon_mc_wait_for_idle(rdev)) {
5630 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5631 }
1c49165d
AD		 5632 evergreen_mc_resume(rdev, &save);
5633 /* we need to own VRAM, so turn off the VGA renderer here
5634 * to stop it overwriting our objects */
5635 rv515_vga_render_disable(rdev);
21a93e13
AD		 5636}
5637
5638/**
1c49165d 5639 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5640 *
5641 * @rdev: radeon_device pointer
21a93e13 5642 *
1c49165d
AD		 5643 * Look up the amount of vram, vram width, and decide how to place
5644 * vram and gart within the GPU's physical address space (CIK).
5645 * Returns 0 for success.
21a93e13 5646 */
1c49165d 5647static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5648{
1c49165d
AD		 5649 u32 tmp;
5650 int chansize, numchan;
21a93e13 5651
1c49165d
AD		 5652 /* Get VRAM informations */
5653 rdev->mc.vram_is_ddr = true;
5654 tmp = RREG32(MC_ARB_RAMCFG);
5655 if (tmp & CHANSIZE_MASK) {
5656 chansize = 64;
21a93e13 5657 } else {
1c49165d 5658 chansize = 32;
21a93e13 5659 }
1c49165d
AD		 5660 tmp = RREG32(MC_SHARED_CHMAP);
5661 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5662 case 0:
5663 default:
5664 numchan = 1;
5665 break;
5666 case 1:
5667 numchan = 2;
5668 break;
5669 case 2:
5670 numchan = 4;
5671 break;
5672 case 3:
5673 numchan = 8;
5674 break;
5675 case 4:
5676 numchan = 3;
5677 break;
5678 case 5:
5679 numchan = 6;
5680 break;
5681 case 6:
5682 numchan = 10;
5683 break;
5684 case 7:
5685 numchan = 12;
5686 break;
5687 case 8:
5688 numchan = 16;
5689 break;
5690 }
5691 rdev->mc.vram_width = numchan * chansize;
5692 /* Could aper size report 0 ? */
5693 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5694 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5695 /* size in MB on si */
13c5bfda
AD		 5696 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5697 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5698 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5699 si_vram_gtt_location(rdev, &rdev->mc);
5700 radeon_update_bandwidth_info(rdev);
5701
5702 return 0;
5703}
5704
5705/*
5706 * GART
5707 * VMID 0 is the physical GPU addresses as used by the kernel.
5708 * VMIDs 1-15 are used for userspace clients and are handled
5709 * by the radeon vm/hsa code.
5710 */
5711/**
5712 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5713 *
5714 * @rdev: radeon_device pointer
5715 *
5716 * Flush the TLB for the VMID 0 page table (CIK).
5717 */
5718void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5719{
5720 /* flush hdp cache */
5721 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5722
5723 /* bits 0-15 are the VM contexts0-15 */
5724 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5725}
5726
08dcc57f
BG		 5727static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5728{
5729 int i;
5730 uint32_t sh_mem_bases, sh_mem_config;
5731
5732 sh_mem_bases = 0x6000 | 0x6000 << 16;
5733 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5734 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5735
5736 mutex_lock(&rdev->srbm_mutex);
5737 for (i = 8; i < 16; i++) {
5738 cik_srbm_select(rdev, 0, 0, 0, i);
5739 /* CP and shaders */
5740 WREG32(SH_MEM_CONFIG, sh_mem_config);
5741 WREG32(SH_MEM_APE1_BASE, 1);
5742 WREG32(SH_MEM_APE1_LIMIT, 0);
5743 WREG32(SH_MEM_BASES, sh_mem_bases);
5744 }
5745 cik_srbm_select(rdev, 0, 0, 0, 0);
5746 mutex_unlock(&rdev->srbm_mutex);
5747}
5748
1c49165d
AD		 5749/**
5750 * cik_pcie_gart_enable - gart enable
5751 *
5752 * @rdev: radeon_device pointer
5753 *
5754 * This sets up the TLBs, programs the page tables for VMID0,
5755 * sets up the hw for VMIDs 1-15 which are allocated on
5756 * demand, and sets up the global locations for the LDS, GDS,
5757 * and GPUVM for FSA64 clients (CIK).
5758 * Returns 0 for success, errors for failure.
5759 */
5760static int cik_pcie_gart_enable(struct radeon_device *rdev)
5761{
5762 int r, i;
5763
5764 if (rdev->gart.robj == NULL) {
5765 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5766 return -EINVAL;
5767 }
5768 r = radeon_gart_table_vram_pin(rdev);
5769 if (r)
5770 return r;
1c49165d
AD		 5771 /* Setup TLB control */
5772 WREG32(MC_VM_MX_L1_TLB_CNTL,
5773 (0xA << 7) |
5774 ENABLE_L1_TLB |
ec3dbbcb 5775 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5776 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5777 ENABLE_ADVANCED_DRIVER_MODEL |
5778 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5779 /* Setup L2 cache */
5780 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5781 ENABLE_L2_FRAGMENT_PROCESSING |
5782 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5783 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5784 EFFECTIVE_L2_QUEUE_SIZE(7) |
5785 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5786 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5787 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5788 BANK_SELECT(4) |
5789 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5790 /* setup context0 */
5791 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5792 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5793 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5794 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5795 (u32)(rdev->dummy_page.addr >> 12));
5796 WREG32(VM_CONTEXT0_CNTL2, 0);
5797 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5798 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5799
5800 WREG32(0x15D4, 0);
5801 WREG32(0x15D8, 0);
5802 WREG32(0x15DC, 0);
5803
054e01d6 5804 /* restore context1-15 */
1c49165d
AD		 5805 /* set vm size, must be a multiple of 4 */
5806 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5807 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5808 for (i = 1; i < 16; i++) {
5809 if (i < 8)
5810 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5811 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5812 else
5813 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5814 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5815 }
5816
5817 /* enable context1-15 */
5818 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5819 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5820 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5821 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5822 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5823 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5824 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5825 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5826 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5827 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5828 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5829 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5830 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5831 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5832 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5833 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5834 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5835
1c49165d
AD		 5836 if (rdev->family == CHIP_KAVERI) {
5837 u32 tmp = RREG32(CHUB_CONTROL);
5838 tmp &= ~BYPASS_VM;
5839 WREG32(CHUB_CONTROL, tmp);
5840 }
5841
5842 /* XXX SH_MEM regs */
5843 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5844 mutex_lock(&rdev->srbm_mutex);
1c49165d 5845 for (i = 0; i < 16; i++) {
b556b12e 5846 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5847 /* CP and shaders */
1c49165d
AD		 5848 WREG32(SH_MEM_CONFIG, 0);
5849 WREG32(SH_MEM_APE1_BASE, 1);
5850 WREG32(SH_MEM_APE1_LIMIT, 0);
5851 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5852 /* SDMA GFX */
5853 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5854 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5855 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5856 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5857 /* XXX SDMA RLC - todo */
1c49165d 5858 }
b556b12e 5859 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5860 mutex_unlock(&rdev->srbm_mutex);
1c49165d 5861
08dcc57f
BG		 5862 cik_pcie_init_compute_vmid(rdev);
5863
1c49165d
AD		 5864 cik_pcie_gart_tlb_flush(rdev);
5865 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5866 (unsigned)(rdev->mc.gtt_size >> 20),
5867 (unsigned long long)rdev->gart.table_addr);
5868 rdev->gart.ready = true;
5869 return 0;
5870}
5871
5872/**
5873 * cik_pcie_gart_disable - gart disable
5874 *
5875 * @rdev: radeon_device pointer
5876 *
5877 * This disables all VM page table (CIK).
5878 */
5879static void cik_pcie_gart_disable(struct radeon_device *rdev)
5880{
054e01d6
CK		 5881 unsigned i;
5882
5883 for (i = 1; i < 16; ++i) {
5884 uint32_t reg;
5885 if (i < 8)
5886 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5887 else
5888 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5889 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5890 }
5891
1c49165d
AD		 5892 /* Disable all tables */
5893 WREG32(VM_CONTEXT0_CNTL, 0);
5894 WREG32(VM_CONTEXT1_CNTL, 0);
5895 /* Setup TLB control */
5896 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5897 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5898 /* Setup L2 cache */
5899 WREG32(VM_L2_CNTL,
5900 ENABLE_L2_FRAGMENT_PROCESSING |
5901 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5902 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5903 EFFECTIVE_L2_QUEUE_SIZE(7) |
5904 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5905 WREG32(VM_L2_CNTL2, 0);
5906 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5907 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5908 radeon_gart_table_vram_unpin(rdev);
5909}
5910
5911/**
5912 * cik_pcie_gart_fini - vm fini callback
5913 *
5914 * @rdev: radeon_device pointer
5915 *
5916 * Tears down the driver GART/VM setup (CIK).
5917 */
5918static void cik_pcie_gart_fini(struct radeon_device *rdev)
5919{
5920 cik_pcie_gart_disable(rdev);
5921 radeon_gart_table_vram_free(rdev);
5922 radeon_gart_fini(rdev);
5923}
5924
5925/* vm parser */
5926/**
5927 * cik_ib_parse - vm ib_parse callback
5928 *
5929 * @rdev: radeon_device pointer
5930 * @ib: indirect buffer pointer
5931 *
5932 * CIK uses hw IB checking so this is a nop (CIK).
5933 */
5934int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5935{
5936 return 0;
5937}
5938
5939/*
5940 * vm
5941 * VMID 0 is the physical GPU addresses as used by the kernel.
5942 * VMIDs 1-15 are used for userspace clients and are handled
5943 * by the radeon vm/hsa code.
5944 */
5945/**
5946 * cik_vm_init - cik vm init callback
5947 *
5948 * @rdev: radeon_device pointer
5949 *
5950 * Inits cik specific vm parameters (number of VMs, base of vram for
5951 * VMIDs 1-15) (CIK).
5952 * Returns 0 for success.
5953 */
5954int cik_vm_init(struct radeon_device *rdev)
5955{
62a7b7fb
OG		 5956 /*
5957 * number of VMs
5958 * VMID 0 is reserved for System
5959 * radeon graphics/compute will use VMIDs 1-7
5960 * amdkfd will use VMIDs 8-15
5961 */
5962 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 5963 /* base offset of vram pages */
5964 if (rdev->flags & RADEON_IS_IGP) {
5965 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5966 tmp <<= 22;
5967 rdev->vm_manager.vram_base_offset = tmp;
5968 } else
5969 rdev->vm_manager.vram_base_offset = 0;
5970
5971 return 0;
5972}
5973
5974/**
5975 * cik_vm_fini - cik vm fini callback
5976 *
5977 * @rdev: radeon_device pointer
5978 *
5979 * Tear down any asic specific VM setup (CIK).
5980 */
5981void cik_vm_fini(struct radeon_device *rdev)
5982{
5983}
5984
3ec7d11b
AD		 5985/**
5986 * cik_vm_decode_fault - print human readable fault info
5987 *
5988 * @rdev: radeon_device pointer
5989 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5990 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5991 *
5992 * Print human readable fault information (CIK).
5993 */
5994static void cik_vm_decode_fault(struct radeon_device *rdev,
5995 u32 status, u32 addr, u32 mc_client)
5996{
939c0d3c 5997 u32 mc_id;
3ec7d11b
AD		 5998 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5999 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 6000 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
6001 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 6002
939c0d3c
AD		 6003 if (rdev->family == CHIP_HAWAII)
6004 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6005 else
6006 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6007
328a50c7 6008 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 6009 protections, vmid, addr,
6010 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 6011 block, mc_client, mc_id);
3ec7d11b
AD		 6012}
6013
f96ab484
AD		 6014/**
6015 * cik_vm_flush - cik vm flush using the CP
6016 *
6017 * @rdev: radeon_device pointer
6018 *
6019 * Update the page table base and flush the VM TLB
6020 * using the CP (CIK).
6021 */
faffaf62
CK		 6022void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6023 unsigned vm_id, uint64_t pd_addr)
f96ab484 6024{
faffaf62 6025 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 6026
6027 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 6028 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 6029 WRITE_DATA_DST_SEL(0)));
faffaf62 6030 if (vm_id < 8) {
f96ab484 6031 radeon_ring_write(ring,
faffaf62 6032 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 6033 } else {
6034 radeon_ring_write(ring,
faffaf62 6035 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 6036 }
6037 radeon_ring_write(ring, 0);
faffaf62 6038 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 6039
6040 /* update SH_MEM_* regs */
6041 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6042 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6043 WRITE_DATA_DST_SEL(0)));
6044 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6045 radeon_ring_write(ring, 0);
faffaf62 6046 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 6047
6048 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 6049 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6050 WRITE_DATA_DST_SEL(0)));
6051 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6052 radeon_ring_write(ring, 0);
6053
6054 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6055 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6056 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6057 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 6058
f96ab484 6059 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6060 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6061 WRITE_DATA_DST_SEL(0)));
6062 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6063 radeon_ring_write(ring, 0);
6064 radeon_ring_write(ring, VMID(0));
6f2043ce 6065
f96ab484 6066 /* HDP flush */
faffaf62 6067 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 6068
6069 /* bits 0-15 are the VM contexts0-15 */
6070 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6071 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6072 WRITE_DATA_DST_SEL(0)));
6073 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6074 radeon_ring_write(ring, 0);
faffaf62 6075 radeon_ring_write(ring, 1 << vm_id);
f96ab484 6076
3a01fd36
AD		 6077 /* wait for the invalidate to complete */
6078 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6079 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6080 WAIT_REG_MEM_FUNCTION(0) | /* always */
6081 WAIT_REG_MEM_ENGINE(0))); /* me */
6082 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6083 radeon_ring_write(ring, 0);
6084 radeon_ring_write(ring, 0); /* ref */
6085 radeon_ring_write(ring, 0); /* mask */
6086 radeon_ring_write(ring, 0x20); /* poll interval */
6087
b07fdd38 6088 /* compute doesn't have PFP */
f1d2a26b 6089 if (usepfp) {
b07fdd38
AD		 6090 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6091 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6092 radeon_ring_write(ring, 0x0);
6093 }
cc066715 6094}
6f2043ce 6095
f6796cae
AD		 6096/*
6097 * RLC
6098 * The RLC is a multi-purpose microengine that handles a
6099 * variety of functions, the most important of which is
6100 * the interrupt controller.
6101 */
866d83de
AD		 6102static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6103 bool enable)
f6796cae 6104{
866d83de 6105 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6106
866d83de
AD		 6107 if (enable)
6108 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6109 else
6110 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6111 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6112}
f6796cae 6113
866d83de 6114static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6115{
cc066715 6116 u32 tmp;
6f2043ce 6117
866d83de
AD		 6118 tmp = RREG32(RLC_LB_CNTL);
6119 if (enable)
6120 tmp |= LOAD_BALANCE_ENABLE;
6121 else
6122 tmp &= ~LOAD_BALANCE_ENABLE;
6123 WREG32(RLC_LB_CNTL, tmp);
6124}
cc066715 6125
866d83de
AD		 6126static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6127{
6128 u32 i, j, k;
6129 u32 mask;
cc066715 6130
1c0a4625 6131 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6132 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6133 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6134 cik_select_se_sh(rdev, i, j);
6135 for (k = 0; k < rdev->usec_timeout; k++) {
6136 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6137 break;
6138 udelay(1);
6139 }
6140 }
6141 }
6142 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6143 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6144
f6796cae
AD		 6145 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6146 for (k = 0; k < rdev->usec_timeout; k++) {
6147 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6148 break;
6149 udelay(1);
6150 }
6151}
cc066715 6152
22c775ce
AD		 6153static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6154{
6155 u32 tmp;
cc066715 6156
22c775ce
AD		 6157 tmp = RREG32(RLC_CNTL);
6158 if (tmp != rlc)
6159 WREG32(RLC_CNTL, rlc);
6160}
cc066715 6161
22c775ce
AD		 6162static u32 cik_halt_rlc(struct radeon_device *rdev)
6163{
6164 u32 data, orig;
cc066715 6165
22c775ce 6166 orig = data = RREG32(RLC_CNTL);
cc066715 6167
22c775ce
AD		 6168 if (data & RLC_ENABLE) {
6169 u32 i;
cc066715 6170
22c775ce
AD		 6171 data &= ~RLC_ENABLE;
6172 WREG32(RLC_CNTL, data);
cc066715 6173
22c775ce
AD		 6174 for (i = 0; i < rdev->usec_timeout; i++) {
6175 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6176 break;
6177 udelay(1);
6178 }
cc066715 6179
22c775ce
AD		 6180 cik_wait_for_rlc_serdes(rdev);
6181 }
cc066715 6182
22c775ce
AD		 6183 return orig;
6184}
cc066715 6185
a412fce0
AD		 6186void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6187{
6188 u32 tmp, i, mask;
6189
6190 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6191 WREG32(RLC_GPR_REG2, tmp);
6192
6193 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6194 for (i = 0; i < rdev->usec_timeout; i++) {
6195 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6196 break;
6197 udelay(1);
6198 }
6199
6200 for (i = 0; i < rdev->usec_timeout; i++) {
6201 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6202 break;
6203 udelay(1);
6204 }
6205}
6206
6207void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6208{
6209 u32 tmp;
6210
6211 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6212 WREG32(RLC_GPR_REG2, tmp);
6213}
6214
866d83de
AD		 6215/**
6216 * cik_rlc_stop - stop the RLC ME
6217 *
6218 * @rdev: radeon_device pointer
6219 *
6220 * Halt the RLC ME (MicroEngine) (CIK).
6221 */
6222static void cik_rlc_stop(struct radeon_device *rdev)
6223{
22c775ce 6224 WREG32(RLC_CNTL, 0);
866d83de
AD		 6225
6226 cik_enable_gui_idle_interrupt(rdev, false);
6227
866d83de
AD		 6228 cik_wait_for_rlc_serdes(rdev);
6229}
6230
f6796cae
AD		 6231/**
6232 * cik_rlc_start - start the RLC ME
6233 *
6234 * @rdev: radeon_device pointer
6235 *
6236 * Unhalt the RLC ME (MicroEngine) (CIK).
6237 */
6238static void cik_rlc_start(struct radeon_device *rdev)
6239{
f6796cae 6240 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6241
866d83de 6242 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6243
f6796cae 6244 udelay(50);
6f2043ce
AD		 6245}
6246
6247/**
f6796cae 6248 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6249 *
6250 * @rdev: radeon_device pointer
6251 *
f6796cae
AD		 6252 * Initialize the RLC registers, load the ucode,
6253 * and start the RLC (CIK).
6254 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6255 */
f6796cae 6256static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6257{
22c775ce 6258 u32 i, size, tmp;
cc066715 6259
f6796cae
AD		 6260 if (!rdev->rlc_fw)
6261 return -EINVAL;
cc066715 6262
cc066715
AD		 6263 cik_rlc_stop(rdev);
6264
22c775ce
AD		 6265 /* disable CG */
6266 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6267 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6268
866d83de 6269 si_rlc_reset(rdev);
6f2043ce 6270
22c775ce 6271 cik_init_pg(rdev);
6f2043ce 6272
22c775ce 6273 cik_init_cg(rdev);
cc066715 6274
f6796cae
AD		 6275 WREG32(RLC_LB_CNTR_INIT, 0);
6276 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6277
1c0a4625 6278 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6279 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6280 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6281 WREG32(RLC_LB_PARAMS, 0x00600408);
6282 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 6283 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6284
f6796cae
AD		 6285 WREG32(RLC_MC_CNTL, 0);
6286 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6287
f2c6b0f4
AD		 6288 if (rdev->new_fw) {
6289 const struct rlc_firmware_header_v1_0 *hdr =
6290 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6291 const __le32 *fw_data = (const __le32 *)
6292 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6293
6294 radeon_ucode_print_rlc_hdr(&hdr->header);
6295
6296 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6297 WREG32(RLC_GPM_UCODE_ADDR, 0);
6298 for (i = 0; i < size; i++)
6299 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6300 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6301 } else {
6302 const __be32 *fw_data;
6303
6304 switch (rdev->family) {
6305 case CHIP_BONAIRE:
6306 case CHIP_HAWAII:
6307 default:
6308 size = BONAIRE_RLC_UCODE_SIZE;
6309 break;
6310 case CHIP_KAVERI:
6311 size = KV_RLC_UCODE_SIZE;
6312 break;
6313 case CHIP_KABINI:
6314 size = KB_RLC_UCODE_SIZE;
6315 break;
6316 case CHIP_MULLINS:
6317 size = ML_RLC_UCODE_SIZE;
6318 break;
6319 }
6320
6321 fw_data = (const __be32 *)rdev->rlc_fw->data;
6322 WREG32(RLC_GPM_UCODE_ADDR, 0);
6323 for (i = 0; i < size; i++)
6324 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6325 WREG32(RLC_GPM_UCODE_ADDR, 0);
6326 }
cc066715 6327
866d83de
AD		 6328 /* XXX - find out what chips support lbpw */
6329 cik_enable_lbpw(rdev, false);
cc066715 6330
22c775ce
AD		 6331 if (rdev->family == CHIP_BONAIRE)
6332 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6333
f6796cae 6334 cik_rlc_start(rdev);
cc066715 6335
f6796cae
AD		 6336 return 0;
6337}
cc066715 6338
22c775ce
AD		 6339static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6340{
6341 u32 data, orig, tmp, tmp2;
cc066715 6342
22c775ce 6343 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6344
473359bc 6345 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6346 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6347
22c775ce 6348 tmp = cik_halt_rlc(rdev);
cc066715 6349
1c0a4625 6350 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6351 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6352 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6353 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6354 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6355 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6356 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6357
22c775ce 6358 cik_update_rlc(rdev, tmp);
cc066715 6359
22c775ce
AD		 6360 data |= CGCG_EN | CGLS_EN;
6361 } else {
ddc76ff6 6362 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6363
22c775ce
AD		 6364 RREG32(CB_CGTT_SCLK_CTRL);
6365 RREG32(CB_CGTT_SCLK_CTRL);
6366 RREG32(CB_CGTT_SCLK_CTRL);
6367 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6368
22c775ce 6369 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6370 }
6f2043ce 6371
22c775ce
AD		 6372 if (orig != data)
6373 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6374
6f2043ce
AD		 6375}
6376
22c775ce 6377static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6378{
22c775ce
AD		 6379 u32 data, orig, tmp = 0;
6380
473359bc
AD		 6381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6382 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6383 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6384 orig = data = RREG32(CP_MEM_SLP_CNTL);
6385 data |= CP_MEM_LS_EN;
6386 if (orig != data)
6387 WREG32(CP_MEM_SLP_CNTL, data);
6388 }
6389 }
cc066715 6390
22c775ce 6391 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6392 data |= 0x00000001;
22c775ce
AD		 6393 data &= 0xfffffffd;
6394 if (orig != data)
6395 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6396
6397 tmp = cik_halt_rlc(rdev);
6398
1c0a4625 6399 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6400 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6401 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6402 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6403 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6404 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6405 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6406
6407 cik_update_rlc(rdev, tmp);
6408
473359bc
AD		 6409 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6410 orig = data = RREG32(CGTS_SM_CTRL_REG);
6411 data &= ~SM_MODE_MASK;
6412 data |= SM_MODE(0x2);
6413 data |= SM_MODE_ENABLE;
6414 data &= ~CGTS_OVERRIDE;
6415 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6416 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6417 data &= ~CGTS_LS_OVERRIDE;
6418 data &= ~ON_MONITOR_ADD_MASK;
6419 data |= ON_MONITOR_ADD_EN;
6420 data |= ON_MONITOR_ADD(0x96);
6421 if (orig != data)
6422 WREG32(CGTS_SM_CTRL_REG, data);
6423 }
22c775ce
AD		 6424 } else {
6425 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6426 data |= 0x00000003;
22c775ce
AD		 6427 if (orig != data)
6428 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6429
6430 data = RREG32(RLC_MEM_SLP_CNTL);
6431 if (data & RLC_MEM_LS_EN) {
6432 data &= ~RLC_MEM_LS_EN;
6433 WREG32(RLC_MEM_SLP_CNTL, data);
6434 }
6f2043ce 6435
22c775ce
AD		 6436 data = RREG32(CP_MEM_SLP_CNTL);
6437 if (data & CP_MEM_LS_EN) {
6438 data &= ~CP_MEM_LS_EN;
6439 WREG32(CP_MEM_SLP_CNTL, data);
6440 }
cc066715 6441
22c775ce
AD		 6442 orig = data = RREG32(CGTS_SM_CTRL_REG);
6443 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6444 if (orig != data)
6445 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6446
22c775ce 6447 tmp = cik_halt_rlc(rdev);
cc066715 6448
1c0a4625 6449 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6450 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6451 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6452 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6453 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6454 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6455 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6456
22c775ce 6457 cik_update_rlc(rdev, tmp);
cc066715 6458 }
6f2043ce 6459}
1c49165d 6460
22c775ce 6461static const u32 mc_cg_registers[] =
21a93e13 6462{
22c775ce
AD		 6463 MC_HUB_MISC_HUB_CG,
6464 MC_HUB_MISC_SIP_CG,
6465 MC_HUB_MISC_VM_CG,
6466 MC_XPB_CLK_GAT,
6467 ATC_MISC_CG,
6468 MC_CITF_MISC_WR_CG,
6469 MC_CITF_MISC_RD_CG,
6470 MC_CITF_MISC_VM_CG,
6471 VM_L2_CG,
6472};
21a93e13 6473
22c775ce
AD		 6474static void cik_enable_mc_ls(struct radeon_device *rdev,
6475 bool enable)
1c49165d 6476{
22c775ce
AD		 6477 int i;
6478 u32 orig, data;
1c49165d 6479
22c775ce
AD		 6480 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6481 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6482 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6483 data |= MC_LS_ENABLE;
6484 else
6485 data &= ~MC_LS_ENABLE;
6486 if (data != orig)
6487 WREG32(mc_cg_registers[i], data);
1c49165d 6488 }
22c775ce 6489}
1c49165d 6490
22c775ce
AD		 6491static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6492 bool enable)
6493{
6494 int i;
6495 u32 orig, data;
6496
6497 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6498 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6499 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6500 data |= MC_CG_ENABLE;
6501 else
6502 data &= ~MC_CG_ENABLE;
6503 if (data != orig)
6504 WREG32(mc_cg_registers[i], data);
1c49165d 6505 }
1c49165d
AD		 6506}
6507
22c775ce
AD		 6508static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6509 bool enable)
1c49165d 6510{
22c775ce 6511 u32 orig, data;
1c49165d 6512
473359bc 6513 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6514 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6515 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6516 } else {
22c775ce
AD		 6517 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6518 data |= 0xff000000;
6519 if (data != orig)
6520 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6521
22c775ce
AD		 6522 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6523 data |= 0xff000000;
6524 if (data != orig)
6525 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6526 }
1c49165d
AD		 6527}
6528
22c775ce
AD		 6529static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6530 bool enable)
1c49165d 6531{
22c775ce
AD		 6532 u32 orig, data;
6533
473359bc 6534 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6535 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6536 data |= 0x100;
6537 if (orig != data)
6538 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6539
6540 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6541 data |= 0x100;
6542 if (orig != data)
6543 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6544 } else {
6545 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6546 data &= ~0x100;
6547 if (orig != data)
6548 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6549
22c775ce
AD		 6550 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6551 data &= ~0x100;
6552 if (orig != data)
6553 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6554 }
1c49165d
AD		 6555}
6556
22c775ce
AD		 6557static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6558 bool enable)
1c49165d 6559{
22c775ce 6560 u32 orig, data;
1c49165d 6561
473359bc 6562 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6563 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6564 data = 0xfff;
6565 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6566
22c775ce
AD		 6567 orig = data = RREG32(UVD_CGC_CTRL);
6568 data |= DCM;
6569 if (orig != data)
6570 WREG32(UVD_CGC_CTRL, data);
6571 } else {
6572 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6573 data &= ~0xfff;
6574 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6575
22c775ce
AD		 6576 orig = data = RREG32(UVD_CGC_CTRL);
6577 data &= ~DCM;
6578 if (orig != data)
6579 WREG32(UVD_CGC_CTRL, data);
1c49165d 6580 }
22c775ce 6581}
1c49165d 6582
473359bc
AD		 6583static void cik_enable_bif_mgls(struct radeon_device *rdev,
6584 bool enable)
6585{
6586 u32 orig, data;
1c49165d 6587
473359bc 6588 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6589
473359bc
AD		 6590 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6591 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6592 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6593 else
6594 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6595 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6596
473359bc
AD		 6597 if (orig != data)
6598 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6599}
1c49165d 6600
22c775ce
AD		 6601static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6602 bool enable)
6603{
6604 u32 orig, data;
1c49165d 6605
22c775ce 6606 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6607
473359bc 6608 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6609 data &= ~CLOCK_GATING_DIS;
6610 else
6611 data |= CLOCK_GATING_DIS;
6612
6613 if (orig != data)
6614 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6615}
6616
22c775ce
AD		 6617static void cik_enable_hdp_ls(struct radeon_device *rdev,
6618 bool enable)
1c49165d 6619{
22c775ce
AD		 6620 u32 orig, data;
6621
6622 orig = data = RREG32(HDP_MEM_POWER_LS);
6623
473359bc 6624 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6625 data |= HDP_LS_ENABLE;
6626 else
6627 data &= ~HDP_LS_ENABLE;
6628
6629 if (orig != data)
6630 WREG32(HDP_MEM_POWER_LS, data);
6631}
6632
6633void cik_update_cg(struct radeon_device *rdev,
6634 u32 block, bool enable)
6635{
4214faf6 6636
22c775ce 6637 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6638 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6639 /* order matters! */
6640 if (enable) {
6641 cik_enable_mgcg(rdev, true);
6642 cik_enable_cgcg(rdev, true);
6643 } else {
6644 cik_enable_cgcg(rdev, false);
6645 cik_enable_mgcg(rdev, false);
6646 }
4214faf6 6647 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6648 }
6649
6650 if (block & RADEON_CG_BLOCK_MC) {
6651 if (!(rdev->flags & RADEON_IS_IGP)) {
6652 cik_enable_mc_mgcg(rdev, enable);
6653 cik_enable_mc_ls(rdev, enable);
6654 }
6655 }
6656
6657 if (block & RADEON_CG_BLOCK_SDMA) {
6658 cik_enable_sdma_mgcg(rdev, enable);
6659 cik_enable_sdma_mgls(rdev, enable);
6660 }
6661
473359bc
AD		 6662 if (block & RADEON_CG_BLOCK_BIF) {
6663 cik_enable_bif_mgls(rdev, enable);
6664 }
6665
22c775ce
AD		 6666 if (block & RADEON_CG_BLOCK_UVD) {
6667 if (rdev->has_uvd)
6668 cik_enable_uvd_mgcg(rdev, enable);
6669 }
6670
6671 if (block & RADEON_CG_BLOCK_HDP) {
6672 cik_enable_hdp_mgcg(rdev, enable);
6673 cik_enable_hdp_ls(rdev, enable);
6674 }
a1d6f97c
AD		 6675
6676 if (block & RADEON_CG_BLOCK_VCE) {
6677 vce_v2_0_enable_mgcg(rdev, enable);
6678 }
1c49165d
AD		 6679}
6680
22c775ce 6681static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6682{
22c775ce 6683
ddc76ff6 6684 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6685
6686 if (rdev->has_uvd)
6687 si_init_uvd_internal_cg(rdev);
6688
6689 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6690 RADEON_CG_BLOCK_SDMA |
473359bc 6691 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6692 RADEON_CG_BLOCK_UVD |
6693 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6694}
6695
473359bc 6696static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6697{
473359bc
AD		 6698 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6699 RADEON_CG_BLOCK_SDMA |
6700 RADEON_CG_BLOCK_BIF |
6701 RADEON_CG_BLOCK_UVD |
6702 RADEON_CG_BLOCK_HDP), false);
6703
6704 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6705}
6706
22c775ce
AD		 6707static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6708 bool enable)
1c49165d 6709{
22c775ce 6710 u32 data, orig;
1c49165d 6711
22c775ce 6712 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6713 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6714 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6715 else
6716 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6717 if (orig != data)
6718 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6719}
6720
22c775ce
AD		 6721static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6722 bool enable)
1c49165d 6723{
22c775ce
AD		 6724 u32 data, orig;
6725
6726 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6727 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6728 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6729 else
6730 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6731 if (orig != data)
6732 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6733}
6734
22c775ce 6735static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6736{
22c775ce 6737 u32 data, orig;
3ec7d11b 6738
22c775ce 6739 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6740 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6741 data &= ~DISABLE_CP_PG;
6742 else
6743 data |= DISABLE_CP_PG;
6744 if (orig != data)
6745 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6746}
6747
22c775ce 6748static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6749{
22c775ce 6750 u32 data, orig;
f96ab484 6751
22c775ce 6752 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6753 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6754 data &= ~DISABLE_GDS_PG;
6755 else
6756 data |= DISABLE_GDS_PG;
6757 if (orig != data)
6758 WREG32(RLC_PG_CNTL, data);
6759}
6760
6761#define CP_ME_TABLE_SIZE 96
6762#define CP_ME_TABLE_OFFSET 2048
6763#define CP_MEC_TABLE_OFFSET 4096
6764
6765void cik_init_cp_pg_table(struct radeon_device *rdev)
6766{
22c775ce
AD		 6767 volatile u32 *dst_ptr;
6768 int me, i, max_me = 4;
6769 u32 bo_offset = 0;
f2c6b0f4 6770 u32 table_offset, table_size;
22c775ce
AD		 6771
6772 if (rdev->family == CHIP_KAVERI)
6773 max_me = 5;
6774
6775 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6776 return;
6777
22c775ce
AD		 6778 /* write the cp table buffer */
6779 dst_ptr = rdev->rlc.cp_table_ptr;
6780 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6781 if (rdev->new_fw) {
6782 const __le32 *fw_data;
6783 const struct gfx_firmware_header_v1_0 *hdr;
6784
6785 if (me == 0) {
6786 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6787 fw_data = (const __le32 *)
6788 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6789 table_offset = le32_to_cpu(hdr->jt_offset);
6790 table_size = le32_to_cpu(hdr->jt_size);
6791 } else if (me == 1) {
6792 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6793 fw_data = (const __le32 *)
6794 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6795 table_offset = le32_to_cpu(hdr->jt_offset);
6796 table_size = le32_to_cpu(hdr->jt_size);
6797 } else if (me == 2) {
6798 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6799 fw_data = (const __le32 *)
6800 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6801 table_offset = le32_to_cpu(hdr->jt_offset);
6802 table_size = le32_to_cpu(hdr->jt_size);
6803 } else if (me == 3) {
6804 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6805 fw_data = (const __le32 *)
6806 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6807 table_offset = le32_to_cpu(hdr->jt_offset);
6808 table_size = le32_to_cpu(hdr->jt_size);
6809 } else {
6810 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6811 fw_data = (const __le32 *)
6812 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6813 table_offset = le32_to_cpu(hdr->jt_offset);
6814 table_size = le32_to_cpu(hdr->jt_size);
6815 }
6816
6817 for (i = 0; i < table_size; i ++) {
6818 dst_ptr[bo_offset + i] =
6819 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6820 }
6821 bo_offset += table_size;
22c775ce 6822 } else {
f2c6b0f4
AD		 6823 const __be32 *fw_data;
6824 table_size = CP_ME_TABLE_SIZE;
6825
6826 if (me == 0) {
6827 fw_data = (const __be32 *)rdev->ce_fw->data;
6828 table_offset = CP_ME_TABLE_OFFSET;
6829 } else if (me == 1) {
6830 fw_data = (const __be32 *)rdev->pfp_fw->data;
6831 table_offset = CP_ME_TABLE_OFFSET;
6832 } else if (me == 2) {
6833 fw_data = (const __be32 *)rdev->me_fw->data;
6834 table_offset = CP_ME_TABLE_OFFSET;
6835 } else {
6836 fw_data = (const __be32 *)rdev->mec_fw->data;
6837 table_offset = CP_MEC_TABLE_OFFSET;
6838 }
22c775ce 6839
f2c6b0f4
AD		 6840 for (i = 0; i < table_size; i ++) {
6841 dst_ptr[bo_offset + i] =
6842 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6843 }
6844 bo_offset += table_size;
22c775ce 6845 }
f96ab484 6846 }
22c775ce 6847}
f96ab484 6848
22c775ce
AD		 6849static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6850 bool enable)
6851{
6852 u32 data, orig;
6853
2b19d17f 6854 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6855 orig = data = RREG32(RLC_PG_CNTL);
6856 data |= GFX_PG_ENABLE;
6857 if (orig != data)
6858 WREG32(RLC_PG_CNTL, data);
6859
6860 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6861 data |= AUTO_PG_EN;
6862 if (orig != data)
6863 WREG32(RLC_AUTO_PG_CTRL, data);
6864 } else {
6865 orig = data = RREG32(RLC_PG_CNTL);
6866 data &= ~GFX_PG_ENABLE;
6867 if (orig != data)
6868 WREG32(RLC_PG_CNTL, data);
f96ab484 6869
22c775ce
AD		 6870 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6871 data &= ~AUTO_PG_EN;
6872 if (orig != data)
6873 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6874
22c775ce
AD		 6875 data = RREG32(DB_RENDER_CONTROL);
6876 }
6877}
f96ab484 6878
22c775ce
AD		 6879static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6880{
6881 u32 mask = 0, tmp, tmp1;
6882 int i;
f96ab484 6883
1c0a4625 6884 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6885 cik_select_se_sh(rdev, se, sh);
6886 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6887 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6888 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6889 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6890
22c775ce 6891 tmp &= 0xffff0000;
f96ab484 6892
22c775ce
AD		 6893 tmp |= tmp1;
6894 tmp >>= 16;
6895
6896 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6897 mask <<= 1;
6898 mask |= 1;
b07fdd38 6899 }
22c775ce
AD		 6900
6901 return (~tmp) & mask;
f96ab484
AD		 6902}
6903
22c775ce 6904static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6905{
22c775ce
AD		 6906 u32 i, j, k, active_cu_number = 0;
6907 u32 mask, counter, cu_bitmap;
6908 u32 tmp = 0;
d0e092d9 6909
22c775ce
AD		 6910 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6911 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6912 mask = 1;
6913 cu_bitmap = 0;
6914 counter = 0;
6915 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6916 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6917 if (counter < 2)
6918 cu_bitmap |= mask;
6919 counter ++;
d0e092d9 6920 }
22c775ce 6921 mask <<= 1;
d0e092d9 6922 }
d0e092d9 6923
22c775ce
AD		 6924 active_cu_number += counter;
6925 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6926 }
d0e092d9 6927 }
22c775ce
AD		 6928
6929 WREG32(RLC_PG_AO_CU_MASK, tmp);
6930
6931 tmp = RREG32(RLC_MAX_PG_CU);
6932 tmp &= ~MAX_PU_CU_MASK;
6933 tmp |= MAX_PU_CU(active_cu_number);
6934 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6935}
6936
22c775ce
AD		 6937static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6938 bool enable)
605de6b9 6939{
22c775ce 6940 u32 data, orig;
605de6b9 6941
22c775ce 6942 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6943 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6944 data |= STATIC_PER_CU_PG_ENABLE;
6945 else
6946 data &= ~STATIC_PER_CU_PG_ENABLE;
6947 if (orig != data)
6948 WREG32(RLC_PG_CNTL, data);
6949}
6950
6951static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6952 bool enable)
6953{
6954 u32 data, orig;
605de6b9 6955
22c775ce 6956 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6957 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6958 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6959 else
22c775ce
AD		 6960 data &= ~DYN_PER_CU_PG_ENABLE;
6961 if (orig != data)
6962 WREG32(RLC_PG_CNTL, data);
6963}
605de6b9 6964
22c775ce
AD		 6965#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6966#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6967
6968static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6969{
6970 u32 data, orig;
6971 u32 i;
6972
6973 if (rdev->rlc.cs_data) {
6974 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6975 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6976 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6977 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6978 } else {
22c775ce
AD		 6979 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6980 for (i = 0; i < 3; i++)
6981 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6982 }
6983 if (rdev->rlc.reg_list) {
6984 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6985 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6986 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6987 }
605de6b9 6988
22c775ce
AD		 6989 orig = data = RREG32(RLC_PG_CNTL);
6990 data |= GFX_PG_SRC;
6991 if (orig != data)
6992 WREG32(RLC_PG_CNTL, data);
605de6b9 6993
22c775ce
AD		 6994 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6995 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6996
22c775ce
AD		 6997 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6998 data &= ~IDLE_POLL_COUNT_MASK;
6999 data |= IDLE_POLL_COUNT(0x60);
7000 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 7001
22c775ce
AD		 7002 data = 0x10101010;
7003 WREG32(RLC_PG_DELAY, data);
605de6b9 7004
22c775ce
AD		 7005 data = RREG32(RLC_PG_DELAY_2);
7006 data &= ~0xff;
7007 data |= 0x3;
7008 WREG32(RLC_PG_DELAY_2, data);
605de6b9 7009
22c775ce
AD		 7010 data = RREG32(RLC_AUTO_PG_CTRL);
7011 data &= ~GRBM_REG_SGIT_MASK;
7012 data |= GRBM_REG_SGIT(0x700);
7013 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 7014
605de6b9
AD		 7015}
7016
22c775ce 7017static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 7018{
473359bc
AD		 7019 cik_enable_gfx_cgpg(rdev, enable);
7020 cik_enable_gfx_static_mgpg(rdev, enable);
7021 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 7022}
f6796cae 7023
a0f38609
AD		 7024u32 cik_get_csb_size(struct radeon_device *rdev)
7025{
7026 u32 count = 0;
7027 const struct cs_section_def *sect = NULL;
7028 const struct cs_extent_def *ext = NULL;
f6796cae 7029
a0f38609
AD		 7030 if (rdev->rlc.cs_data == NULL)
7031 return 0;
f6796cae 7032
a0f38609
AD		 7033 /* begin clear state */
7034 count += 2;
7035 /* context control state */
7036 count += 3;
7037
7038 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7039 for (ext = sect->section; ext->extent != NULL; ++ext) {
7040 if (sect->id == SECT_CONTEXT)
7041 count += 2 + ext->reg_count;
7042 else
7043 return 0;
f6796cae
AD		 7044 }
7045 }
a0f38609
AD		 7046 /* pa_sc_raster_config/pa_sc_raster_config1 */
7047 count += 4;
7048 /* end clear state */
7049 count += 2;
7050 /* clear state */
7051 count += 2;
f6796cae 7052
a0f38609 7053 return count;
f6796cae
AD		 7054}
7055
a0f38609 7056void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 7057{
a0f38609
AD		 7058 u32 count = 0, i;
7059 const struct cs_section_def *sect = NULL;
7060 const struct cs_extent_def *ext = NULL;
f6796cae 7061
a0f38609
AD		 7062 if (rdev->rlc.cs_data == NULL)
7063 return;
7064 if (buffer == NULL)
7065 return;
f6796cae 7066
6ba81e53
AD		 7067 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7068 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 7069
6ba81e53
AD		 7070 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7071 buffer[count++] = cpu_to_le32(0x80000000);
7072 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 7073
7074 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7075 for (ext = sect->section; ext->extent != NULL; ++ext) {
7076 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 7077 buffer[count++] =
7078 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7079 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 7080 for (i = 0; i < ext->reg_count; i++)
6ba81e53 7081 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 7082 } else {
7083 return;
7084 }
7085 }
7086 }
f6796cae 7087
6ba81e53
AD		 7088 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7089 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 7090 switch (rdev->family) {
7091 case CHIP_BONAIRE:
6ba81e53
AD		 7092 buffer[count++] = cpu_to_le32(0x16000012);
7093 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7094 break;
7095 case CHIP_KAVERI:
6ba81e53
AD		 7096 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7097 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7098 break;
7099 case CHIP_KABINI:
f73a9e83 7100 case CHIP_MULLINS:
6ba81e53
AD		 7101 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7102 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7103 break;
bbfe90bd 7104 case CHIP_HAWAII:
a8947f57
AD		 7105 buffer[count++] = cpu_to_le32(0x3a00161a);
7106 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7107 break;
a0f38609 7108 default:
6ba81e53
AD		 7109 buffer[count++] = cpu_to_le32(0x00000000);
7110 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7111 break;
7112 }
7113
6ba81e53
AD		 7114 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7115 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7116
6ba81e53
AD		 7117 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7118 buffer[count++] = cpu_to_le32(0);
a0f38609 7119}
f6796cae 7120
473359bc 7121static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7122{
473359bc 7123 if (rdev->pg_flags) {
22c775ce
AD		 7124 cik_enable_sck_slowdown_on_pu(rdev, true);
7125 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7126 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7127 cik_init_gfx_cgpg(rdev);
7128 cik_enable_cp_pg(rdev, true);
7129 cik_enable_gds_pg(rdev, true);
7130 }
22c775ce
AD		 7131 cik_init_ao_cu_mask(rdev);
7132 cik_update_gfx_pg(rdev, true);
7133 }
7134}
f6796cae 7135
473359bc
AD		 7136static void cik_fini_pg(struct radeon_device *rdev)
7137{
7138 if (rdev->pg_flags) {
7139 cik_update_gfx_pg(rdev, false);
2b19d17f 7140 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7141 cik_enable_cp_pg(rdev, false);
7142 cik_enable_gds_pg(rdev, false);
7143 }
7144 }
f6796cae 7145}
a59781bb
AD		 7146
7147/*
7148 * Interrupts
7149 * Starting with r6xx, interrupts are handled via a ring buffer.
7150 * Ring buffers are areas of GPU accessible memory that the GPU
7151 * writes interrupt vectors into and the host reads vectors out of.
7152 * There is a rptr (read pointer) that determines where the
7153 * host is currently reading, and a wptr (write pointer)
7154 * which determines where the GPU has written. When the
7155 * pointers are equal, the ring is idle. When the GPU
7156 * writes vectors to the ring buffer, it increments the
7157 * wptr. When there is an interrupt, the host then starts
7158 * fetching commands and processing them until the pointers are
7159 * equal again at which point it updates the rptr.
7160 */
7161
7162/**
7163 * cik_enable_interrupts - Enable the interrupt ring buffer
7164 *
7165 * @rdev: radeon_device pointer
7166 *
7167 * Enable the interrupt ring buffer (CIK).
7168 */
7169static void cik_enable_interrupts(struct radeon_device *rdev)
7170{
7171 u32 ih_cntl = RREG32(IH_CNTL);
7172 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7173
7174 ih_cntl |= ENABLE_INTR;
7175 ih_rb_cntl |= IH_RB_ENABLE;
7176 WREG32(IH_CNTL, ih_cntl);
7177 WREG32(IH_RB_CNTL, ih_rb_cntl);
7178 rdev->ih.enabled = true;
7179}
7180
7181/**
7182 * cik_disable_interrupts - Disable the interrupt ring buffer
7183 *
7184 * @rdev: radeon_device pointer
7185 *
7186 * Disable the interrupt ring buffer (CIK).
7187 */
7188static void cik_disable_interrupts(struct radeon_device *rdev)
7189{
7190 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7191 u32 ih_cntl = RREG32(IH_CNTL);
7192
7193 ih_rb_cntl &= ~IH_RB_ENABLE;
7194 ih_cntl &= ~ENABLE_INTR;
7195 WREG32(IH_RB_CNTL, ih_rb_cntl);
7196 WREG32(IH_CNTL, ih_cntl);
7197 /* set rptr, wptr to 0 */
7198 WREG32(IH_RB_RPTR, 0);
7199 WREG32(IH_RB_WPTR, 0);
7200 rdev->ih.enabled = false;
7201 rdev->ih.rptr = 0;
7202}
7203
7204/**
7205 * cik_disable_interrupt_state - Disable all interrupt sources
7206 *
7207 * @rdev: radeon_device pointer
7208 *
7209 * Clear all interrupt enable bits used by the driver (CIK).
7210 */
7211static void cik_disable_interrupt_state(struct radeon_device *rdev)
7212{
7213 u32 tmp;
7214
7215 /* gfx ring */
4214faf6
AD		 7216 tmp = RREG32(CP_INT_CNTL_RING0) &
7217 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7218 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7219 /* sdma */
7220 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7221 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7222 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7223 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7224 /* compute queues */
7225 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7226 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7227 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7228 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7229 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7230 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7231 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7232 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7233 /* grbm */
7234 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 7235 /* SRBM */
7236 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 7237 /* vline/vblank, etc. */
7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7239 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7240 if (rdev->num_crtc >= 4) {
7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7243 }
7244 if (rdev->num_crtc >= 6) {
7245 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7246 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7247 }
f5d636d2
CK		 7248 /* pflip */
7249 if (rdev->num_crtc >= 2) {
7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7251 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7252 }
7253 if (rdev->num_crtc >= 4) {
7254 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7255 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7256 }
7257 if (rdev->num_crtc >= 6) {
7258 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7259 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7260 }
a59781bb
AD		 7261
7262 /* dac hotplug */
7263 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7264
7265 /* digital hotplug */
7266 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7267 WREG32(DC_HPD1_INT_CONTROL, tmp);
7268 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7269 WREG32(DC_HPD2_INT_CONTROL, tmp);
7270 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7271 WREG32(DC_HPD3_INT_CONTROL, tmp);
7272 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7273 WREG32(DC_HPD4_INT_CONTROL, tmp);
7274 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7275 WREG32(DC_HPD5_INT_CONTROL, tmp);
7276 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7277 WREG32(DC_HPD6_INT_CONTROL, tmp);
7278
7279}
7280
7281/**
7282 * cik_irq_init - init and enable the interrupt ring
7283 *
7284 * @rdev: radeon_device pointer
7285 *
7286 * Allocate a ring buffer for the interrupt controller,
7287 * enable the RLC, disable interrupts, enable the IH
7288 * ring buffer and enable it (CIK).
7289 * Called at device load and reume.
7290 * Returns 0 for success, errors for failure.
7291 */
7292static int cik_irq_init(struct radeon_device *rdev)
7293{
7294 int ret = 0;
7295 int rb_bufsz;
7296 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7297
7298 /* allocate ring */
7299 ret = r600_ih_ring_alloc(rdev);
7300 if (ret)
7301 return ret;
7302
7303 /* disable irqs */
7304 cik_disable_interrupts(rdev);
7305
7306 /* init rlc */
7307 ret = cik_rlc_resume(rdev);
7308 if (ret) {
7309 r600_ih_ring_fini(rdev);
7310 return ret;
7311 }
7312
7313 /* setup interrupt control */
7314 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7315 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7316 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7317 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7318 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7319 */
7320 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7321 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7322 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7323 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7324
7325 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7326 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7327
7328 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7329 IH_WPTR_OVERFLOW_CLEAR |
7330 (rb_bufsz << 1));
7331
7332 if (rdev->wb.enabled)
7333 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7334
7335 /* set the writeback address whether it's enabled or not */
7336 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7337 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7338
7339 WREG32(IH_RB_CNTL, ih_rb_cntl);
7340
7341 /* set rptr, wptr to 0 */
7342 WREG32(IH_RB_RPTR, 0);
7343 WREG32(IH_RB_WPTR, 0);
7344
7345 /* Default settings for IH_CNTL (disabled at first) */
7346 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7347 /* RPTR_REARM only works if msi's are enabled */
7348 if (rdev->msi_enabled)
7349 ih_cntl |= RPTR_REARM;
7350 WREG32(IH_CNTL, ih_cntl);
7351
7352 /* force the active interrupt state to all disabled */
7353 cik_disable_interrupt_state(rdev);
7354
7355 pci_set_master(rdev->pdev);
7356
7357 /* enable irqs */
7358 cik_enable_interrupts(rdev);
7359
7360 return ret;
7361}
7362
7363/**
7364 * cik_irq_set - enable/disable interrupt sources
7365 *
7366 * @rdev: radeon_device pointer
7367 *
7368 * Enable interrupt sources on the GPU (vblanks, hpd,
7369 * etc.) (CIK).
7370 * Returns 0 for success, errors for failure.
7371 */
7372int cik_irq_set(struct radeon_device *rdev)
7373{
4214faf6 7374 u32 cp_int_cntl;
28b57b85 7375 u32 cp_m1p0;
a59781bb
AD		 7376 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7377 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7378 u32 grbm_int_cntl = 0;
21a93e13 7379 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7380
7381 if (!rdev->irq.installed) {
7382 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7383 return -EINVAL;
7384 }
7385 /* don't enable anything if the ih is disabled */
7386 if (!rdev->ih.enabled) {
7387 cik_disable_interrupts(rdev);
7388 /* force the active interrupt state to all disabled */
7389 cik_disable_interrupt_state(rdev);
7390 return 0;
7391 }
7392
4214faf6
AD		 7393 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7394 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7395 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7396
a59781bb
AD		 7397 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7398 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7399 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7400 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7401 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7402 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7403
21a93e13
AD		 7404 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7405 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7406
2b0781a6 7407 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7408
a59781bb
AD		 7409 /* enable CP interrupts on all rings */
7410 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7411 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7412 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7413 }
2b0781a6
AD		 7414 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7415 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7416 DRM_DEBUG("si_irq_set: sw int cp1\n");
7417 if (ring->me == 1) {
7418 switch (ring->pipe) {
7419 case 0:
7420 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7421 break;
2b0781a6
AD		 7422 default:
7423 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7424 break;
7425 }
7426 } else {
7427 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7428 }
7429 }
7430 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7431 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7432 DRM_DEBUG("si_irq_set: sw int cp2\n");
7433 if (ring->me == 1) {
7434 switch (ring->pipe) {
7435 case 0:
7436 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7437 break;
2b0781a6
AD		 7438 default:
7439 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7440 break;
7441 }
7442 } else {
7443 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7444 }
7445 }
a59781bb 7446
21a93e13
AD		 7447 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7448 DRM_DEBUG("cik_irq_set: sw int dma\n");
7449 dma_cntl |= TRAP_ENABLE;
7450 }
7451
7452 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7453 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7454 dma_cntl1 |= TRAP_ENABLE;
7455 }
7456
a59781bb
AD		 7457 if (rdev->irq.crtc_vblank_int[0] ||
7458 atomic_read(&rdev->irq.pflip[0])) {
7459 DRM_DEBUG("cik_irq_set: vblank 0\n");
7460 crtc1 |= VBLANK_INTERRUPT_MASK;
7461 }
7462 if (rdev->irq.crtc_vblank_int[1] ||
7463 atomic_read(&rdev->irq.pflip[1])) {
7464 DRM_DEBUG("cik_irq_set: vblank 1\n");
7465 crtc2 |= VBLANK_INTERRUPT_MASK;
7466 }
7467 if (rdev->irq.crtc_vblank_int[2] ||
7468 atomic_read(&rdev->irq.pflip[2])) {
7469 DRM_DEBUG("cik_irq_set: vblank 2\n");
7470 crtc3 |= VBLANK_INTERRUPT_MASK;
7471 }
7472 if (rdev->irq.crtc_vblank_int[3] ||
7473 atomic_read(&rdev->irq.pflip[3])) {
7474 DRM_DEBUG("cik_irq_set: vblank 3\n");
7475 crtc4 |= VBLANK_INTERRUPT_MASK;
7476 }
7477 if (rdev->irq.crtc_vblank_int[4] ||
7478 atomic_read(&rdev->irq.pflip[4])) {
7479 DRM_DEBUG("cik_irq_set: vblank 4\n");
7480 crtc5 |= VBLANK_INTERRUPT_MASK;
7481 }
7482 if (rdev->irq.crtc_vblank_int[5] ||
7483 atomic_read(&rdev->irq.pflip[5])) {
7484 DRM_DEBUG("cik_irq_set: vblank 5\n");
7485 crtc6 |= VBLANK_INTERRUPT_MASK;
7486 }
7487 if (rdev->irq.hpd[0]) {
7488 DRM_DEBUG("cik_irq_set: hpd 1\n");
7489 hpd1 |= DC_HPDx_INT_EN;
7490 }
7491 if (rdev->irq.hpd[1]) {
7492 DRM_DEBUG("cik_irq_set: hpd 2\n");
7493 hpd2 |= DC_HPDx_INT_EN;
7494 }
7495 if (rdev->irq.hpd[2]) {
7496 DRM_DEBUG("cik_irq_set: hpd 3\n");
7497 hpd3 |= DC_HPDx_INT_EN;
7498 }
7499 if (rdev->irq.hpd[3]) {
7500 DRM_DEBUG("cik_irq_set: hpd 4\n");
7501 hpd4 |= DC_HPDx_INT_EN;
7502 }
7503 if (rdev->irq.hpd[4]) {
7504 DRM_DEBUG("cik_irq_set: hpd 5\n");
7505 hpd5 |= DC_HPDx_INT_EN;
7506 }
7507 if (rdev->irq.hpd[5]) {
7508 DRM_DEBUG("cik_irq_set: hpd 6\n");
7509 hpd6 |= DC_HPDx_INT_EN;
7510 }
7511
7512 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7513
21a93e13
AD		 7514 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7515 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7516
2b0781a6 7517 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7518
a59781bb
AD		 7519 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7520
7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7523 if (rdev->num_crtc >= 4) {
7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7526 }
7527 if (rdev->num_crtc >= 6) {
7528 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7529 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7530 }
7531
f5d636d2
CK		 7532 if (rdev->num_crtc >= 2) {
7533 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7534 GRPH_PFLIP_INT_MASK);
7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7536 GRPH_PFLIP_INT_MASK);
7537 }
7538 if (rdev->num_crtc >= 4) {
7539 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7540 GRPH_PFLIP_INT_MASK);
7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7542 GRPH_PFLIP_INT_MASK);
7543 }
7544 if (rdev->num_crtc >= 6) {
7545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7546 GRPH_PFLIP_INT_MASK);
7547 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7548 GRPH_PFLIP_INT_MASK);
7549 }
7550
a59781bb
AD		 7551 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7552 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7553 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7554 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7555 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7556 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7557
cffefd9b
AD		 7558 /* posting read */
7559 RREG32(SRBM_STATUS);
7560
a59781bb
AD		 7561 return 0;
7562}
7563
7564/**
7565 * cik_irq_ack - ack interrupt sources
7566 *
7567 * @rdev: radeon_device pointer
7568 *
7569 * Ack interrupt sources on the GPU (vblanks, hpd,
7570 * etc.) (CIK). Certain interrupts sources are sw
7571 * generated and do not require an explicit ack.
7572 */
7573static inline void cik_irq_ack(struct radeon_device *rdev)
7574{
7575 u32 tmp;
7576
7577 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7578 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7579 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7580 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7581 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7582 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7583 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7584
f5d636d2
CK		 7585 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7586 EVERGREEN_CRTC0_REGISTER_OFFSET);
7587 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7588 EVERGREEN_CRTC1_REGISTER_OFFSET);
7589 if (rdev->num_crtc >= 4) {
7590 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7591 EVERGREEN_CRTC2_REGISTER_OFFSET);
7592 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7593 EVERGREEN_CRTC3_REGISTER_OFFSET);
7594 }
7595 if (rdev->num_crtc >= 6) {
7596 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7597 EVERGREEN_CRTC4_REGISTER_OFFSET);
7598 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7599 EVERGREEN_CRTC5_REGISTER_OFFSET);
7600 }
7601
7602 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7604 GRPH_PFLIP_INT_CLEAR);
7605 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7606 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7607 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7608 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7610 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7612 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7613 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7614 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7615 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7616
7617 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7618 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7623 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7624 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7625 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7627 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7628 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7629 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7630 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7631 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7632 }
7633
7634 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7635 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7640 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7641 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7642 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7643 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7644 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7645 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7646 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7647 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7648 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7649 }
7650
7651 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7652 tmp = RREG32(DC_HPD1_INT_CONTROL);
7653 tmp |= DC_HPDx_INT_ACK;
7654 WREG32(DC_HPD1_INT_CONTROL, tmp);
7655 }
7656 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7657 tmp = RREG32(DC_HPD2_INT_CONTROL);
7658 tmp |= DC_HPDx_INT_ACK;
7659 WREG32(DC_HPD2_INT_CONTROL, tmp);
7660 }
7661 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7662 tmp = RREG32(DC_HPD3_INT_CONTROL);
7663 tmp |= DC_HPDx_INT_ACK;
7664 WREG32(DC_HPD3_INT_CONTROL, tmp);
7665 }
7666 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7667 tmp = RREG32(DC_HPD4_INT_CONTROL);
7668 tmp |= DC_HPDx_INT_ACK;
7669 WREG32(DC_HPD4_INT_CONTROL, tmp);
7670 }
7671 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7672 tmp = RREG32(DC_HPD5_INT_CONTROL);
7673 tmp |= DC_HPDx_INT_ACK;
7674 WREG32(DC_HPD5_INT_CONTROL, tmp);
7675 }
7676 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7677 tmp = RREG32(DC_HPD5_INT_CONTROL);
7678 tmp |= DC_HPDx_INT_ACK;
7679 WREG32(DC_HPD6_INT_CONTROL, tmp);
7680 }
7681}
7682
7683/**
7684 * cik_irq_disable - disable interrupts
7685 *
7686 * @rdev: radeon_device pointer
7687 *
7688 * Disable interrupts on the hw (CIK).
7689 */
7690static void cik_irq_disable(struct radeon_device *rdev)
7691{
7692 cik_disable_interrupts(rdev);
7693 /* Wait and acknowledge irq */
7694 mdelay(1);
7695 cik_irq_ack(rdev);
7696 cik_disable_interrupt_state(rdev);
7697}
7698
7699/**
7700 * cik_irq_disable - disable interrupts for suspend
7701 *
7702 * @rdev: radeon_device pointer
7703 *
7704 * Disable interrupts and stop the RLC (CIK).
7705 * Used for suspend.
7706 */
7707static void cik_irq_suspend(struct radeon_device *rdev)
7708{
7709 cik_irq_disable(rdev);
7710 cik_rlc_stop(rdev);
7711}
7712
7713/**
7714 * cik_irq_fini - tear down interrupt support
7715 *
7716 * @rdev: radeon_device pointer
7717 *
7718 * Disable interrupts on the hw and free the IH ring
7719 * buffer (CIK).
7720 * Used for driver unload.
7721 */
7722static void cik_irq_fini(struct radeon_device *rdev)
7723{
7724 cik_irq_suspend(rdev);
7725 r600_ih_ring_fini(rdev);
7726}
7727
7728/**
7729 * cik_get_ih_wptr - get the IH ring buffer wptr
7730 *
7731 * @rdev: radeon_device pointer
7732 *
7733 * Get the IH ring buffer wptr from either the register
7734 * or the writeback memory buffer (CIK). Also check for
7735 * ring buffer overflow and deal with it.
7736 * Used by cik_irq_process().
7737 * Returns the value of the wptr.
7738 */
7739static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7740{
7741 u32 wptr, tmp;
7742
7743 if (rdev->wb.enabled)
7744 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7745 else
7746 wptr = RREG32(IH_RB_WPTR);
7747
7748 if (wptr & RB_OVERFLOW) {
11bab0ae 7749 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7750 /* When a ring buffer overflow happen start parsing interrupt
7751 * from the last not overwritten vector (wptr + 16). Hopefully
7752 * this should allow us to catchup.
7753 */
6cc2fda2
MD		 7754 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7755 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7756 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7757 tmp = RREG32(IH_RB_CNTL);
7758 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7759 WREG32(IH_RB_CNTL, tmp);
7760 }
7761 return (wptr & rdev->ih.ptr_mask);
7762}
7763
7764/* CIK IV Ring
7765 * Each IV ring entry is 128 bits:
7766 * [7:0] - interrupt source id
7767 * [31:8] - reserved
7768 * [59:32] - interrupt source data
7769 * [63:60] - reserved
21a93e13
AD		 7770 * [71:64] - RINGID
7771 * CP:
7772 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7773 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7774 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7775 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7776 * PIPE_ID - ME0 0=3D
7777 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7778 * SDMA:
7779 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7780 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7781 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7782 * [79:72] - VMID
7783 * [95:80] - PASID
7784 * [127:96] - reserved
7785 */
7786/**
7787 * cik_irq_process - interrupt handler
7788 *
7789 * @rdev: radeon_device pointer
7790 *
7791 * Interrupt hander (CIK). Walk the IH ring,
7792 * ack interrupts and schedule work to handle
7793 * interrupt events.
7794 * Returns irq process return code.
7795 */
7796int cik_irq_process(struct radeon_device *rdev)
7797{
2b0781a6
AD		 7798 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7799 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7800 u32 wptr;
7801 u32 rptr;
7802 u32 src_id, src_data, ring_id;
7803 u8 me_id, pipe_id, queue_id;
7804 u32 ring_index;
7805 bool queue_hotplug = false;
7806 bool queue_reset = false;
3ec7d11b 7807 u32 addr, status, mc_client;
41a524ab 7808 bool queue_thermal = false;
a59781bb
AD		 7809
7810 if (!rdev->ih.enabled || rdev->shutdown)
7811 return IRQ_NONE;
7812
7813 wptr = cik_get_ih_wptr(rdev);
7814
7815restart_ih:
7816 /* is somebody else already processing irqs? */
7817 if (atomic_xchg(&rdev->ih.lock, 1))
7818 return IRQ_NONE;
7819
7820 rptr = rdev->ih.rptr;
7821 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7822
7823 /* Order reading of wptr vs. reading of IH ring data */
7824 rmb();
7825
7826 /* display interrupts */
7827 cik_irq_ack(rdev);
7828
7829 while (rptr != wptr) {
7830 /* wptr/rptr are in bytes! */
7831 ring_index = rptr / 4;
e28740ec
OG		 7832
7833 radeon_kfd_interrupt(rdev,
7834 (const void *) &rdev->ih.ring[ring_index]);
7835
a59781bb
AD		 7836 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7837 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7838 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7839
7840 switch (src_id) {
7841 case 1: /* D1 vblank/vline */
7842 switch (src_data) {
7843 case 0: /* D1 vblank */
7844 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7845 if (rdev->irq.crtc_vblank_int[0]) {
7846 drm_handle_vblank(rdev->ddev, 0);
7847 rdev->pm.vblank_sync = true;
7848 wake_up(&rdev->irq.vblank_queue);
7849 }
7850 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 7851 radeon_crtc_handle_vblank(rdev, 0);
a59781bb
AD		 7852 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7853 DRM_DEBUG("IH: D1 vblank\n");
7854 }
7855 break;
7856 case 1: /* D1 vline */
7857 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7858 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7859 DRM_DEBUG("IH: D1 vline\n");
7860 }
7861 break;
7862 default:
7863 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7864 break;
7865 }
7866 break;
7867 case 2: /* D2 vblank/vline */
7868 switch (src_data) {
7869 case 0: /* D2 vblank */
7870 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7871 if (rdev->irq.crtc_vblank_int[1]) {
7872 drm_handle_vblank(rdev->ddev, 1);
7873 rdev->pm.vblank_sync = true;
7874 wake_up(&rdev->irq.vblank_queue);
7875 }
7876 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 7877 radeon_crtc_handle_vblank(rdev, 1);
a59781bb
AD		 7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7879 DRM_DEBUG("IH: D2 vblank\n");
7880 }
7881 break;
7882 case 1: /* D2 vline */
7883 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7884 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7885 DRM_DEBUG("IH: D2 vline\n");
7886 }
7887 break;
7888 default:
7889 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7890 break;
7891 }
7892 break;
7893 case 3: /* D3 vblank/vline */
7894 switch (src_data) {
7895 case 0: /* D3 vblank */
7896 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7897 if (rdev->irq.crtc_vblank_int[2]) {
7898 drm_handle_vblank(rdev->ddev, 2);
7899 rdev->pm.vblank_sync = true;
7900 wake_up(&rdev->irq.vblank_queue);
7901 }
7902 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 7903 radeon_crtc_handle_vblank(rdev, 2);
a59781bb
AD		 7904 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7905 DRM_DEBUG("IH: D3 vblank\n");
7906 }
7907 break;
7908 case 1: /* D3 vline */
7909 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7910 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7911 DRM_DEBUG("IH: D3 vline\n");
7912 }
7913 break;
7914 default:
7915 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7916 break;
7917 }
7918 break;
7919 case 4: /* D4 vblank/vline */
7920 switch (src_data) {
7921 case 0: /* D4 vblank */
7922 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7923 if (rdev->irq.crtc_vblank_int[3]) {
7924 drm_handle_vblank(rdev->ddev, 3);
7925 rdev->pm.vblank_sync = true;
7926 wake_up(&rdev->irq.vblank_queue);
7927 }
7928 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 7929 radeon_crtc_handle_vblank(rdev, 3);
a59781bb
AD		 7930 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7931 DRM_DEBUG("IH: D4 vblank\n");
7932 }
7933 break;
7934 case 1: /* D4 vline */
7935 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7936 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7937 DRM_DEBUG("IH: D4 vline\n");
7938 }
7939 break;
7940 default:
7941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7942 break;
7943 }
7944 break;
7945 case 5: /* D5 vblank/vline */
7946 switch (src_data) {
7947 case 0: /* D5 vblank */
7948 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7949 if (rdev->irq.crtc_vblank_int[4]) {
7950 drm_handle_vblank(rdev->ddev, 4);
7951 rdev->pm.vblank_sync = true;
7952 wake_up(&rdev->irq.vblank_queue);
7953 }
7954 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 7955 radeon_crtc_handle_vblank(rdev, 4);
a59781bb
AD		 7956 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7957 DRM_DEBUG("IH: D5 vblank\n");
7958 }
7959 break;
7960 case 1: /* D5 vline */
7961 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7962 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7963 DRM_DEBUG("IH: D5 vline\n");
7964 }
7965 break;
7966 default:
7967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7968 break;
7969 }
7970 break;
7971 case 6: /* D6 vblank/vline */
7972 switch (src_data) {
7973 case 0: /* D6 vblank */
7974 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7975 if (rdev->irq.crtc_vblank_int[5]) {
7976 drm_handle_vblank(rdev->ddev, 5);
7977 rdev->pm.vblank_sync = true;
7978 wake_up(&rdev->irq.vblank_queue);
7979 }
7980 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 7981 radeon_crtc_handle_vblank(rdev, 5);
a59781bb
AD		 7982 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7983 DRM_DEBUG("IH: D6 vblank\n");
7984 }
7985 break;
7986 case 1: /* D6 vline */
7987 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7988 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7989 DRM_DEBUG("IH: D6 vline\n");
7990 }
7991 break;
7992 default:
7993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7994 break;
7995 }
7996 break;
f5d636d2
CK		 7997 case 8: /* D1 page flip */
7998 case 10: /* D2 page flip */
7999 case 12: /* D3 page flip */
8000 case 14: /* D4 page flip */
8001 case 16: /* D5 page flip */
8002 case 18: /* D6 page flip */
8003 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 8004 if (radeon_use_pflipirq > 0)
8005 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 8006 break;
a59781bb
AD		 8007 case 42: /* HPD hotplug */
8008 switch (src_data) {
8009 case 0:
8010 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8011 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8012 queue_hotplug = true;
8013 DRM_DEBUG("IH: HPD1\n");
8014 }
8015 break;
8016 case 1:
8017 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8018 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8019 queue_hotplug = true;
8020 DRM_DEBUG("IH: HPD2\n");
8021 }
8022 break;
8023 case 2:
8024 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8025 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8026 queue_hotplug = true;
8027 DRM_DEBUG("IH: HPD3\n");
8028 }
8029 break;
8030 case 3:
8031 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8032 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8033 queue_hotplug = true;
8034 DRM_DEBUG("IH: HPD4\n");
8035 }
8036 break;
8037 case 4:
8038 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8039 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8040 queue_hotplug = true;
8041 DRM_DEBUG("IH: HPD5\n");
8042 }
8043 break;
8044 case 5:
8045 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8046 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8047 queue_hotplug = true;
8048 DRM_DEBUG("IH: HPD6\n");
8049 }
8050 break;
8051 default:
8052 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8053 break;
8054 }
8055 break;
dc12a3ec
LL		 8056 case 96:
8057 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
8058 WREG32(SRBM_INT_ACK, 0x1);
8059 break;
6a3808b8
CK		 8060 case 124: /* UVD */
8061 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8062 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8063 break;
9d97c99b
AD		 8064 case 146:
8065 case 147:
3ec7d11b
AD		 8066 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8067 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8068 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8069 /* reset addr and status */
8070 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8071 if (addr == 0x0 && status == 0x0)
8072 break;
9d97c99b
AD		 8073 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8074 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8075 addr);
9d97c99b 8076 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8077 status);
8078 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8079 break;
d93f7937
CK		 8080 case 167: /* VCE */
8081 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8082 switch (src_data) {
8083 case 0:
8084 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8085 break;
8086 case 1:
8087 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8088 break;
8089 default:
8090 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8091 break;
8092 }
8093 break;
a59781bb
AD		 8094 case 176: /* GFX RB CP_INT */
8095 case 177: /* GFX IB CP_INT */
8096 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8097 break;
8098 case 181: /* CP EOP event */
8099 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8100 /* XXX check the bitfield order! */
8101 me_id = (ring_id & 0x60) >> 5;
8102 pipe_id = (ring_id & 0x18) >> 3;
8103 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8104 switch (me_id) {
8105 case 0:
8106 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8107 break;
8108 case 1:
a59781bb 8109 case 2:
2b0781a6
AD		 8110 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8111 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8112 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8113 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8114 break;
8115 }
8116 break;
8117 case 184: /* CP Privileged reg access */
8118 DRM_ERROR("Illegal register access in command stream\n");
8119 /* XXX check the bitfield order! */
8120 me_id = (ring_id & 0x60) >> 5;
8121 pipe_id = (ring_id & 0x18) >> 3;
8122 queue_id = (ring_id & 0x7) >> 0;
8123 switch (me_id) {
8124 case 0:
8125 /* This results in a full GPU reset, but all we need to do is soft
8126 * reset the CP for gfx
8127 */
8128 queue_reset = true;
8129 break;
8130 case 1:
8131 /* XXX compute */
2b0781a6 8132 queue_reset = true;
a59781bb
AD		 8133 break;
8134 case 2:
8135 /* XXX compute */
2b0781a6 8136 queue_reset = true;
a59781bb
AD		 8137 break;
8138 }
8139 break;
8140 case 185: /* CP Privileged inst */
8141 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8142 /* XXX check the bitfield order! */
8143 me_id = (ring_id & 0x60) >> 5;
8144 pipe_id = (ring_id & 0x18) >> 3;
8145 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8146 switch (me_id) {
8147 case 0:
8148 /* This results in a full GPU reset, but all we need to do is soft
8149 * reset the CP for gfx
8150 */
8151 queue_reset = true;
8152 break;
8153 case 1:
8154 /* XXX compute */
2b0781a6 8155 queue_reset = true;
a59781bb
AD		 8156 break;
8157 case 2:
8158 /* XXX compute */
2b0781a6 8159 queue_reset = true;
a59781bb
AD		 8160 break;
8161 }
8162 break;
21a93e13
AD		 8163 case 224: /* SDMA trap event */
8164 /* XXX check the bitfield order! */
8165 me_id = (ring_id & 0x3) >> 0;
8166 queue_id = (ring_id & 0xc) >> 2;
8167 DRM_DEBUG("IH: SDMA trap\n");
8168 switch (me_id) {
8169 case 0:
8170 switch (queue_id) {
8171 case 0:
8172 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8173 break;
8174 case 1:
8175 /* XXX compute */
8176 break;
8177 case 2:
8178 /* XXX compute */
8179 break;
8180 }
8181 break;
8182 case 1:
8183 switch (queue_id) {
8184 case 0:
8185 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8186 break;
8187 case 1:
8188 /* XXX compute */
8189 break;
8190 case 2:
8191 /* XXX compute */
8192 break;
8193 }
8194 break;
8195 }
8196 break;
41a524ab
AD		 8197 case 230: /* thermal low to high */
8198 DRM_DEBUG("IH: thermal low to high\n");
8199 rdev->pm.dpm.thermal.high_to_low = false;
8200 queue_thermal = true;
8201 break;
8202 case 231: /* thermal high to low */
8203 DRM_DEBUG("IH: thermal high to low\n");
8204 rdev->pm.dpm.thermal.high_to_low = true;
8205 queue_thermal = true;
8206 break;
8207 case 233: /* GUI IDLE */
8208 DRM_DEBUG("IH: GUI idle\n");
8209 break;
21a93e13
AD		 8210 case 241: /* SDMA Privileged inst */
8211 case 247: /* SDMA Privileged inst */
8212 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8213 /* XXX check the bitfield order! */
8214 me_id = (ring_id & 0x3) >> 0;
8215 queue_id = (ring_id & 0xc) >> 2;
8216 switch (me_id) {
8217 case 0:
8218 switch (queue_id) {
8219 case 0:
8220 queue_reset = true;
8221 break;
8222 case 1:
8223 /* XXX compute */
8224 queue_reset = true;
8225 break;
8226 case 2:
8227 /* XXX compute */
8228 queue_reset = true;
8229 break;
8230 }
8231 break;
8232 case 1:
8233 switch (queue_id) {
8234 case 0:
8235 queue_reset = true;
8236 break;
8237 case 1:
8238 /* XXX compute */
8239 queue_reset = true;
8240 break;
8241 case 2:
8242 /* XXX compute */
8243 queue_reset = true;
8244 break;
8245 }
8246 break;
8247 }
8248 break;
a59781bb
AD		 8249 default:
8250 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8251 break;
8252 }
8253
8254 /* wptr/rptr are in bytes! */
8255 rptr += 16;
8256 rptr &= rdev->ih.ptr_mask;
f55e03b9 8257 WREG32(IH_RB_RPTR, rptr);
a59781bb
AD		 8258 }
8259 if (queue_hotplug)
8260 schedule_work(&rdev->hotplug_work);
3c036389
CK		 8261 if (queue_reset) {
8262 rdev->needs_reset = true;
8263 wake_up_all(&rdev->fence_queue);
8264 }
41a524ab
AD		 8265 if (queue_thermal)
8266 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8267 rdev->ih.rptr = rptr;
a59781bb
AD		 8268 atomic_set(&rdev->ih.lock, 0);
8269
8270 /* make sure wptr hasn't changed while processing */
8271 wptr = cik_get_ih_wptr(rdev);
8272 if (wptr != rptr)
8273 goto restart_ih;
8274
8275 return IRQ_HANDLED;
8276}
7bf94a2c
AD		 8277
8278/*
8279 * startup/shutdown callbacks
8280 */
8281/**
8282 * cik_startup - program the asic to a functional state
8283 *
8284 * @rdev: radeon_device pointer
8285 *
8286 * Programs the asic to a functional state (CIK).
8287 * Called by cik_init() and cik_resume().
8288 * Returns 0 for success, error for failure.
8289 */
8290static int cik_startup(struct radeon_device *rdev)
8291{
8292 struct radeon_ring *ring;
0e16e4cf 8293 u32 nop;
7bf94a2c
AD		 8294 int r;
8295
8a7cd276
AD		 8296 /* enable pcie gen2/3 link */
8297 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8298 /* enable aspm */
8299 cik_program_aspm(rdev);
8a7cd276 8300
e5903d39
AD		 8301 /* scratch needs to be initialized before MC */
8302 r = r600_vram_scratch_init(rdev);
8303 if (r)
8304 return r;
8305
6fab3feb
AD		 8306 cik_mc_program(rdev);
8307
6c7bccea 8308 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8309 r = ci_mc_load_microcode(rdev);
8310 if (r) {
8311 DRM_ERROR("Failed to load MC firmware!\n");
8312 return r;
8313 }
8314 }
8315
7bf94a2c
AD		 8316 r = cik_pcie_gart_enable(rdev);
8317 if (r)
8318 return r;
8319 cik_gpu_init(rdev);
8320
8321 /* allocate rlc buffers */
22c775ce
AD		 8322 if (rdev->flags & RADEON_IS_IGP) {
8323 if (rdev->family == CHIP_KAVERI) {
8324 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8325 rdev->rlc.reg_list_size =
8326 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8327 } else {
8328 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8329 rdev->rlc.reg_list_size =
8330 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8331 }
8332 }
8333 rdev->rlc.cs_data = ci_cs_data;
8334 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8335 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8336 if (r) {
8337 DRM_ERROR("Failed to init rlc BOs!\n");
8338 return r;
8339 }
8340
8341 /* allocate wb buffer */
8342 r = radeon_wb_init(rdev);
8343 if (r)
8344 return r;
8345
963e81f9
AD		 8346 /* allocate mec buffers */
8347 r = cik_mec_init(rdev);
8348 if (r) {
8349 DRM_ERROR("Failed to init MEC BOs!\n");
8350 return r;
8351 }
8352
7bf94a2c
AD		 8353 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8354 if (r) {
8355 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8356 return r;
8357 }
8358
963e81f9
AD		 8359 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8360 if (r) {
8361 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8362 return r;
8363 }
8364
8365 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8366 if (r) {
8367 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8368 return r;
8369 }
8370
7bf94a2c
AD		 8371 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8372 if (r) {
8373 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8374 return r;
8375 }
8376
8377 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8378 if (r) {
8379 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8380 return r;
8381 }
8382
2ce529da 8383 r = radeon_uvd_resume(rdev);
87167bb1 8384 if (!r) {
2ce529da
AD		 8385 r = uvd_v4_2_resume(rdev);
8386 if (!r) {
8387 r = radeon_fence_driver_start_ring(rdev,
8388 R600_RING_TYPE_UVD_INDEX);
8389 if (r)
8390 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8391 }
87167bb1
CK		 8392 }
8393 if (r)
8394 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8395
d93f7937
CK		 8396 r = radeon_vce_resume(rdev);
8397 if (!r) {
8398 r = vce_v2_0_resume(rdev);
8399 if (!r)
8400 r = radeon_fence_driver_start_ring(rdev,
8401 TN_RING_TYPE_VCE1_INDEX);
8402 if (!r)
8403 r = radeon_fence_driver_start_ring(rdev,
8404 TN_RING_TYPE_VCE2_INDEX);
8405 }
8406 if (r) {
8407 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8408 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8409 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8410 }
8411
7bf94a2c
AD		 8412 /* Enable IRQ */
8413 if (!rdev->irq.installed) {
8414 r = radeon_irq_kms_init(rdev);
8415 if (r)
8416 return r;
8417 }
8418
8419 r = cik_irq_init(rdev);
8420 if (r) {
8421 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8422 radeon_irq_kms_fini(rdev);
8423 return r;
8424 }
8425 cik_irq_set(rdev);
8426
0e16e4cf 8427 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8428 if (rdev->new_fw)
8429 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8430 else
8431 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8432 } else {
8433 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8434 }
8435
7bf94a2c
AD		 8436 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8437 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8438 nop);
7bf94a2c
AD		 8439 if (r)
8440 return r;
8441
963e81f9 8442 /* set up the compute queues */
2615b53a 8443 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8444 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8445 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8446 nop);
963e81f9
AD		 8447 if (r)
8448 return r;
8449 ring->me = 1; /* first MEC */
8450 ring->pipe = 0; /* first pipe */
8451 ring->queue = 0; /* first queue */
8452 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8453
2615b53a 8454 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8455 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8456 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8457 nop);
963e81f9
AD		 8458 if (r)
8459 return r;
8460 /* dGPU only have 1 MEC */
8461 ring->me = 1; /* first MEC */
8462 ring->pipe = 0; /* first pipe */
8463 ring->queue = 1; /* second queue */
8464 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8465
7bf94a2c
AD		 8466 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8467 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8468 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8469 if (r)
8470 return r;
8471
8472 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8473 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8474 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8475 if (r)
8476 return r;
8477
8478 r = cik_cp_resume(rdev);
8479 if (r)
8480 return r;
8481
8482 r = cik_sdma_resume(rdev);
8483 if (r)
8484 return r;
8485
87167bb1
CK		 8486 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8487 if (ring->ring_size) {
02c9f7fa 8488 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8489 RADEON_CP_PACKET2);
87167bb1 8490 if (!r)
e409b128 8491 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8492 if (r)
8493 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8494 }
8495
d93f7937
CK		 8496 r = -ENOENT;
8497
8498 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8499 if (ring->ring_size)
8500 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8501 VCE_CMD_NO_OP);
8502
8503 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8504 if (ring->ring_size)
8505 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8506 VCE_CMD_NO_OP);
8507
8508 if (!r)
8509 r = vce_v1_0_init(rdev);
8510 else if (r != -ENOENT)
8511 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8512
7bf94a2c
AD		 8513 r = radeon_ib_pool_init(rdev);
8514 if (r) {
8515 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8516 return r;
8517 }
8518
8519 r = radeon_vm_manager_init(rdev);
8520 if (r) {
8521 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8522 return r;
8523 }
8524
bfc1f97d 8525 r = radeon_audio_init(rdev);
b530602f
AD		 8526 if (r)
8527 return r;
8528
e28740ec
OG		 8529 r = radeon_kfd_resume(rdev);
8530 if (r)
8531 return r;
8532
7bf94a2c
AD		 8533 return 0;
8534}
8535
8536/**
8537 * cik_resume - resume the asic to a functional state
8538 *
8539 * @rdev: radeon_device pointer
8540 *
8541 * Programs the asic to a functional state (CIK).
8542 * Called at resume.
8543 * Returns 0 for success, error for failure.
8544 */
8545int cik_resume(struct radeon_device *rdev)
8546{
8547 int r;
8548
8549 /* post card */
8550 atom_asic_init(rdev->mode_info.atom_context);
8551
0aafd313
AD		 8552 /* init golden registers */
8553 cik_init_golden_registers(rdev);
8554
bc6a6295
AD		 8555 if (rdev->pm.pm_method == PM_METHOD_DPM)
8556 radeon_pm_resume(rdev);
6c7bccea 8557
7bf94a2c
AD		 8558 rdev->accel_working = true;
8559 r = cik_startup(rdev);
8560 if (r) {
8561 DRM_ERROR("cik startup failed on resume\n");
8562 rdev->accel_working = false;
8563 return r;
8564 }
8565
8566 return r;
8567
8568}
8569
8570/**
8571 * cik_suspend - suspend the asic
8572 *
8573 * @rdev: radeon_device pointer
8574 *
8575 * Bring the chip into a state suitable for suspend (CIK).
8576 * Called at suspend.
8577 * Returns 0 for success.
8578 */
8579int cik_suspend(struct radeon_device *rdev)
8580{
e28740ec 8581 radeon_kfd_suspend(rdev);
6c7bccea 8582 radeon_pm_suspend(rdev);
7991d665 8583 radeon_audio_fini(rdev);
7bf94a2c
AD		 8584 radeon_vm_manager_fini(rdev);
8585 cik_cp_enable(rdev, false);
8586 cik_sdma_enable(rdev, false);
e409b128 8587 uvd_v1_0_fini(rdev);
87167bb1 8588 radeon_uvd_suspend(rdev);
d93f7937 8589 radeon_vce_suspend(rdev);
473359bc
AD		 8590 cik_fini_pg(rdev);
8591 cik_fini_cg(rdev);
7bf94a2c
AD		 8592 cik_irq_suspend(rdev);
8593 radeon_wb_disable(rdev);
8594 cik_pcie_gart_disable(rdev);
8595 return 0;
8596}
8597
8598/* Plan is to move initialization in that function and use
8599 * helper function so that radeon_device_init pretty much
8600 * do nothing more than calling asic specific function. This
8601 * should also allow to remove a bunch of callback function
8602 * like vram_info.
8603 */
8604/**
8605 * cik_init - asic specific driver and hw init
8606 *
8607 * @rdev: radeon_device pointer
8608 *
8609 * Setup asic specific driver variables and program the hw
8610 * to a functional state (CIK).
8611 * Called at driver startup.
8612 * Returns 0 for success, errors for failure.
8613 */
8614int cik_init(struct radeon_device *rdev)
8615{
8616 struct radeon_ring *ring;
8617 int r;
8618
8619 /* Read BIOS */
8620 if (!radeon_get_bios(rdev)) {
8621 if (ASIC_IS_AVIVO(rdev))
8622 return -EINVAL;
8623 }
8624 /* Must be an ATOMBIOS */
8625 if (!rdev->is_atom_bios) {
8626 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8627 return -EINVAL;
8628 }
8629 r = radeon_atombios_init(rdev);
8630 if (r)
8631 return r;
8632
8633 /* Post card if necessary */
8634 if (!radeon_card_posted(rdev)) {
8635 if (!rdev->bios) {
8636 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8637 return -EINVAL;
8638 }
8639 DRM_INFO("GPU not posted. posting now...\n");
8640 atom_asic_init(rdev->mode_info.atom_context);
8641 }
0aafd313
AD		 8642 /* init golden registers */
8643 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8644 /* Initialize scratch registers */
8645 cik_scratch_init(rdev);
8646 /* Initialize surface registers */
8647 radeon_surface_init(rdev);
8648 /* Initialize clocks */
8649 radeon_get_clock_info(rdev->ddev);
8650
8651 /* Fence driver */
8652 r = radeon_fence_driver_init(rdev);
8653 if (r)
8654 return r;
8655
8656 /* initialize memory controller */
8657 r = cik_mc_init(rdev);
8658 if (r)
8659 return r;
8660 /* Memory manager */
8661 r = radeon_bo_init(rdev);
8662 if (r)
8663 return r;
8664
01ac8794
AD		 8665 if (rdev->flags & RADEON_IS_IGP) {
8666 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8667 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8668 r = cik_init_microcode(rdev);
8669 if (r) {
8670 DRM_ERROR("Failed to load firmware!\n");
8671 return r;
8672 }
8673 }
8674 } else {
8675 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8676 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8677 !rdev->mc_fw) {
8678 r = cik_init_microcode(rdev);
8679 if (r) {
8680 DRM_ERROR("Failed to load firmware!\n");
8681 return r;
8682 }
8683 }
8684 }
8685
6c7bccea
AD		 8686 /* Initialize power management */
8687 radeon_pm_init(rdev);
8688
7bf94a2c
AD		 8689 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 1024 * 1024);
8692
963e81f9
AD		 8693 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8694 ring->ring_obj = NULL;
8695 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8696 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8697 if (r)
8698 return r;
8699
8700 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8701 ring->ring_obj = NULL;
8702 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8703 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8704 if (r)
8705 return r;
8706
7bf94a2c
AD		 8707 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8708 ring->ring_obj = NULL;
8709 r600_ring_init(rdev, ring, 256 * 1024);
8710
8711 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8712 ring->ring_obj = NULL;
8713 r600_ring_init(rdev, ring, 256 * 1024);
8714
87167bb1
CK		 8715 r = radeon_uvd_init(rdev);
8716 if (!r) {
8717 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8718 ring->ring_obj = NULL;
8719 r600_ring_init(rdev, ring, 4096);
8720 }
8721
d93f7937
CK		 8722 r = radeon_vce_init(rdev);
8723 if (!r) {
8724 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8725 ring->ring_obj = NULL;
8726 r600_ring_init(rdev, ring, 4096);
8727
8728 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8729 ring->ring_obj = NULL;
8730 r600_ring_init(rdev, ring, 4096);
8731 }
8732
7bf94a2c
AD		 8733 rdev->ih.ring_obj = NULL;
8734 r600_ih_ring_init(rdev, 64 * 1024);
8735
8736 r = r600_pcie_gart_init(rdev);
8737 if (r)
8738 return r;
8739
8740 rdev->accel_working = true;
8741 r = cik_startup(rdev);
8742 if (r) {
8743 dev_err(rdev->dev, "disabling GPU acceleration\n");
8744 cik_cp_fini(rdev);
8745 cik_sdma_fini(rdev);
8746 cik_irq_fini(rdev);
1fd11777 8747 sumo_rlc_fini(rdev);
963e81f9 8748 cik_mec_fini(rdev);
7bf94a2c
AD		 8749 radeon_wb_fini(rdev);
8750 radeon_ib_pool_fini(rdev);
8751 radeon_vm_manager_fini(rdev);
8752 radeon_irq_kms_fini(rdev);
8753 cik_pcie_gart_fini(rdev);
8754 rdev->accel_working = false;
8755 }
8756
8757 /* Don't start up if the MC ucode is missing.
8758 * The default clocks and voltages before the MC ucode
8759 * is loaded are not suffient for advanced operations.
8760 */
8761 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8762 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8763 return -EINVAL;
8764 }
8765
8766 return 0;
8767}
8768
8769/**
8770 * cik_fini - asic specific driver and hw fini
8771 *
8772 * @rdev: radeon_device pointer
8773 *
8774 * Tear down the asic specific driver variables and program the hw
8775 * to an idle state (CIK).
8776 * Called at driver unload.
8777 */
8778void cik_fini(struct radeon_device *rdev)
8779{
6c7bccea 8780 radeon_pm_fini(rdev);
7bf94a2c
AD		 8781 cik_cp_fini(rdev);
8782 cik_sdma_fini(rdev);
473359bc
AD		 8783 cik_fini_pg(rdev);
8784 cik_fini_cg(rdev);
7bf94a2c 8785 cik_irq_fini(rdev);
1fd11777 8786 sumo_rlc_fini(rdev);
963e81f9 8787 cik_mec_fini(rdev);
7bf94a2c
AD		 8788 radeon_wb_fini(rdev);
8789 radeon_vm_manager_fini(rdev);
8790 radeon_ib_pool_fini(rdev);
8791 radeon_irq_kms_fini(rdev);
e409b128 8792 uvd_v1_0_fini(rdev);
87167bb1 8793 radeon_uvd_fini(rdev);
d93f7937 8794 radeon_vce_fini(rdev);
7bf94a2c
AD		 8795 cik_pcie_gart_fini(rdev);
8796 r600_vram_scratch_fini(rdev);
8797 radeon_gem_fini(rdev);
8798 radeon_fence_driver_fini(rdev);
8799 radeon_bo_fini(rdev);
8800 radeon_atombios_fini(rdev);
8801 kfree(rdev->bios);
8802 rdev->bios = NULL;
8803}
cd84a27d 8804
134b480f
AD		 8805void dce8_program_fmt(struct drm_encoder *encoder)
8806{
8807 struct drm_device *dev = encoder->dev;
8808 struct radeon_device *rdev = dev->dev_private;
8809 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8810 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8811 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8812 int bpc = 0;
8813 u32 tmp = 0;
6214bb74 8814 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8815
6214bb74
AD		 8816 if (connector) {
8817 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8818 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8819 dither = radeon_connector->dither;
8820 }
134b480f
AD		 8821
8822 /* LVDS/eDP FMT is set up by atom */
8823 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8824 return;
8825
8826 /* not needed for analog */
8827 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8828 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8829 return;
8830
8831 if (bpc == 0)
8832 return;
8833
8834 switch (bpc) {
8835 case 6:
6214bb74 8836 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8837 /* XXX sort out optimal dither settings */
8838 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8839 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8840 else
8841 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8842 break;
8843 case 8:
6214bb74 8844 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8845 /* XXX sort out optimal dither settings */
8846 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8847 FMT_RGB_RANDOM_ENABLE |
8848 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8849 else
8850 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8851 break;
8852 case 10:
6214bb74 8853 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8854 /* XXX sort out optimal dither settings */
8855 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8856 FMT_RGB_RANDOM_ENABLE |
8857 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8858 else
8859 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8860 break;
8861 default:
8862 /* not needed */
8863 break;
8864 }
8865
8866 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8867}
8868
cd84a27d
AD		 8869/* display watermark setup */
8870/**
8871 * dce8_line_buffer_adjust - Set up the line buffer
8872 *
8873 * @rdev: radeon_device pointer
8874 * @radeon_crtc: the selected display controller
8875 * @mode: the current display mode on the selected display
8876 * controller
8877 *
8878 * Setup up the line buffer allocation for
8879 * the selected display controller (CIK).
8880 * Returns the line buffer size in pixels.
8881 */
8882static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8883 struct radeon_crtc *radeon_crtc,
8884 struct drm_display_mode *mode)
8885{
bc01a8c7
AD		 8886 u32 tmp, buffer_alloc, i;
8887 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8888 /*
8889 * Line Buffer Setup
8890 * There are 6 line buffers, one for each display controllers.
8891 * There are 3 partitions per LB. Select the number of partitions
8892 * to enable based on the display width. For display widths larger
8893 * than 4096, you need use to use 2 display controllers and combine
8894 * them using the stereo blender.
8895 */
8896 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8897 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8898 tmp = 1;
bc01a8c7
AD		 8899 buffer_alloc = 2;
8900 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8901 tmp = 2;
bc01a8c7
AD		 8902 buffer_alloc = 2;
8903 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8904 tmp = 0;
bc01a8c7
AD		 8905 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8906 } else {
cd84a27d
AD		 8907 DRM_DEBUG_KMS("Mode too big for LB!\n");
8908 tmp = 0;
bc01a8c7 8909 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8910 }
bc01a8c7 8911 } else {
cd84a27d 8912 tmp = 1;
bc01a8c7
AD		 8913 buffer_alloc = 0;
8914 }
cd84a27d
AD		 8915
8916 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8917 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8918
bc01a8c7
AD		 8919 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8920 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8921 for (i = 0; i < rdev->usec_timeout; i++) {
8922 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8923 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8924 break;
8925 udelay(1);
8926 }
8927
cd84a27d
AD		 8928 if (radeon_crtc->base.enabled && mode) {
8929 switch (tmp) {
8930 case 0:
8931 default:
8932 return 4096 * 2;
8933 case 1:
8934 return 1920 * 2;
8935 case 2:
8936 return 2560 * 2;
8937 }
8938 }
8939
8940 /* controller not enabled, so no lb used */
8941 return 0;
8942}
8943
8944/**
8945 * cik_get_number_of_dram_channels - get the number of dram channels
8946 *
8947 * @rdev: radeon_device pointer
8948 *
8949 * Look up the number of video ram channels (CIK).
8950 * Used for display watermark bandwidth calculations
8951 * Returns the number of dram channels
8952 */
8953static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8954{
8955 u32 tmp = RREG32(MC_SHARED_CHMAP);
8956
8957 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8958 case 0:
8959 default:
8960 return 1;
8961 case 1:
8962 return 2;
8963 case 2:
8964 return 4;
8965 case 3:
8966 return 8;
8967 case 4:
8968 return 3;
8969 case 5:
8970 return 6;
8971 case 6:
8972 return 10;
8973 case 7:
8974 return 12;
8975 case 8:
8976 return 16;
8977 }
8978}
8979
8980struct dce8_wm_params {
8981 u32 dram_channels; /* number of dram channels */
8982 u32 yclk; /* bandwidth per dram data pin in kHz */
8983 u32 sclk; /* engine clock in kHz */
8984 u32 disp_clk; /* display clock in kHz */
8985 u32 src_width; /* viewport width */
8986 u32 active_time; /* active display time in ns */
8987 u32 blank_time; /* blank time in ns */
8988 bool interlaced; /* mode is interlaced */
8989 fixed20_12 vsc; /* vertical scale ratio */
8990 u32 num_heads; /* number of active crtcs */
8991 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8992 u32 lb_size; /* line buffer allocated to pipe */
8993 u32 vtaps; /* vertical scaler taps */
8994};
8995
8996/**
8997 * dce8_dram_bandwidth - get the dram bandwidth
8998 *
8999 * @wm: watermark calculation data
9000 *
9001 * Calculate the raw dram bandwidth (CIK).
9002 * Used for display watermark bandwidth calculations
9003 * Returns the dram bandwidth in MBytes/s
9004 */
9005static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9006{
9007 /* Calculate raw DRAM Bandwidth */
9008 fixed20_12 dram_efficiency; /* 0.7 */
9009 fixed20_12 yclk, dram_channels, bandwidth;
9010 fixed20_12 a;
9011
9012 a.full = dfixed_const(1000);
9013 yclk.full = dfixed_const(wm->yclk);
9014 yclk.full = dfixed_div(yclk, a);
9015 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9016 a.full = dfixed_const(10);
9017 dram_efficiency.full = dfixed_const(7);
9018 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9019 bandwidth.full = dfixed_mul(dram_channels, yclk);
9020 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9021
9022 return dfixed_trunc(bandwidth);
9023}
9024
9025/**
9026 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9027 *
9028 * @wm: watermark calculation data
9029 *
9030 * Calculate the dram bandwidth used for display (CIK).
9031 * Used for display watermark bandwidth calculations
9032 * Returns the dram bandwidth for display in MBytes/s
9033 */
9034static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9035{
9036 /* Calculate DRAM Bandwidth and the part allocated to display. */
9037 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9038 fixed20_12 yclk, dram_channels, bandwidth;
9039 fixed20_12 a;
9040
9041 a.full = dfixed_const(1000);
9042 yclk.full = dfixed_const(wm->yclk);
9043 yclk.full = dfixed_div(yclk, a);
9044 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9045 a.full = dfixed_const(10);
9046 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9047 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9048 bandwidth.full = dfixed_mul(dram_channels, yclk);
9049 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9050
9051 return dfixed_trunc(bandwidth);
9052}
9053
9054/**
9055 * dce8_data_return_bandwidth - get the data return bandwidth
9056 *
9057 * @wm: watermark calculation data
9058 *
9059 * Calculate the data return bandwidth used for display (CIK).
9060 * Used for display watermark bandwidth calculations
9061 * Returns the data return bandwidth in MBytes/s
9062 */
9063static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9064{
9065 /* Calculate the display Data return Bandwidth */
9066 fixed20_12 return_efficiency; /* 0.8 */
9067 fixed20_12 sclk, bandwidth;
9068 fixed20_12 a;
9069
9070 a.full = dfixed_const(1000);
9071 sclk.full = dfixed_const(wm->sclk);
9072 sclk.full = dfixed_div(sclk, a);
9073 a.full = dfixed_const(10);
9074 return_efficiency.full = dfixed_const(8);
9075 return_efficiency.full = dfixed_div(return_efficiency, a);
9076 a.full = dfixed_const(32);
9077 bandwidth.full = dfixed_mul(a, sclk);
9078 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9079
9080 return dfixed_trunc(bandwidth);
9081}
9082
9083/**
9084 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9085 *
9086 * @wm: watermark calculation data
9087 *
9088 * Calculate the dmif bandwidth used for display (CIK).
9089 * Used for display watermark bandwidth calculations
9090 * Returns the dmif bandwidth in MBytes/s
9091 */
9092static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9093{
9094 /* Calculate the DMIF Request Bandwidth */
9095 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9096 fixed20_12 disp_clk, bandwidth;
9097 fixed20_12 a, b;
9098
9099 a.full = dfixed_const(1000);
9100 disp_clk.full = dfixed_const(wm->disp_clk);
9101 disp_clk.full = dfixed_div(disp_clk, a);
9102 a.full = dfixed_const(32);
9103 b.full = dfixed_mul(a, disp_clk);
9104
9105 a.full = dfixed_const(10);
9106 disp_clk_request_efficiency.full = dfixed_const(8);
9107 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9108
9109 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9110
9111 return dfixed_trunc(bandwidth);
9112}
9113
9114/**
9115 * dce8_available_bandwidth - get the min available bandwidth
9116 *
9117 * @wm: watermark calculation data
9118 *
9119 * Calculate the min available bandwidth used for display (CIK).
9120 * Used for display watermark bandwidth calculations
9121 * Returns the min available bandwidth in MBytes/s
9122 */
9123static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9124{
9125 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9126 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9127 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9128 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9129
9130 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9131}
9132
9133/**
9134 * dce8_average_bandwidth - get the average available bandwidth
9135 *
9136 * @wm: watermark calculation data
9137 *
9138 * Calculate the average available bandwidth used for display (CIK).
9139 * Used for display watermark bandwidth calculations
9140 * Returns the average available bandwidth in MBytes/s
9141 */
9142static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9143{
9144 /* Calculate the display mode Average Bandwidth
9145 * DisplayMode should contain the source and destination dimensions,
9146 * timing, etc.
9147 */
9148 fixed20_12 bpp;
9149 fixed20_12 line_time;
9150 fixed20_12 src_width;
9151 fixed20_12 bandwidth;
9152 fixed20_12 a;
9153
9154 a.full = dfixed_const(1000);
9155 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9156 line_time.full = dfixed_div(line_time, a);
9157 bpp.full = dfixed_const(wm->bytes_per_pixel);
9158 src_width.full = dfixed_const(wm->src_width);
9159 bandwidth.full = dfixed_mul(src_width, bpp);
9160 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9161 bandwidth.full = dfixed_div(bandwidth, line_time);
9162
9163 return dfixed_trunc(bandwidth);
9164}
9165
9166/**
9167 * dce8_latency_watermark - get the latency watermark
9168 *
9169 * @wm: watermark calculation data
9170 *
9171 * Calculate the latency watermark (CIK).
9172 * Used for display watermark bandwidth calculations
9173 * Returns the latency watermark in ns
9174 */
9175static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9176{
9177 /* First calculate the latency in ns */
9178 u32 mc_latency = 2000; /* 2000 ns. */
9179 u32 available_bandwidth = dce8_available_bandwidth(wm);
9180 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9181 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9182 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9183 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9184 (wm->num_heads * cursor_line_pair_return_time);
9185 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9186 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9187 u32 tmp, dmif_size = 12288;
9188 fixed20_12 a, b, c;
9189
9190 if (wm->num_heads == 0)
9191 return 0;
9192
9193 a.full = dfixed_const(2);
9194 b.full = dfixed_const(1);
9195 if ((wm->vsc.full > a.full) ||
9196 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9197 (wm->vtaps >= 5) ||
9198 ((wm->vsc.full >= a.full) && wm->interlaced))
9199 max_src_lines_per_dst_line = 4;
9200 else
9201 max_src_lines_per_dst_line = 2;
9202
9203 a.full = dfixed_const(available_bandwidth);
9204 b.full = dfixed_const(wm->num_heads);
9205 a.full = dfixed_div(a, b);
9206
9207 b.full = dfixed_const(mc_latency + 512);
9208 c.full = dfixed_const(wm->disp_clk);
9209 b.full = dfixed_div(b, c);
9210
9211 c.full = dfixed_const(dmif_size);
9212 b.full = dfixed_div(c, b);
9213
9214 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9215
9216 b.full = dfixed_const(1000);
9217 c.full = dfixed_const(wm->disp_clk);
9218 b.full = dfixed_div(c, b);
9219 c.full = dfixed_const(wm->bytes_per_pixel);
9220 b.full = dfixed_mul(b, c);
9221
9222 lb_fill_bw = min(tmp, dfixed_trunc(b));
9223
9224 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9225 b.full = dfixed_const(1000);
9226 c.full = dfixed_const(lb_fill_bw);
9227 b.full = dfixed_div(c, b);
9228 a.full = dfixed_div(a, b);
9229 line_fill_time = dfixed_trunc(a);
9230
9231 if (line_fill_time < wm->active_time)
9232 return latency;
9233 else
9234 return latency + (line_fill_time - wm->active_time);
9235
9236}
9237
9238/**
9239 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9240 * average and available dram bandwidth
9241 *
9242 * @wm: watermark calculation data
9243 *
9244 * Check if the display average bandwidth fits in the display
9245 * dram bandwidth (CIK).
9246 * Used for display watermark bandwidth calculations
9247 * Returns true if the display fits, false if not.
9248 */
9249static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9250{
9251 if (dce8_average_bandwidth(wm) <=
9252 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9253 return true;
9254 else
9255 return false;
9256}
9257
9258/**
9259 * dce8_average_bandwidth_vs_available_bandwidth - check
9260 * average and available bandwidth
9261 *
9262 * @wm: watermark calculation data
9263 *
9264 * Check if the display average bandwidth fits in the display
9265 * available bandwidth (CIK).
9266 * Used for display watermark bandwidth calculations
9267 * Returns true if the display fits, false if not.
9268 */
9269static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9270{
9271 if (dce8_average_bandwidth(wm) <=
9272 (dce8_available_bandwidth(wm) / wm->num_heads))
9273 return true;
9274 else
9275 return false;
9276}
9277
9278/**
9279 * dce8_check_latency_hiding - check latency hiding
9280 *
9281 * @wm: watermark calculation data
9282 *
9283 * Check latency hiding (CIK).
9284 * Used for display watermark bandwidth calculations
9285 * Returns true if the display fits, false if not.
9286 */
9287static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9288{
9289 u32 lb_partitions = wm->lb_size / wm->src_width;
9290 u32 line_time = wm->active_time + wm->blank_time;
9291 u32 latency_tolerant_lines;
9292 u32 latency_hiding;
9293 fixed20_12 a;
9294
9295 a.full = dfixed_const(1);
9296 if (wm->vsc.full > a.full)
9297 latency_tolerant_lines = 1;
9298 else {
9299 if (lb_partitions <= (wm->vtaps + 1))
9300 latency_tolerant_lines = 1;
9301 else
9302 latency_tolerant_lines = 2;
9303 }
9304
9305 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9306
9307 if (dce8_latency_watermark(wm) <= latency_hiding)
9308 return true;
9309 else
9310 return false;
9311}
9312
9313/**
9314 * dce8_program_watermarks - program display watermarks
9315 *
9316 * @rdev: radeon_device pointer
9317 * @radeon_crtc: the selected display controller
9318 * @lb_size: line buffer size
9319 * @num_heads: number of display controllers in use
9320 *
9321 * Calculate and program the display watermarks for the
9322 * selected display controller (CIK).
9323 */
9324static void dce8_program_watermarks(struct radeon_device *rdev,
9325 struct radeon_crtc *radeon_crtc,
9326 u32 lb_size, u32 num_heads)
9327{
9328 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9329 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9330 u32 pixel_period;
9331 u32 line_time = 0;
9332 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9333 u32 tmp, wm_mask;
9334
9335 if (radeon_crtc->base.enabled && num_heads && mode) {
9336 pixel_period = 1000000 / (u32)mode->clock;
9337 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9338
58ea2dea
AD		 9339 /* watermark for high clocks */
9340 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9341 rdev->pm.dpm_enabled) {
9342 wm_high.yclk =
9343 radeon_dpm_get_mclk(rdev, false) * 10;
9344 wm_high.sclk =
9345 radeon_dpm_get_sclk(rdev, false) * 10;
9346 } else {
9347 wm_high.yclk = rdev->pm.current_mclk * 10;
9348 wm_high.sclk = rdev->pm.current_sclk * 10;
9349 }
9350
9351 wm_high.disp_clk = mode->clock;
9352 wm_high.src_width = mode->crtc_hdisplay;
9353 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9354 wm_high.blank_time = line_time - wm_high.active_time;
9355 wm_high.interlaced = false;
cd84a27d 9356 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9357 wm_high.interlaced = true;
9358 wm_high.vsc = radeon_crtc->vsc;
9359 wm_high.vtaps = 1;
cd84a27d 9360 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9361 wm_high.vtaps = 2;
9362 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9363 wm_high.lb_size = lb_size;
9364 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9365 wm_high.num_heads = num_heads;
cd84a27d
AD		 9366
9367 /* set for high clocks */
58ea2dea
AD		 9368 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9369
9370 /* possibly force display priority to high */
9371 /* should really do this at mode validation time... */
9372 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9373 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9374 !dce8_check_latency_hiding(&wm_high) ||
9375 (rdev->disp_priority == 2)) {
9376 DRM_DEBUG_KMS("force priority to high\n");
9377 }
9378
9379 /* watermark for low clocks */
9380 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9381 rdev->pm.dpm_enabled) {
9382 wm_low.yclk =
9383 radeon_dpm_get_mclk(rdev, true) * 10;
9384 wm_low.sclk =
9385 radeon_dpm_get_sclk(rdev, true) * 10;
9386 } else {
9387 wm_low.yclk = rdev->pm.current_mclk * 10;
9388 wm_low.sclk = rdev->pm.current_sclk * 10;
9389 }
9390
9391 wm_low.disp_clk = mode->clock;
9392 wm_low.src_width = mode->crtc_hdisplay;
9393 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9394 wm_low.blank_time = line_time - wm_low.active_time;
9395 wm_low.interlaced = false;
9396 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9397 wm_low.interlaced = true;
9398 wm_low.vsc = radeon_crtc->vsc;
9399 wm_low.vtaps = 1;
9400 if (radeon_crtc->rmx_type != RMX_OFF)
9401 wm_low.vtaps = 2;
9402 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9403 wm_low.lb_size = lb_size;
9404 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9405 wm_low.num_heads = num_heads;
9406
cd84a27d 9407 /* set for low clocks */
58ea2dea 9408 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9409
9410 /* possibly force display priority to high */
9411 /* should really do this at mode validation time... */
58ea2dea
AD		 9412 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9413 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9414 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9415 (rdev->disp_priority == 2)) {
9416 DRM_DEBUG_KMS("force priority to high\n");
9417 }
9418 }
9419
9420 /* select wm A */
9421 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9422 tmp = wm_mask;
9423 tmp &= ~LATENCY_WATERMARK_MASK(3);
9424 tmp |= LATENCY_WATERMARK_MASK(1);
9425 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9426 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9427 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9428 LATENCY_HIGH_WATERMARK(line_time)));
9429 /* select wm B */
9430 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9431 tmp &= ~LATENCY_WATERMARK_MASK(3);
9432 tmp |= LATENCY_WATERMARK_MASK(2);
9433 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9434 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9435 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9436 LATENCY_HIGH_WATERMARK(line_time)));
9437 /* restore original selection */
9438 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9439
9440 /* save values for DPM */
9441 radeon_crtc->line_time = line_time;
9442 radeon_crtc->wm_high = latency_watermark_a;
9443 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9444}
9445
9446/**
9447 * dce8_bandwidth_update - program display watermarks
9448 *
9449 * @rdev: radeon_device pointer
9450 *
9451 * Calculate and program the display watermarks and line
9452 * buffer allocation (CIK).
9453 */
9454void dce8_bandwidth_update(struct radeon_device *rdev)
9455{
9456 struct drm_display_mode *mode = NULL;
9457 u32 num_heads = 0, lb_size;
9458 int i;
9459
8efe82ca
AD		 9460 if (!rdev->mode_info.mode_config_initialized)
9461 return;
9462
cd84a27d
AD		 9463 radeon_update_display_priority(rdev);
9464
9465 for (i = 0; i < rdev->num_crtc; i++) {
9466 if (rdev->mode_info.crtcs[i]->base.enabled)
9467 num_heads++;
9468 }
9469 for (i = 0; i < rdev->num_crtc; i++) {
9470 mode = &rdev->mode_info.crtcs[i]->base.mode;
9471 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9472 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9473 }
9474}
44fa346f
AD		 9475
9476/**
9477 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9478 *
9479 * @rdev: radeon_device pointer
9480 *
9481 * Fetches a GPU clock counter snapshot (SI).
9482 * Returns the 64 bit clock counter snapshot.
9483 */
9484uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9485{
9486 uint64_t clock;
9487
9488 mutex_lock(&rdev->gpu_clock_mutex);
9489 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9490 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9491 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9492 mutex_unlock(&rdev->gpu_clock_mutex);
9493 return clock;
9494}
9495
87167bb1
CK		 9496static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9497 u32 cntl_reg, u32 status_reg)
9498{
9499 int r, i;
9500 struct atom_clock_dividers dividers;
9501 uint32_t tmp;
9502
9503 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9504 clock, false, &dividers);
9505 if (r)
9506 return r;
9507
9508 tmp = RREG32_SMC(cntl_reg);
9509 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9510 tmp |= dividers.post_divider;
9511 WREG32_SMC(cntl_reg, tmp);
9512
9513 for (i = 0; i < 100; i++) {
9514 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9515 break;
9516 mdelay(10);
9517 }
9518 if (i == 100)
9519 return -ETIMEDOUT;
9520
9521 return 0;
9522}
9523
9524int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9525{
9526 int r = 0;
9527
9528 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9529 if (r)
9530 return r;
9531
9532 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9533 return r;
9534}
9535
5ad6bf91
AD		 9536int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9537{
9538 int r, i;
9539 struct atom_clock_dividers dividers;
9540 u32 tmp;
9541
9542 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9543 ecclk, false, &dividers);
9544 if (r)
9545 return r;
9546
9547 for (i = 0; i < 100; i++) {
9548 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9549 break;
9550 mdelay(10);
9551 }
9552 if (i == 100)
9553 return -ETIMEDOUT;
9554
9555 tmp = RREG32_SMC(CG_ECLK_CNTL);
9556 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9557 tmp |= dividers.post_divider;
9558 WREG32_SMC(CG_ECLK_CNTL, tmp);
9559
9560 for (i = 0; i < 100; i++) {
9561 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9562 break;
9563 mdelay(10);
9564 }
9565 if (i == 100)
9566 return -ETIMEDOUT;
9567
9568 return 0;
9569}
9570
8a7cd276 9571static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9572{
8a7cd276
AD		 9573 struct pci_dev *root = rdev->pdev->bus->self;
9574 int bridge_pos, gpu_pos;
9575 u32 speed_cntl, mask, current_data_rate;
9576 int ret, i;
9577 u16 tmp16;
87167bb1 9578
0bd252de
AW		 9579 if (pci_is_root_bus(rdev->pdev->bus))
9580 return;
9581
8a7cd276
AD		 9582 if (radeon_pcie_gen2 == 0)
9583 return;
87167bb1 9584
8a7cd276
AD		 9585 if (rdev->flags & RADEON_IS_IGP)
9586 return;
87167bb1 9587
8a7cd276
AD		 9588 if (!(rdev->flags & RADEON_IS_PCIE))
9589 return;
87167bb1 9590
8a7cd276
AD		 9591 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9592 if (ret != 0)
9593 return;
87167bb1 9594
8a7cd276
AD		 9595 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9596 return;
87167bb1 9597
8a7cd276
AD		 9598 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9599 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9600 LC_CURRENT_DATA_RATE_SHIFT;
9601 if (mask & DRM_PCIE_SPEED_80) {
9602 if (current_data_rate == 2) {
9603 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9604 return;
9605 }
9606 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9607 } else if (mask & DRM_PCIE_SPEED_50) {
9608 if (current_data_rate == 1) {
9609 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9610 return;
9611 }
9612 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9613 }
87167bb1 9614
8a7cd276
AD		 9615 bridge_pos = pci_pcie_cap(root);
9616 if (!bridge_pos)
9617 return;
9618
9619 gpu_pos = pci_pcie_cap(rdev->pdev);
9620 if (!gpu_pos)
9621 return;
9622
9623 if (mask & DRM_PCIE_SPEED_80) {
9624 /* re-try equalization if gen3 is not already enabled */
9625 if (current_data_rate != 2) {
9626 u16 bridge_cfg, gpu_cfg;
9627 u16 bridge_cfg2, gpu_cfg2;
9628 u32 max_lw, current_lw, tmp;
9629
9630 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9631 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9632
9633 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9634 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9635
9636 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9637 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9638
9639 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9640 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9641 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9642
9643 if (current_lw < max_lw) {
9644 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9645 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9646 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9647 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9648 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9649 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9650 }
9651 }
9652
9653 for (i = 0; i < 10; i++) {
9654 /* check status */
9655 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9656 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9657 break;
9658
9659 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9661
9662 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9663 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9664
9665 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9666 tmp |= LC_SET_QUIESCE;
9667 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9668
9669 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9670 tmp |= LC_REDO_EQ;
9671 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9672
9673 mdelay(100);
9674
9675 /* linkctl */
9676 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9677 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9678 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9679 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9680
9681 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9682 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9683 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9684 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9685
9686 /* linkctl2 */
9687 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9688 tmp16 &= ~((1 << 4) | (7 << 9));
9689 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9690 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9691
9692 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9693 tmp16 &= ~((1 << 4) | (7 << 9));
9694 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9695 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9696
9697 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9698 tmp &= ~LC_SET_QUIESCE;
9699 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9700 }
9701 }
9702 }
9703
9704 /* set the link speed */
9705 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9706 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9707 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9708
9709 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9710 tmp16 &= ~0xf;
9711 if (mask & DRM_PCIE_SPEED_80)
9712 tmp16 |= 3; /* gen3 */
9713 else if (mask & DRM_PCIE_SPEED_50)
9714 tmp16 |= 2; /* gen2 */
9715 else
9716 tmp16 |= 1; /* gen1 */
9717 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9718
9719 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9720 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9721 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9722
9723 for (i = 0; i < rdev->usec_timeout; i++) {
9724 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9725 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9726 break;
9727 udelay(1);
9728 }
9729}
7235711a
AD		 9730
9731static void cik_program_aspm(struct radeon_device *rdev)
9732{
9733 u32 data, orig;
9734 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9735 bool disable_clkreq = false;
9736
9737 if (radeon_aspm == 0)
9738 return;
9739
9740 /* XXX double check IGPs */
9741 if (rdev->flags & RADEON_IS_IGP)
9742 return;
9743
9744 if (!(rdev->flags & RADEON_IS_PCIE))
9745 return;
9746
9747 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9748 data &= ~LC_XMIT_N_FTS_MASK;
9749 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9750 if (orig != data)
9751 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9752
9753 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9754 data |= LC_GO_TO_RECOVERY;
9755 if (orig != data)
9756 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9757
9758 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9759 data |= P_IGNORE_EDB_ERR;
9760 if (orig != data)
9761 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9762
9763 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9764 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9765 data |= LC_PMI_TO_L1_DIS;
9766 if (!disable_l0s)
9767 data |= LC_L0S_INACTIVITY(7);
9768
9769 if (!disable_l1) {
9770 data |= LC_L1_INACTIVITY(7);
9771 data &= ~LC_PMI_TO_L1_DIS;
9772 if (orig != data)
9773 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9774
9775 if (!disable_plloff_in_l1) {
9776 bool clk_req_support;
9777
9778 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9779 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9780 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9781 if (orig != data)
9782 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9783
9784 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9785 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9786 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9787 if (orig != data)
9788 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9789
9790 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9791 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9792 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9793 if (orig != data)
9794 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9795
9796 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9797 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9798 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9799 if (orig != data)
9800 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9801
9802 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9803 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9804 data |= LC_DYN_LANES_PWR_STATE(3);
9805 if (orig != data)
9806 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9807
0bd252de
AW		 9808 if (!disable_clkreq &&
9809 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9810 struct pci_dev *root = rdev->pdev->bus->self;
9811 u32 lnkcap;
9812
9813 clk_req_support = false;
9814 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9815 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9816 clk_req_support = true;
9817 } else {
9818 clk_req_support = false;
9819 }
9820
9821 if (clk_req_support) {
9822 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9823 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9824 if (orig != data)
9825 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9826
9827 orig = data = RREG32_SMC(THM_CLK_CNTL);
9828 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9829 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9830 if (orig != data)
9831 WREG32_SMC(THM_CLK_CNTL, data);
9832
9833 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9834 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9835 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9836 if (orig != data)
9837 WREG32_SMC(MISC_CLK_CTRL, data);
9838
9839 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9840 data &= ~BCLK_AS_XCLK;
9841 if (orig != data)
9842 WREG32_SMC(CG_CLKPIN_CNTL, data);
9843
9844 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9845 data &= ~FORCE_BIF_REFCLK_EN;
9846 if (orig != data)
9847 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9848
9849 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9850 data &= ~MPLL_CLKOUT_SEL_MASK;
9851 data |= MPLL_CLKOUT_SEL(4);
9852 if (orig != data)
9853 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9854 }
9855 }
9856 } else {
9857 if (orig != data)
9858 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9859 }
9860
9861 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9862 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9863 if (orig != data)
9864 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9865
9866 if (!disable_l0s) {
9867 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9868 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9869 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9870 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9871 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9872 data &= ~LC_L0S_INACTIVITY_MASK;
9873 if (orig != data)
9874 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9875 }
9876 }
9877 }
87167bb1 9878}
Linux kernel - Deliverables
This page took 1.098659 seconds and 5 git commands to generate.