projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fix VM_CONTEXT*_PAGE_TABLE_END_ADDR handling
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
e28740ec 36#include "radeon_kfd.h"
02c81327
AD		 37
38MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 43MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 44MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 45MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 46MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 47
48MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49MODULE_FIRMWARE("radeon/bonaire_me.bin");
50MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55MODULE_FIRMWARE("radeon/bonaire_smc.bin");
56
d4775655
AD		 57MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 62MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 63MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 66
67MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68MODULE_FIRMWARE("radeon/hawaii_me.bin");
69MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74MODULE_FIRMWARE("radeon/hawaii_smc.bin");
75
02c81327
AD		 76MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 81MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 82
83MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84MODULE_FIRMWARE("radeon/kaveri_me.bin");
85MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
90
02c81327
AD		 91MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92MODULE_FIRMWARE("radeon/KABINI_me.bin");
93MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 96MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 97
98MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99MODULE_FIRMWARE("radeon/kabini_me.bin");
100MODULE_FIRMWARE("radeon/kabini_ce.bin");
101MODULE_FIRMWARE("radeon/kabini_mec.bin");
102MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103MODULE_FIRMWARE("radeon/kabini_sdma.bin");
104
f73a9e83
SL		 105MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 111
f2c6b0f4
AD		 112MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113MODULE_FIRMWARE("radeon/mullins_me.bin");
114MODULE_FIRMWARE("radeon/mullins_ce.bin");
115MODULE_FIRMWARE("radeon/mullins_mec.bin");
116MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117MODULE_FIRMWARE("radeon/mullins_sdma.bin");
118
a59781bb
AD		 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 123extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 124extern void sumo_rlc_fini(struct radeon_device *rdev);
125extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 126extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 127extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 128extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 129static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 130extern int cik_sdma_resume(struct radeon_device *rdev);
131extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 133extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 134static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 135static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 136static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 137static void cik_init_pg(struct radeon_device *rdev);
138static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 139static void cik_fini_pg(struct radeon_device *rdev);
140static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 141static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 bool enable);
6f2043ce 143
353eec2a
AD		 144/**
145 * cik_get_allowed_info_register - fetch the register for the info ioctl
146 *
147 * @rdev: radeon_device pointer
148 * @reg: register offset in bytes
149 * @val: register value
150 *
151 * Returns 0 for success or -EINVAL for an invalid register
152 *
153 */
154int cik_get_allowed_info_register(struct radeon_device *rdev,
155 u32 reg, u32 *val)
156{
157 switch (reg) {
158 case GRBM_STATUS:
159 case GRBM_STATUS2:
160 case GRBM_STATUS_SE0:
161 case GRBM_STATUS_SE1:
162 case GRBM_STATUS_SE2:
163 case GRBM_STATUS_SE3:
164 case SRBM_STATUS:
165 case SRBM_STATUS2:
166 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
167 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
168 case UVD_STATUS:
169 /* TODO VCE */
170 *val = RREG32(reg);
171 return 0;
172 default:
173 return -EINVAL;
174 }
175}
176
286d9cc6
AD		 177/* get temperature in millidegrees */
178int ci_get_temp(struct radeon_device *rdev)
179{
180 u32 temp;
181 int actual_temp = 0;
182
183 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
184 CTF_TEMP_SHIFT;
185
186 if (temp & 0x200)
187 actual_temp = 255;
188 else
189 actual_temp = temp & 0x1ff;
190
191 actual_temp = actual_temp * 1000;
192
193 return actual_temp;
194}
195
196/* get temperature in millidegrees */
197int kv_get_temp(struct radeon_device *rdev)
198{
199 u32 temp;
200 int actual_temp = 0;
201
202 temp = RREG32_SMC(0xC0300E0C);
203
204 if (temp)
205 actual_temp = (temp / 8) - 49;
206 else
207 actual_temp = 0;
208
209 actual_temp = actual_temp * 1000;
210
211 return actual_temp;
212}
6f2043ce 213
6e2c3c0a
AD		 214/*
215 * Indirect registers accessor
216 */
217u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
218{
0a5b7b0b 219 unsigned long flags;
6e2c3c0a
AD		 220 u32 r;
221
0a5b7b0b 222 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 223 WREG32(PCIE_INDEX, reg);
224 (void)RREG32(PCIE_INDEX);
225 r = RREG32(PCIE_DATA);
0a5b7b0b 226 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 227 return r;
228}
229
230void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
231{
0a5b7b0b
AD		 232 unsigned long flags;
233
234 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 235 WREG32(PCIE_INDEX, reg);
236 (void)RREG32(PCIE_INDEX);
237 WREG32(PCIE_DATA, v);
238 (void)RREG32(PCIE_DATA);
0a5b7b0b 239 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 240}
241
22c775ce
AD		 242static const u32 spectre_rlc_save_restore_register_list[] =
243{
244 (0x0e00 << 16) | (0xc12c >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc140 >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc150 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc15c >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc168 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc170 >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc178 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc204 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc2b4 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc2b8 >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2bc >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2c0 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0x8228 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0x829c >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0x869c >> 2),
273 0x00000000,
274 (0x0600 << 16) | (0x98f4 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x98f8 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x9900 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0xc260 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x90e8 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3c000 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0x3c00c >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0x8c1c >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0x9700 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xcd20 >> 2),
293 0x00000000,
294 (0x4e00 << 16) | (0xcd20 >> 2),
295 0x00000000,
296 (0x5e00 << 16) | (0xcd20 >> 2),
297 0x00000000,
298 (0x6e00 << 16) | (0xcd20 >> 2),
299 0x00000000,
300 (0x7e00 << 16) | (0xcd20 >> 2),
301 0x00000000,
302 (0x8e00 << 16) | (0xcd20 >> 2),
303 0x00000000,
304 (0x9e00 << 16) | (0xcd20 >> 2),
305 0x00000000,
306 (0xae00 << 16) | (0xcd20 >> 2),
307 0x00000000,
308 (0xbe00 << 16) | (0xcd20 >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0x89bc >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0x8900 >> 2),
313 0x00000000,
314 0x3,
315 (0x0e00 << 16) | (0xc130 >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0xc134 >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0xc1fc >> 2),
320 0x00000000,
321 (0x0e00 << 16) | (0xc208 >> 2),
322 0x00000000,
323 (0x0e00 << 16) | (0xc264 >> 2),
324 0x00000000,
325 (0x0e00 << 16) | (0xc268 >> 2),
326 0x00000000,
327 (0x0e00 << 16) | (0xc26c >> 2),
328 0x00000000,
329 (0x0e00 << 16) | (0xc270 >> 2),
330 0x00000000,
331 (0x0e00 << 16) | (0xc274 >> 2),
332 0x00000000,
333 (0x0e00 << 16) | (0xc278 >> 2),
334 0x00000000,
335 (0x0e00 << 16) | (0xc27c >> 2),
336 0x00000000,
337 (0x0e00 << 16) | (0xc280 >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0xc284 >> 2),
340 0x00000000,
341 (0x0e00 << 16) | (0xc288 >> 2),
342 0x00000000,
343 (0x0e00 << 16) | (0xc28c >> 2),
344 0x00000000,
345 (0x0e00 << 16) | (0xc290 >> 2),
346 0x00000000,
347 (0x0e00 << 16) | (0xc294 >> 2),
348 0x00000000,
349 (0x0e00 << 16) | (0xc298 >> 2),
350 0x00000000,
351 (0x0e00 << 16) | (0xc29c >> 2),
352 0x00000000,
353 (0x0e00 << 16) | (0xc2a0 >> 2),
354 0x00000000,
355 (0x0e00 << 16) | (0xc2a4 >> 2),
356 0x00000000,
357 (0x0e00 << 16) | (0xc2a8 >> 2),
358 0x00000000,
359 (0x0e00 << 16) | (0xc2ac >> 2),
360 0x00000000,
361 (0x0e00 << 16) | (0xc2b0 >> 2),
362 0x00000000,
363 (0x0e00 << 16) | (0x301d0 >> 2),
364 0x00000000,
365 (0x0e00 << 16) | (0x30238 >> 2),
366 0x00000000,
367 (0x0e00 << 16) | (0x30250 >> 2),
368 0x00000000,
369 (0x0e00 << 16) | (0x30254 >> 2),
370 0x00000000,
371 (0x0e00 << 16) | (0x30258 >> 2),
372 0x00000000,
373 (0x0e00 << 16) | (0x3025c >> 2),
374 0x00000000,
375 (0x4e00 << 16) | (0xc900 >> 2),
376 0x00000000,
377 (0x5e00 << 16) | (0xc900 >> 2),
378 0x00000000,
379 (0x6e00 << 16) | (0xc900 >> 2),
380 0x00000000,
381 (0x7e00 << 16) | (0xc900 >> 2),
382 0x00000000,
383 (0x8e00 << 16) | (0xc900 >> 2),
384 0x00000000,
385 (0x9e00 << 16) | (0xc900 >> 2),
386 0x00000000,
387 (0xae00 << 16) | (0xc900 >> 2),
388 0x00000000,
389 (0xbe00 << 16) | (0xc900 >> 2),
390 0x00000000,
391 (0x4e00 << 16) | (0xc904 >> 2),
392 0x00000000,
393 (0x5e00 << 16) | (0xc904 >> 2),
394 0x00000000,
395 (0x6e00 << 16) | (0xc904 >> 2),
396 0x00000000,
397 (0x7e00 << 16) | (0xc904 >> 2),
398 0x00000000,
399 (0x8e00 << 16) | (0xc904 >> 2),
400 0x00000000,
401 (0x9e00 << 16) | (0xc904 >> 2),
402 0x00000000,
403 (0xae00 << 16) | (0xc904 >> 2),
404 0x00000000,
405 (0xbe00 << 16) | (0xc904 >> 2),
406 0x00000000,
407 (0x4e00 << 16) | (0xc908 >> 2),
408 0x00000000,
409 (0x5e00 << 16) | (0xc908 >> 2),
410 0x00000000,
411 (0x6e00 << 16) | (0xc908 >> 2),
412 0x00000000,
413 (0x7e00 << 16) | (0xc908 >> 2),
414 0x00000000,
415 (0x8e00 << 16) | (0xc908 >> 2),
416 0x00000000,
417 (0x9e00 << 16) | (0xc908 >> 2),
418 0x00000000,
419 (0xae00 << 16) | (0xc908 >> 2),
420 0x00000000,
421 (0xbe00 << 16) | (0xc908 >> 2),
422 0x00000000,
423 (0x4e00 << 16) | (0xc90c >> 2),
424 0x00000000,
425 (0x5e00 << 16) | (0xc90c >> 2),
426 0x00000000,
427 (0x6e00 << 16) | (0xc90c >> 2),
428 0x00000000,
429 (0x7e00 << 16) | (0xc90c >> 2),
430 0x00000000,
431 (0x8e00 << 16) | (0xc90c >> 2),
432 0x00000000,
433 (0x9e00 << 16) | (0xc90c >> 2),
434 0x00000000,
435 (0xae00 << 16) | (0xc90c >> 2),
436 0x00000000,
437 (0xbe00 << 16) | (0xc90c >> 2),
438 0x00000000,
439 (0x4e00 << 16) | (0xc910 >> 2),
440 0x00000000,
441 (0x5e00 << 16) | (0xc910 >> 2),
442 0x00000000,
443 (0x6e00 << 16) | (0xc910 >> 2),
444 0x00000000,
445 (0x7e00 << 16) | (0xc910 >> 2),
446 0x00000000,
447 (0x8e00 << 16) | (0xc910 >> 2),
448 0x00000000,
449 (0x9e00 << 16) | (0xc910 >> 2),
450 0x00000000,
451 (0xae00 << 16) | (0xc910 >> 2),
452 0x00000000,
453 (0xbe00 << 16) | (0xc910 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0xc99c >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x9834 >> 2),
458 0x00000000,
459 (0x0000 << 16) | (0x30f00 >> 2),
460 0x00000000,
461 (0x0001 << 16) | (0x30f00 >> 2),
462 0x00000000,
463 (0x0000 << 16) | (0x30f04 >> 2),
464 0x00000000,
465 (0x0001 << 16) | (0x30f04 >> 2),
466 0x00000000,
467 (0x0000 << 16) | (0x30f08 >> 2),
468 0x00000000,
469 (0x0001 << 16) | (0x30f08 >> 2),
470 0x00000000,
471 (0x0000 << 16) | (0x30f0c >> 2),
472 0x00000000,
473 (0x0001 << 16) | (0x30f0c >> 2),
474 0x00000000,
475 (0x0600 << 16) | (0x9b7c >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8a14 >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0x8a18 >> 2),
480 0x00000000,
481 (0x0600 << 16) | (0x30a00 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0x8bf0 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0x8bcc >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0x8b24 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0x30a04 >> 2),
490 0x00000000,
491 (0x0600 << 16) | (0x30a10 >> 2),
492 0x00000000,
493 (0x0600 << 16) | (0x30a14 >> 2),
494 0x00000000,
495 (0x0600 << 16) | (0x30a18 >> 2),
496 0x00000000,
497 (0x0600 << 16) | (0x30a2c >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xc700 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xc704 >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xc708 >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xc768 >> 2),
506 0x00000000,
507 (0x0400 << 16) | (0xc770 >> 2),
508 0x00000000,
509 (0x0400 << 16) | (0xc774 >> 2),
510 0x00000000,
511 (0x0400 << 16) | (0xc778 >> 2),
512 0x00000000,
513 (0x0400 << 16) | (0xc77c >> 2),
514 0x00000000,
515 (0x0400 << 16) | (0xc780 >> 2),
516 0x00000000,
517 (0x0400 << 16) | (0xc784 >> 2),
518 0x00000000,
519 (0x0400 << 16) | (0xc788 >> 2),
520 0x00000000,
521 (0x0400 << 16) | (0xc78c >> 2),
522 0x00000000,
523 (0x0400 << 16) | (0xc798 >> 2),
524 0x00000000,
525 (0x0400 << 16) | (0xc79c >> 2),
526 0x00000000,
527 (0x0400 << 16) | (0xc7a0 >> 2),
528 0x00000000,
529 (0x0400 << 16) | (0xc7a4 >> 2),
530 0x00000000,
531 (0x0400 << 16) | (0xc7a8 >> 2),
532 0x00000000,
533 (0x0400 << 16) | (0xc7ac >> 2),
534 0x00000000,
535 (0x0400 << 16) | (0xc7b0 >> 2),
536 0x00000000,
537 (0x0400 << 16) | (0xc7b4 >> 2),
538 0x00000000,
539 (0x0e00 << 16) | (0x9100 >> 2),
540 0x00000000,
541 (0x0e00 << 16) | (0x3c010 >> 2),
542 0x00000000,
543 (0x0e00 << 16) | (0x92a8 >> 2),
544 0x00000000,
545 (0x0e00 << 16) | (0x92ac >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0x92b4 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0x92b8 >> 2),
550 0x00000000,
551 (0x0e00 << 16) | (0x92bc >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x92c0 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x92c4 >> 2),
556 0x00000000,
557 (0x0e00 << 16) | (0x92c8 >> 2),
558 0x00000000,
559 (0x0e00 << 16) | (0x92cc >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x92d0 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x8c00 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x8c04 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x8c20 >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x8c38 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x8c3c >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0xae00 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x9604 >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0xac08 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0xac0c >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0xac10 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0xac14 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0xac58 >> 2),
586 0x00000000,
587 (0x0e00 << 16) | (0xac68 >> 2),
588 0x00000000,
589 (0x0e00 << 16) | (0xac6c >> 2),
590 0x00000000,
591 (0x0e00 << 16) | (0xac70 >> 2),
592 0x00000000,
593 (0x0e00 << 16) | (0xac74 >> 2),
594 0x00000000,
595 (0x0e00 << 16) | (0xac78 >> 2),
596 0x00000000,
597 (0x0e00 << 16) | (0xac7c >> 2),
598 0x00000000,
599 (0x0e00 << 16) | (0xac80 >> 2),
600 0x00000000,
601 (0x0e00 << 16) | (0xac84 >> 2),
602 0x00000000,
603 (0x0e00 << 16) | (0xac88 >> 2),
604 0x00000000,
605 (0x0e00 << 16) | (0xac8c >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0x970c >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0x9714 >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0x9718 >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0x971c >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0x31068 >> 2),
616 0x00000000,
617 (0x4e00 << 16) | (0x31068 >> 2),
618 0x00000000,
619 (0x5e00 << 16) | (0x31068 >> 2),
620 0x00000000,
621 (0x6e00 << 16) | (0x31068 >> 2),
622 0x00000000,
623 (0x7e00 << 16) | (0x31068 >> 2),
624 0x00000000,
625 (0x8e00 << 16) | (0x31068 >> 2),
626 0x00000000,
627 (0x9e00 << 16) | (0x31068 >> 2),
628 0x00000000,
629 (0xae00 << 16) | (0x31068 >> 2),
630 0x00000000,
631 (0xbe00 << 16) | (0x31068 >> 2),
632 0x00000000,
633 (0x0e00 << 16) | (0xcd10 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0xcd14 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x88b0 >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0x88b4 >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x88b8 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0x88bc >> 2),
644 0x00000000,
645 (0x0400 << 16) | (0x89c0 >> 2),
646 0x00000000,
647 (0x0e00 << 16) | (0x88c4 >> 2),
648 0x00000000,
649 (0x0e00 << 16) | (0x88c8 >> 2),
650 0x00000000,
651 (0x0e00 << 16) | (0x88d0 >> 2),
652 0x00000000,
653 (0x0e00 << 16) | (0x88d4 >> 2),
654 0x00000000,
655 (0x0e00 << 16) | (0x88d8 >> 2),
656 0x00000000,
657 (0x0e00 << 16) | (0x8980 >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0x30938 >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0x3093c >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0x30940 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0x89a0 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0x30900 >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0x30904 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0x89b4 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0x3c210 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0x3c214 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0x3c218 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0x8904 >> 2),
680 0x00000000,
681 0x5,
682 (0x0e00 << 16) | (0x8c28 >> 2),
683 (0x0e00 << 16) | (0x8c2c >> 2),
684 (0x0e00 << 16) | (0x8c30 >> 2),
685 (0x0e00 << 16) | (0x8c34 >> 2),
686 (0x0e00 << 16) | (0x9600 >> 2),
687};
688
689static const u32 kalindi_rlc_save_restore_register_list[] =
690{
691 (0x0e00 << 16) | (0xc12c >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc140 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc150 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0xc15c >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0xc168 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0xc170 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0xc204 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0xc2b4 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0xc2b8 >> 2),
708 0x00000000,
709 (0x0e00 << 16) | (0xc2bc >> 2),
710 0x00000000,
711 (0x0e00 << 16) | (0xc2c0 >> 2),
712 0x00000000,
713 (0x0e00 << 16) | (0x8228 >> 2),
714 0x00000000,
715 (0x0e00 << 16) | (0x829c >> 2),
716 0x00000000,
717 (0x0e00 << 16) | (0x869c >> 2),
718 0x00000000,
719 (0x0600 << 16) | (0x98f4 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0x98f8 >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0x9900 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc260 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0x90e8 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0x3c000 >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0x3c00c >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0x8c1c >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0x9700 >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xcd20 >> 2),
738 0x00000000,
739 (0x4e00 << 16) | (0xcd20 >> 2),
740 0x00000000,
741 (0x5e00 << 16) | (0xcd20 >> 2),
742 0x00000000,
743 (0x6e00 << 16) | (0xcd20 >> 2),
744 0x00000000,
745 (0x7e00 << 16) | (0xcd20 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0x89bc >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0x8900 >> 2),
750 0x00000000,
751 0x3,
752 (0x0e00 << 16) | (0xc130 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0xc134 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0xc1fc >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0xc208 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0xc264 >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0xc268 >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0xc26c >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0xc270 >> 2),
767 0x00000000,
768 (0x0e00 << 16) | (0xc274 >> 2),
769 0x00000000,
770 (0x0e00 << 16) | (0xc28c >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0xc290 >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0xc294 >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0xc298 >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0xc2a0 >> 2),
779 0x00000000,
780 (0x0e00 << 16) | (0xc2a4 >> 2),
781 0x00000000,
782 (0x0e00 << 16) | (0xc2a8 >> 2),
783 0x00000000,
784 (0x0e00 << 16) | (0xc2ac >> 2),
785 0x00000000,
786 (0x0e00 << 16) | (0x301d0 >> 2),
787 0x00000000,
788 (0x0e00 << 16) | (0x30238 >> 2),
789 0x00000000,
790 (0x0e00 << 16) | (0x30250 >> 2),
791 0x00000000,
792 (0x0e00 << 16) | (0x30254 >> 2),
793 0x00000000,
794 (0x0e00 << 16) | (0x30258 >> 2),
795 0x00000000,
796 (0x0e00 << 16) | (0x3025c >> 2),
797 0x00000000,
798 (0x4e00 << 16) | (0xc900 >> 2),
799 0x00000000,
800 (0x5e00 << 16) | (0xc900 >> 2),
801 0x00000000,
802 (0x6e00 << 16) | (0xc900 >> 2),
803 0x00000000,
804 (0x7e00 << 16) | (0xc900 >> 2),
805 0x00000000,
806 (0x4e00 << 16) | (0xc904 >> 2),
807 0x00000000,
808 (0x5e00 << 16) | (0xc904 >> 2),
809 0x00000000,
810 (0x6e00 << 16) | (0xc904 >> 2),
811 0x00000000,
812 (0x7e00 << 16) | (0xc904 >> 2),
813 0x00000000,
814 (0x4e00 << 16) | (0xc908 >> 2),
815 0x00000000,
816 (0x5e00 << 16) | (0xc908 >> 2),
817 0x00000000,
818 (0x6e00 << 16) | (0xc908 >> 2),
819 0x00000000,
820 (0x7e00 << 16) | (0xc908 >> 2),
821 0x00000000,
822 (0x4e00 << 16) | (0xc90c >> 2),
823 0x00000000,
824 (0x5e00 << 16) | (0xc90c >> 2),
825 0x00000000,
826 (0x6e00 << 16) | (0xc90c >> 2),
827 0x00000000,
828 (0x7e00 << 16) | (0xc90c >> 2),
829 0x00000000,
830 (0x4e00 << 16) | (0xc910 >> 2),
831 0x00000000,
832 (0x5e00 << 16) | (0xc910 >> 2),
833 0x00000000,
834 (0x6e00 << 16) | (0xc910 >> 2),
835 0x00000000,
836 (0x7e00 << 16) | (0xc910 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xc99c >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0x9834 >> 2),
841 0x00000000,
842 (0x0000 << 16) | (0x30f00 >> 2),
843 0x00000000,
844 (0x0000 << 16) | (0x30f04 >> 2),
845 0x00000000,
846 (0x0000 << 16) | (0x30f08 >> 2),
847 0x00000000,
848 (0x0000 << 16) | (0x30f0c >> 2),
849 0x00000000,
850 (0x0600 << 16) | (0x9b7c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x8a14 >> 2),
853 0x00000000,
854 (0x0e00 << 16) | (0x8a18 >> 2),
855 0x00000000,
856 (0x0600 << 16) | (0x30a00 >> 2),
857 0x00000000,
858 (0x0e00 << 16) | (0x8bf0 >> 2),
859 0x00000000,
860 (0x0e00 << 16) | (0x8bcc >> 2),
861 0x00000000,
862 (0x0e00 << 16) | (0x8b24 >> 2),
863 0x00000000,
864 (0x0e00 << 16) | (0x30a04 >> 2),
865 0x00000000,
866 (0x0600 << 16) | (0x30a10 >> 2),
867 0x00000000,
868 (0x0600 << 16) | (0x30a14 >> 2),
869 0x00000000,
870 (0x0600 << 16) | (0x30a18 >> 2),
871 0x00000000,
872 (0x0600 << 16) | (0x30a2c >> 2),
873 0x00000000,
874 (0x0e00 << 16) | (0xc700 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0xc704 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0xc708 >> 2),
879 0x00000000,
880 (0x0e00 << 16) | (0xc768 >> 2),
881 0x00000000,
882 (0x0400 << 16) | (0xc770 >> 2),
883 0x00000000,
884 (0x0400 << 16) | (0xc774 >> 2),
885 0x00000000,
886 (0x0400 << 16) | (0xc798 >> 2),
887 0x00000000,
888 (0x0400 << 16) | (0xc79c >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x9100 >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x3c010 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x8c00 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x8c04 >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x8c20 >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x8c38 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x8c3c >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0xae00 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x9604 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0xac08 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0xac0c >> 2),
911 0x00000000,
912 (0x0e00 << 16) | (0xac10 >> 2),
913 0x00000000,
914 (0x0e00 << 16) | (0xac14 >> 2),
915 0x00000000,
916 (0x0e00 << 16) | (0xac58 >> 2),
917 0x00000000,
918 (0x0e00 << 16) | (0xac68 >> 2),
919 0x00000000,
920 (0x0e00 << 16) | (0xac6c >> 2),
921 0x00000000,
922 (0x0e00 << 16) | (0xac70 >> 2),
923 0x00000000,
924 (0x0e00 << 16) | (0xac74 >> 2),
925 0x00000000,
926 (0x0e00 << 16) | (0xac78 >> 2),
927 0x00000000,
928 (0x0e00 << 16) | (0xac7c >> 2),
929 0x00000000,
930 (0x0e00 << 16) | (0xac80 >> 2),
931 0x00000000,
932 (0x0e00 << 16) | (0xac84 >> 2),
933 0x00000000,
934 (0x0e00 << 16) | (0xac88 >> 2),
935 0x00000000,
936 (0x0e00 << 16) | (0xac8c >> 2),
937 0x00000000,
938 (0x0e00 << 16) | (0x970c >> 2),
939 0x00000000,
940 (0x0e00 << 16) | (0x9714 >> 2),
941 0x00000000,
942 (0x0e00 << 16) | (0x9718 >> 2),
943 0x00000000,
944 (0x0e00 << 16) | (0x971c >> 2),
945 0x00000000,
946 (0x0e00 << 16) | (0x31068 >> 2),
947 0x00000000,
948 (0x4e00 << 16) | (0x31068 >> 2),
949 0x00000000,
950 (0x5e00 << 16) | (0x31068 >> 2),
951 0x00000000,
952 (0x6e00 << 16) | (0x31068 >> 2),
953 0x00000000,
954 (0x7e00 << 16) | (0x31068 >> 2),
955 0x00000000,
956 (0x0e00 << 16) | (0xcd10 >> 2),
957 0x00000000,
958 (0x0e00 << 16) | (0xcd14 >> 2),
959 0x00000000,
960 (0x0e00 << 16) | (0x88b0 >> 2),
961 0x00000000,
962 (0x0e00 << 16) | (0x88b4 >> 2),
963 0x00000000,
964 (0x0e00 << 16) | (0x88b8 >> 2),
965 0x00000000,
966 (0x0e00 << 16) | (0x88bc >> 2),
967 0x00000000,
968 (0x0400 << 16) | (0x89c0 >> 2),
969 0x00000000,
970 (0x0e00 << 16) | (0x88c4 >> 2),
971 0x00000000,
972 (0x0e00 << 16) | (0x88c8 >> 2),
973 0x00000000,
974 (0x0e00 << 16) | (0x88d0 >> 2),
975 0x00000000,
976 (0x0e00 << 16) | (0x88d4 >> 2),
977 0x00000000,
978 (0x0e00 << 16) | (0x88d8 >> 2),
979 0x00000000,
980 (0x0e00 << 16) | (0x8980 >> 2),
981 0x00000000,
982 (0x0e00 << 16) | (0x30938 >> 2),
983 0x00000000,
984 (0x0e00 << 16) | (0x3093c >> 2),
985 0x00000000,
986 (0x0e00 << 16) | (0x30940 >> 2),
987 0x00000000,
988 (0x0e00 << 16) | (0x89a0 >> 2),
989 0x00000000,
990 (0x0e00 << 16) | (0x30900 >> 2),
991 0x00000000,
992 (0x0e00 << 16) | (0x30904 >> 2),
993 0x00000000,
994 (0x0e00 << 16) | (0x89b4 >> 2),
995 0x00000000,
996 (0x0e00 << 16) | (0x3e1fc >> 2),
997 0x00000000,
998 (0x0e00 << 16) | (0x3c210 >> 2),
999 0x00000000,
1000 (0x0e00 << 16) | (0x3c214 >> 2),
1001 0x00000000,
1002 (0x0e00 << 16) | (0x3c218 >> 2),
1003 0x00000000,
1004 (0x0e00 << 16) | (0x8904 >> 2),
1005 0x00000000,
1006 0x5,
1007 (0x0e00 << 16) | (0x8c28 >> 2),
1008 (0x0e00 << 16) | (0x8c2c >> 2),
1009 (0x0e00 << 16) | (0x8c30 >> 2),
1010 (0x0e00 << 16) | (0x8c34 >> 2),
1011 (0x0e00 << 16) | (0x9600 >> 2),
1012};
1013
0aafd313
AD		 1014static const u32 bonaire_golden_spm_registers[] =
1015{
1016 0x30800, 0xe0ffffff, 0xe0000000
1017};
1018
1019static const u32 bonaire_golden_common_registers[] =
1020{
1021 0xc770, 0xffffffff, 0x00000800,
1022 0xc774, 0xffffffff, 0x00000800,
1023 0xc798, 0xffffffff, 0x00007fbf,
1024 0xc79c, 0xffffffff, 0x00007faf
1025};
1026
1027static const u32 bonaire_golden_registers[] =
1028{
1029 0x3354, 0x00000333, 0x00000333,
1030 0x3350, 0x000c0fc0, 0x00040200,
1031 0x9a10, 0x00010000, 0x00058208,
1032 0x3c000, 0xffff1fff, 0x00140000,
1033 0x3c200, 0xfdfc0fff, 0x00000100,
1034 0x3c234, 0x40000000, 0x40000200,
1035 0x9830, 0xffffffff, 0x00000000,
1036 0x9834, 0xf00fffff, 0x00000400,
1037 0x9838, 0x0002021c, 0x00020200,
1038 0xc78, 0x00000080, 0x00000000,
1039 0x5bb0, 0x000000f0, 0x00000070,
1040 0x5bc0, 0xf0311fff, 0x80300000,
1041 0x98f8, 0x73773777, 0x12010001,
1042 0x350c, 0x00810000, 0x408af000,
1043 0x7030, 0x31000111, 0x00000011,
1044 0x2f48, 0x73773777, 0x12010001,
1045 0x220c, 0x00007fb6, 0x0021a1b1,
1046 0x2210, 0x00007fb6, 0x002021b1,
1047 0x2180, 0x00007fb6, 0x00002191,
1048 0x2218, 0x00007fb6, 0x002121b1,
1049 0x221c, 0x00007fb6, 0x002021b1,
1050 0x21dc, 0x00007fb6, 0x00002191,
1051 0x21e0, 0x00007fb6, 0x00002191,
1052 0x3628, 0x0000003f, 0x0000000a,
1053 0x362c, 0x0000003f, 0x0000000a,
1054 0x2ae4, 0x00073ffe, 0x000022a2,
1055 0x240c, 0x000007ff, 0x00000000,
1056 0x8a14, 0xf000003f, 0x00000007,
1057 0x8bf0, 0x00002001, 0x00000001,
1058 0x8b24, 0xffffffff, 0x00ffffff,
1059 0x30a04, 0x0000ff0f, 0x00000000,
1060 0x28a4c, 0x07ffffff, 0x06000000,
1061 0x4d8, 0x00000fff, 0x00000100,
1062 0x3e78, 0x00000001, 0x00000002,
1063 0x9100, 0x03000000, 0x0362c688,
1064 0x8c00, 0x000000ff, 0x00000001,
1065 0xe40, 0x00001fff, 0x00001fff,
1066 0x9060, 0x0000007f, 0x00000020,
1067 0x9508, 0x00010000, 0x00010000,
1068 0xac14, 0x000003ff, 0x000000f3,
1069 0xac0c, 0xffffffff, 0x00001032
1070};
1071
1072static const u32 bonaire_mgcg_cgcg_init[] =
1073{
1074 0xc420, 0xffffffff, 0xfffffffc,
1075 0x30800, 0xffffffff, 0xe0000000,
1076 0x3c2a0, 0xffffffff, 0x00000100,
1077 0x3c208, 0xffffffff, 0x00000100,
1078 0x3c2c0, 0xffffffff, 0xc0000100,
1079 0x3c2c8, 0xffffffff, 0xc0000100,
1080 0x3c2c4, 0xffffffff, 0xc0000100,
1081 0x55e4, 0xffffffff, 0x00600100,
1082 0x3c280, 0xffffffff, 0x00000100,
1083 0x3c214, 0xffffffff, 0x06000100,
1084 0x3c220, 0xffffffff, 0x00000100,
1085 0x3c218, 0xffffffff, 0x06000100,
1086 0x3c204, 0xffffffff, 0x00000100,
1087 0x3c2e0, 0xffffffff, 0x00000100,
1088 0x3c224, 0xffffffff, 0x00000100,
1089 0x3c200, 0xffffffff, 0x00000100,
1090 0x3c230, 0xffffffff, 0x00000100,
1091 0x3c234, 0xffffffff, 0x00000100,
1092 0x3c250, 0xffffffff, 0x00000100,
1093 0x3c254, 0xffffffff, 0x00000100,
1094 0x3c258, 0xffffffff, 0x00000100,
1095 0x3c25c, 0xffffffff, 0x00000100,
1096 0x3c260, 0xffffffff, 0x00000100,
1097 0x3c27c, 0xffffffff, 0x00000100,
1098 0x3c278, 0xffffffff, 0x00000100,
1099 0x3c210, 0xffffffff, 0x06000100,
1100 0x3c290, 0xffffffff, 0x00000100,
1101 0x3c274, 0xffffffff, 0x00000100,
1102 0x3c2b4, 0xffffffff, 0x00000100,
1103 0x3c2b0, 0xffffffff, 0x00000100,
1104 0x3c270, 0xffffffff, 0x00000100,
1105 0x30800, 0xffffffff, 0xe0000000,
1106 0x3c020, 0xffffffff, 0x00010000,
1107 0x3c024, 0xffffffff, 0x00030002,
1108 0x3c028, 0xffffffff, 0x00040007,
1109 0x3c02c, 0xffffffff, 0x00060005,
1110 0x3c030, 0xffffffff, 0x00090008,
1111 0x3c034, 0xffffffff, 0x00010000,
1112 0x3c038, 0xffffffff, 0x00030002,
1113 0x3c03c, 0xffffffff, 0x00040007,
1114 0x3c040, 0xffffffff, 0x00060005,
1115 0x3c044, 0xffffffff, 0x00090008,
1116 0x3c048, 0xffffffff, 0x00010000,
1117 0x3c04c, 0xffffffff, 0x00030002,
1118 0x3c050, 0xffffffff, 0x00040007,
1119 0x3c054, 0xffffffff, 0x00060005,
1120 0x3c058, 0xffffffff, 0x00090008,
1121 0x3c05c, 0xffffffff, 0x00010000,
1122 0x3c060, 0xffffffff, 0x00030002,
1123 0x3c064, 0xffffffff, 0x00040007,
1124 0x3c068, 0xffffffff, 0x00060005,
1125 0x3c06c, 0xffffffff, 0x00090008,
1126 0x3c070, 0xffffffff, 0x00010000,
1127 0x3c074, 0xffffffff, 0x00030002,
1128 0x3c078, 0xffffffff, 0x00040007,
1129 0x3c07c, 0xffffffff, 0x00060005,
1130 0x3c080, 0xffffffff, 0x00090008,
1131 0x3c084, 0xffffffff, 0x00010000,
1132 0x3c088, 0xffffffff, 0x00030002,
1133 0x3c08c, 0xffffffff, 0x00040007,
1134 0x3c090, 0xffffffff, 0x00060005,
1135 0x3c094, 0xffffffff, 0x00090008,
1136 0x3c098, 0xffffffff, 0x00010000,
1137 0x3c09c, 0xffffffff, 0x00030002,
1138 0x3c0a0, 0xffffffff, 0x00040007,
1139 0x3c0a4, 0xffffffff, 0x00060005,
1140 0x3c0a8, 0xffffffff, 0x00090008,
1141 0x3c000, 0xffffffff, 0x96e00200,
1142 0x8708, 0xffffffff, 0x00900100,
1143 0xc424, 0xffffffff, 0x0020003f,
1144 0x38, 0xffffffff, 0x0140001c,
1145 0x3c, 0x000f0000, 0x000f0000,
1146 0x220, 0xffffffff, 0xC060000C,
1147 0x224, 0xc0000fff, 0x00000100,
1148 0xf90, 0xffffffff, 0x00000100,
1149 0xf98, 0x00000101, 0x00000000,
1150 0x20a8, 0xffffffff, 0x00000104,
1151 0x55e4, 0xff000fff, 0x00000100,
1152 0x30cc, 0xc0000fff, 0x00000104,
1153 0xc1e4, 0x00000001, 0x00000001,
1154 0xd00c, 0xff000ff0, 0x00000100,
1155 0xd80c, 0xff000ff0, 0x00000100
1156};
1157
1158static const u32 spectre_golden_spm_registers[] =
1159{
1160 0x30800, 0xe0ffffff, 0xe0000000
1161};
1162
1163static const u32 spectre_golden_common_registers[] =
1164{
1165 0xc770, 0xffffffff, 0x00000800,
1166 0xc774, 0xffffffff, 0x00000800,
1167 0xc798, 0xffffffff, 0x00007fbf,
1168 0xc79c, 0xffffffff, 0x00007faf
1169};
1170
1171static const u32 spectre_golden_registers[] =
1172{
1173 0x3c000, 0xffff1fff, 0x96940200,
1174 0x3c00c, 0xffff0001, 0xff000000,
1175 0x3c200, 0xfffc0fff, 0x00000100,
1176 0x6ed8, 0x00010101, 0x00010000,
1177 0x9834, 0xf00fffff, 0x00000400,
1178 0x9838, 0xfffffffc, 0x00020200,
1179 0x5bb0, 0x000000f0, 0x00000070,
1180 0x5bc0, 0xf0311fff, 0x80300000,
1181 0x98f8, 0x73773777, 0x12010001,
1182 0x9b7c, 0x00ff0000, 0x00fc0000,
1183 0x2f48, 0x73773777, 0x12010001,
1184 0x8a14, 0xf000003f, 0x00000007,
1185 0x8b24, 0xffffffff, 0x00ffffff,
1186 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1187 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1188 0x3e78, 0x00000001, 0x00000002,
1189 0x913c, 0xffff03df, 0x00000004,
1190 0xc768, 0x00000008, 0x00000008,
1191 0x8c00, 0x000008ff, 0x00000800,
1192 0x9508, 0x00010000, 0x00010000,
1193 0xac0c, 0xffffffff, 0x54763210,
1194 0x214f8, 0x01ff01ff, 0x00000002,
1195 0x21498, 0x007ff800, 0x00200000,
1196 0x2015c, 0xffffffff, 0x00000f40,
1197 0x30934, 0xffffffff, 0x00000001
1198};
1199
1200static const u32 spectre_mgcg_cgcg_init[] =
1201{
1202 0xc420, 0xffffffff, 0xfffffffc,
1203 0x30800, 0xffffffff, 0xe0000000,
1204 0x3c2a0, 0xffffffff, 0x00000100,
1205 0x3c208, 0xffffffff, 0x00000100,
1206 0x3c2c0, 0xffffffff, 0x00000100,
1207 0x3c2c8, 0xffffffff, 0x00000100,
1208 0x3c2c4, 0xffffffff, 0x00000100,
1209 0x55e4, 0xffffffff, 0x00600100,
1210 0x3c280, 0xffffffff, 0x00000100,
1211 0x3c214, 0xffffffff, 0x06000100,
1212 0x3c220, 0xffffffff, 0x00000100,
1213 0x3c218, 0xffffffff, 0x06000100,
1214 0x3c204, 0xffffffff, 0x00000100,
1215 0x3c2e0, 0xffffffff, 0x00000100,
1216 0x3c224, 0xffffffff, 0x00000100,
1217 0x3c200, 0xffffffff, 0x00000100,
1218 0x3c230, 0xffffffff, 0x00000100,
1219 0x3c234, 0xffffffff, 0x00000100,
1220 0x3c250, 0xffffffff, 0x00000100,
1221 0x3c254, 0xffffffff, 0x00000100,
1222 0x3c258, 0xffffffff, 0x00000100,
1223 0x3c25c, 0xffffffff, 0x00000100,
1224 0x3c260, 0xffffffff, 0x00000100,
1225 0x3c27c, 0xffffffff, 0x00000100,
1226 0x3c278, 0xffffffff, 0x00000100,
1227 0x3c210, 0xffffffff, 0x06000100,
1228 0x3c290, 0xffffffff, 0x00000100,
1229 0x3c274, 0xffffffff, 0x00000100,
1230 0x3c2b4, 0xffffffff, 0x00000100,
1231 0x3c2b0, 0xffffffff, 0x00000100,
1232 0x3c270, 0xffffffff, 0x00000100,
1233 0x30800, 0xffffffff, 0xe0000000,
1234 0x3c020, 0xffffffff, 0x00010000,
1235 0x3c024, 0xffffffff, 0x00030002,
1236 0x3c028, 0xffffffff, 0x00040007,
1237 0x3c02c, 0xffffffff, 0x00060005,
1238 0x3c030, 0xffffffff, 0x00090008,
1239 0x3c034, 0xffffffff, 0x00010000,
1240 0x3c038, 0xffffffff, 0x00030002,
1241 0x3c03c, 0xffffffff, 0x00040007,
1242 0x3c040, 0xffffffff, 0x00060005,
1243 0x3c044, 0xffffffff, 0x00090008,
1244 0x3c048, 0xffffffff, 0x00010000,
1245 0x3c04c, 0xffffffff, 0x00030002,
1246 0x3c050, 0xffffffff, 0x00040007,
1247 0x3c054, 0xffffffff, 0x00060005,
1248 0x3c058, 0xffffffff, 0x00090008,
1249 0x3c05c, 0xffffffff, 0x00010000,
1250 0x3c060, 0xffffffff, 0x00030002,
1251 0x3c064, 0xffffffff, 0x00040007,
1252 0x3c068, 0xffffffff, 0x00060005,
1253 0x3c06c, 0xffffffff, 0x00090008,
1254 0x3c070, 0xffffffff, 0x00010000,
1255 0x3c074, 0xffffffff, 0x00030002,
1256 0x3c078, 0xffffffff, 0x00040007,
1257 0x3c07c, 0xffffffff, 0x00060005,
1258 0x3c080, 0xffffffff, 0x00090008,
1259 0x3c084, 0xffffffff, 0x00010000,
1260 0x3c088, 0xffffffff, 0x00030002,
1261 0x3c08c, 0xffffffff, 0x00040007,
1262 0x3c090, 0xffffffff, 0x00060005,
1263 0x3c094, 0xffffffff, 0x00090008,
1264 0x3c098, 0xffffffff, 0x00010000,
1265 0x3c09c, 0xffffffff, 0x00030002,
1266 0x3c0a0, 0xffffffff, 0x00040007,
1267 0x3c0a4, 0xffffffff, 0x00060005,
1268 0x3c0a8, 0xffffffff, 0x00090008,
1269 0x3c0ac, 0xffffffff, 0x00010000,
1270 0x3c0b0, 0xffffffff, 0x00030002,
1271 0x3c0b4, 0xffffffff, 0x00040007,
1272 0x3c0b8, 0xffffffff, 0x00060005,
1273 0x3c0bc, 0xffffffff, 0x00090008,
1274 0x3c000, 0xffffffff, 0x96e00200,
1275 0x8708, 0xffffffff, 0x00900100,
1276 0xc424, 0xffffffff, 0x0020003f,
1277 0x38, 0xffffffff, 0x0140001c,
1278 0x3c, 0x000f0000, 0x000f0000,
1279 0x220, 0xffffffff, 0xC060000C,
1280 0x224, 0xc0000fff, 0x00000100,
1281 0xf90, 0xffffffff, 0x00000100,
1282 0xf98, 0x00000101, 0x00000000,
1283 0x20a8, 0xffffffff, 0x00000104,
1284 0x55e4, 0xff000fff, 0x00000100,
1285 0x30cc, 0xc0000fff, 0x00000104,
1286 0xc1e4, 0x00000001, 0x00000001,
1287 0xd00c, 0xff000ff0, 0x00000100,
1288 0xd80c, 0xff000ff0, 0x00000100
1289};
1290
1291static const u32 kalindi_golden_spm_registers[] =
1292{
1293 0x30800, 0xe0ffffff, 0xe0000000
1294};
1295
1296static const u32 kalindi_golden_common_registers[] =
1297{
1298 0xc770, 0xffffffff, 0x00000800,
1299 0xc774, 0xffffffff, 0x00000800,
1300 0xc798, 0xffffffff, 0x00007fbf,
1301 0xc79c, 0xffffffff, 0x00007faf
1302};
1303
1304static const u32 kalindi_golden_registers[] =
1305{
1306 0x3c000, 0xffffdfff, 0x6e944040,
1307 0x55e4, 0xff607fff, 0xfc000100,
1308 0x3c220, 0xff000fff, 0x00000100,
1309 0x3c224, 0xff000fff, 0x00000100,
1310 0x3c200, 0xfffc0fff, 0x00000100,
1311 0x6ed8, 0x00010101, 0x00010000,
1312 0x9830, 0xffffffff, 0x00000000,
1313 0x9834, 0xf00fffff, 0x00000400,
1314 0x5bb0, 0x000000f0, 0x00000070,
1315 0x5bc0, 0xf0311fff, 0x80300000,
1316 0x98f8, 0x73773777, 0x12010001,
1317 0x98fc, 0xffffffff, 0x00000010,
1318 0x9b7c, 0x00ff0000, 0x00fc0000,
1319 0x8030, 0x00001f0f, 0x0000100a,
1320 0x2f48, 0x73773777, 0x12010001,
1321 0x2408, 0x000fffff, 0x000c007f,
1322 0x8a14, 0xf000003f, 0x00000007,
1323 0x8b24, 0x3fff3fff, 0x00ffcfff,
1324 0x30a04, 0x0000ff0f, 0x00000000,
1325 0x28a4c, 0x07ffffff, 0x06000000,
1326 0x4d8, 0x00000fff, 0x00000100,
1327 0x3e78, 0x00000001, 0x00000002,
1328 0xc768, 0x00000008, 0x00000008,
1329 0x8c00, 0x000000ff, 0x00000003,
1330 0x214f8, 0x01ff01ff, 0x00000002,
1331 0x21498, 0x007ff800, 0x00200000,
1332 0x2015c, 0xffffffff, 0x00000f40,
1333 0x88c4, 0x001f3ae3, 0x00000082,
1334 0x88d4, 0x0000001f, 0x00000010,
1335 0x30934, 0xffffffff, 0x00000000
1336};
1337
1338static const u32 kalindi_mgcg_cgcg_init[] =
1339{
1340 0xc420, 0xffffffff, 0xfffffffc,
1341 0x30800, 0xffffffff, 0xe0000000,
1342 0x3c2a0, 0xffffffff, 0x00000100,
1343 0x3c208, 0xffffffff, 0x00000100,
1344 0x3c2c0, 0xffffffff, 0x00000100,
1345 0x3c2c8, 0xffffffff, 0x00000100,
1346 0x3c2c4, 0xffffffff, 0x00000100,
1347 0x55e4, 0xffffffff, 0x00600100,
1348 0x3c280, 0xffffffff, 0x00000100,
1349 0x3c214, 0xffffffff, 0x06000100,
1350 0x3c220, 0xffffffff, 0x00000100,
1351 0x3c218, 0xffffffff, 0x06000100,
1352 0x3c204, 0xffffffff, 0x00000100,
1353 0x3c2e0, 0xffffffff, 0x00000100,
1354 0x3c224, 0xffffffff, 0x00000100,
1355 0x3c200, 0xffffffff, 0x00000100,
1356 0x3c230, 0xffffffff, 0x00000100,
1357 0x3c234, 0xffffffff, 0x00000100,
1358 0x3c250, 0xffffffff, 0x00000100,
1359 0x3c254, 0xffffffff, 0x00000100,
1360 0x3c258, 0xffffffff, 0x00000100,
1361 0x3c25c, 0xffffffff, 0x00000100,
1362 0x3c260, 0xffffffff, 0x00000100,
1363 0x3c27c, 0xffffffff, 0x00000100,
1364 0x3c278, 0xffffffff, 0x00000100,
1365 0x3c210, 0xffffffff, 0x06000100,
1366 0x3c290, 0xffffffff, 0x00000100,
1367 0x3c274, 0xffffffff, 0x00000100,
1368 0x3c2b4, 0xffffffff, 0x00000100,
1369 0x3c2b0, 0xffffffff, 0x00000100,
1370 0x3c270, 0xffffffff, 0x00000100,
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x3c020, 0xffffffff, 0x00010000,
1373 0x3c024, 0xffffffff, 0x00030002,
1374 0x3c028, 0xffffffff, 0x00040007,
1375 0x3c02c, 0xffffffff, 0x00060005,
1376 0x3c030, 0xffffffff, 0x00090008,
1377 0x3c034, 0xffffffff, 0x00010000,
1378 0x3c038, 0xffffffff, 0x00030002,
1379 0x3c03c, 0xffffffff, 0x00040007,
1380 0x3c040, 0xffffffff, 0x00060005,
1381 0x3c044, 0xffffffff, 0x00090008,
1382 0x3c000, 0xffffffff, 0x96e00200,
1383 0x8708, 0xffffffff, 0x00900100,
1384 0xc424, 0xffffffff, 0x0020003f,
1385 0x38, 0xffffffff, 0x0140001c,
1386 0x3c, 0x000f0000, 0x000f0000,
1387 0x220, 0xffffffff, 0xC060000C,
1388 0x224, 0xc0000fff, 0x00000100,
1389 0x20a8, 0xffffffff, 0x00000104,
1390 0x55e4, 0xff000fff, 0x00000100,
1391 0x30cc, 0xc0000fff, 0x00000104,
1392 0xc1e4, 0x00000001, 0x00000001,
1393 0xd00c, 0xff000ff0, 0x00000100,
1394 0xd80c, 0xff000ff0, 0x00000100
1395};
1396
8efff337
AD		 1397static const u32 hawaii_golden_spm_registers[] =
1398{
1399 0x30800, 0xe0ffffff, 0xe0000000
1400};
1401
1402static const u32 hawaii_golden_common_registers[] =
1403{
1404 0x30800, 0xffffffff, 0xe0000000,
1405 0x28350, 0xffffffff, 0x3a00161a,
1406 0x28354, 0xffffffff, 0x0000002e,
1407 0x9a10, 0xffffffff, 0x00018208,
1408 0x98f8, 0xffffffff, 0x12011003
1409};
1410
1411static const u32 hawaii_golden_registers[] =
1412{
1413 0x3354, 0x00000333, 0x00000333,
1414 0x9a10, 0x00010000, 0x00058208,
1415 0x9830, 0xffffffff, 0x00000000,
1416 0x9834, 0xf00fffff, 0x00000400,
1417 0x9838, 0x0002021c, 0x00020200,
1418 0xc78, 0x00000080, 0x00000000,
1419 0x5bb0, 0x000000f0, 0x00000070,
1420 0x5bc0, 0xf0311fff, 0x80300000,
1421 0x350c, 0x00810000, 0x408af000,
1422 0x7030, 0x31000111, 0x00000011,
1423 0x2f48, 0x73773777, 0x12010001,
1424 0x2120, 0x0000007f, 0x0000001b,
1425 0x21dc, 0x00007fb6, 0x00002191,
1426 0x3628, 0x0000003f, 0x0000000a,
1427 0x362c, 0x0000003f, 0x0000000a,
1428 0x2ae4, 0x00073ffe, 0x000022a2,
1429 0x240c, 0x000007ff, 0x00000000,
1430 0x8bf0, 0x00002001, 0x00000001,
1431 0x8b24, 0xffffffff, 0x00ffffff,
1432 0x30a04, 0x0000ff0f, 0x00000000,
1433 0x28a4c, 0x07ffffff, 0x06000000,
1434 0x3e78, 0x00000001, 0x00000002,
1435 0xc768, 0x00000008, 0x00000008,
1436 0xc770, 0x00000f00, 0x00000800,
1437 0xc774, 0x00000f00, 0x00000800,
1438 0xc798, 0x00ffffff, 0x00ff7fbf,
1439 0xc79c, 0x00ffffff, 0x00ff7faf,
1440 0x8c00, 0x000000ff, 0x00000800,
1441 0xe40, 0x00001fff, 0x00001fff,
1442 0x9060, 0x0000007f, 0x00000020,
1443 0x9508, 0x00010000, 0x00010000,
1444 0xae00, 0x00100000, 0x000ff07c,
1445 0xac14, 0x000003ff, 0x0000000f,
1446 0xac10, 0xffffffff, 0x7564fdec,
1447 0xac0c, 0xffffffff, 0x3120b9a8,
1448 0xac08, 0x20000000, 0x0f9c0000
1449};
1450
1451static const u32 hawaii_mgcg_cgcg_init[] =
1452{
1453 0xc420, 0xffffffff, 0xfffffffd,
1454 0x30800, 0xffffffff, 0xe0000000,
1455 0x3c2a0, 0xffffffff, 0x00000100,
1456 0x3c208, 0xffffffff, 0x00000100,
1457 0x3c2c0, 0xffffffff, 0x00000100,
1458 0x3c2c8, 0xffffffff, 0x00000100,
1459 0x3c2c4, 0xffffffff, 0x00000100,
1460 0x55e4, 0xffffffff, 0x00200100,
1461 0x3c280, 0xffffffff, 0x00000100,
1462 0x3c214, 0xffffffff, 0x06000100,
1463 0x3c220, 0xffffffff, 0x00000100,
1464 0x3c218, 0xffffffff, 0x06000100,
1465 0x3c204, 0xffffffff, 0x00000100,
1466 0x3c2e0, 0xffffffff, 0x00000100,
1467 0x3c224, 0xffffffff, 0x00000100,
1468 0x3c200, 0xffffffff, 0x00000100,
1469 0x3c230, 0xffffffff, 0x00000100,
1470 0x3c234, 0xffffffff, 0x00000100,
1471 0x3c250, 0xffffffff, 0x00000100,
1472 0x3c254, 0xffffffff, 0x00000100,
1473 0x3c258, 0xffffffff, 0x00000100,
1474 0x3c25c, 0xffffffff, 0x00000100,
1475 0x3c260, 0xffffffff, 0x00000100,
1476 0x3c27c, 0xffffffff, 0x00000100,
1477 0x3c278, 0xffffffff, 0x00000100,
1478 0x3c210, 0xffffffff, 0x06000100,
1479 0x3c290, 0xffffffff, 0x00000100,
1480 0x3c274, 0xffffffff, 0x00000100,
1481 0x3c2b4, 0xffffffff, 0x00000100,
1482 0x3c2b0, 0xffffffff, 0x00000100,
1483 0x3c270, 0xffffffff, 0x00000100,
1484 0x30800, 0xffffffff, 0xe0000000,
1485 0x3c020, 0xffffffff, 0x00010000,
1486 0x3c024, 0xffffffff, 0x00030002,
1487 0x3c028, 0xffffffff, 0x00040007,
1488 0x3c02c, 0xffffffff, 0x00060005,
1489 0x3c030, 0xffffffff, 0x00090008,
1490 0x3c034, 0xffffffff, 0x00010000,
1491 0x3c038, 0xffffffff, 0x00030002,
1492 0x3c03c, 0xffffffff, 0x00040007,
1493 0x3c040, 0xffffffff, 0x00060005,
1494 0x3c044, 0xffffffff, 0x00090008,
1495 0x3c048, 0xffffffff, 0x00010000,
1496 0x3c04c, 0xffffffff, 0x00030002,
1497 0x3c050, 0xffffffff, 0x00040007,
1498 0x3c054, 0xffffffff, 0x00060005,
1499 0x3c058, 0xffffffff, 0x00090008,
1500 0x3c05c, 0xffffffff, 0x00010000,
1501 0x3c060, 0xffffffff, 0x00030002,
1502 0x3c064, 0xffffffff, 0x00040007,
1503 0x3c068, 0xffffffff, 0x00060005,
1504 0x3c06c, 0xffffffff, 0x00090008,
1505 0x3c070, 0xffffffff, 0x00010000,
1506 0x3c074, 0xffffffff, 0x00030002,
1507 0x3c078, 0xffffffff, 0x00040007,
1508 0x3c07c, 0xffffffff, 0x00060005,
1509 0x3c080, 0xffffffff, 0x00090008,
1510 0x3c084, 0xffffffff, 0x00010000,
1511 0x3c088, 0xffffffff, 0x00030002,
1512 0x3c08c, 0xffffffff, 0x00040007,
1513 0x3c090, 0xffffffff, 0x00060005,
1514 0x3c094, 0xffffffff, 0x00090008,
1515 0x3c098, 0xffffffff, 0x00010000,
1516 0x3c09c, 0xffffffff, 0x00030002,
1517 0x3c0a0, 0xffffffff, 0x00040007,
1518 0x3c0a4, 0xffffffff, 0x00060005,
1519 0x3c0a8, 0xffffffff, 0x00090008,
1520 0x3c0ac, 0xffffffff, 0x00010000,
1521 0x3c0b0, 0xffffffff, 0x00030002,
1522 0x3c0b4, 0xffffffff, 0x00040007,
1523 0x3c0b8, 0xffffffff, 0x00060005,
1524 0x3c0bc, 0xffffffff, 0x00090008,
1525 0x3c0c0, 0xffffffff, 0x00010000,
1526 0x3c0c4, 0xffffffff, 0x00030002,
1527 0x3c0c8, 0xffffffff, 0x00040007,
1528 0x3c0cc, 0xffffffff, 0x00060005,
1529 0x3c0d0, 0xffffffff, 0x00090008,
1530 0x3c0d4, 0xffffffff, 0x00010000,
1531 0x3c0d8, 0xffffffff, 0x00030002,
1532 0x3c0dc, 0xffffffff, 0x00040007,
1533 0x3c0e0, 0xffffffff, 0x00060005,
1534 0x3c0e4, 0xffffffff, 0x00090008,
1535 0x3c0e8, 0xffffffff, 0x00010000,
1536 0x3c0ec, 0xffffffff, 0x00030002,
1537 0x3c0f0, 0xffffffff, 0x00040007,
1538 0x3c0f4, 0xffffffff, 0x00060005,
1539 0x3c0f8, 0xffffffff, 0x00090008,
1540 0xc318, 0xffffffff, 0x00020200,
1541 0x3350, 0xffffffff, 0x00000200,
1542 0x15c0, 0xffffffff, 0x00000400,
1543 0x55e8, 0xffffffff, 0x00000000,
1544 0x2f50, 0xffffffff, 0x00000902,
1545 0x3c000, 0xffffffff, 0x96940200,
1546 0x8708, 0xffffffff, 0x00900100,
1547 0xc424, 0xffffffff, 0x0020003f,
1548 0x38, 0xffffffff, 0x0140001c,
1549 0x3c, 0x000f0000, 0x000f0000,
1550 0x220, 0xffffffff, 0xc060000c,
1551 0x224, 0xc0000fff, 0x00000100,
1552 0xf90, 0xffffffff, 0x00000100,
1553 0xf98, 0x00000101, 0x00000000,
1554 0x20a8, 0xffffffff, 0x00000104,
1555 0x55e4, 0xff000fff, 0x00000100,
1556 0x30cc, 0xc0000fff, 0x00000104,
1557 0xc1e4, 0x00000001, 0x00000001,
1558 0xd00c, 0xff000ff0, 0x00000100,
1559 0xd80c, 0xff000ff0, 0x00000100
1560};
1561
f73a9e83
SL		 1562static const u32 godavari_golden_registers[] =
1563{
1564 0x55e4, 0xff607fff, 0xfc000100,
1565 0x6ed8, 0x00010101, 0x00010000,
1566 0x9830, 0xffffffff, 0x00000000,
1567 0x98302, 0xf00fffff, 0x00000400,
1568 0x6130, 0xffffffff, 0x00010000,
1569 0x5bb0, 0x000000f0, 0x00000070,
1570 0x5bc0, 0xf0311fff, 0x80300000,
1571 0x98f8, 0x73773777, 0x12010001,
1572 0x98fc, 0xffffffff, 0x00000010,
1573 0x8030, 0x00001f0f, 0x0000100a,
1574 0x2f48, 0x73773777, 0x12010001,
1575 0x2408, 0x000fffff, 0x000c007f,
1576 0x8a14, 0xf000003f, 0x00000007,
1577 0x8b24, 0xffffffff, 0x00ff0fff,
1578 0x30a04, 0x0000ff0f, 0x00000000,
1579 0x28a4c, 0x07ffffff, 0x06000000,
1580 0x4d8, 0x00000fff, 0x00000100,
1581 0xd014, 0x00010000, 0x00810001,
1582 0xd814, 0x00010000, 0x00810001,
1583 0x3e78, 0x00000001, 0x00000002,
1584 0xc768, 0x00000008, 0x00000008,
1585 0xc770, 0x00000f00, 0x00000800,
1586 0xc774, 0x00000f00, 0x00000800,
1587 0xc798, 0x00ffffff, 0x00ff7fbf,
1588 0xc79c, 0x00ffffff, 0x00ff7faf,
1589 0x8c00, 0x000000ff, 0x00000001,
1590 0x214f8, 0x01ff01ff, 0x00000002,
1591 0x21498, 0x007ff800, 0x00200000,
1592 0x2015c, 0xffffffff, 0x00000f40,
1593 0x88c4, 0x001f3ae3, 0x00000082,
1594 0x88d4, 0x0000001f, 0x00000010,
1595 0x30934, 0xffffffff, 0x00000000
1596};
1597
1598
0aafd313
AD		 1599static void cik_init_golden_registers(struct radeon_device *rdev)
1600{
1c0a4625
OG		 1601 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1602 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1603 switch (rdev->family) {
1604 case CHIP_BONAIRE:
1605 radeon_program_register_sequence(rdev,
1606 bonaire_mgcg_cgcg_init,
1607 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1608 radeon_program_register_sequence(rdev,
1609 bonaire_golden_registers,
1610 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1611 radeon_program_register_sequence(rdev,
1612 bonaire_golden_common_registers,
1613 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1614 radeon_program_register_sequence(rdev,
1615 bonaire_golden_spm_registers,
1616 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1617 break;
1618 case CHIP_KABINI:
1619 radeon_program_register_sequence(rdev,
1620 kalindi_mgcg_cgcg_init,
1621 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1622 radeon_program_register_sequence(rdev,
1623 kalindi_golden_registers,
1624 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1625 radeon_program_register_sequence(rdev,
1626 kalindi_golden_common_registers,
1627 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1628 radeon_program_register_sequence(rdev,
1629 kalindi_golden_spm_registers,
1630 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1631 break;
f73a9e83
SL		 1632 case CHIP_MULLINS:
1633 radeon_program_register_sequence(rdev,
1634 kalindi_mgcg_cgcg_init,
1635 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1636 radeon_program_register_sequence(rdev,
1637 godavari_golden_registers,
1638 (const u32)ARRAY_SIZE(godavari_golden_registers));
1639 radeon_program_register_sequence(rdev,
1640 kalindi_golden_common_registers,
1641 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1642 radeon_program_register_sequence(rdev,
1643 kalindi_golden_spm_registers,
1644 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1645 break;
0aafd313
AD		 1646 case CHIP_KAVERI:
1647 radeon_program_register_sequence(rdev,
1648 spectre_mgcg_cgcg_init,
1649 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1650 radeon_program_register_sequence(rdev,
1651 spectre_golden_registers,
1652 (const u32)ARRAY_SIZE(spectre_golden_registers));
1653 radeon_program_register_sequence(rdev,
1654 spectre_golden_common_registers,
1655 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1656 radeon_program_register_sequence(rdev,
1657 spectre_golden_spm_registers,
1658 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1659 break;
8efff337
AD		 1660 case CHIP_HAWAII:
1661 radeon_program_register_sequence(rdev,
1662 hawaii_mgcg_cgcg_init,
1663 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1664 radeon_program_register_sequence(rdev,
1665 hawaii_golden_registers,
1666 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1667 radeon_program_register_sequence(rdev,
1668 hawaii_golden_common_registers,
1669 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1670 radeon_program_register_sequence(rdev,
1671 hawaii_golden_spm_registers,
1672 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1673 break;
0aafd313
AD		 1674 default:
1675 break;
1676 }
1c0a4625 1677 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1678}
1679
2c67912c
AD		 1680/**
1681 * cik_get_xclk - get the xclk
1682 *
1683 * @rdev: radeon_device pointer
1684 *
1685 * Returns the reference clock used by the gfx engine
1686 * (CIK).
1687 */
1688u32 cik_get_xclk(struct radeon_device *rdev)
1689{
1690 u32 reference_clock = rdev->clock.spll.reference_freq;
1691
1692 if (rdev->flags & RADEON_IS_IGP) {
1693 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1694 return reference_clock / 2;
1695 } else {
1696 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1697 return reference_clock / 4;
1698 }
1699 return reference_clock;
1700}
1701
75efdee1
AD		 1702/**
1703 * cik_mm_rdoorbell - read a doorbell dword
1704 *
1705 * @rdev: radeon_device pointer
d5754ab8 1706 * @index: doorbell index
75efdee1
AD		 1707 *
1708 * Returns the value in the doorbell aperture at the
d5754ab8 1709 * requested doorbell index (CIK).
75efdee1 1710 */
d5754ab8 1711u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1712{
d5754ab8
AL		 1713 if (index < rdev->doorbell.num_doorbells) {
1714 return readl(rdev->doorbell.ptr + index);
75efdee1 1715 } else {
d5754ab8 1716 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1717 return 0;
1718 }
1719}
1720
1721/**
1722 * cik_mm_wdoorbell - write a doorbell dword
1723 *
1724 * @rdev: radeon_device pointer
d5754ab8 1725 * @index: doorbell index
75efdee1
AD		 1726 * @v: value to write
1727 *
1728 * Writes @v to the doorbell aperture at the
d5754ab8 1729 * requested doorbell index (CIK).
75efdee1 1730 */
d5754ab8 1731void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1732{
d5754ab8
AL		 1733 if (index < rdev->doorbell.num_doorbells) {
1734 writel(v, rdev->doorbell.ptr + index);
75efdee1 1735 } else {
d5754ab8 1736 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1737 }
1738}
1739
bc8273fe
AD		 1740#define BONAIRE_IO_MC_REGS_SIZE 36
1741
1742static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1743{
1744 {0x00000070, 0x04400000},
1745 {0x00000071, 0x80c01803},
1746 {0x00000072, 0x00004004},
1747 {0x00000073, 0x00000100},
1748 {0x00000074, 0x00ff0000},
1749 {0x00000075, 0x34000000},
1750 {0x00000076, 0x08000014},
1751 {0x00000077, 0x00cc08ec},
1752 {0x00000078, 0x00000400},
1753 {0x00000079, 0x00000000},
1754 {0x0000007a, 0x04090000},
1755 {0x0000007c, 0x00000000},
1756 {0x0000007e, 0x4408a8e8},
1757 {0x0000007f, 0x00000304},
1758 {0x00000080, 0x00000000},
1759 {0x00000082, 0x00000001},
1760 {0x00000083, 0x00000002},
1761 {0x00000084, 0xf3e4f400},
1762 {0x00000085, 0x052024e3},
1763 {0x00000087, 0x00000000},
1764 {0x00000088, 0x01000000},
1765 {0x0000008a, 0x1c0a0000},
1766 {0x0000008b, 0xff010000},
1767 {0x0000008d, 0xffffefff},
1768 {0x0000008e, 0xfff3efff},
1769 {0x0000008f, 0xfff3efbf},
1770 {0x00000092, 0xf7ffffff},
1771 {0x00000093, 0xffffff7f},
1772 {0x00000095, 0x00101101},
1773 {0x00000096, 0x00000fff},
1774 {0x00000097, 0x00116fff},
1775 {0x00000098, 0x60010000},
1776 {0x00000099, 0x10010000},
1777 {0x0000009a, 0x00006000},
1778 {0x0000009b, 0x00001000},
1779 {0x0000009f, 0x00b48000}
1780};
1781
d4775655
AD		 1782#define HAWAII_IO_MC_REGS_SIZE 22
1783
1784static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1785{
1786 {0x0000007d, 0x40000000},
1787 {0x0000007e, 0x40180304},
1788 {0x0000007f, 0x0000ff00},
1789 {0x00000081, 0x00000000},
1790 {0x00000083, 0x00000800},
1791 {0x00000086, 0x00000000},
1792 {0x00000087, 0x00000100},
1793 {0x00000088, 0x00020100},
1794 {0x00000089, 0x00000000},
1795 {0x0000008b, 0x00040000},
1796 {0x0000008c, 0x00000100},
1797 {0x0000008e, 0xff010000},
1798 {0x00000090, 0xffffefff},
1799 {0x00000091, 0xfff3efff},
1800 {0x00000092, 0xfff3efbf},
1801 {0x00000093, 0xf7ffffff},
1802 {0x00000094, 0xffffff7f},
1803 {0x00000095, 0x00000fff},
1804 {0x00000096, 0x00116fff},
1805 {0x00000097, 0x60010000},
1806 {0x00000098, 0x10010000},
1807 {0x0000009f, 0x00c79000}
1808};
1809
1810
b556b12e
AD		 1811/**
1812 * cik_srbm_select - select specific register instances
1813 *
1814 * @rdev: radeon_device pointer
1815 * @me: selected ME (micro engine)
1816 * @pipe: pipe
1817 * @queue: queue
1818 * @vmid: VMID
1819 *
1820 * Switches the currently active registers instances. Some
1821 * registers are instanced per VMID, others are instanced per
1822 * me/pipe/queue combination.
1823 */
1824static void cik_srbm_select(struct radeon_device *rdev,
1825 u32 me, u32 pipe, u32 queue, u32 vmid)
1826{
1827 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1828 MEID(me & 0x3) |
1829 VMID(vmid & 0xf) |
1830 QUEUEID(queue & 0x7));
1831 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1832}
1833
bc8273fe
AD		 1834/* ucode loading */
1835/**
1836 * ci_mc_load_microcode - load MC ucode into the hw
1837 *
1838 * @rdev: radeon_device pointer
1839 *
1840 * Load the GDDR MC ucode into the hw (CIK).
1841 * Returns 0 on success, error on failure.
1842 */
6c7bccea 1843int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1844{
f2c6b0f4
AD		 1845 const __be32 *fw_data = NULL;
1846 const __le32 *new_fw_data = NULL;
9feb3dda 1847 u32 running, blackout = 0, tmp;
f2c6b0f4
AD		 1848 u32 *io_mc_regs = NULL;
1849 const __le32 *new_io_mc_regs = NULL;
bcddee29 1850 int i, regs_size, ucode_size;
bc8273fe
AD		 1851
1852 if (!rdev->mc_fw)
1853 return -EINVAL;
1854
f2c6b0f4
AD		 1855 if (rdev->new_fw) {
1856 const struct mc_firmware_header_v1_0 *hdr =
1857 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1858
f2c6b0f4
AD		 1859 radeon_ucode_print_mc_hdr(&hdr->header);
1860
1861 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1862 new_io_mc_regs = (const __le32 *)
1863 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1864 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1865 new_fw_data = (const __le32 *)
1866 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1867 } else {
1868 ucode_size = rdev->mc_fw->size / 4;
1869
1870 switch (rdev->family) {
1871 case CHIP_BONAIRE:
1872 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1873 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1874 break;
1875 case CHIP_HAWAII:
1876 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1877 regs_size = HAWAII_IO_MC_REGS_SIZE;
1878 break;
1879 default:
1880 return -EINVAL;
1881 }
1882 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1883 }
1884
1885 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1886
1887 if (running == 0) {
1888 if (running) {
1889 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1890 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1891 }
1892
1893 /* reset the engine and set to writable */
1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1895 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1896
1897 /* load mc io regs */
1898 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1899 if (rdev->new_fw) {
1900 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1901 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1902 } else {
1903 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1904 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1905 }
bc8273fe 1906 }
9feb3dda
AD		 1907
1908 tmp = RREG32(MC_SEQ_MISC0);
1909 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1910 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1911 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1912 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1913 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1914 }
1915
bc8273fe 1916 /* load the MC ucode */
f2c6b0f4
AD		 1917 for (i = 0; i < ucode_size; i++) {
1918 if (rdev->new_fw)
1919 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1920 else
1921 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1922 }
bc8273fe
AD		 1923
1924 /* put the engine back into the active state */
1925 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1926 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1927 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1928
1929 /* wait for training to complete */
1930 for (i = 0; i < rdev->usec_timeout; i++) {
1931 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1932 break;
1933 udelay(1);
1934 }
1935 for (i = 0; i < rdev->usec_timeout; i++) {
1936 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1937 break;
1938 udelay(1);
1939 }
1940
1941 if (running)
1942 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1943 }
1944
1945 return 0;
1946}
1947
02c81327
AD		 1948/**
1949 * cik_init_microcode - load ucode images from disk
1950 *
1951 * @rdev: radeon_device pointer
1952 *
1953 * Use the firmware interface to load the ucode images into
1954 * the driver (not loaded into hw).
1955 * Returns 0 on success, error on failure.
1956 */
1957static int cik_init_microcode(struct radeon_device *rdev)
1958{
02c81327 1959 const char *chip_name;
f2c6b0f4 1960 const char *new_chip_name;
02c81327 1961 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1962 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1963 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1964 char fw_name[30];
f2c6b0f4 1965 int new_fw = 0;
02c81327 1966 int err;
f2c6b0f4 1967 int num_fw;
02c81327
AD		 1968
1969 DRM_DEBUG("\n");
1970
02c81327
AD		 1971 switch (rdev->family) {
1972 case CHIP_BONAIRE:
1973 chip_name = "BONAIRE";
f2c6b0f4 1974 new_chip_name = "bonaire";
02c81327
AD		 1975 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1976 me_req_size = CIK_ME_UCODE_SIZE * 4;
1977 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1978 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1979 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1980 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1981 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 1982 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1983 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 1984 num_fw = 8;
02c81327 1985 break;
d4775655
AD		 1986 case CHIP_HAWAII:
1987 chip_name = "HAWAII";
f2c6b0f4 1988 new_chip_name = "hawaii";
d4775655
AD		 1989 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1990 me_req_size = CIK_ME_UCODE_SIZE * 4;
1991 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1992 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1993 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1994 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 1995 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 1996 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1997 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 1998 num_fw = 8;
d4775655 1999 break;
02c81327
AD		 2000 case CHIP_KAVERI:
2001 chip_name = "KAVERI";
f2c6b0f4 2002 new_chip_name = "kaveri";
02c81327
AD		 2003 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2004 me_req_size = CIK_ME_UCODE_SIZE * 4;
2005 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2006 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2007 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2008 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2009 num_fw = 7;
02c81327
AD		 2010 break;
2011 case CHIP_KABINI:
2012 chip_name = "KABINI";
f2c6b0f4 2013 new_chip_name = "kabini";
02c81327
AD		 2014 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2015 me_req_size = CIK_ME_UCODE_SIZE * 4;
2016 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2017 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2018 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2019 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2020 num_fw = 6;
02c81327 2021 break;
f73a9e83
SL		 2022 case CHIP_MULLINS:
2023 chip_name = "MULLINS";
f2c6b0f4 2024 new_chip_name = "mullins";
f73a9e83
SL		 2025 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2026 me_req_size = CIK_ME_UCODE_SIZE * 4;
2027 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2028 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2029 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2030 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2031 num_fw = 6;
f73a9e83 2032 break;
02c81327
AD		 2033 default: BUG();
2034 }
2035
f2c6b0f4 2036 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2037
f2c6b0f4 2038 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2039 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2040 if (err) {
2041 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2042 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2043 if (err)
2044 goto out;
2045 if (rdev->pfp_fw->size != pfp_req_size) {
2046 printk(KERN_ERR
2047 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2048 rdev->pfp_fw->size, fw_name);
2049 err = -EINVAL;
2050 goto out;
2051 }
2052 } else {
2053 err = radeon_ucode_validate(rdev->pfp_fw);
2054 if (err) {
2055 printk(KERN_ERR
2056 "cik_fw: validation failed for firmware \"%s\"\n",
2057 fw_name);
2058 goto out;
2059 } else {
2060 new_fw++;
2061 }
02c81327
AD		 2062 }
2063
f2c6b0f4 2064 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2065 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2066 if (err) {
2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2068 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2069 if (err)
2070 goto out;
2071 if (rdev->me_fw->size != me_req_size) {
2072 printk(KERN_ERR
2073 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2074 rdev->me_fw->size, fw_name);
2075 err = -EINVAL;
2076 }
2077 } else {
2078 err = radeon_ucode_validate(rdev->me_fw);
2079 if (err) {
2080 printk(KERN_ERR
2081 "cik_fw: validation failed for firmware \"%s\"\n",
2082 fw_name);
2083 goto out;
2084 } else {
2085 new_fw++;
2086 }
02c81327
AD		 2087 }
2088
f2c6b0f4 2089 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2090 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2091 if (err) {
2092 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2093 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2094 if (err)
2095 goto out;
2096 if (rdev->ce_fw->size != ce_req_size) {
2097 printk(KERN_ERR
2098 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2099 rdev->ce_fw->size, fw_name);
2100 err = -EINVAL;
2101 }
2102 } else {
2103 err = radeon_ucode_validate(rdev->ce_fw);
2104 if (err) {
2105 printk(KERN_ERR
2106 "cik_fw: validation failed for firmware \"%s\"\n",
2107 fw_name);
2108 goto out;
2109 } else {
2110 new_fw++;
2111 }
02c81327
AD		 2112 }
2113
f2c6b0f4 2114 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2115 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2116 if (err) {
2117 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2118 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2119 if (err)
2120 goto out;
2121 if (rdev->mec_fw->size != mec_req_size) {
2122 printk(KERN_ERR
2123 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2124 rdev->mec_fw->size, fw_name);
2125 err = -EINVAL;
2126 }
2127 } else {
2128 err = radeon_ucode_validate(rdev->mec_fw);
2129 if (err) {
2130 printk(KERN_ERR
2131 "cik_fw: validation failed for firmware \"%s\"\n",
2132 fw_name);
2133 goto out;
2134 } else {
2135 new_fw++;
2136 }
2137 }
2138
2139 if (rdev->family == CHIP_KAVERI) {
2140 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2141 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2142 if (err) {
2143 goto out;
2144 } else {
2145 err = radeon_ucode_validate(rdev->mec2_fw);
2146 if (err) {
2147 goto out;
2148 } else {
2149 new_fw++;
2150 }
2151 }
02c81327
AD		 2152 }
2153
f2c6b0f4 2154 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2155 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2156 if (err) {
2157 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2158 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2159 if (err)
2160 goto out;
2161 if (rdev->rlc_fw->size != rlc_req_size) {
2162 printk(KERN_ERR
2163 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2164 rdev->rlc_fw->size, fw_name);
2165 err = -EINVAL;
2166 }
2167 } else {
2168 err = radeon_ucode_validate(rdev->rlc_fw);
2169 if (err) {
2170 printk(KERN_ERR
2171 "cik_fw: validation failed for firmware \"%s\"\n",
2172 fw_name);
2173 goto out;
2174 } else {
2175 new_fw++;
2176 }
02c81327
AD		 2177 }
2178
f2c6b0f4 2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2180 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2181 if (err) {
2182 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2183 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2184 if (err)
2185 goto out;
2186 if (rdev->sdma_fw->size != sdma_req_size) {
2187 printk(KERN_ERR
2188 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2189 rdev->sdma_fw->size, fw_name);
2190 err = -EINVAL;
2191 }
2192 } else {
2193 err = radeon_ucode_validate(rdev->sdma_fw);
2194 if (err) {
2195 printk(KERN_ERR
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2197 fw_name);
2198 goto out;
2199 } else {
2200 new_fw++;
2201 }
21a93e13
AD		 2202 }
2203
cc8dbbb4 2204 /* No SMC, MC ucode on APUs */
02c81327 2205 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2206 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2207 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2208 if (err) {
f2c6b0f4 2209 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2210 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2211 if (err) {
2212 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2213 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2214 if (err)
2215 goto out;
2216 }
2217 if ((rdev->mc_fw->size != mc_req_size) &&
2218 (rdev->mc_fw->size != mc2_req_size)){
2219 printk(KERN_ERR
2220 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2221 rdev->mc_fw->size, fw_name);
2222 err = -EINVAL;
2223 }
2224 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2225 } else {
2226 err = radeon_ucode_validate(rdev->mc_fw);
2227 if (err) {
2228 printk(KERN_ERR
2229 "cik_fw: validation failed for firmware \"%s\"\n",
2230 fw_name);
277babc3 2231 goto out;
f2c6b0f4
AD		 2232 } else {
2233 new_fw++;
2234 }
277babc3 2235 }
cc8dbbb4 2236
f2c6b0f4 2237 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2238 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2239 if (err) {
f2c6b0f4
AD		 2240 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2241 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2242 if (err) {
2243 printk(KERN_ERR
2244 "smc: error loading firmware \"%s\"\n",
2245 fw_name);
2246 release_firmware(rdev->smc_fw);
2247 rdev->smc_fw = NULL;
2248 err = 0;
2249 } else if (rdev->smc_fw->size != smc_req_size) {
2250 printk(KERN_ERR
2251 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2252 rdev->smc_fw->size, fw_name);
2253 err = -EINVAL;
2254 }
2255 } else {
2256 err = radeon_ucode_validate(rdev->smc_fw);
2257 if (err) {
2258 printk(KERN_ERR
2259 "cik_fw: validation failed for firmware \"%s\"\n",
2260 fw_name);
2261 goto out;
2262 } else {
2263 new_fw++;
2264 }
cc8dbbb4 2265 }
02c81327
AD		 2266 }
2267
f2c6b0f4
AD		 2268 if (new_fw == 0) {
2269 rdev->new_fw = false;
2270 } else if (new_fw < num_fw) {
2271 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2272 err = -EINVAL;
2273 } else {
2274 rdev->new_fw = true;
2275 }
2276
02c81327 2277out:
02c81327
AD		 2278 if (err) {
2279 if (err != -EINVAL)
2280 printk(KERN_ERR
2281 "cik_cp: Failed to load firmware \"%s\"\n",
2282 fw_name);
2283 release_firmware(rdev->pfp_fw);
2284 rdev->pfp_fw = NULL;
2285 release_firmware(rdev->me_fw);
2286 rdev->me_fw = NULL;
2287 release_firmware(rdev->ce_fw);
2288 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2289 release_firmware(rdev->mec_fw);
2290 rdev->mec_fw = NULL;
2291 release_firmware(rdev->mec2_fw);
2292 rdev->mec2_fw = NULL;
02c81327
AD		 2293 release_firmware(rdev->rlc_fw);
2294 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2295 release_firmware(rdev->sdma_fw);
2296 rdev->sdma_fw = NULL;
02c81327
AD		 2297 release_firmware(rdev->mc_fw);
2298 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2299 release_firmware(rdev->smc_fw);
2300 rdev->smc_fw = NULL;
02c81327
AD		 2301 }
2302 return err;
2303}
2304
8cc1a532
AD		 2305/*
2306 * Core functions
2307 */
2308/**
2309 * cik_tiling_mode_table_init - init the hw tiling table
2310 *
2311 * @rdev: radeon_device pointer
2312 *
2313 * Starting with SI, the tiling setup is done globally in a
2314 * set of 32 tiling modes. Rather than selecting each set of
2315 * parameters per surface as on older asics, we just select
2316 * which index in the tiling table we want to use, and the
2317 * surface uses those parameters (CIK).
2318 */
2319static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2320{
2321 const u32 num_tile_mode_states = 32;
2322 const u32 num_secondary_tile_mode_states = 16;
2323 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2324 u32 num_pipe_configs;
2325 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2326 rdev->config.cik.max_shader_engines;
2327
2328 switch (rdev->config.cik.mem_row_size_in_kb) {
2329 case 1:
2330 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2331 break;
2332 case 2:
2333 default:
2334 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2335 break;
2336 case 4:
2337 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2338 break;
2339 }
2340
2341 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2342 if (num_pipe_configs > 8)
21e438af 2343 num_pipe_configs = 16;
8cc1a532 2344
21e438af
AD		 2345 if (num_pipe_configs == 16) {
2346 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2347 switch (reg_offset) {
2348 case 0:
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2350 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2351 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2352 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2353 break;
2354 case 1:
2355 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2356 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2357 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2358 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2359 break;
2360 case 2:
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2363 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2364 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2365 break;
2366 case 3:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2370 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2371 break;
2372 case 4:
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2376 TILE_SPLIT(split_equal_to_row_size));
2377 break;
2378 case 5:
2379 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2381 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2382 break;
2383 case 6:
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2388 break;
2389 case 7:
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2391 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2392 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2393 TILE_SPLIT(split_equal_to_row_size));
2394 break;
2395 case 8:
2396 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2398 break;
2399 case 9:
2400 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2401 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2402 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2403 break;
2404 case 10:
2405 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409 break;
2410 case 11:
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2412 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2414 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2415 break;
2416 case 12:
2417 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2421 break;
2422 case 13:
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2424 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2425 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2426 break;
2427 case 14:
2428 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 break;
2433 case 16:
2434 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2435 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2436 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2437 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2438 break;
2439 case 17:
2440 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2441 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2442 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2443 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2444 break;
2445 case 27:
2446 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2447 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2448 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2449 break;
2450 case 28:
2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2454 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2455 break;
2456 case 29:
2457 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2459 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2460 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2461 break;
2462 case 30:
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2466 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2467 break;
2468 default:
2469 gb_tile_moden = 0;
2470 break;
2471 }
2472 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2473 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2474 }
2475 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2476 switch (reg_offset) {
2477 case 0:
2478 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2480 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2481 NUM_BANKS(ADDR_SURF_16_BANK));
2482 break;
2483 case 1:
2484 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 break;
2489 case 2:
2490 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2493 NUM_BANKS(ADDR_SURF_16_BANK));
2494 break;
2495 case 3:
2496 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_16_BANK));
2500 break;
2501 case 4:
2502 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2503 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2504 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2505 NUM_BANKS(ADDR_SURF_8_BANK));
2506 break;
2507 case 5:
2508 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2511 NUM_BANKS(ADDR_SURF_4_BANK));
2512 break;
2513 case 6:
2514 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2517 NUM_BANKS(ADDR_SURF_2_BANK));
2518 break;
2519 case 8:
2520 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2523 NUM_BANKS(ADDR_SURF_16_BANK));
2524 break;
2525 case 9:
2526 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2528 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2529 NUM_BANKS(ADDR_SURF_16_BANK));
2530 break;
2531 case 10:
2532 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2533 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2534 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2535 NUM_BANKS(ADDR_SURF_16_BANK));
2536 break;
2537 case 11:
2538 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2540 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2541 NUM_BANKS(ADDR_SURF_8_BANK));
2542 break;
2543 case 12:
2544 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2545 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2546 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2547 NUM_BANKS(ADDR_SURF_4_BANK));
2548 break;
2549 case 13:
2550 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2551 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2552 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2553 NUM_BANKS(ADDR_SURF_2_BANK));
2554 break;
2555 case 14:
2556 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2558 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2559 NUM_BANKS(ADDR_SURF_2_BANK));
2560 break;
2561 default:
2562 gb_tile_moden = 0;
2563 break;
2564 }
1b2c4869 2565 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2566 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2567 }
2568 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2569 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2570 switch (reg_offset) {
2571 case 0:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2576 break;
2577 case 1:
2578 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2580 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2581 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2582 break;
2583 case 2:
2584 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2585 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2586 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2587 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2588 break;
2589 case 3:
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2592 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2593 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2594 break;
2595 case 4:
2596 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2597 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2598 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2599 TILE_SPLIT(split_equal_to_row_size));
2600 break;
2601 case 5:
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2605 break;
2606 case 6:
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2610 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2611 break;
2612 case 7:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2616 TILE_SPLIT(split_equal_to_row_size));
2617 break;
2618 case 8:
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2621 break;
2622 case 9:
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2626 break;
2627 case 10:
2628 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2629 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2630 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2631 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2632 break;
2633 case 11:
2634 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2635 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2636 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2637 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2638 break;
2639 case 12:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 13:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2647 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2648 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2649 break;
2650 case 14:
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 break;
2656 case 16:
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2660 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2661 break;
2662 case 17:
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2666 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2667 break;
2668 case 27:
2669 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2670 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2671 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2672 break;
2673 case 28:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 case 29:
2680 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2683 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2684 break;
2685 case 30:
2686 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2687 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2688 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2689 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2690 break;
2691 default:
2692 gb_tile_moden = 0;
2693 break;
2694 }
39aee490 2695 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2696 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2697 }
2698 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2699 switch (reg_offset) {
2700 case 0:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 1:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_16_BANK));
2711 break;
2712 case 2:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716 NUM_BANKS(ADDR_SURF_16_BANK));
2717 break;
2718 case 3:
2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722 NUM_BANKS(ADDR_SURF_16_BANK));
2723 break;
2724 case 4:
2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728 NUM_BANKS(ADDR_SURF_8_BANK));
2729 break;
2730 case 5:
2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2734 NUM_BANKS(ADDR_SURF_4_BANK));
2735 break;
2736 case 6:
2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2740 NUM_BANKS(ADDR_SURF_2_BANK));
2741 break;
2742 case 8:
2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746 NUM_BANKS(ADDR_SURF_16_BANK));
2747 break;
2748 case 9:
2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752 NUM_BANKS(ADDR_SURF_16_BANK));
2753 break;
2754 case 10:
2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2758 NUM_BANKS(ADDR_SURF_16_BANK));
2759 break;
2760 case 11:
2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2764 NUM_BANKS(ADDR_SURF_16_BANK));
2765 break;
2766 case 12:
2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2770 NUM_BANKS(ADDR_SURF_8_BANK));
2771 break;
2772 case 13:
2773 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2776 NUM_BANKS(ADDR_SURF_4_BANK));
2777 break;
2778 case 14:
2779 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2782 NUM_BANKS(ADDR_SURF_2_BANK));
2783 break;
2784 default:
2785 gb_tile_moden = 0;
2786 break;
2787 }
32f79a8a 2788 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2789 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2790 }
2791 } else if (num_pipe_configs == 4) {
2792 if (num_rbs == 4) {
2793 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2794 switch (reg_offset) {
2795 case 0:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2800 break;
2801 case 1:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2806 break;
2807 case 2:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2811 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2812 break;
2813 case 3:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2816 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2817 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2818 break;
2819 case 4:
2820 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2821 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2822 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2823 TILE_SPLIT(split_equal_to_row_size));
2824 break;
2825 case 5:
2826 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2827 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2828 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2829 break;
2830 case 6:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2834 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2835 break;
2836 case 7:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2840 TILE_SPLIT(split_equal_to_row_size));
2841 break;
2842 case 8:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2844 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2845 break;
2846 case 9:
2847 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2848 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2849 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2850 break;
2851 case 10:
2852 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2854 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2855 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2856 break;
2857 case 11:
2858 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2859 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2860 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2861 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2862 break;
2863 case 12:
2864 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2865 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2866 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2867 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2868 break;
2869 case 13:
2870 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2871 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2873 break;
2874 case 14:
2875 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2876 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2877 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2878 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2879 break;
2880 case 16:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 17:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 27:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2894 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2895 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2896 break;
2897 case 28:
2898 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2900 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2901 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2902 break;
2903 case 29:
2904 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2905 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2906 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2907 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2908 break;
2909 case 30:
2910 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2912 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2913 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2914 break;
2915 default:
2916 gb_tile_moden = 0;
2917 break;
2918 }
39aee490 2919 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2920 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2921 }
2922 } else if (num_rbs < 4) {
2923 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2924 switch (reg_offset) {
2925 case 0:
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2927 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2928 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2929 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2930 break;
2931 case 1:
2932 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2933 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2934 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2935 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2936 break;
2937 case 2:
2938 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2939 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2940 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2941 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2942 break;
2943 case 3:
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2945 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2946 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2947 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2948 break;
2949 case 4:
2950 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2952 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2953 TILE_SPLIT(split_equal_to_row_size));
2954 break;
2955 case 5:
2956 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2958 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2959 break;
2960 case 6:
2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2964 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2965 break;
2966 case 7:
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2968 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2969 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2970 TILE_SPLIT(split_equal_to_row_size));
2971 break;
2972 case 8:
2973 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2974 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2975 break;
2976 case 9:
2977 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2978 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2979 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2980 break;
2981 case 10:
2982 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2983 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2984 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2985 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2986 break;
2987 case 11:
2988 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2989 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2990 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2991 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2992 break;
2993 case 12:
2994 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2995 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2996 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2997 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2998 break;
2999 case 13:
3000 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3001 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 3002 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3003 break;
3004 case 14:
3005 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3006 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3007 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3008 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3009 break;
3010 case 16:
3011 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3012 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3013 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3014 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3015 break;
3016 case 17:
3017 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3018 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3019 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3020 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3021 break;
3022 case 27:
3023 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3024 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 3025 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
3026 break;
3027 case 28:
3028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3029 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3030 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3031 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3032 break;
3033 case 29:
3034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3035 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3036 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3037 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3038 break;
3039 case 30:
3040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3041 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3042 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3043 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3044 break;
3045 default:
3046 gb_tile_moden = 0;
3047 break;
3048 }
39aee490 3049 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3050 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3051 }
3052 }
3053 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3054 switch (reg_offset) {
3055 case 0:
3056 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3057 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3058 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3059 NUM_BANKS(ADDR_SURF_16_BANK));
3060 break;
3061 case 1:
3062 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3063 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3064 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3065 NUM_BANKS(ADDR_SURF_16_BANK));
3066 break;
3067 case 2:
3068 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3069 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3070 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3071 NUM_BANKS(ADDR_SURF_16_BANK));
3072 break;
3073 case 3:
3074 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3077 NUM_BANKS(ADDR_SURF_16_BANK));
3078 break;
3079 case 4:
3080 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3081 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3082 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3083 NUM_BANKS(ADDR_SURF_16_BANK));
3084 break;
3085 case 5:
3086 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3087 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3088 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3089 NUM_BANKS(ADDR_SURF_8_BANK));
3090 break;
3091 case 6:
3092 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3093 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3094 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3095 NUM_BANKS(ADDR_SURF_4_BANK));
3096 break;
3097 case 8:
3098 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3099 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3100 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3101 NUM_BANKS(ADDR_SURF_16_BANK));
3102 break;
3103 case 9:
3104 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3105 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3106 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3107 NUM_BANKS(ADDR_SURF_16_BANK));
3108 break;
3109 case 10:
3110 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3111 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3112 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3113 NUM_BANKS(ADDR_SURF_16_BANK));
3114 break;
3115 case 11:
3116 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3117 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3118 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3119 NUM_BANKS(ADDR_SURF_16_BANK));
3120 break;
3121 case 12:
3122 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3123 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3124 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3125 NUM_BANKS(ADDR_SURF_16_BANK));
3126 break;
3127 case 13:
3128 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3129 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3130 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3131 NUM_BANKS(ADDR_SURF_8_BANK));
3132 break;
3133 case 14:
3134 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3135 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3136 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3137 NUM_BANKS(ADDR_SURF_4_BANK));
3138 break;
3139 default:
3140 gb_tile_moden = 0;
3141 break;
3142 }
32f79a8a 3143 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3144 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3145 }
3146 } else if (num_pipe_configs == 2) {
3147 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3148 switch (reg_offset) {
3149 case 0:
3150 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3151 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3152 PIPE_CONFIG(ADDR_SURF_P2) |
3153 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3154 break;
3155 case 1:
3156 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3157 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3158 PIPE_CONFIG(ADDR_SURF_P2) |
3159 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3160 break;
3161 case 2:
3162 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3163 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3164 PIPE_CONFIG(ADDR_SURF_P2) |
3165 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3166 break;
3167 case 3:
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2) |
3171 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3172 break;
3173 case 4:
3174 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3175 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3176 PIPE_CONFIG(ADDR_SURF_P2) |
3177 TILE_SPLIT(split_equal_to_row_size));
3178 break;
3179 case 5:
3180 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3181 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3182 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3183 break;
3184 case 6:
3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3187 PIPE_CONFIG(ADDR_SURF_P2) |
3188 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3189 break;
3190 case 7:
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3192 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3193 PIPE_CONFIG(ADDR_SURF_P2) |
3194 TILE_SPLIT(split_equal_to_row_size));
3195 break;
3196 case 8:
020ff546
MO		 3197 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3198 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3199 break;
3200 case 9:
3201 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3202 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3203 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3204 break;
3205 case 10:
3206 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3207 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3208 PIPE_CONFIG(ADDR_SURF_P2) |
3209 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3210 break;
3211 case 11:
3212 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3213 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3214 PIPE_CONFIG(ADDR_SURF_P2) |
3215 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3216 break;
3217 case 12:
3218 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3219 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3220 PIPE_CONFIG(ADDR_SURF_P2) |
3221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3222 break;
3223 case 13:
3224 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3225 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3226 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3227 break;
3228 case 14:
3229 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3230 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3231 PIPE_CONFIG(ADDR_SURF_P2) |
3232 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3233 break;
3234 case 16:
3235 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3236 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3237 PIPE_CONFIG(ADDR_SURF_P2) |
3238 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3239 break;
3240 case 17:
3241 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3242 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3243 PIPE_CONFIG(ADDR_SURF_P2) |
3244 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3245 break;
3246 case 27:
3247 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3248 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3249 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3250 break;
3251 case 28:
3252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3253 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3254 PIPE_CONFIG(ADDR_SURF_P2) |
3255 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3256 break;
3257 case 29:
3258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3259 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3260 PIPE_CONFIG(ADDR_SURF_P2) |
3261 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3262 break;
3263 case 30:
3264 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3265 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3266 PIPE_CONFIG(ADDR_SURF_P2) |
3267 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3268 break;
3269 default:
3270 gb_tile_moden = 0;
3271 break;
3272 }
39aee490 3273 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3274 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3275 }
3276 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3277 switch (reg_offset) {
3278 case 0:
3279 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3282 NUM_BANKS(ADDR_SURF_16_BANK));
3283 break;
3284 case 1:
3285 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3288 NUM_BANKS(ADDR_SURF_16_BANK));
3289 break;
3290 case 2:
3291 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3292 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3293 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3294 NUM_BANKS(ADDR_SURF_16_BANK));
3295 break;
3296 case 3:
3297 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3298 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3299 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3300 NUM_BANKS(ADDR_SURF_16_BANK));
3301 break;
3302 case 4:
3303 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3306 NUM_BANKS(ADDR_SURF_16_BANK));
3307 break;
3308 case 5:
3309 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3310 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3311 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3312 NUM_BANKS(ADDR_SURF_16_BANK));
3313 break;
3314 case 6:
3315 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3316 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3317 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3318 NUM_BANKS(ADDR_SURF_8_BANK));
3319 break;
3320 case 8:
3321 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3322 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3323 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3324 NUM_BANKS(ADDR_SURF_16_BANK));
3325 break;
3326 case 9:
3327 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3328 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3329 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3330 NUM_BANKS(ADDR_SURF_16_BANK));
3331 break;
3332 case 10:
3333 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3336 NUM_BANKS(ADDR_SURF_16_BANK));
3337 break;
3338 case 11:
3339 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3340 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3341 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3342 NUM_BANKS(ADDR_SURF_16_BANK));
3343 break;
3344 case 12:
3345 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3348 NUM_BANKS(ADDR_SURF_16_BANK));
3349 break;
3350 case 13:
3351 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3352 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3353 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3354 NUM_BANKS(ADDR_SURF_16_BANK));
3355 break;
3356 case 14:
3357 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3358 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3359 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3360 NUM_BANKS(ADDR_SURF_8_BANK));
3361 break;
3362 default:
3363 gb_tile_moden = 0;
3364 break;
3365 }
32f79a8a 3366 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3367 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3368 }
3369 } else
3370 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3371}
3372
3373/**
3374 * cik_select_se_sh - select which SE, SH to address
3375 *
3376 * @rdev: radeon_device pointer
3377 * @se_num: shader engine to address
3378 * @sh_num: sh block to address
3379 *
3380 * Select which SE, SH combinations to address. Certain
3381 * registers are instanced per SE or SH. 0xffffffff means
3382 * broadcast to all SEs or SHs (CIK).
3383 */
3384static void cik_select_se_sh(struct radeon_device *rdev,
3385 u32 se_num, u32 sh_num)
3386{
3387 u32 data = INSTANCE_BROADCAST_WRITES;
3388
3389 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3390 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3391 else if (se_num == 0xffffffff)
3392 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3393 else if (sh_num == 0xffffffff)
3394 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3395 else
3396 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3397 WREG32(GRBM_GFX_INDEX, data);
3398}
3399
3400/**
3401 * cik_create_bitmask - create a bitmask
3402 *
3403 * @bit_width: length of the mask
3404 *
3405 * create a variable length bit mask (CIK).
3406 * Returns the bitmask.
3407 */
3408static u32 cik_create_bitmask(u32 bit_width)
3409{
3410 u32 i, mask = 0;
3411
3412 for (i = 0; i < bit_width; i++) {
3413 mask <<= 1;
3414 mask |= 1;
3415 }
3416 return mask;
3417}
3418
3419/**
972c5ddb 3420 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3421 *
3422 * @rdev: radeon_device pointer
3423 * @max_rb_num: max RBs (render backends) for the asic
3424 * @se_num: number of SEs (shader engines) for the asic
3425 * @sh_per_se: number of SH blocks per SE for the asic
3426 *
3427 * Calculates the bitmask of disabled RBs (CIK).
3428 * Returns the disabled RB bitmask.
3429 */
3430static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3431 u32 max_rb_num_per_se,
8cc1a532
AD		 3432 u32 sh_per_se)
3433{
3434 u32 data, mask;
3435
3436 data = RREG32(CC_RB_BACKEND_DISABLE);
3437 if (data & 1)
3438 data &= BACKEND_DISABLE_MASK;
3439 else
3440 data = 0;
3441 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3442
3443 data >>= BACKEND_DISABLE_SHIFT;
3444
9fadb352 3445 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3446
3447 return data & mask;
3448}
3449
3450/**
3451 * cik_setup_rb - setup the RBs on the asic
3452 *
3453 * @rdev: radeon_device pointer
3454 * @se_num: number of SEs (shader engines) for the asic
3455 * @sh_per_se: number of SH blocks per SE for the asic
3456 * @max_rb_num: max RBs (render backends) for the asic
3457 *
3458 * Configures per-SE/SH RB registers (CIK).
3459 */
3460static void cik_setup_rb(struct radeon_device *rdev,
3461 u32 se_num, u32 sh_per_se,
9fadb352 3462 u32 max_rb_num_per_se)
8cc1a532
AD		 3463{
3464 int i, j;
3465 u32 data, mask;
3466 u32 disabled_rbs = 0;
3467 u32 enabled_rbs = 0;
3468
1c0a4625 3469 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3470 for (i = 0; i < se_num; i++) {
3471 for (j = 0; j < sh_per_se; j++) {
3472 cik_select_se_sh(rdev, i, j);
9fadb352 3473 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3474 if (rdev->family == CHIP_HAWAII)
3475 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3476 else
3477 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3478 }
3479 }
3480 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3481 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3482
3483 mask = 1;
9fadb352 3484 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3485 if (!(disabled_rbs & mask))
3486 enabled_rbs |= mask;
3487 mask <<= 1;
3488 }
3489
439a1cff
MO		 3490 rdev->config.cik.backend_enable_mask = enabled_rbs;
3491
1c0a4625 3492 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3493 for (i = 0; i < se_num; i++) {
3494 cik_select_se_sh(rdev, i, 0xffffffff);
3495 data = 0;
3496 for (j = 0; j < sh_per_se; j++) {
3497 switch (enabled_rbs & 3) {
fc821b70
AD		 3498 case 0:
3499 if (j == 0)
3500 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3501 else
3502 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3503 break;
8cc1a532
AD		 3504 case 1:
3505 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3506 break;
3507 case 2:
3508 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3509 break;
3510 case 3:
3511 default:
3512 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3513 break;
3514 }
3515 enabled_rbs >>= 2;
3516 }
3517 WREG32(PA_SC_RASTER_CONFIG, data);
3518 }
3519 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3520 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3521}
3522
3523/**
3524 * cik_gpu_init - setup the 3D engine
3525 *
3526 * @rdev: radeon_device pointer
3527 *
3528 * Configures the 3D engine and tiling configuration
3529 * registers so that the 3D engine is usable.
3530 */
3531static void cik_gpu_init(struct radeon_device *rdev)
3532{
3533 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3534 u32 mc_shared_chmap, mc_arb_ramcfg;
3535 u32 hdp_host_path_cntl;
3536 u32 tmp;
6101b3ae 3537 int i, j;
8cc1a532
AD		 3538
3539 switch (rdev->family) {
3540 case CHIP_BONAIRE:
3541 rdev->config.cik.max_shader_engines = 2;
3542 rdev->config.cik.max_tile_pipes = 4;
3543 rdev->config.cik.max_cu_per_sh = 7;
3544 rdev->config.cik.max_sh_per_se = 1;
3545 rdev->config.cik.max_backends_per_se = 2;
3546 rdev->config.cik.max_texture_channel_caches = 4;
3547 rdev->config.cik.max_gprs = 256;
3548 rdev->config.cik.max_gs_threads = 32;
3549 rdev->config.cik.max_hw_contexts = 8;
3550
3551 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3552 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3553 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3554 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3555 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3556 break;
b496038b
AD		 3557 case CHIP_HAWAII:
3558 rdev->config.cik.max_shader_engines = 4;
3559 rdev->config.cik.max_tile_pipes = 16;
3560 rdev->config.cik.max_cu_per_sh = 11;
3561 rdev->config.cik.max_sh_per_se = 1;
3562 rdev->config.cik.max_backends_per_se = 4;
3563 rdev->config.cik.max_texture_channel_caches = 16;
3564 rdev->config.cik.max_gprs = 256;
3565 rdev->config.cik.max_gs_threads = 32;
3566 rdev->config.cik.max_hw_contexts = 8;
3567
3568 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3569 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3570 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3571 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3572 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3573 break;
8cc1a532 3574 case CHIP_KAVERI:
b2e4c70a
AD		 3575 rdev->config.cik.max_shader_engines = 1;
3576 rdev->config.cik.max_tile_pipes = 4;
3577 if ((rdev->pdev->device == 0x1304) ||
3578 (rdev->pdev->device == 0x1305) ||
3579 (rdev->pdev->device == 0x130C) ||
3580 (rdev->pdev->device == 0x130F) ||
3581 (rdev->pdev->device == 0x1310) ||
3582 (rdev->pdev->device == 0x1311) ||
3583 (rdev->pdev->device == 0x131C)) {
3584 rdev->config.cik.max_cu_per_sh = 8;
3585 rdev->config.cik.max_backends_per_se = 2;
3586 } else if ((rdev->pdev->device == 0x1309) ||
3587 (rdev->pdev->device == 0x130A) ||
3588 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3589 (rdev->pdev->device == 0x1313) ||
3590 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3591 rdev->config.cik.max_cu_per_sh = 6;
3592 rdev->config.cik.max_backends_per_se = 2;
3593 } else if ((rdev->pdev->device == 0x1306) ||
3594 (rdev->pdev->device == 0x1307) ||
3595 (rdev->pdev->device == 0x130B) ||
3596 (rdev->pdev->device == 0x130E) ||
3597 (rdev->pdev->device == 0x1315) ||
6dc14baf 3598 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3599 (rdev->pdev->device == 0x131B)) {
3600 rdev->config.cik.max_cu_per_sh = 4;
3601 rdev->config.cik.max_backends_per_se = 1;
3602 } else {
3603 rdev->config.cik.max_cu_per_sh = 3;
3604 rdev->config.cik.max_backends_per_se = 1;
3605 }
3606 rdev->config.cik.max_sh_per_se = 1;
3607 rdev->config.cik.max_texture_channel_caches = 4;
3608 rdev->config.cik.max_gprs = 256;
3609 rdev->config.cik.max_gs_threads = 16;
3610 rdev->config.cik.max_hw_contexts = 8;
3611
3612 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3613 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3614 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3615 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3616 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3617 break;
3618 case CHIP_KABINI:
f73a9e83 3619 case CHIP_MULLINS:
8cc1a532
AD		 3620 default:
3621 rdev->config.cik.max_shader_engines = 1;
3622 rdev->config.cik.max_tile_pipes = 2;
3623 rdev->config.cik.max_cu_per_sh = 2;
3624 rdev->config.cik.max_sh_per_se = 1;
3625 rdev->config.cik.max_backends_per_se = 1;
3626 rdev->config.cik.max_texture_channel_caches = 2;
3627 rdev->config.cik.max_gprs = 256;
3628 rdev->config.cik.max_gs_threads = 16;
3629 rdev->config.cik.max_hw_contexts = 8;
3630
3631 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3632 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3633 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3634 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3635 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3636 break;
3637 }
3638
3639 /* Initialize HDP */
3640 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3641 WREG32((0x2c14 + j), 0x00000000);
3642 WREG32((0x2c18 + j), 0x00000000);
3643 WREG32((0x2c1c + j), 0x00000000);
3644 WREG32((0x2c20 + j), 0x00000000);
3645 WREG32((0x2c24 + j), 0x00000000);
3646 }
3647
3648 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3649 WREG32(SRBM_INT_CNTL, 0x1);
3650 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3651
3652 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3653
3654 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3655 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3656
3657 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3658 rdev->config.cik.mem_max_burst_length_bytes = 256;
3659 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3660 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3661 if (rdev->config.cik.mem_row_size_in_kb > 4)
3662 rdev->config.cik.mem_row_size_in_kb = 4;
3663 /* XXX use MC settings? */
3664 rdev->config.cik.shader_engine_tile_size = 32;
3665 rdev->config.cik.num_gpus = 1;
3666 rdev->config.cik.multi_gpu_tile_size = 64;
3667
3668 /* fix up row size */
3669 gb_addr_config &= ~ROW_SIZE_MASK;
3670 switch (rdev->config.cik.mem_row_size_in_kb) {
3671 case 1:
3672 default:
3673 gb_addr_config |= ROW_SIZE(0);
3674 break;
3675 case 2:
3676 gb_addr_config |= ROW_SIZE(1);
3677 break;
3678 case 4:
3679 gb_addr_config |= ROW_SIZE(2);
3680 break;
3681 }
3682
3683 /* setup tiling info dword. gb_addr_config is not adequate since it does
3684 * not have bank info, so create a custom tiling dword.
3685 * bits 3:0 num_pipes
3686 * bits 7:4 num_banks
3687 * bits 11:8 group_size
3688 * bits 15:12 row_size
3689 */
3690 rdev->config.cik.tile_config = 0;
3691 switch (rdev->config.cik.num_tile_pipes) {
3692 case 1:
3693 rdev->config.cik.tile_config |= (0 << 0);
3694 break;
3695 case 2:
3696 rdev->config.cik.tile_config |= (1 << 0);
3697 break;
3698 case 4:
3699 rdev->config.cik.tile_config |= (2 << 0);
3700 break;
3701 case 8:
3702 default:
3703 /* XXX what about 12? */
3704 rdev->config.cik.tile_config |= (3 << 0);
3705 break;
3706 }
a537314e
MD		 3707 rdev->config.cik.tile_config |=
3708 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3709 rdev->config.cik.tile_config |=
3710 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3711 rdev->config.cik.tile_config |=
3712 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3713
3714 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3715 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3716 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3717 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3718 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3719 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3720 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3721 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3722
3723 cik_tiling_mode_table_init(rdev);
3724
3725 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3726 rdev->config.cik.max_sh_per_se,
3727 rdev->config.cik.max_backends_per_se);
3728
52da51f0 3729 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3730 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3731 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3732 rdev->config.cik.active_cus +=
3733 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3734 }
3735 }
3736
8cc1a532
AD		 3737 /* set HW defaults for 3D engine */
3738 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3739
1c0a4625
OG		 3740 mutex_lock(&rdev->grbm_idx_mutex);
3741 /*
3742 * making sure that the following register writes will be broadcasted
3743 * to all the shaders
3744 */
3745 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3746 WREG32(SX_DEBUG_1, 0x20);
3747
3748 WREG32(TA_CNTL_AUX, 0x00010000);
3749
3750 tmp = RREG32(SPI_CONFIG_CNTL);
3751 tmp |= 0x03000000;
3752 WREG32(SPI_CONFIG_CNTL, tmp);
3753
3754 WREG32(SQ_CONFIG, 1);
3755
3756 WREG32(DB_DEBUG, 0);
3757
3758 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3759 tmp |= 0x00000400;
3760 WREG32(DB_DEBUG2, tmp);
3761
3762 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3763 tmp |= 0x00020200;
3764 WREG32(DB_DEBUG3, tmp);
3765
3766 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3767 tmp |= 0x00018208;
3768 WREG32(CB_HW_CONTROL, tmp);
3769
3770 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3771
3772 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3773 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3774 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3775 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3776
3777 WREG32(VGT_NUM_INSTANCES, 1);
3778
3779 WREG32(CP_PERFMON_CNTL, 0);
3780
3781 WREG32(SQ_CONFIG, 0);
3782
3783 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3784 FORCE_EOV_MAX_REZ_CNT(255)));
3785
3786 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3787 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3788
3789 WREG32(VGT_GS_VERTEX_REUSE, 16);
3790 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3791
3792 tmp = RREG32(HDP_MISC_CNTL);
3793 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3794 WREG32(HDP_MISC_CNTL, tmp);
3795
3796 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3797 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3798
3799 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3800 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3801 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3802
3803 udelay(50);
3804}
3805
2cae3bc3
AD		 3806/*
3807 * GPU scratch registers helpers function.
3808 */
3809/**
3810 * cik_scratch_init - setup driver info for CP scratch regs
3811 *
3812 * @rdev: radeon_device pointer
3813 *
3814 * Set up the number and offset of the CP scratch registers.
3815 * NOTE: use of CP scratch registers is a legacy inferface and
3816 * is not used by default on newer asics (r6xx+). On newer asics,
3817 * memory buffers are used for fences rather than scratch regs.
3818 */
3819static void cik_scratch_init(struct radeon_device *rdev)
3820{
3821 int i;
3822
3823 rdev->scratch.num_reg = 7;
3824 rdev->scratch.reg_base = SCRATCH_REG0;
3825 for (i = 0; i < rdev->scratch.num_reg; i++) {
3826 rdev->scratch.free[i] = true;
3827 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3828 }
3829}
3830
fbc832c7
AD		 3831/**
3832 * cik_ring_test - basic gfx ring test
3833 *
3834 * @rdev: radeon_device pointer
3835 * @ring: radeon_ring structure holding ring information
3836 *
3837 * Allocate a scratch register and write to it using the gfx ring (CIK).
3838 * Provides a basic gfx ring test to verify that the ring is working.
3839 * Used by cik_cp_gfx_resume();
3840 * Returns 0 on success, error on failure.
3841 */
3842int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3843{
3844 uint32_t scratch;
3845 uint32_t tmp = 0;
3846 unsigned i;
3847 int r;
3848
3849 r = radeon_scratch_get(rdev, &scratch);
3850 if (r) {
3851 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3852 return r;
3853 }
3854 WREG32(scratch, 0xCAFEDEAD);
3855 r = radeon_ring_lock(rdev, ring, 3);
3856 if (r) {
3857 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3858 radeon_scratch_free(rdev, scratch);
3859 return r;
3860 }
3861 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3862 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3863 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3864 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3865
fbc832c7
AD		 3866 for (i = 0; i < rdev->usec_timeout; i++) {
3867 tmp = RREG32(scratch);
3868 if (tmp == 0xDEADBEEF)
3869 break;
3870 DRM_UDELAY(1);
3871 }
3872 if (i < rdev->usec_timeout) {
3873 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3874 } else {
3875 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3876 ring->idx, scratch, tmp);
3877 r = -EINVAL;
3878 }
3879 radeon_scratch_free(rdev, scratch);
3880 return r;
3881}
3882
780f5ddd
AD		 3883/**
3884 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3885 *
3886 * @rdev: radeon_device pointer
3887 * @ridx: radeon ring index
3888 *
3889 * Emits an hdp flush on the cp.
3890 */
3891static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3892 int ridx)
3893{
3894 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3895 u32 ref_and_mask;
780f5ddd 3896
5d259067
AD		 3897 switch (ring->idx) {
3898 case CAYMAN_RING_TYPE_CP1_INDEX:
3899 case CAYMAN_RING_TYPE_CP2_INDEX:
3900 default:
3901 switch (ring->me) {
3902 case 0:
3903 ref_and_mask = CP2 << ring->pipe;
3904 break;
3905 case 1:
3906 ref_and_mask = CP6 << ring->pipe;
3907 break;
3908 default:
3909 return;
3910 }
3911 break;
3912 case RADEON_RING_TYPE_GFX_INDEX:
3913 ref_and_mask = CP0;
3914 break;
3915 }
3916
3917 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3918 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3919 WAIT_REG_MEM_FUNCTION(3) | /* == */
3920 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3921 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3922 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3923 radeon_ring_write(ring, ref_and_mask);
3924 radeon_ring_write(ring, ref_and_mask);
3925 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3926}
3927
2cae3bc3 3928/**
b07fdd38 3929 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3930 *
3931 * @rdev: radeon_device pointer
3932 * @fence: radeon fence object
3933 *
3934 * Emits a fence sequnce number on the gfx ring and flushes
3935 * GPU caches.
3936 */
b07fdd38
AD		 3937void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3938 struct radeon_fence *fence)
2cae3bc3
AD		 3939{
3940 struct radeon_ring *ring = &rdev->ring[fence->ring];
3941 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3942
a9c73a0e
CK		 3943 /* Workaround for cache flush problems. First send a dummy EOP
3944 * event down the pipe with seq one below.
3945 */
3946 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3947 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3948 EOP_TC_ACTION_EN |
3949 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3950 EVENT_INDEX(5)));
3951 radeon_ring_write(ring, addr & 0xfffffffc);
3952 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3953 DATA_SEL(1) | INT_SEL(0));
3954 radeon_ring_write(ring, fence->seq - 1);
3955 radeon_ring_write(ring, 0);
3956
3957 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3958 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3959 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3960 EOP_TC_ACTION_EN |
3961 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3962 EVENT_INDEX(5)));
3963 radeon_ring_write(ring, addr & 0xfffffffc);
3964 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3965 radeon_ring_write(ring, fence->seq);
3966 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3967}
3968
b07fdd38
AD		 3969/**
3970 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3971 *
3972 * @rdev: radeon_device pointer
3973 * @fence: radeon fence object
3974 *
3975 * Emits a fence sequnce number on the compute ring and flushes
3976 * GPU caches.
3977 */
3978void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3979 struct radeon_fence *fence)
3980{
3981 struct radeon_ring *ring = &rdev->ring[fence->ring];
3982 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3983
3984 /* RELEASE_MEM - flush caches, send int */
3985 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3986 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3987 EOP_TC_ACTION_EN |
3988 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3989 EVENT_INDEX(5)));
3990 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3991 radeon_ring_write(ring, addr & 0xfffffffc);
3992 radeon_ring_write(ring, upper_32_bits(addr));
3993 radeon_ring_write(ring, fence->seq);
3994 radeon_ring_write(ring, 0);
b07fdd38
AD		 3995}
3996
86302eea
CK		 3997/**
3998 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3999 *
4000 * @rdev: radeon_device pointer
4001 * @ring: radeon ring buffer object
4002 * @semaphore: radeon semaphore object
4003 * @emit_wait: Is this a sempahore wait?
4004 *
4005 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
4006 * from running ahead of semaphore waits.
4007 */
1654b817 4008bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 4009 struct radeon_ring *ring,
4010 struct radeon_semaphore *semaphore,
4011 bool emit_wait)
4012{
4013 uint64_t addr = semaphore->gpu_addr;
4014 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
4015
4016 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 4017 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 4018 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 4019
86302eea
CK		 4020 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
4021 /* Prevent the PFP from running ahead of the semaphore wait */
4022 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4023 radeon_ring_write(ring, 0x0);
4024 }
4025
1654b817 4026 return true;
2cae3bc3
AD		 4027}
4028
c9dbd705
AD		 4029/**
4030 * cik_copy_cpdma - copy pages using the CP DMA engine
4031 *
4032 * @rdev: radeon_device pointer
4033 * @src_offset: src GPU address
4034 * @dst_offset: dst GPU address
4035 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 4036 * @resv: reservation object to sync to
c9dbd705
AD		 4037 *
4038 * Copy GPU paging using the CP DMA engine (CIK+).
4039 * Used by the radeon ttm implementation to move pages if
4040 * registered as the asic copy callback.
4041 */
57d20a43
CK		 4042struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4043 uint64_t src_offset, uint64_t dst_offset,
4044 unsigned num_gpu_pages,
4045 struct reservation_object *resv)
c9dbd705 4046{
57d20a43 4047 struct radeon_fence *fence;
975700d2 4048 struct radeon_sync sync;
c9dbd705
AD		 4049 int ring_index = rdev->asic->copy.blit_ring_index;
4050 struct radeon_ring *ring = &rdev->ring[ring_index];
4051 u32 size_in_bytes, cur_size_in_bytes, control;
4052 int i, num_loops;
4053 int r = 0;
4054
975700d2 4055 radeon_sync_create(&sync);
c9dbd705
AD		 4056
4057 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4058 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4059 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4060 if (r) {
4061 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 4062 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4063 return ERR_PTR(r);
c9dbd705
AD		 4064 }
4065
975700d2
CK		 4066 radeon_sync_resv(rdev, &sync, resv, false);
4067 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 4068
4069 for (i = 0; i < num_loops; i++) {
4070 cur_size_in_bytes = size_in_bytes;
4071 if (cur_size_in_bytes > 0x1fffff)
4072 cur_size_in_bytes = 0x1fffff;
4073 size_in_bytes -= cur_size_in_bytes;
4074 control = 0;
4075 if (size_in_bytes == 0)
4076 control |= PACKET3_DMA_DATA_CP_SYNC;
4077 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4078 radeon_ring_write(ring, control);
4079 radeon_ring_write(ring, lower_32_bits(src_offset));
4080 radeon_ring_write(ring, upper_32_bits(src_offset));
4081 radeon_ring_write(ring, lower_32_bits(dst_offset));
4082 radeon_ring_write(ring, upper_32_bits(dst_offset));
4083 radeon_ring_write(ring, cur_size_in_bytes);
4084 src_offset += cur_size_in_bytes;
4085 dst_offset += cur_size_in_bytes;
4086 }
4087
57d20a43 4088 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 4089 if (r) {
4090 radeon_ring_unlock_undo(rdev, ring);
975700d2 4091 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4092 return ERR_PTR(r);
c9dbd705
AD		 4093 }
4094
1538a9e0 4095 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 4096 radeon_sync_free(rdev, &sync, fence);
c9dbd705 4097
57d20a43 4098 return fence;
c9dbd705
AD		 4099}
4100
2cae3bc3
AD		 4101/*
4102 * IB stuff
4103 */
4104/**
4105 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4106 *
4107 * @rdev: radeon_device pointer
4108 * @ib: radeon indirect buffer object
4109 *
4110 * Emits an DE (drawing engine) or CE (constant engine) IB
4111 * on the gfx ring. IBs are usually generated by userspace
4112 * acceleration drivers and submitted to the kernel for
4113 * sheduling on the ring. This function schedules the IB
4114 * on the gfx ring for execution by the GPU.
4115 */
4116void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4117{
4118 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 4119 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 4120 u32 header, control = INDIRECT_BUFFER_VALID;
4121
4122 if (ib->is_const_ib) {
4123 /* set switch buffer packet before const IB */
4124 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4125 radeon_ring_write(ring, 0);
4126
4127 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4128 } else {
4129 u32 next_rptr;
4130 if (ring->rptr_save_reg) {
4131 next_rptr = ring->wptr + 3 + 4;
4132 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4133 radeon_ring_write(ring, ((ring->rptr_save_reg -
4134 PACKET3_SET_UCONFIG_REG_START) >> 2));
4135 radeon_ring_write(ring, next_rptr);
4136 } else if (rdev->wb.enabled) {
4137 next_rptr = ring->wptr + 5 + 4;
4138 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4139 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4140 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4141 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4142 radeon_ring_write(ring, next_rptr);
4143 }
4144
4145 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4146 }
4147
7c42bc1a 4148 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 4149
4150 radeon_ring_write(ring, header);
4151 radeon_ring_write(ring,
4152#ifdef __BIG_ENDIAN
4153 (2 << 0) |
4154#endif
4155 (ib->gpu_addr & 0xFFFFFFFC));
4156 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4157 radeon_ring_write(ring, control);
4158}
4159
fbc832c7
AD		 4160/**
4161 * cik_ib_test - basic gfx ring IB test
4162 *
4163 * @rdev: radeon_device pointer
4164 * @ring: radeon_ring structure holding ring information
4165 *
4166 * Allocate an IB and execute it on the gfx ring (CIK).
4167 * Provides a basic gfx ring test to verify that IBs are working.
4168 * Returns 0 on success, error on failure.
4169 */
4170int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4171{
4172 struct radeon_ib ib;
4173 uint32_t scratch;
4174 uint32_t tmp = 0;
4175 unsigned i;
4176 int r;
4177
4178 r = radeon_scratch_get(rdev, &scratch);
4179 if (r) {
4180 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4181 return r;
4182 }
4183 WREG32(scratch, 0xCAFEDEAD);
4184 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4185 if (r) {
4186 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4187 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4188 return r;
4189 }
4190 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4191 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4192 ib.ptr[2] = 0xDEADBEEF;
4193 ib.length_dw = 3;
1538a9e0 4194 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 4195 if (r) {
4196 radeon_scratch_free(rdev, scratch);
4197 radeon_ib_free(rdev, &ib);
4198 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4199 return r;
4200 }
4201 r = radeon_fence_wait(ib.fence, false);
4202 if (r) {
4203 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4204 radeon_scratch_free(rdev, scratch);
4205 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4206 return r;
4207 }
4208 for (i = 0; i < rdev->usec_timeout; i++) {
4209 tmp = RREG32(scratch);
4210 if (tmp == 0xDEADBEEF)
4211 break;
4212 DRM_UDELAY(1);
4213 }
4214 if (i < rdev->usec_timeout) {
4215 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4216 } else {
4217 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4218 scratch, tmp);
4219 r = -EINVAL;
4220 }
4221 radeon_scratch_free(rdev, scratch);
4222 radeon_ib_free(rdev, &ib);
4223 return r;
4224}
4225
841cf442
AD		 4226/*
4227 * CP.
4228 * On CIK, gfx and compute now have independant command processors.
4229 *
4230 * GFX
4231 * Gfx consists of a single ring and can process both gfx jobs and
4232 * compute jobs. The gfx CP consists of three microengines (ME):
4233 * PFP - Pre-Fetch Parser
4234 * ME - Micro Engine
4235 * CE - Constant Engine
4236 * The PFP and ME make up what is considered the Drawing Engine (DE).
4237 * The CE is an asynchronous engine used for updating buffer desciptors
4238 * used by the DE so that they can be loaded into cache in parallel
4239 * while the DE is processing state update packets.
4240 *
4241 * Compute
4242 * The compute CP consists of two microengines (ME):
4243 * MEC1 - Compute MicroEngine 1
4244 * MEC2 - Compute MicroEngine 2
4245 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4246 * The queues are exposed to userspace and are programmed directly
4247 * by the compute runtime.
4248 */
4249/**
4250 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4251 *
4252 * @rdev: radeon_device pointer
4253 * @enable: enable or disable the MEs
4254 *
4255 * Halts or unhalts the gfx MEs.
4256 */
4257static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4258{
4259 if (enable)
4260 WREG32(CP_ME_CNTL, 0);
4261 else {
50efa51a
AD		 4262 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4263 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4264 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4265 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4266 }
4267 udelay(50);
4268}
4269
4270/**
4271 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4272 *
4273 * @rdev: radeon_device pointer
4274 *
4275 * Loads the gfx PFP, ME, and CE ucode.
4276 * Returns 0 for success, -EINVAL if the ucode is not available.
4277 */
4278static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4279{
841cf442
AD		 4280 int i;
4281
4282 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4283 return -EINVAL;
4284
4285 cik_cp_gfx_enable(rdev, false);
4286
f2c6b0f4
AD		 4287 if (rdev->new_fw) {
4288 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4289 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4290 const struct gfx_firmware_header_v1_0 *ce_hdr =
4291 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4292 const struct gfx_firmware_header_v1_0 *me_hdr =
4293 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4294 const __le32 *fw_data;
4295 u32 fw_size;
4296
4297 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4298 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4299 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4300
4301 /* PFP */
4302 fw_data = (const __le32 *)
4303 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4304 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4305 WREG32(CP_PFP_UCODE_ADDR, 0);
4306 for (i = 0; i < fw_size; i++)
4307 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4308 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 4309
4310 /* CE */
4311 fw_data = (const __le32 *)
4312 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4313 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4314 WREG32(CP_CE_UCODE_ADDR, 0);
4315 for (i = 0; i < fw_size; i++)
4316 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4317 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 4318
4319 /* ME */
4320 fw_data = (const __be32 *)
4321 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4322 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4323 WREG32(CP_ME_RAM_WADDR, 0);
4324 for (i = 0; i < fw_size; i++)
4325 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4326 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4327 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4328 } else {
4329 const __be32 *fw_data;
4330
4331 /* PFP */
4332 fw_data = (const __be32 *)rdev->pfp_fw->data;
4333 WREG32(CP_PFP_UCODE_ADDR, 0);
4334 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4335 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4336 WREG32(CP_PFP_UCODE_ADDR, 0);
4337
4338 /* CE */
4339 fw_data = (const __be32 *)rdev->ce_fw->data;
4340 WREG32(CP_CE_UCODE_ADDR, 0);
4341 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4342 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4343 WREG32(CP_CE_UCODE_ADDR, 0);
4344
4345 /* ME */
4346 fw_data = (const __be32 *)rdev->me_fw->data;
4347 WREG32(CP_ME_RAM_WADDR, 0);
4348 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4349 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4350 WREG32(CP_ME_RAM_WADDR, 0);
4351 }
841cf442 4352
841cf442
AD		 4353 return 0;
4354}
4355
4356/**
4357 * cik_cp_gfx_start - start the gfx ring
4358 *
4359 * @rdev: radeon_device pointer
4360 *
4361 * Enables the ring and loads the clear state context and other
4362 * packets required to init the ring.
4363 * Returns 0 for success, error for failure.
4364 */
4365static int cik_cp_gfx_start(struct radeon_device *rdev)
4366{
4367 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4368 int r, i;
4369
4370 /* init the CP */
4371 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4372 WREG32(CP_ENDIAN_SWAP, 0);
4373 WREG32(CP_DEVICE_ID, 1);
4374
4375 cik_cp_gfx_enable(rdev, true);
4376
4377 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4378 if (r) {
4379 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4380 return r;
4381 }
4382
4383 /* init the CE partitions. CE only used for gfx on CIK */
4384 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4385 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4386 radeon_ring_write(ring, 0x8000);
4387 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4388
4389 /* setup clear context state */
4390 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4391 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4392
4393 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4394 radeon_ring_write(ring, 0x80000000);
4395 radeon_ring_write(ring, 0x80000000);
4396
4397 for (i = 0; i < cik_default_size; i++)
4398 radeon_ring_write(ring, cik_default_state[i]);
4399
4400 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4401 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4402
4403 /* set clear context state */
4404 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4405 radeon_ring_write(ring, 0);
4406
4407 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4408 radeon_ring_write(ring, 0x00000316);
4409 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4410 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4411
1538a9e0 4412 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4413
4414 return 0;
4415}
4416
4417/**
4418 * cik_cp_gfx_fini - stop the gfx ring
4419 *
4420 * @rdev: radeon_device pointer
4421 *
4422 * Stop the gfx ring and tear down the driver ring
4423 * info.
4424 */
4425static void cik_cp_gfx_fini(struct radeon_device *rdev)
4426{
4427 cik_cp_gfx_enable(rdev, false);
4428 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4429}
4430
4431/**
4432 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4433 *
4434 * @rdev: radeon_device pointer
4435 *
4436 * Program the location and size of the gfx ring buffer
4437 * and test it to make sure it's working.
4438 * Returns 0 for success, error for failure.
4439 */
4440static int cik_cp_gfx_resume(struct radeon_device *rdev)
4441{
4442 struct radeon_ring *ring;
4443 u32 tmp;
4444 u32 rb_bufsz;
4445 u64 rb_addr;
4446 int r;
4447
4448 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4449 if (rdev->family != CHIP_HAWAII)
4450 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4451
4452 /* Set the write pointer delay */
4453 WREG32(CP_RB_WPTR_DELAY, 0);
4454
4455 /* set the RB to use vmid 0 */
4456 WREG32(CP_RB_VMID, 0);
4457
4458 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4459
4460 /* ring 0 - compute and gfx */
4461 /* Set ring buffer size */
4462 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4463 rb_bufsz = order_base_2(ring->ring_size / 8);
4464 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4465#ifdef __BIG_ENDIAN
4466 tmp |= BUF_SWAP_32BIT;
4467#endif
4468 WREG32(CP_RB0_CNTL, tmp);
4469
4470 /* Initialize the ring buffer's read and write pointers */
4471 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4472 ring->wptr = 0;
4473 WREG32(CP_RB0_WPTR, ring->wptr);
4474
4475 /* set the wb address wether it's enabled or not */
4476 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4477 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4478
4479 /* scratch register shadowing is no longer supported */
4480 WREG32(SCRATCH_UMSK, 0);
4481
4482 if (!rdev->wb.enabled)
4483 tmp |= RB_NO_UPDATE;
4484
4485 mdelay(1);
4486 WREG32(CP_RB0_CNTL, tmp);
4487
4488 rb_addr = ring->gpu_addr >> 8;
4489 WREG32(CP_RB0_BASE, rb_addr);
4490 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4491
841cf442
AD		 4492 /* start the ring */
4493 cik_cp_gfx_start(rdev);
4494 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4495 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4496 if (r) {
4497 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4498 return r;
4499 }
50efa51a
AD		 4500
4501 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4502 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4503
841cf442
AD		 4504 return 0;
4505}
4506
ea31bf69
AD		 4507u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4508 struct radeon_ring *ring)
963e81f9
AD		 4509{
4510 u32 rptr;
4511
ea31bf69
AD		 4512 if (rdev->wb.enabled)
4513 rptr = rdev->wb.wb[ring->rptr_offs/4];
4514 else
4515 rptr = RREG32(CP_RB0_RPTR);
4516
4517 return rptr;
4518}
4519
4520u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4521 struct radeon_ring *ring)
4522{
4523 u32 wptr;
4524
4525 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4526
ea31bf69
AD		 4527 return wptr;
4528}
4529
4530void cik_gfx_set_wptr(struct radeon_device *rdev,
4531 struct radeon_ring *ring)
4532{
4533 WREG32(CP_RB0_WPTR, ring->wptr);
4534 (void)RREG32(CP_RB0_WPTR);
4535}
4536
4537u32 cik_compute_get_rptr(struct radeon_device *rdev,
4538 struct radeon_ring *ring)
4539{
4540 u32 rptr;
963e81f9
AD		 4541
4542 if (rdev->wb.enabled) {
ea31bf69 4543 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4544 } else {
f61d5b46 4545 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4546 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4547 rptr = RREG32(CP_HQD_PQ_RPTR);
4548 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4549 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4550 }
963e81f9
AD		 4551
4552 return rptr;
4553}
4554
ea31bf69
AD		 4555u32 cik_compute_get_wptr(struct radeon_device *rdev,
4556 struct radeon_ring *ring)
963e81f9
AD		 4557{
4558 u32 wptr;
4559
4560 if (rdev->wb.enabled) {
ea31bf69
AD		 4561 /* XXX check if swapping is necessary on BE */
4562 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4563 } else {
f61d5b46 4564 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4565 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4566 wptr = RREG32(CP_HQD_PQ_WPTR);
4567 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4568 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4569 }
963e81f9
AD		 4570
4571 return wptr;
4572}
4573
ea31bf69
AD		 4574void cik_compute_set_wptr(struct radeon_device *rdev,
4575 struct radeon_ring *ring)
963e81f9 4576{
ea31bf69
AD		 4577 /* XXX check if swapping is necessary on BE */
4578 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4579 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4580}
4581
841cf442
AD		 4582/**
4583 * cik_cp_compute_enable - enable/disable the compute CP MEs
4584 *
4585 * @rdev: radeon_device pointer
4586 * @enable: enable or disable the MEs
4587 *
4588 * Halts or unhalts the compute MEs.
4589 */
4590static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4591{
4592 if (enable)
4593 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4594 else {
841cf442 4595 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4596 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4597 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4598 }
841cf442
AD		 4599 udelay(50);
4600}
4601
4602/**
4603 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4604 *
4605 * @rdev: radeon_device pointer
4606 *
4607 * Loads the compute MEC1&2 ucode.
4608 * Returns 0 for success, -EINVAL if the ucode is not available.
4609 */
4610static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4611{
841cf442
AD		 4612 int i;
4613
4614 if (!rdev->mec_fw)
4615 return -EINVAL;
4616
4617 cik_cp_compute_enable(rdev, false);
4618
f2c6b0f4
AD		 4619 if (rdev->new_fw) {
4620 const struct gfx_firmware_header_v1_0 *mec_hdr =
4621 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4622 const __le32 *fw_data;
4623 u32 fw_size;
4624
4625 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4626
4627 /* MEC1 */
4628 fw_data = (const __le32 *)
4629 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4630 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4631 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4632 for (i = 0; i < fw_size; i++)
4633 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4634 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4635
841cf442 4636 /* MEC2 */
f2c6b0f4
AD		 4637 if (rdev->family == CHIP_KAVERI) {
4638 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4639 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4640
4641 fw_data = (const __le32 *)
4642 (rdev->mec2_fw->data +
4643 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4644 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4645 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4646 for (i = 0; i < fw_size; i++)
4647 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4648 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4649 }
4650 } else {
4651 const __be32 *fw_data;
4652
4653 /* MEC1 */
841cf442 4654 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4655 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4656 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4657 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4658 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4659
4660 if (rdev->family == CHIP_KAVERI) {
4661 /* MEC2 */
4662 fw_data = (const __be32 *)rdev->mec_fw->data;
4663 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4664 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4665 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4666 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4667 }
841cf442
AD		 4668 }
4669
4670 return 0;
4671}
4672
4673/**
4674 * cik_cp_compute_start - start the compute queues
4675 *
4676 * @rdev: radeon_device pointer
4677 *
4678 * Enable the compute queues.
4679 * Returns 0 for success, error for failure.
4680 */
4681static int cik_cp_compute_start(struct radeon_device *rdev)
4682{
963e81f9
AD		 4683 cik_cp_compute_enable(rdev, true);
4684
841cf442
AD		 4685 return 0;
4686}
4687
4688/**
4689 * cik_cp_compute_fini - stop the compute queues
4690 *
4691 * @rdev: radeon_device pointer
4692 *
4693 * Stop the compute queues and tear down the driver queue
4694 * info.
4695 */
4696static void cik_cp_compute_fini(struct radeon_device *rdev)
4697{
963e81f9
AD		 4698 int i, idx, r;
4699
841cf442 4700 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4701
4702 for (i = 0; i < 2; i++) {
4703 if (i == 0)
4704 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4705 else
4706 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4707
4708 if (rdev->ring[idx].mqd_obj) {
4709 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4710 if (unlikely(r != 0))
4711 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4712
4713 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4714 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4715
4716 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4717 rdev->ring[idx].mqd_obj = NULL;
4718 }
4719 }
841cf442
AD		 4720}
4721
963e81f9
AD		 4722static void cik_mec_fini(struct radeon_device *rdev)
4723{
4724 int r;
4725
4726 if (rdev->mec.hpd_eop_obj) {
4727 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4728 if (unlikely(r != 0))
4729 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4730 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4731 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4732
4733 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4734 rdev->mec.hpd_eop_obj = NULL;
4735 }
4736}
4737
4738#define MEC_HPD_SIZE 2048
4739
4740static int cik_mec_init(struct radeon_device *rdev)
4741{
4742 int r;
4743 u32 *hpd;
4744
4745 /*
4746 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4747 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4748 * Nonetheless, we assign only 1 pipe because all other pipes will
4749 * be handled by KFD
963e81f9 4750 */
62a7b7fb
OG		 4751 rdev->mec.num_mec = 1;
4752 rdev->mec.num_pipe = 1;
963e81f9
AD		 4753 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4754
4755 if (rdev->mec.hpd_eop_obj == NULL) {
4756 r = radeon_bo_create(rdev,
4757 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4758 PAGE_SIZE, true,
831b6966 4759 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4760 &rdev->mec.hpd_eop_obj);
4761 if (r) {
4762 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4763 return r;
4764 }
4765 }
4766
4767 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4768 if (unlikely(r != 0)) {
4769 cik_mec_fini(rdev);
4770 return r;
4771 }
4772 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4773 &rdev->mec.hpd_eop_gpu_addr);
4774 if (r) {
4775 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4776 cik_mec_fini(rdev);
4777 return r;
4778 }
4779 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4780 if (r) {
4781 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4782 cik_mec_fini(rdev);
4783 return r;
4784 }
4785
4786 /* clear memory. Not sure if this is required or not */
4787 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4788
4789 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4790 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4791
4792 return 0;
4793}
4794
4795struct hqd_registers
4796{
4797 u32 cp_mqd_base_addr;
4798 u32 cp_mqd_base_addr_hi;
4799 u32 cp_hqd_active;
4800 u32 cp_hqd_vmid;
4801 u32 cp_hqd_persistent_state;
4802 u32 cp_hqd_pipe_priority;
4803 u32 cp_hqd_queue_priority;
4804 u32 cp_hqd_quantum;
4805 u32 cp_hqd_pq_base;
4806 u32 cp_hqd_pq_base_hi;
4807 u32 cp_hqd_pq_rptr;
4808 u32 cp_hqd_pq_rptr_report_addr;
4809 u32 cp_hqd_pq_rptr_report_addr_hi;
4810 u32 cp_hqd_pq_wptr_poll_addr;
4811 u32 cp_hqd_pq_wptr_poll_addr_hi;
4812 u32 cp_hqd_pq_doorbell_control;
4813 u32 cp_hqd_pq_wptr;
4814 u32 cp_hqd_pq_control;
4815 u32 cp_hqd_ib_base_addr;
4816 u32 cp_hqd_ib_base_addr_hi;
4817 u32 cp_hqd_ib_rptr;
4818 u32 cp_hqd_ib_control;
4819 u32 cp_hqd_iq_timer;
4820 u32 cp_hqd_iq_rptr;
4821 u32 cp_hqd_dequeue_request;
4822 u32 cp_hqd_dma_offload;
4823 u32 cp_hqd_sema_cmd;
4824 u32 cp_hqd_msg_type;
4825 u32 cp_hqd_atomic0_preop_lo;
4826 u32 cp_hqd_atomic0_preop_hi;
4827 u32 cp_hqd_atomic1_preop_lo;
4828 u32 cp_hqd_atomic1_preop_hi;
4829 u32 cp_hqd_hq_scheduler0;
4830 u32 cp_hqd_hq_scheduler1;
4831 u32 cp_mqd_control;
4832};
4833
4834struct bonaire_mqd
4835{
4836 u32 header;
4837 u32 dispatch_initiator;
4838 u32 dimensions[3];
4839 u32 start_idx[3];
4840 u32 num_threads[3];
4841 u32 pipeline_stat_enable;
4842 u32 perf_counter_enable;
4843 u32 pgm[2];
4844 u32 tba[2];
4845 u32 tma[2];
4846 u32 pgm_rsrc[2];
4847 u32 vmid;
4848 u32 resource_limits;
4849 u32 static_thread_mgmt01[2];
4850 u32 tmp_ring_size;
4851 u32 static_thread_mgmt23[2];
4852 u32 restart[3];
4853 u32 thread_trace_enable;
4854 u32 reserved1;
4855 u32 user_data[16];
4856 u32 vgtcs_invoke_count[2];
4857 struct hqd_registers queue_state;
4858 u32 dequeue_cntr;
4859 u32 interrupt_queue[64];
4860};
4861
841cf442
AD		 4862/**
4863 * cik_cp_compute_resume - setup the compute queue registers
4864 *
4865 * @rdev: radeon_device pointer
4866 *
4867 * Program the compute queues and test them to make sure they
4868 * are working.
4869 * Returns 0 for success, error for failure.
4870 */
4871static int cik_cp_compute_resume(struct radeon_device *rdev)
4872{
370ce45b 4873 int r, i, j, idx;
963e81f9
AD		 4874 u32 tmp;
4875 bool use_doorbell = true;
4876 u64 hqd_gpu_addr;
4877 u64 mqd_gpu_addr;
4878 u64 eop_gpu_addr;
4879 u64 wb_gpu_addr;
4880 u32 *buf;
4881 struct bonaire_mqd *mqd;
841cf442 4882
841cf442
AD		 4883 r = cik_cp_compute_start(rdev);
4884 if (r)
4885 return r;
963e81f9
AD		 4886
4887 /* fix up chicken bits */
4888 tmp = RREG32(CP_CPF_DEBUG);
4889 tmp |= (1 << 23);
4890 WREG32(CP_CPF_DEBUG, tmp);
4891
4892 /* init the pipes */
f61d5b46 4893 mutex_lock(&rdev->srbm_mutex);
963e81f9 4894
62a7b7fb 4895 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4896
62a7b7fb 4897 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4898
62a7b7fb
OG		 4899 /* write the EOP addr */
4900 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4901 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4902
62a7b7fb
OG		 4903 /* set the VMID assigned */
4904 WREG32(CP_HPD_EOP_VMID, 0);
4905
4906 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4907 tmp = RREG32(CP_HPD_EOP_CONTROL);
4908 tmp &= ~EOP_SIZE_MASK;
4909 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4910 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4911
f61d5b46 4912 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4913
4914 /* init the queues. Just two for now. */
4915 for (i = 0; i < 2; i++) {
4916 if (i == 0)
4917 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4918 else
4919 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4920
4921 if (rdev->ring[idx].mqd_obj == NULL) {
4922 r = radeon_bo_create(rdev,
4923 sizeof(struct bonaire_mqd),
4924 PAGE_SIZE, true,
02376d82 4925 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4926 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4927 if (r) {
4928 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4929 return r;
4930 }
4931 }
4932
4933 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4934 if (unlikely(r != 0)) {
4935 cik_cp_compute_fini(rdev);
4936 return r;
4937 }
4938 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4939 &mqd_gpu_addr);
4940 if (r) {
4941 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4942 cik_cp_compute_fini(rdev);
4943 return r;
4944 }
4945 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4946 if (r) {
4947 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4948 cik_cp_compute_fini(rdev);
4949 return r;
4950 }
4951
963e81f9
AD		 4952 /* init the mqd struct */
4953 memset(buf, 0, sizeof(struct bonaire_mqd));
4954
4955 mqd = (struct bonaire_mqd *)buf;
4956 mqd->header = 0xC0310800;
4957 mqd->static_thread_mgmt01[0] = 0xffffffff;
4958 mqd->static_thread_mgmt01[1] = 0xffffffff;
4959 mqd->static_thread_mgmt23[0] = 0xffffffff;
4960 mqd->static_thread_mgmt23[1] = 0xffffffff;
4961
f61d5b46 4962 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4963 cik_srbm_select(rdev, rdev->ring[idx].me,
4964 rdev->ring[idx].pipe,
4965 rdev->ring[idx].queue, 0);
4966
4967 /* disable wptr polling */
4968 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4969 tmp &= ~WPTR_POLL_EN;
4970 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4971
4972 /* enable doorbell? */
4973 mqd->queue_state.cp_hqd_pq_doorbell_control =
4974 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4975 if (use_doorbell)
4976 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4977 else
4978 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4979 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4980 mqd->queue_state.cp_hqd_pq_doorbell_control);
4981
4982 /* disable the queue if it's active */
4983 mqd->queue_state.cp_hqd_dequeue_request = 0;
4984 mqd->queue_state.cp_hqd_pq_rptr = 0;
4985 mqd->queue_state.cp_hqd_pq_wptr= 0;
4986 if (RREG32(CP_HQD_ACTIVE) & 1) {
4987 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4988 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4989 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4990 break;
4991 udelay(1);
4992 }
4993 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4994 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4995 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4996 }
4997
4998 /* set the pointer to the MQD */
4999 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
5000 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
5001 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
5002 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
5003 /* set MQD vmid to 0 */
5004 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
5005 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
5006 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
5007
5008 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
5009 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
5010 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
5011 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
5012 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
5013 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
5014
5015 /* set up the HQD, this is similar to CP_RB0_CNTL */
5016 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
5017 mqd->queue_state.cp_hqd_pq_control &=
5018 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
5019
5020 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 5021 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 5022 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 5023 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 5024#ifdef __BIG_ENDIAN
5025 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
5026#endif
5027 mqd->queue_state.cp_hqd_pq_control &=
5028 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
5029 mqd->queue_state.cp_hqd_pq_control |=
5030 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
5031 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
5032
5033 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5034 if (i == 0)
5035 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5036 else
5037 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5038 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5039 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5040 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5041 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5042 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5043
5044 /* set the wb address wether it's enabled or not */
5045 if (i == 0)
5046 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5047 else
5048 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5049 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5050 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5051 upper_32_bits(wb_gpu_addr) & 0xffff;
5052 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5053 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5054 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5055 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5056
5057 /* enable the doorbell if requested */
5058 if (use_doorbell) {
5059 mqd->queue_state.cp_hqd_pq_doorbell_control =
5060 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5061 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5062 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 5063 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 5064 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5065 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5066 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5067
5068 } else {
5069 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5070 }
5071 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5072 mqd->queue_state.cp_hqd_pq_doorbell_control);
5073
5074 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5075 rdev->ring[idx].wptr = 0;
5076 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5077 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5078 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5079
5080 /* set the vmid for the queue */
5081 mqd->queue_state.cp_hqd_vmid = 0;
5082 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5083
5084 /* activate the queue */
5085 mqd->queue_state.cp_hqd_active = 1;
5086 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5087
5088 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5089 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5090
5091 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5092 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5093
5094 rdev->ring[idx].ready = true;
5095 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5096 if (r)
5097 rdev->ring[idx].ready = false;
5098 }
5099
841cf442
AD		 5100 return 0;
5101}
5102
841cf442
AD		 5103static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5104{
5105 cik_cp_gfx_enable(rdev, enable);
5106 cik_cp_compute_enable(rdev, enable);
5107}
5108
841cf442
AD		 5109static int cik_cp_load_microcode(struct radeon_device *rdev)
5110{
5111 int r;
5112
5113 r = cik_cp_gfx_load_microcode(rdev);
5114 if (r)
5115 return r;
5116 r = cik_cp_compute_load_microcode(rdev);
5117 if (r)
5118 return r;
5119
5120 return 0;
5121}
5122
841cf442
AD		 5123static void cik_cp_fini(struct radeon_device *rdev)
5124{
5125 cik_cp_gfx_fini(rdev);
5126 cik_cp_compute_fini(rdev);
5127}
5128
841cf442
AD		 5129static int cik_cp_resume(struct radeon_device *rdev)
5130{
5131 int r;
5132
4214faf6
AD		 5133 cik_enable_gui_idle_interrupt(rdev, false);
5134
841cf442
AD		 5135 r = cik_cp_load_microcode(rdev);
5136 if (r)
5137 return r;
5138
5139 r = cik_cp_gfx_resume(rdev);
5140 if (r)
5141 return r;
5142 r = cik_cp_compute_resume(rdev);
5143 if (r)
5144 return r;
5145
4214faf6
AD		 5146 cik_enable_gui_idle_interrupt(rdev, true);
5147
841cf442
AD		 5148 return 0;
5149}
5150
cc066715 5151static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5152{
6f2043ce
AD		 5153 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5154 RREG32(GRBM_STATUS));
5155 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5156 RREG32(GRBM_STATUS2));
5157 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5158 RREG32(GRBM_STATUS_SE0));
5159 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5160 RREG32(GRBM_STATUS_SE1));
5161 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5162 RREG32(GRBM_STATUS_SE2));
5163 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5164 RREG32(GRBM_STATUS_SE3));
5165 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5166 RREG32(SRBM_STATUS));
5167 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5168 RREG32(SRBM_STATUS2));
cc066715
AD		 5169 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5170 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5171 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5172 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5173 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5174 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5175 RREG32(CP_STALLED_STAT1));
5176 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5177 RREG32(CP_STALLED_STAT2));
5178 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5179 RREG32(CP_STALLED_STAT3));
5180 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5181 RREG32(CP_CPF_BUSY_STAT));
5182 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5183 RREG32(CP_CPF_STALLED_STAT1));
5184 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5185 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5186 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5187 RREG32(CP_CPC_STALLED_STAT1));
5188 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5189}
6f2043ce 5190
21a93e13 5191/**
cc066715 5192 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5193 *
5194 * @rdev: radeon_device pointer
21a93e13 5195 *
cc066715
AD		 5196 * Check which blocks are busy and return the relevant reset
5197 * mask to be used by cik_gpu_soft_reset().
5198 * Returns a mask of the blocks to be reset.
21a93e13 5199 */
2483b4ea 5200u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5201{
cc066715
AD		 5202 u32 reset_mask = 0;
5203 u32 tmp;
21a93e13 5204
cc066715
AD		 5205 /* GRBM_STATUS */
5206 tmp = RREG32(GRBM_STATUS);
5207 if (tmp & (PA_BUSY | SC_BUSY |
5208 BCI_BUSY | SX_BUSY |
5209 TA_BUSY | VGT_BUSY |
5210 DB_BUSY | CB_BUSY |
5211 GDS_BUSY | SPI_BUSY |
5212 IA_BUSY | IA_BUSY_NO_DMA))
5213 reset_mask |= RADEON_RESET_GFX;
21a93e13 5214
cc066715
AD		 5215 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5216 reset_mask |= RADEON_RESET_CP;
21a93e13 5217
cc066715
AD		 5218 /* GRBM_STATUS2 */
5219 tmp = RREG32(GRBM_STATUS2);
5220 if (tmp & RLC_BUSY)
5221 reset_mask |= RADEON_RESET_RLC;
21a93e13 5222
cc066715
AD		 5223 /* SDMA0_STATUS_REG */
5224 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5225 if (!(tmp & SDMA_IDLE))
5226 reset_mask |= RADEON_RESET_DMA;
21a93e13 5227
cc066715
AD		 5228 /* SDMA1_STATUS_REG */
5229 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5230 if (!(tmp & SDMA_IDLE))
5231 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5232
cc066715
AD		 5233 /* SRBM_STATUS2 */
5234 tmp = RREG32(SRBM_STATUS2);
5235 if (tmp & SDMA_BUSY)
5236 reset_mask |= RADEON_RESET_DMA;
21a93e13 5237
cc066715
AD		 5238 if (tmp & SDMA1_BUSY)
5239 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5240
cc066715
AD		 5241 /* SRBM_STATUS */
5242 tmp = RREG32(SRBM_STATUS);
21a93e13 5243
cc066715
AD		 5244 if (tmp & IH_BUSY)
5245 reset_mask |= RADEON_RESET_IH;
21a93e13 5246
cc066715
AD		 5247 if (tmp & SEM_BUSY)
5248 reset_mask |= RADEON_RESET_SEM;
21a93e13 5249
cc066715
AD		 5250 if (tmp & GRBM_RQ_PENDING)
5251 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5252
cc066715
AD		 5253 if (tmp & VMC_BUSY)
5254 reset_mask |= RADEON_RESET_VMC;
21a93e13 5255
cc066715
AD		 5256 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5257 MCC_BUSY | MCD_BUSY))
5258 reset_mask |= RADEON_RESET_MC;
21a93e13 5259
cc066715
AD		 5260 if (evergreen_is_display_hung(rdev))
5261 reset_mask |= RADEON_RESET_DISPLAY;
5262
5263 /* Skip MC reset as it's mostly likely not hung, just busy */
5264 if (reset_mask & RADEON_RESET_MC) {
5265 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5266 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5267 }
cc066715
AD		 5268
5269 return reset_mask;
21a93e13
AD		 5270}
5271
5272/**
cc066715 5273 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5274 *
5275 * @rdev: radeon_device pointer
cc066715 5276 * @reset_mask: mask of which blocks to reset
21a93e13 5277 *
cc066715 5278 * Soft reset the blocks specified in @reset_mask.
21a93e13 5279 */
cc066715 5280static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5281{
6f2043ce 5282 struct evergreen_mc_save save;
cc066715
AD		 5283 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5284 u32 tmp;
21a93e13 5285
cc066715
AD		 5286 if (reset_mask == 0)
5287 return;
21a93e13 5288
cc066715 5289 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5290
cc066715
AD		 5291 cik_print_gpu_status_regs(rdev);
5292 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5293 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5294 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5295 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5296
fb2c7f4d
AD		 5297 /* disable CG/PG */
5298 cik_fini_pg(rdev);
5299 cik_fini_cg(rdev);
5300
cc066715
AD		 5301 /* stop the rlc */
5302 cik_rlc_stop(rdev);
21a93e13 5303
cc066715
AD		 5304 /* Disable GFX parsing/prefetching */
5305 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5306
cc066715
AD		 5307 /* Disable MEC parsing/prefetching */
5308 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5309
cc066715
AD		 5310 if (reset_mask & RADEON_RESET_DMA) {
5311 /* sdma0 */
5312 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5313 tmp |= SDMA_HALT;
5314 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5315 }
5316 if (reset_mask & RADEON_RESET_DMA1) {
5317 /* sdma1 */
5318 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5319 tmp |= SDMA_HALT;
5320 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5321 }
21a93e13 5322
6f2043ce 5323 evergreen_mc_stop(rdev, &save);
cc066715 5324 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5325 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5326 }
21a93e13 5327
cc066715
AD		 5328 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5329 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5330
cc066715
AD		 5331 if (reset_mask & RADEON_RESET_CP) {
5332 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5333
cc066715
AD		 5334 srbm_soft_reset |= SOFT_RESET_GRBM;
5335 }
21a93e13 5336
cc066715
AD		 5337 if (reset_mask & RADEON_RESET_DMA)
5338 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5339
cc066715
AD		 5340 if (reset_mask & RADEON_RESET_DMA1)
5341 srbm_soft_reset |= SOFT_RESET_SDMA1;
5342
5343 if (reset_mask & RADEON_RESET_DISPLAY)
5344 srbm_soft_reset |= SOFT_RESET_DC;
5345
5346 if (reset_mask & RADEON_RESET_RLC)
5347 grbm_soft_reset |= SOFT_RESET_RLC;
5348
5349 if (reset_mask & RADEON_RESET_SEM)
5350 srbm_soft_reset |= SOFT_RESET_SEM;
5351
5352 if (reset_mask & RADEON_RESET_IH)
5353 srbm_soft_reset |= SOFT_RESET_IH;
5354
5355 if (reset_mask & RADEON_RESET_GRBM)
5356 srbm_soft_reset |= SOFT_RESET_GRBM;
5357
5358 if (reset_mask & RADEON_RESET_VMC)
5359 srbm_soft_reset |= SOFT_RESET_VMC;
5360
5361 if (!(rdev->flags & RADEON_IS_IGP)) {
5362 if (reset_mask & RADEON_RESET_MC)
5363 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5364 }
5365
cc066715
AD		 5366 if (grbm_soft_reset) {
5367 tmp = RREG32(GRBM_SOFT_RESET);
5368 tmp |= grbm_soft_reset;
5369 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5370 WREG32(GRBM_SOFT_RESET, tmp);
5371 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5372
cc066715 5373 udelay(50);
21a93e13 5374
cc066715
AD		 5375 tmp &= ~grbm_soft_reset;
5376 WREG32(GRBM_SOFT_RESET, tmp);
5377 tmp = RREG32(GRBM_SOFT_RESET);
5378 }
21a93e13 5379
cc066715
AD		 5380 if (srbm_soft_reset) {
5381 tmp = RREG32(SRBM_SOFT_RESET);
5382 tmp |= srbm_soft_reset;
5383 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5384 WREG32(SRBM_SOFT_RESET, tmp);
5385 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5386
cc066715 5387 udelay(50);
21a93e13 5388
cc066715
AD		 5389 tmp &= ~srbm_soft_reset;
5390 WREG32(SRBM_SOFT_RESET, tmp);
5391 tmp = RREG32(SRBM_SOFT_RESET);
5392 }
21a93e13 5393
6f2043ce
AD		 5394 /* Wait a little for things to settle down */
5395 udelay(50);
21a93e13 5396
6f2043ce 5397 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5398 udelay(50);
5399
5400 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5401}
5402
0279ed19
AD		 5403struct kv_reset_save_regs {
5404 u32 gmcon_reng_execute;
5405 u32 gmcon_misc;
5406 u32 gmcon_misc3;
5407};
5408
5409static void kv_save_regs_for_reset(struct radeon_device *rdev,
5410 struct kv_reset_save_regs *save)
5411{
5412 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5413 save->gmcon_misc = RREG32(GMCON_MISC);
5414 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5415
5416 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5417 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5418 STCTRL_STUTTER_EN));
5419}
5420
5421static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5422 struct kv_reset_save_regs *save)
5423{
5424 int i;
5425
5426 WREG32(GMCON_PGFSM_WRITE, 0);
5427 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5428
5429 for (i = 0; i < 5; i++)
5430 WREG32(GMCON_PGFSM_WRITE, 0);
5431
5432 WREG32(GMCON_PGFSM_WRITE, 0);
5433 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5434
5435 for (i = 0; i < 5; i++)
5436 WREG32(GMCON_PGFSM_WRITE, 0);
5437
5438 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5439 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5440
5441 for (i = 0; i < 5; i++)
5442 WREG32(GMCON_PGFSM_WRITE, 0);
5443
5444 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5445 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5446
5447 for (i = 0; i < 5; i++)
5448 WREG32(GMCON_PGFSM_WRITE, 0);
5449
5450 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5451 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5452
5453 for (i = 0; i < 5; i++)
5454 WREG32(GMCON_PGFSM_WRITE, 0);
5455
5456 WREG32(GMCON_PGFSM_WRITE, 0);
5457 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5458
5459 for (i = 0; i < 5; i++)
5460 WREG32(GMCON_PGFSM_WRITE, 0);
5461
5462 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5463 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5464
5465 for (i = 0; i < 5; i++)
5466 WREG32(GMCON_PGFSM_WRITE, 0);
5467
5468 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5469 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5470
5471 for (i = 0; i < 5; i++)
5472 WREG32(GMCON_PGFSM_WRITE, 0);
5473
5474 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5475 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5476
5477 for (i = 0; i < 5; i++)
5478 WREG32(GMCON_PGFSM_WRITE, 0);
5479
5480 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5481 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5482
5483 for (i = 0; i < 5; i++)
5484 WREG32(GMCON_PGFSM_WRITE, 0);
5485
5486 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5487 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5488
5489 WREG32(GMCON_MISC3, save->gmcon_misc3);
5490 WREG32(GMCON_MISC, save->gmcon_misc);
5491 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5492}
5493
5494static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5495{
5496 struct evergreen_mc_save save;
5497 struct kv_reset_save_regs kv_save = { 0 };
5498 u32 tmp, i;
5499
5500 dev_info(rdev->dev, "GPU pci config reset\n");
5501
5502 /* disable dpm? */
5503
5504 /* disable cg/pg */
5505 cik_fini_pg(rdev);
5506 cik_fini_cg(rdev);
5507
5508 /* Disable GFX parsing/prefetching */
5509 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5510
5511 /* Disable MEC parsing/prefetching */
5512 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5513
5514 /* sdma0 */
5515 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5516 tmp |= SDMA_HALT;
5517 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5518 /* sdma1 */
5519 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5520 tmp |= SDMA_HALT;
5521 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5522 /* XXX other engines? */
5523
5524 /* halt the rlc, disable cp internal ints */
5525 cik_rlc_stop(rdev);
5526
5527 udelay(50);
5528
5529 /* disable mem access */
5530 evergreen_mc_stop(rdev, &save);
5531 if (evergreen_mc_wait_for_idle(rdev)) {
5532 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5533 }
5534
5535 if (rdev->flags & RADEON_IS_IGP)
5536 kv_save_regs_for_reset(rdev, &kv_save);
5537
5538 /* disable BM */
5539 pci_clear_master(rdev->pdev);
5540 /* reset */
5541 radeon_pci_config_reset(rdev);
5542
5543 udelay(100);
5544
5545 /* wait for asic to come out of reset */
5546 for (i = 0; i < rdev->usec_timeout; i++) {
5547 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5548 break;
5549 udelay(1);
5550 }
5551
5552 /* does asic init need to be run first??? */
5553 if (rdev->flags & RADEON_IS_IGP)
5554 kv_restore_regs_for_reset(rdev, &kv_save);
5555}
5556
21a93e13 5557/**
cc066715 5558 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5559 *
5560 * @rdev: radeon_device pointer
5561 *
cc066715
AD		 5562 * Look up which blocks are hung and attempt
5563 * to reset them.
6f2043ce 5564 * Returns 0 for success.
21a93e13 5565 */
6f2043ce 5566int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5567{
cc066715 5568 u32 reset_mask;
21a93e13 5569
cc066715 5570 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5571
cc066715
AD		 5572 if (reset_mask)
5573 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5574
0279ed19 5575 /* try soft reset */
cc066715 5576 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5577
cc066715
AD		 5578 reset_mask = cik_gpu_check_soft_reset(rdev);
5579
0279ed19
AD		 5580 /* try pci config reset */
5581 if (reset_mask && radeon_hard_reset)
5582 cik_gpu_pci_config_reset(rdev);
5583
5584 reset_mask = cik_gpu_check_soft_reset(rdev);
5585
cc066715
AD		 5586 if (!reset_mask)
5587 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5588
5589 return 0;
5590}
5591
5592/**
cc066715 5593 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5594 *
5595 * @rdev: radeon_device pointer
cc066715 5596 * @ring: radeon_ring structure holding ring information
21a93e13 5597 *
cc066715
AD		 5598 * Check if the 3D engine is locked up (CIK).
5599 * Returns true if the engine is locked, false if not.
21a93e13 5600 */
cc066715 5601bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5602{
cc066715 5603 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5604
cc066715
AD		 5605 if (!(reset_mask & (RADEON_RESET_GFX |
5606 RADEON_RESET_COMPUTE |
5607 RADEON_RESET_CP))) {
ff212f25 5608 radeon_ring_lockup_update(rdev, ring);
cc066715 5609 return false;
21a93e13 5610 }
cc066715 5611 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5612}
5613
1c49165d 5614/* MC */
21a93e13 5615/**
1c49165d 5616 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5617 *
5618 * @rdev: radeon_device pointer
21a93e13 5619 *
1c49165d
AD		 5620 * Set the location of vram, gart, and AGP in the GPU's
5621 * physical address space (CIK).
21a93e13 5622 */
1c49165d 5623static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5624{
1c49165d 5625 struct evergreen_mc_save save;
21a93e13 5626 u32 tmp;
1c49165d 5627 int i, j;
21a93e13 5628
1c49165d
AD		 5629 /* Initialize HDP */
5630 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5631 WREG32((0x2c14 + j), 0x00000000);
5632 WREG32((0x2c18 + j), 0x00000000);
5633 WREG32((0x2c1c + j), 0x00000000);
5634 WREG32((0x2c20 + j), 0x00000000);
5635 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5636 }
1c49165d 5637 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5638
1c49165d
AD		 5639 evergreen_mc_stop(rdev, &save);
5640 if (radeon_mc_wait_for_idle(rdev)) {
5641 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5642 }
1c49165d
AD		 5643 /* Lockout access through VGA aperture*/
5644 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5645 /* Update configuration */
5646 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5647 rdev->mc.vram_start >> 12);
5648 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5649 rdev->mc.vram_end >> 12);
5650 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5651 rdev->vram_scratch.gpu_addr >> 12);
5652 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5653 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5654 WREG32(MC_VM_FB_LOCATION, tmp);
5655 /* XXX double check these! */
5656 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5657 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5658 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5659 WREG32(MC_VM_AGP_BASE, 0);
5660 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5661 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5662 if (radeon_mc_wait_for_idle(rdev)) {
5663 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5664 }
1c49165d
AD		 5665 evergreen_mc_resume(rdev, &save);
5666 /* we need to own VRAM, so turn off the VGA renderer here
5667 * to stop it overwriting our objects */
5668 rv515_vga_render_disable(rdev);
21a93e13
AD		 5669}
5670
5671/**
1c49165d 5672 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5673 *
5674 * @rdev: radeon_device pointer
21a93e13 5675 *
1c49165d
AD		 5676 * Look up the amount of vram, vram width, and decide how to place
5677 * vram and gart within the GPU's physical address space (CIK).
5678 * Returns 0 for success.
21a93e13 5679 */
1c49165d 5680static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5681{
1c49165d
AD		 5682 u32 tmp;
5683 int chansize, numchan;
21a93e13 5684
1c49165d
AD		 5685 /* Get VRAM informations */
5686 rdev->mc.vram_is_ddr = true;
5687 tmp = RREG32(MC_ARB_RAMCFG);
5688 if (tmp & CHANSIZE_MASK) {
5689 chansize = 64;
21a93e13 5690 } else {
1c49165d 5691 chansize = 32;
21a93e13 5692 }
1c49165d
AD		 5693 tmp = RREG32(MC_SHARED_CHMAP);
5694 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5695 case 0:
5696 default:
5697 numchan = 1;
5698 break;
5699 case 1:
5700 numchan = 2;
5701 break;
5702 case 2:
5703 numchan = 4;
5704 break;
5705 case 3:
5706 numchan = 8;
5707 break;
5708 case 4:
5709 numchan = 3;
5710 break;
5711 case 5:
5712 numchan = 6;
5713 break;
5714 case 6:
5715 numchan = 10;
5716 break;
5717 case 7:
5718 numchan = 12;
5719 break;
5720 case 8:
5721 numchan = 16;
5722 break;
5723 }
5724 rdev->mc.vram_width = numchan * chansize;
5725 /* Could aper size report 0 ? */
5726 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5727 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5728 /* size in MB on si */
13c5bfda
AD		 5729 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5730 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5731 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5732 si_vram_gtt_location(rdev, &rdev->mc);
5733 radeon_update_bandwidth_info(rdev);
5734
5735 return 0;
5736}
5737
5738/*
5739 * GART
5740 * VMID 0 is the physical GPU addresses as used by the kernel.
5741 * VMIDs 1-15 are used for userspace clients and are handled
5742 * by the radeon vm/hsa code.
5743 */
5744/**
5745 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5746 *
5747 * @rdev: radeon_device pointer
5748 *
5749 * Flush the TLB for the VMID 0 page table (CIK).
5750 */
5751void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5752{
5753 /* flush hdp cache */
5754 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5755
5756 /* bits 0-15 are the VM contexts0-15 */
5757 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5758}
5759
08dcc57f
BG		 5760static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5761{
5762 int i;
5763 uint32_t sh_mem_bases, sh_mem_config;
5764
5765 sh_mem_bases = 0x6000 | 0x6000 << 16;
5766 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5767 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5768
5769 mutex_lock(&rdev->srbm_mutex);
5770 for (i = 8; i < 16; i++) {
5771 cik_srbm_select(rdev, 0, 0, 0, i);
5772 /* CP and shaders */
5773 WREG32(SH_MEM_CONFIG, sh_mem_config);
5774 WREG32(SH_MEM_APE1_BASE, 1);
5775 WREG32(SH_MEM_APE1_LIMIT, 0);
5776 WREG32(SH_MEM_BASES, sh_mem_bases);
5777 }
5778 cik_srbm_select(rdev, 0, 0, 0, 0);
5779 mutex_unlock(&rdev->srbm_mutex);
5780}
5781
1c49165d
AD		 5782/**
5783 * cik_pcie_gart_enable - gart enable
5784 *
5785 * @rdev: radeon_device pointer
5786 *
5787 * This sets up the TLBs, programs the page tables for VMID0,
5788 * sets up the hw for VMIDs 1-15 which are allocated on
5789 * demand, and sets up the global locations for the LDS, GDS,
5790 * and GPUVM for FSA64 clients (CIK).
5791 * Returns 0 for success, errors for failure.
5792 */
5793static int cik_pcie_gart_enable(struct radeon_device *rdev)
5794{
5795 int r, i;
5796
5797 if (rdev->gart.robj == NULL) {
5798 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5799 return -EINVAL;
5800 }
5801 r = radeon_gart_table_vram_pin(rdev);
5802 if (r)
5803 return r;
1c49165d
AD		 5804 /* Setup TLB control */
5805 WREG32(MC_VM_MX_L1_TLB_CNTL,
5806 (0xA << 7) |
5807 ENABLE_L1_TLB |
ec3dbbcb 5808 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5809 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5810 ENABLE_ADVANCED_DRIVER_MODEL |
5811 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5812 /* Setup L2 cache */
5813 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5814 ENABLE_L2_FRAGMENT_PROCESSING |
5815 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5816 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5817 EFFECTIVE_L2_QUEUE_SIZE(7) |
5818 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5819 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5820 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5821 BANK_SELECT(4) |
5822 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5823 /* setup context0 */
5824 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
607d4806 5825 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, (rdev->mc.gtt_end >> 12) - 1);
1c49165d
AD		 5826 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5827 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5828 (u32)(rdev->dummy_page.addr >> 12));
5829 WREG32(VM_CONTEXT0_CNTL2, 0);
5830 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5831 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5832
5833 WREG32(0x15D4, 0);
5834 WREG32(0x15D8, 0);
5835 WREG32(0x15DC, 0);
5836
054e01d6 5837 /* restore context1-15 */
1c49165d
AD		 5838 /* set vm size, must be a multiple of 4 */
5839 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5840 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5841 for (i = 1; i < 16; i++) {
5842 if (i < 8)
5843 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5844 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5845 else
5846 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5847 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5848 }
5849
5850 /* enable context1-15 */
5851 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5852 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5853 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5854 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5855 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5856 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5857 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5858 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5859 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5860 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5861 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5862 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5863 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5864 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5865 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5866 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5867 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5868
1c49165d
AD		 5869 if (rdev->family == CHIP_KAVERI) {
5870 u32 tmp = RREG32(CHUB_CONTROL);
5871 tmp &= ~BYPASS_VM;
5872 WREG32(CHUB_CONTROL, tmp);
5873 }
5874
5875 /* XXX SH_MEM regs */
5876 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5877 mutex_lock(&rdev->srbm_mutex);
1c49165d 5878 for (i = 0; i < 16; i++) {
b556b12e 5879 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5880 /* CP and shaders */
1c49165d
AD		 5881 WREG32(SH_MEM_CONFIG, 0);
5882 WREG32(SH_MEM_APE1_BASE, 1);
5883 WREG32(SH_MEM_APE1_LIMIT, 0);
5884 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5885 /* SDMA GFX */
5886 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5887 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5888 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5889 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5890 /* XXX SDMA RLC - todo */
1c49165d 5891 }
b556b12e 5892 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5893 mutex_unlock(&rdev->srbm_mutex);
1c49165d 5894
08dcc57f
BG		 5895 cik_pcie_init_compute_vmid(rdev);
5896
1c49165d
AD		 5897 cik_pcie_gart_tlb_flush(rdev);
5898 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5899 (unsigned)(rdev->mc.gtt_size >> 20),
5900 (unsigned long long)rdev->gart.table_addr);
5901 rdev->gart.ready = true;
5902 return 0;
5903}
5904
5905/**
5906 * cik_pcie_gart_disable - gart disable
5907 *
5908 * @rdev: radeon_device pointer
5909 *
5910 * This disables all VM page table (CIK).
5911 */
5912static void cik_pcie_gart_disable(struct radeon_device *rdev)
5913{
054e01d6
CK		 5914 unsigned i;
5915
5916 for (i = 1; i < 16; ++i) {
5917 uint32_t reg;
5918 if (i < 8)
5919 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5920 else
5921 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5922 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5923 }
5924
1c49165d
AD		 5925 /* Disable all tables */
5926 WREG32(VM_CONTEXT0_CNTL, 0);
5927 WREG32(VM_CONTEXT1_CNTL, 0);
5928 /* Setup TLB control */
5929 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5930 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5931 /* Setup L2 cache */
5932 WREG32(VM_L2_CNTL,
5933 ENABLE_L2_FRAGMENT_PROCESSING |
5934 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5935 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5936 EFFECTIVE_L2_QUEUE_SIZE(7) |
5937 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5938 WREG32(VM_L2_CNTL2, 0);
5939 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5940 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5941 radeon_gart_table_vram_unpin(rdev);
5942}
5943
5944/**
5945 * cik_pcie_gart_fini - vm fini callback
5946 *
5947 * @rdev: radeon_device pointer
5948 *
5949 * Tears down the driver GART/VM setup (CIK).
5950 */
5951static void cik_pcie_gart_fini(struct radeon_device *rdev)
5952{
5953 cik_pcie_gart_disable(rdev);
5954 radeon_gart_table_vram_free(rdev);
5955 radeon_gart_fini(rdev);
5956}
5957
5958/* vm parser */
5959/**
5960 * cik_ib_parse - vm ib_parse callback
5961 *
5962 * @rdev: radeon_device pointer
5963 * @ib: indirect buffer pointer
5964 *
5965 * CIK uses hw IB checking so this is a nop (CIK).
5966 */
5967int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5968{
5969 return 0;
5970}
5971
5972/*
5973 * vm
5974 * VMID 0 is the physical GPU addresses as used by the kernel.
5975 * VMIDs 1-15 are used for userspace clients and are handled
5976 * by the radeon vm/hsa code.
5977 */
5978/**
5979 * cik_vm_init - cik vm init callback
5980 *
5981 * @rdev: radeon_device pointer
5982 *
5983 * Inits cik specific vm parameters (number of VMs, base of vram for
5984 * VMIDs 1-15) (CIK).
5985 * Returns 0 for success.
5986 */
5987int cik_vm_init(struct radeon_device *rdev)
5988{
62a7b7fb
OG		 5989 /*
5990 * number of VMs
5991 * VMID 0 is reserved for System
5992 * radeon graphics/compute will use VMIDs 1-7
5993 * amdkfd will use VMIDs 8-15
5994 */
5995 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 5996 /* base offset of vram pages */
5997 if (rdev->flags & RADEON_IS_IGP) {
5998 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5999 tmp <<= 22;
6000 rdev->vm_manager.vram_base_offset = tmp;
6001 } else
6002 rdev->vm_manager.vram_base_offset = 0;
6003
6004 return 0;
6005}
6006
6007/**
6008 * cik_vm_fini - cik vm fini callback
6009 *
6010 * @rdev: radeon_device pointer
6011 *
6012 * Tear down any asic specific VM setup (CIK).
6013 */
6014void cik_vm_fini(struct radeon_device *rdev)
6015{
6016}
6017
3ec7d11b
AD		 6018/**
6019 * cik_vm_decode_fault - print human readable fault info
6020 *
6021 * @rdev: radeon_device pointer
6022 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
6023 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
6024 *
6025 * Print human readable fault information (CIK).
6026 */
6027static void cik_vm_decode_fault(struct radeon_device *rdev,
6028 u32 status, u32 addr, u32 mc_client)
6029{
939c0d3c 6030 u32 mc_id;
3ec7d11b
AD		 6031 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
6032 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 6033 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
6034 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 6035
939c0d3c
AD		 6036 if (rdev->family == CHIP_HAWAII)
6037 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6038 else
6039 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6040
328a50c7 6041 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 6042 protections, vmid, addr,
6043 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 6044 block, mc_client, mc_id);
3ec7d11b
AD		 6045}
6046
f96ab484
AD		 6047/**
6048 * cik_vm_flush - cik vm flush using the CP
6049 *
6050 * @rdev: radeon_device pointer
6051 *
6052 * Update the page table base and flush the VM TLB
6053 * using the CP (CIK).
6054 */
faffaf62
CK		 6055void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6056 unsigned vm_id, uint64_t pd_addr)
f96ab484 6057{
faffaf62 6058 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 6059
6060 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 6061 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 6062 WRITE_DATA_DST_SEL(0)));
faffaf62 6063 if (vm_id < 8) {
f96ab484 6064 radeon_ring_write(ring,
faffaf62 6065 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 6066 } else {
6067 radeon_ring_write(ring,
faffaf62 6068 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 6069 }
6070 radeon_ring_write(ring, 0);
faffaf62 6071 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 6072
6073 /* update SH_MEM_* regs */
6074 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6075 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6076 WRITE_DATA_DST_SEL(0)));
6077 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6078 radeon_ring_write(ring, 0);
faffaf62 6079 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 6080
6081 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 6082 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6083 WRITE_DATA_DST_SEL(0)));
6084 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6085 radeon_ring_write(ring, 0);
6086
6087 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6088 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6089 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6090 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 6091
f96ab484 6092 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6093 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6094 WRITE_DATA_DST_SEL(0)));
6095 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6096 radeon_ring_write(ring, 0);
6097 radeon_ring_write(ring, VMID(0));
6f2043ce 6098
f96ab484 6099 /* HDP flush */
faffaf62 6100 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 6101
6102 /* bits 0-15 are the VM contexts0-15 */
6103 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6104 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6105 WRITE_DATA_DST_SEL(0)));
6106 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6107 radeon_ring_write(ring, 0);
faffaf62 6108 radeon_ring_write(ring, 1 << vm_id);
f96ab484 6109
3a01fd36
AD		 6110 /* wait for the invalidate to complete */
6111 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6112 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6113 WAIT_REG_MEM_FUNCTION(0) | /* always */
6114 WAIT_REG_MEM_ENGINE(0))); /* me */
6115 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6116 radeon_ring_write(ring, 0);
6117 radeon_ring_write(ring, 0); /* ref */
6118 radeon_ring_write(ring, 0); /* mask */
6119 radeon_ring_write(ring, 0x20); /* poll interval */
6120
b07fdd38 6121 /* compute doesn't have PFP */
f1d2a26b 6122 if (usepfp) {
b07fdd38
AD		 6123 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6124 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6125 radeon_ring_write(ring, 0x0);
6126 }
cc066715 6127}
6f2043ce 6128
f6796cae
AD		 6129/*
6130 * RLC
6131 * The RLC is a multi-purpose microengine that handles a
6132 * variety of functions, the most important of which is
6133 * the interrupt controller.
6134 */
866d83de
AD		 6135static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6136 bool enable)
f6796cae 6137{
866d83de 6138 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6139
866d83de
AD		 6140 if (enable)
6141 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6142 else
6143 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6144 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6145}
f6796cae 6146
866d83de 6147static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6148{
cc066715 6149 u32 tmp;
6f2043ce 6150
866d83de
AD		 6151 tmp = RREG32(RLC_LB_CNTL);
6152 if (enable)
6153 tmp |= LOAD_BALANCE_ENABLE;
6154 else
6155 tmp &= ~LOAD_BALANCE_ENABLE;
6156 WREG32(RLC_LB_CNTL, tmp);
6157}
cc066715 6158
866d83de
AD		 6159static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6160{
6161 u32 i, j, k;
6162 u32 mask;
cc066715 6163
1c0a4625 6164 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6165 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6166 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6167 cik_select_se_sh(rdev, i, j);
6168 for (k = 0; k < rdev->usec_timeout; k++) {
6169 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6170 break;
6171 udelay(1);
6172 }
6173 }
6174 }
6175 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6176 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6177
f6796cae
AD		 6178 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6179 for (k = 0; k < rdev->usec_timeout; k++) {
6180 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6181 break;
6182 udelay(1);
6183 }
6184}
cc066715 6185
22c775ce
AD		 6186static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6187{
6188 u32 tmp;
cc066715 6189
22c775ce
AD		 6190 tmp = RREG32(RLC_CNTL);
6191 if (tmp != rlc)
6192 WREG32(RLC_CNTL, rlc);
6193}
cc066715 6194
22c775ce
AD		 6195static u32 cik_halt_rlc(struct radeon_device *rdev)
6196{
6197 u32 data, orig;
cc066715 6198
22c775ce 6199 orig = data = RREG32(RLC_CNTL);
cc066715 6200
22c775ce
AD		 6201 if (data & RLC_ENABLE) {
6202 u32 i;
cc066715 6203
22c775ce
AD		 6204 data &= ~RLC_ENABLE;
6205 WREG32(RLC_CNTL, data);
cc066715 6206
22c775ce
AD		 6207 for (i = 0; i < rdev->usec_timeout; i++) {
6208 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6209 break;
6210 udelay(1);
6211 }
cc066715 6212
22c775ce
AD		 6213 cik_wait_for_rlc_serdes(rdev);
6214 }
cc066715 6215
22c775ce
AD		 6216 return orig;
6217}
cc066715 6218
a412fce0
AD		 6219void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6220{
6221 u32 tmp, i, mask;
6222
6223 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6224 WREG32(RLC_GPR_REG2, tmp);
6225
6226 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6227 for (i = 0; i < rdev->usec_timeout; i++) {
6228 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6229 break;
6230 udelay(1);
6231 }
6232
6233 for (i = 0; i < rdev->usec_timeout; i++) {
6234 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6235 break;
6236 udelay(1);
6237 }
6238}
6239
6240void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6241{
6242 u32 tmp;
6243
6244 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6245 WREG32(RLC_GPR_REG2, tmp);
6246}
6247
866d83de
AD		 6248/**
6249 * cik_rlc_stop - stop the RLC ME
6250 *
6251 * @rdev: radeon_device pointer
6252 *
6253 * Halt the RLC ME (MicroEngine) (CIK).
6254 */
6255static void cik_rlc_stop(struct radeon_device *rdev)
6256{
22c775ce 6257 WREG32(RLC_CNTL, 0);
866d83de
AD		 6258
6259 cik_enable_gui_idle_interrupt(rdev, false);
6260
866d83de
AD		 6261 cik_wait_for_rlc_serdes(rdev);
6262}
6263
f6796cae
AD		 6264/**
6265 * cik_rlc_start - start the RLC ME
6266 *
6267 * @rdev: radeon_device pointer
6268 *
6269 * Unhalt the RLC ME (MicroEngine) (CIK).
6270 */
6271static void cik_rlc_start(struct radeon_device *rdev)
6272{
f6796cae 6273 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6274
866d83de 6275 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6276
f6796cae 6277 udelay(50);
6f2043ce
AD		 6278}
6279
6280/**
f6796cae 6281 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6282 *
6283 * @rdev: radeon_device pointer
6284 *
f6796cae
AD		 6285 * Initialize the RLC registers, load the ucode,
6286 * and start the RLC (CIK).
6287 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6288 */
f6796cae 6289static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6290{
22c775ce 6291 u32 i, size, tmp;
cc066715 6292
f6796cae
AD		 6293 if (!rdev->rlc_fw)
6294 return -EINVAL;
cc066715 6295
cc066715
AD		 6296 cik_rlc_stop(rdev);
6297
22c775ce
AD		 6298 /* disable CG */
6299 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6300 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6301
866d83de 6302 si_rlc_reset(rdev);
6f2043ce 6303
22c775ce 6304 cik_init_pg(rdev);
6f2043ce 6305
22c775ce 6306 cik_init_cg(rdev);
cc066715 6307
f6796cae
AD		 6308 WREG32(RLC_LB_CNTR_INIT, 0);
6309 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6310
1c0a4625 6311 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6312 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6313 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6314 WREG32(RLC_LB_PARAMS, 0x00600408);
6315 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 6316 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6317
f6796cae
AD		 6318 WREG32(RLC_MC_CNTL, 0);
6319 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6320
f2c6b0f4
AD		 6321 if (rdev->new_fw) {
6322 const struct rlc_firmware_header_v1_0 *hdr =
6323 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6324 const __le32 *fw_data = (const __le32 *)
6325 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6326
6327 radeon_ucode_print_rlc_hdr(&hdr->header);
6328
6329 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6330 WREG32(RLC_GPM_UCODE_ADDR, 0);
6331 for (i = 0; i < size; i++)
6332 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6333 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6334 } else {
6335 const __be32 *fw_data;
6336
6337 switch (rdev->family) {
6338 case CHIP_BONAIRE:
6339 case CHIP_HAWAII:
6340 default:
6341 size = BONAIRE_RLC_UCODE_SIZE;
6342 break;
6343 case CHIP_KAVERI:
6344 size = KV_RLC_UCODE_SIZE;
6345 break;
6346 case CHIP_KABINI:
6347 size = KB_RLC_UCODE_SIZE;
6348 break;
6349 case CHIP_MULLINS:
6350 size = ML_RLC_UCODE_SIZE;
6351 break;
6352 }
6353
6354 fw_data = (const __be32 *)rdev->rlc_fw->data;
6355 WREG32(RLC_GPM_UCODE_ADDR, 0);
6356 for (i = 0; i < size; i++)
6357 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6358 WREG32(RLC_GPM_UCODE_ADDR, 0);
6359 }
cc066715 6360
866d83de
AD		 6361 /* XXX - find out what chips support lbpw */
6362 cik_enable_lbpw(rdev, false);
cc066715 6363
22c775ce
AD		 6364 if (rdev->family == CHIP_BONAIRE)
6365 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6366
f6796cae 6367 cik_rlc_start(rdev);
cc066715 6368
f6796cae
AD		 6369 return 0;
6370}
cc066715 6371
22c775ce
AD		 6372static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6373{
6374 u32 data, orig, tmp, tmp2;
cc066715 6375
22c775ce 6376 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6377
473359bc 6378 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6379 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6380
22c775ce 6381 tmp = cik_halt_rlc(rdev);
cc066715 6382
1c0a4625 6383 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6384 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6385 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6386 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6387 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6388 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6389 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6390
22c775ce 6391 cik_update_rlc(rdev, tmp);
cc066715 6392
22c775ce
AD		 6393 data |= CGCG_EN | CGLS_EN;
6394 } else {
ddc76ff6 6395 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6396
22c775ce
AD		 6397 RREG32(CB_CGTT_SCLK_CTRL);
6398 RREG32(CB_CGTT_SCLK_CTRL);
6399 RREG32(CB_CGTT_SCLK_CTRL);
6400 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6401
22c775ce 6402 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6403 }
6f2043ce 6404
22c775ce
AD		 6405 if (orig != data)
6406 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6407
6f2043ce
AD		 6408}
6409
22c775ce 6410static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6411{
22c775ce
AD		 6412 u32 data, orig, tmp = 0;
6413
473359bc
AD		 6414 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6415 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6416 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6417 orig = data = RREG32(CP_MEM_SLP_CNTL);
6418 data |= CP_MEM_LS_EN;
6419 if (orig != data)
6420 WREG32(CP_MEM_SLP_CNTL, data);
6421 }
6422 }
cc066715 6423
22c775ce 6424 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6425 data |= 0x00000001;
22c775ce
AD		 6426 data &= 0xfffffffd;
6427 if (orig != data)
6428 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6429
6430 tmp = cik_halt_rlc(rdev);
6431
1c0a4625 6432 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6433 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6434 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6435 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6436 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6437 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6438 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6439
6440 cik_update_rlc(rdev, tmp);
6441
473359bc
AD		 6442 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6443 orig = data = RREG32(CGTS_SM_CTRL_REG);
6444 data &= ~SM_MODE_MASK;
6445 data |= SM_MODE(0x2);
6446 data |= SM_MODE_ENABLE;
6447 data &= ~CGTS_OVERRIDE;
6448 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6449 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6450 data &= ~CGTS_LS_OVERRIDE;
6451 data &= ~ON_MONITOR_ADD_MASK;
6452 data |= ON_MONITOR_ADD_EN;
6453 data |= ON_MONITOR_ADD(0x96);
6454 if (orig != data)
6455 WREG32(CGTS_SM_CTRL_REG, data);
6456 }
22c775ce
AD		 6457 } else {
6458 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6459 data |= 0x00000003;
22c775ce
AD		 6460 if (orig != data)
6461 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6462
6463 data = RREG32(RLC_MEM_SLP_CNTL);
6464 if (data & RLC_MEM_LS_EN) {
6465 data &= ~RLC_MEM_LS_EN;
6466 WREG32(RLC_MEM_SLP_CNTL, data);
6467 }
6f2043ce 6468
22c775ce
AD		 6469 data = RREG32(CP_MEM_SLP_CNTL);
6470 if (data & CP_MEM_LS_EN) {
6471 data &= ~CP_MEM_LS_EN;
6472 WREG32(CP_MEM_SLP_CNTL, data);
6473 }
cc066715 6474
22c775ce
AD		 6475 orig = data = RREG32(CGTS_SM_CTRL_REG);
6476 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6477 if (orig != data)
6478 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6479
22c775ce 6480 tmp = cik_halt_rlc(rdev);
cc066715 6481
1c0a4625 6482 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6483 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6484 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6485 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6486 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6487 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6488 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6489
22c775ce 6490 cik_update_rlc(rdev, tmp);
cc066715 6491 }
6f2043ce 6492}
1c49165d 6493
22c775ce 6494static const u32 mc_cg_registers[] =
21a93e13 6495{
22c775ce
AD		 6496 MC_HUB_MISC_HUB_CG,
6497 MC_HUB_MISC_SIP_CG,
6498 MC_HUB_MISC_VM_CG,
6499 MC_XPB_CLK_GAT,
6500 ATC_MISC_CG,
6501 MC_CITF_MISC_WR_CG,
6502 MC_CITF_MISC_RD_CG,
6503 MC_CITF_MISC_VM_CG,
6504 VM_L2_CG,
6505};
21a93e13 6506
22c775ce
AD		 6507static void cik_enable_mc_ls(struct radeon_device *rdev,
6508 bool enable)
1c49165d 6509{
22c775ce
AD		 6510 int i;
6511 u32 orig, data;
1c49165d 6512
22c775ce
AD		 6513 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6514 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6515 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6516 data |= MC_LS_ENABLE;
6517 else
6518 data &= ~MC_LS_ENABLE;
6519 if (data != orig)
6520 WREG32(mc_cg_registers[i], data);
1c49165d 6521 }
22c775ce 6522}
1c49165d 6523
22c775ce
AD		 6524static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6525 bool enable)
6526{
6527 int i;
6528 u32 orig, data;
6529
6530 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6531 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6532 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6533 data |= MC_CG_ENABLE;
6534 else
6535 data &= ~MC_CG_ENABLE;
6536 if (data != orig)
6537 WREG32(mc_cg_registers[i], data);
1c49165d 6538 }
1c49165d
AD		 6539}
6540
22c775ce
AD		 6541static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6542 bool enable)
1c49165d 6543{
22c775ce 6544 u32 orig, data;
1c49165d 6545
473359bc 6546 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6547 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6548 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6549 } else {
22c775ce
AD		 6550 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6551 data |= 0xff000000;
6552 if (data != orig)
6553 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6554
22c775ce
AD		 6555 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6556 data |= 0xff000000;
6557 if (data != orig)
6558 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6559 }
1c49165d
AD		 6560}
6561
22c775ce
AD		 6562static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6563 bool enable)
1c49165d 6564{
22c775ce
AD		 6565 u32 orig, data;
6566
473359bc 6567 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6568 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6569 data |= 0x100;
6570 if (orig != data)
6571 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6572
6573 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6574 data |= 0x100;
6575 if (orig != data)
6576 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6577 } else {
6578 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6579 data &= ~0x100;
6580 if (orig != data)
6581 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6582
22c775ce
AD		 6583 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6584 data &= ~0x100;
6585 if (orig != data)
6586 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6587 }
1c49165d
AD		 6588}
6589
22c775ce
AD		 6590static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6591 bool enable)
1c49165d 6592{
22c775ce 6593 u32 orig, data;
1c49165d 6594
473359bc 6595 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6596 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6597 data = 0xfff;
6598 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6599
22c775ce
AD		 6600 orig = data = RREG32(UVD_CGC_CTRL);
6601 data |= DCM;
6602 if (orig != data)
6603 WREG32(UVD_CGC_CTRL, data);
6604 } else {
6605 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6606 data &= ~0xfff;
6607 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6608
22c775ce
AD		 6609 orig = data = RREG32(UVD_CGC_CTRL);
6610 data &= ~DCM;
6611 if (orig != data)
6612 WREG32(UVD_CGC_CTRL, data);
1c49165d 6613 }
22c775ce 6614}
1c49165d 6615
473359bc
AD		 6616static void cik_enable_bif_mgls(struct radeon_device *rdev,
6617 bool enable)
6618{
6619 u32 orig, data;
1c49165d 6620
473359bc 6621 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6622
473359bc
AD		 6623 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6624 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6625 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6626 else
6627 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6628 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6629
473359bc
AD		 6630 if (orig != data)
6631 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6632}
1c49165d 6633
22c775ce
AD		 6634static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6635 bool enable)
6636{
6637 u32 orig, data;
1c49165d 6638
22c775ce 6639 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6640
473359bc 6641 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6642 data &= ~CLOCK_GATING_DIS;
6643 else
6644 data |= CLOCK_GATING_DIS;
6645
6646 if (orig != data)
6647 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6648}
6649
22c775ce
AD		 6650static void cik_enable_hdp_ls(struct radeon_device *rdev,
6651 bool enable)
1c49165d 6652{
22c775ce
AD		 6653 u32 orig, data;
6654
6655 orig = data = RREG32(HDP_MEM_POWER_LS);
6656
473359bc 6657 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6658 data |= HDP_LS_ENABLE;
6659 else
6660 data &= ~HDP_LS_ENABLE;
6661
6662 if (orig != data)
6663 WREG32(HDP_MEM_POWER_LS, data);
6664}
6665
6666void cik_update_cg(struct radeon_device *rdev,
6667 u32 block, bool enable)
6668{
4214faf6 6669
22c775ce 6670 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6671 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6672 /* order matters! */
6673 if (enable) {
6674 cik_enable_mgcg(rdev, true);
6675 cik_enable_cgcg(rdev, true);
6676 } else {
6677 cik_enable_cgcg(rdev, false);
6678 cik_enable_mgcg(rdev, false);
6679 }
4214faf6 6680 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6681 }
6682
6683 if (block & RADEON_CG_BLOCK_MC) {
6684 if (!(rdev->flags & RADEON_IS_IGP)) {
6685 cik_enable_mc_mgcg(rdev, enable);
6686 cik_enable_mc_ls(rdev, enable);
6687 }
6688 }
6689
6690 if (block & RADEON_CG_BLOCK_SDMA) {
6691 cik_enable_sdma_mgcg(rdev, enable);
6692 cik_enable_sdma_mgls(rdev, enable);
6693 }
6694
473359bc
AD		 6695 if (block & RADEON_CG_BLOCK_BIF) {
6696 cik_enable_bif_mgls(rdev, enable);
6697 }
6698
22c775ce
AD		 6699 if (block & RADEON_CG_BLOCK_UVD) {
6700 if (rdev->has_uvd)
6701 cik_enable_uvd_mgcg(rdev, enable);
6702 }
6703
6704 if (block & RADEON_CG_BLOCK_HDP) {
6705 cik_enable_hdp_mgcg(rdev, enable);
6706 cik_enable_hdp_ls(rdev, enable);
6707 }
a1d6f97c
AD		 6708
6709 if (block & RADEON_CG_BLOCK_VCE) {
6710 vce_v2_0_enable_mgcg(rdev, enable);
6711 }
1c49165d
AD		 6712}
6713
22c775ce 6714static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6715{
22c775ce 6716
ddc76ff6 6717 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6718
6719 if (rdev->has_uvd)
6720 si_init_uvd_internal_cg(rdev);
6721
6722 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6723 RADEON_CG_BLOCK_SDMA |
473359bc 6724 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6725 RADEON_CG_BLOCK_UVD |
6726 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6727}
6728
473359bc 6729static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6730{
473359bc
AD		 6731 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6732 RADEON_CG_BLOCK_SDMA |
6733 RADEON_CG_BLOCK_BIF |
6734 RADEON_CG_BLOCK_UVD |
6735 RADEON_CG_BLOCK_HDP), false);
6736
6737 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6738}
6739
22c775ce
AD		 6740static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6741 bool enable)
1c49165d 6742{
22c775ce 6743 u32 data, orig;
1c49165d 6744
22c775ce 6745 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6746 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6747 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6748 else
6749 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6750 if (orig != data)
6751 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6752}
6753
22c775ce
AD		 6754static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6755 bool enable)
1c49165d 6756{
22c775ce
AD		 6757 u32 data, orig;
6758
6759 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6760 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6761 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6762 else
6763 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6764 if (orig != data)
6765 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6766}
6767
22c775ce 6768static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6769{
22c775ce 6770 u32 data, orig;
3ec7d11b 6771
22c775ce 6772 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6773 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6774 data &= ~DISABLE_CP_PG;
6775 else
6776 data |= DISABLE_CP_PG;
6777 if (orig != data)
6778 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6779}
6780
22c775ce 6781static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6782{
22c775ce 6783 u32 data, orig;
f96ab484 6784
22c775ce 6785 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6786 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6787 data &= ~DISABLE_GDS_PG;
6788 else
6789 data |= DISABLE_GDS_PG;
6790 if (orig != data)
6791 WREG32(RLC_PG_CNTL, data);
6792}
6793
6794#define CP_ME_TABLE_SIZE 96
6795#define CP_ME_TABLE_OFFSET 2048
6796#define CP_MEC_TABLE_OFFSET 4096
6797
6798void cik_init_cp_pg_table(struct radeon_device *rdev)
6799{
22c775ce
AD		 6800 volatile u32 *dst_ptr;
6801 int me, i, max_me = 4;
6802 u32 bo_offset = 0;
f2c6b0f4 6803 u32 table_offset, table_size;
22c775ce
AD		 6804
6805 if (rdev->family == CHIP_KAVERI)
6806 max_me = 5;
6807
6808 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6809 return;
6810
22c775ce
AD		 6811 /* write the cp table buffer */
6812 dst_ptr = rdev->rlc.cp_table_ptr;
6813 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6814 if (rdev->new_fw) {
6815 const __le32 *fw_data;
6816 const struct gfx_firmware_header_v1_0 *hdr;
6817
6818 if (me == 0) {
6819 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6820 fw_data = (const __le32 *)
6821 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6822 table_offset = le32_to_cpu(hdr->jt_offset);
6823 table_size = le32_to_cpu(hdr->jt_size);
6824 } else if (me == 1) {
6825 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6826 fw_data = (const __le32 *)
6827 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6828 table_offset = le32_to_cpu(hdr->jt_offset);
6829 table_size = le32_to_cpu(hdr->jt_size);
6830 } else if (me == 2) {
6831 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6832 fw_data = (const __le32 *)
6833 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6834 table_offset = le32_to_cpu(hdr->jt_offset);
6835 table_size = le32_to_cpu(hdr->jt_size);
6836 } else if (me == 3) {
6837 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6838 fw_data = (const __le32 *)
6839 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6840 table_offset = le32_to_cpu(hdr->jt_offset);
6841 table_size = le32_to_cpu(hdr->jt_size);
6842 } else {
6843 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6844 fw_data = (const __le32 *)
6845 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6846 table_offset = le32_to_cpu(hdr->jt_offset);
6847 table_size = le32_to_cpu(hdr->jt_size);
6848 }
6849
6850 for (i = 0; i < table_size; i ++) {
6851 dst_ptr[bo_offset + i] =
6852 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6853 }
6854 bo_offset += table_size;
22c775ce 6855 } else {
f2c6b0f4
AD		 6856 const __be32 *fw_data;
6857 table_size = CP_ME_TABLE_SIZE;
6858
6859 if (me == 0) {
6860 fw_data = (const __be32 *)rdev->ce_fw->data;
6861 table_offset = CP_ME_TABLE_OFFSET;
6862 } else if (me == 1) {
6863 fw_data = (const __be32 *)rdev->pfp_fw->data;
6864 table_offset = CP_ME_TABLE_OFFSET;
6865 } else if (me == 2) {
6866 fw_data = (const __be32 *)rdev->me_fw->data;
6867 table_offset = CP_ME_TABLE_OFFSET;
6868 } else {
6869 fw_data = (const __be32 *)rdev->mec_fw->data;
6870 table_offset = CP_MEC_TABLE_OFFSET;
6871 }
22c775ce 6872
f2c6b0f4
AD		 6873 for (i = 0; i < table_size; i ++) {
6874 dst_ptr[bo_offset + i] =
6875 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6876 }
6877 bo_offset += table_size;
22c775ce 6878 }
f96ab484 6879 }
22c775ce 6880}
f96ab484 6881
22c775ce
AD		 6882static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6883 bool enable)
6884{
6885 u32 data, orig;
6886
2b19d17f 6887 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6888 orig = data = RREG32(RLC_PG_CNTL);
6889 data |= GFX_PG_ENABLE;
6890 if (orig != data)
6891 WREG32(RLC_PG_CNTL, data);
6892
6893 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6894 data |= AUTO_PG_EN;
6895 if (orig != data)
6896 WREG32(RLC_AUTO_PG_CTRL, data);
6897 } else {
6898 orig = data = RREG32(RLC_PG_CNTL);
6899 data &= ~GFX_PG_ENABLE;
6900 if (orig != data)
6901 WREG32(RLC_PG_CNTL, data);
f96ab484 6902
22c775ce
AD		 6903 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6904 data &= ~AUTO_PG_EN;
6905 if (orig != data)
6906 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6907
22c775ce
AD		 6908 data = RREG32(DB_RENDER_CONTROL);
6909 }
6910}
f96ab484 6911
22c775ce
AD		 6912static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6913{
6914 u32 mask = 0, tmp, tmp1;
6915 int i;
f96ab484 6916
1c0a4625 6917 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6918 cik_select_se_sh(rdev, se, sh);
6919 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6920 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6921 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6922 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6923
22c775ce 6924 tmp &= 0xffff0000;
f96ab484 6925
22c775ce
AD		 6926 tmp |= tmp1;
6927 tmp >>= 16;
6928
6929 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6930 mask <<= 1;
6931 mask |= 1;
b07fdd38 6932 }
22c775ce
AD		 6933
6934 return (~tmp) & mask;
f96ab484
AD		 6935}
6936
22c775ce 6937static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6938{
22c775ce
AD		 6939 u32 i, j, k, active_cu_number = 0;
6940 u32 mask, counter, cu_bitmap;
6941 u32 tmp = 0;
d0e092d9 6942
22c775ce
AD		 6943 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6944 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6945 mask = 1;
6946 cu_bitmap = 0;
6947 counter = 0;
6948 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6949 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6950 if (counter < 2)
6951 cu_bitmap |= mask;
6952 counter ++;
d0e092d9 6953 }
22c775ce 6954 mask <<= 1;
d0e092d9 6955 }
d0e092d9 6956
22c775ce
AD		 6957 active_cu_number += counter;
6958 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6959 }
d0e092d9 6960 }
22c775ce
AD		 6961
6962 WREG32(RLC_PG_AO_CU_MASK, tmp);
6963
6964 tmp = RREG32(RLC_MAX_PG_CU);
6965 tmp &= ~MAX_PU_CU_MASK;
6966 tmp |= MAX_PU_CU(active_cu_number);
6967 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6968}
6969
22c775ce
AD		 6970static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6971 bool enable)
605de6b9 6972{
22c775ce 6973 u32 data, orig;
605de6b9 6974
22c775ce 6975 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6976 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6977 data |= STATIC_PER_CU_PG_ENABLE;
6978 else
6979 data &= ~STATIC_PER_CU_PG_ENABLE;
6980 if (orig != data)
6981 WREG32(RLC_PG_CNTL, data);
6982}
6983
6984static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6985 bool enable)
6986{
6987 u32 data, orig;
605de6b9 6988
22c775ce 6989 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6990 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6991 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6992 else
22c775ce
AD		 6993 data &= ~DYN_PER_CU_PG_ENABLE;
6994 if (orig != data)
6995 WREG32(RLC_PG_CNTL, data);
6996}
605de6b9 6997
22c775ce
AD		 6998#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6999#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
7000
7001static void cik_init_gfx_cgpg(struct radeon_device *rdev)
7002{
7003 u32 data, orig;
7004 u32 i;
7005
7006 if (rdev->rlc.cs_data) {
7007 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7008 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 7009 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 7010 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 7011 } else {
22c775ce
AD		 7012 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7013 for (i = 0; i < 3; i++)
7014 WREG32(RLC_GPM_SCRATCH_DATA, 0);
7015 }
7016 if (rdev->rlc.reg_list) {
7017 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
7018 for (i = 0; i < rdev->rlc.reg_list_size; i++)
7019 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 7020 }
605de6b9 7021
22c775ce
AD		 7022 orig = data = RREG32(RLC_PG_CNTL);
7023 data |= GFX_PG_SRC;
7024 if (orig != data)
7025 WREG32(RLC_PG_CNTL, data);
605de6b9 7026
22c775ce
AD		 7027 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
7028 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 7029
22c775ce
AD		 7030 data = RREG32(CP_RB_WPTR_POLL_CNTL);
7031 data &= ~IDLE_POLL_COUNT_MASK;
7032 data |= IDLE_POLL_COUNT(0x60);
7033 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 7034
22c775ce
AD		 7035 data = 0x10101010;
7036 WREG32(RLC_PG_DELAY, data);
605de6b9 7037
22c775ce
AD		 7038 data = RREG32(RLC_PG_DELAY_2);
7039 data &= ~0xff;
7040 data |= 0x3;
7041 WREG32(RLC_PG_DELAY_2, data);
605de6b9 7042
22c775ce
AD		 7043 data = RREG32(RLC_AUTO_PG_CTRL);
7044 data &= ~GRBM_REG_SGIT_MASK;
7045 data |= GRBM_REG_SGIT(0x700);
7046 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 7047
605de6b9
AD		 7048}
7049
22c775ce 7050static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 7051{
473359bc
AD		 7052 cik_enable_gfx_cgpg(rdev, enable);
7053 cik_enable_gfx_static_mgpg(rdev, enable);
7054 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 7055}
f6796cae 7056
a0f38609
AD		 7057u32 cik_get_csb_size(struct radeon_device *rdev)
7058{
7059 u32 count = 0;
7060 const struct cs_section_def *sect = NULL;
7061 const struct cs_extent_def *ext = NULL;
f6796cae 7062
a0f38609
AD		 7063 if (rdev->rlc.cs_data == NULL)
7064 return 0;
f6796cae 7065
a0f38609
AD		 7066 /* begin clear state */
7067 count += 2;
7068 /* context control state */
7069 count += 3;
7070
7071 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7072 for (ext = sect->section; ext->extent != NULL; ++ext) {
7073 if (sect->id == SECT_CONTEXT)
7074 count += 2 + ext->reg_count;
7075 else
7076 return 0;
f6796cae
AD		 7077 }
7078 }
a0f38609
AD		 7079 /* pa_sc_raster_config/pa_sc_raster_config1 */
7080 count += 4;
7081 /* end clear state */
7082 count += 2;
7083 /* clear state */
7084 count += 2;
f6796cae 7085
a0f38609 7086 return count;
f6796cae
AD		 7087}
7088
a0f38609 7089void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 7090{
a0f38609
AD		 7091 u32 count = 0, i;
7092 const struct cs_section_def *sect = NULL;
7093 const struct cs_extent_def *ext = NULL;
f6796cae 7094
a0f38609
AD		 7095 if (rdev->rlc.cs_data == NULL)
7096 return;
7097 if (buffer == NULL)
7098 return;
f6796cae 7099
6ba81e53
AD		 7100 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7101 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 7102
6ba81e53
AD		 7103 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7104 buffer[count++] = cpu_to_le32(0x80000000);
7105 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 7106
7107 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7108 for (ext = sect->section; ext->extent != NULL; ++ext) {
7109 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 7110 buffer[count++] =
7111 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7112 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 7113 for (i = 0; i < ext->reg_count; i++)
6ba81e53 7114 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 7115 } else {
7116 return;
7117 }
7118 }
7119 }
f6796cae 7120
6ba81e53
AD		 7121 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7122 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 7123 switch (rdev->family) {
7124 case CHIP_BONAIRE:
6ba81e53
AD		 7125 buffer[count++] = cpu_to_le32(0x16000012);
7126 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7127 break;
7128 case CHIP_KAVERI:
6ba81e53
AD		 7129 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7130 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7131 break;
7132 case CHIP_KABINI:
f73a9e83 7133 case CHIP_MULLINS:
6ba81e53
AD		 7134 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7135 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7136 break;
bbfe90bd 7137 case CHIP_HAWAII:
a8947f57
AD		 7138 buffer[count++] = cpu_to_le32(0x3a00161a);
7139 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7140 break;
a0f38609 7141 default:
6ba81e53
AD		 7142 buffer[count++] = cpu_to_le32(0x00000000);
7143 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7144 break;
7145 }
7146
6ba81e53
AD		 7147 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7148 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7149
6ba81e53
AD		 7150 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7151 buffer[count++] = cpu_to_le32(0);
a0f38609 7152}
f6796cae 7153
473359bc 7154static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7155{
473359bc 7156 if (rdev->pg_flags) {
22c775ce
AD		 7157 cik_enable_sck_slowdown_on_pu(rdev, true);
7158 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7159 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7160 cik_init_gfx_cgpg(rdev);
7161 cik_enable_cp_pg(rdev, true);
7162 cik_enable_gds_pg(rdev, true);
7163 }
22c775ce
AD		 7164 cik_init_ao_cu_mask(rdev);
7165 cik_update_gfx_pg(rdev, true);
7166 }
7167}
f6796cae 7168
473359bc
AD		 7169static void cik_fini_pg(struct radeon_device *rdev)
7170{
7171 if (rdev->pg_flags) {
7172 cik_update_gfx_pg(rdev, false);
2b19d17f 7173 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7174 cik_enable_cp_pg(rdev, false);
7175 cik_enable_gds_pg(rdev, false);
7176 }
7177 }
f6796cae 7178}
a59781bb
AD		 7179
7180/*
7181 * Interrupts
7182 * Starting with r6xx, interrupts are handled via a ring buffer.
7183 * Ring buffers are areas of GPU accessible memory that the GPU
7184 * writes interrupt vectors into and the host reads vectors out of.
7185 * There is a rptr (read pointer) that determines where the
7186 * host is currently reading, and a wptr (write pointer)
7187 * which determines where the GPU has written. When the
7188 * pointers are equal, the ring is idle. When the GPU
7189 * writes vectors to the ring buffer, it increments the
7190 * wptr. When there is an interrupt, the host then starts
7191 * fetching commands and processing them until the pointers are
7192 * equal again at which point it updates the rptr.
7193 */
7194
7195/**
7196 * cik_enable_interrupts - Enable the interrupt ring buffer
7197 *
7198 * @rdev: radeon_device pointer
7199 *
7200 * Enable the interrupt ring buffer (CIK).
7201 */
7202static void cik_enable_interrupts(struct radeon_device *rdev)
7203{
7204 u32 ih_cntl = RREG32(IH_CNTL);
7205 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7206
7207 ih_cntl |= ENABLE_INTR;
7208 ih_rb_cntl |= IH_RB_ENABLE;
7209 WREG32(IH_CNTL, ih_cntl);
7210 WREG32(IH_RB_CNTL, ih_rb_cntl);
7211 rdev->ih.enabled = true;
7212}
7213
7214/**
7215 * cik_disable_interrupts - Disable the interrupt ring buffer
7216 *
7217 * @rdev: radeon_device pointer
7218 *
7219 * Disable the interrupt ring buffer (CIK).
7220 */
7221static void cik_disable_interrupts(struct radeon_device *rdev)
7222{
7223 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7224 u32 ih_cntl = RREG32(IH_CNTL);
7225
7226 ih_rb_cntl &= ~IH_RB_ENABLE;
7227 ih_cntl &= ~ENABLE_INTR;
7228 WREG32(IH_RB_CNTL, ih_rb_cntl);
7229 WREG32(IH_CNTL, ih_cntl);
7230 /* set rptr, wptr to 0 */
7231 WREG32(IH_RB_RPTR, 0);
7232 WREG32(IH_RB_WPTR, 0);
7233 rdev->ih.enabled = false;
7234 rdev->ih.rptr = 0;
7235}
7236
7237/**
7238 * cik_disable_interrupt_state - Disable all interrupt sources
7239 *
7240 * @rdev: radeon_device pointer
7241 *
7242 * Clear all interrupt enable bits used by the driver (CIK).
7243 */
7244static void cik_disable_interrupt_state(struct radeon_device *rdev)
7245{
7246 u32 tmp;
7247
7248 /* gfx ring */
4214faf6
AD		 7249 tmp = RREG32(CP_INT_CNTL_RING0) &
7250 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7251 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7252 /* sdma */
7253 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7254 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7255 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7256 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7257 /* compute queues */
7258 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7259 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7260 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7261 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7262 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7263 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7264 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7265 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7266 /* grbm */
7267 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 7268 /* SRBM */
7269 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 7270 /* vline/vblank, etc. */
7271 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7272 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7273 if (rdev->num_crtc >= 4) {
7274 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7276 }
7277 if (rdev->num_crtc >= 6) {
7278 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7279 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7280 }
f5d636d2
CK		 7281 /* pflip */
7282 if (rdev->num_crtc >= 2) {
7283 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7284 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7285 }
7286 if (rdev->num_crtc >= 4) {
7287 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7288 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7289 }
7290 if (rdev->num_crtc >= 6) {
7291 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7292 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7293 }
a59781bb
AD		 7294
7295 /* dac hotplug */
7296 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7297
7298 /* digital hotplug */
7299 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7300 WREG32(DC_HPD1_INT_CONTROL, tmp);
7301 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7302 WREG32(DC_HPD2_INT_CONTROL, tmp);
7303 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7304 WREG32(DC_HPD3_INT_CONTROL, tmp);
7305 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7306 WREG32(DC_HPD4_INT_CONTROL, tmp);
7307 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7308 WREG32(DC_HPD5_INT_CONTROL, tmp);
7309 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7310 WREG32(DC_HPD6_INT_CONTROL, tmp);
7311
7312}
7313
7314/**
7315 * cik_irq_init - init and enable the interrupt ring
7316 *
7317 * @rdev: radeon_device pointer
7318 *
7319 * Allocate a ring buffer for the interrupt controller,
7320 * enable the RLC, disable interrupts, enable the IH
7321 * ring buffer and enable it (CIK).
7322 * Called at device load and reume.
7323 * Returns 0 for success, errors for failure.
7324 */
7325static int cik_irq_init(struct radeon_device *rdev)
7326{
7327 int ret = 0;
7328 int rb_bufsz;
7329 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7330
7331 /* allocate ring */
7332 ret = r600_ih_ring_alloc(rdev);
7333 if (ret)
7334 return ret;
7335
7336 /* disable irqs */
7337 cik_disable_interrupts(rdev);
7338
7339 /* init rlc */
7340 ret = cik_rlc_resume(rdev);
7341 if (ret) {
7342 r600_ih_ring_fini(rdev);
7343 return ret;
7344 }
7345
7346 /* setup interrupt control */
7347 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7348 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7349 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7350 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7351 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7352 */
7353 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7354 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7355 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7356 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7357
7358 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7359 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7360
7361 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7362 IH_WPTR_OVERFLOW_CLEAR |
7363 (rb_bufsz << 1));
7364
7365 if (rdev->wb.enabled)
7366 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7367
7368 /* set the writeback address whether it's enabled or not */
7369 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7370 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7371
7372 WREG32(IH_RB_CNTL, ih_rb_cntl);
7373
7374 /* set rptr, wptr to 0 */
7375 WREG32(IH_RB_RPTR, 0);
7376 WREG32(IH_RB_WPTR, 0);
7377
7378 /* Default settings for IH_CNTL (disabled at first) */
7379 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7380 /* RPTR_REARM only works if msi's are enabled */
7381 if (rdev->msi_enabled)
7382 ih_cntl |= RPTR_REARM;
7383 WREG32(IH_CNTL, ih_cntl);
7384
7385 /* force the active interrupt state to all disabled */
7386 cik_disable_interrupt_state(rdev);
7387
7388 pci_set_master(rdev->pdev);
7389
7390 /* enable irqs */
7391 cik_enable_interrupts(rdev);
7392
7393 return ret;
7394}
7395
7396/**
7397 * cik_irq_set - enable/disable interrupt sources
7398 *
7399 * @rdev: radeon_device pointer
7400 *
7401 * Enable interrupt sources on the GPU (vblanks, hpd,
7402 * etc.) (CIK).
7403 * Returns 0 for success, errors for failure.
7404 */
7405int cik_irq_set(struct radeon_device *rdev)
7406{
4214faf6 7407 u32 cp_int_cntl;
28b57b85 7408 u32 cp_m1p0;
a59781bb
AD		 7409 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7410 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7411 u32 grbm_int_cntl = 0;
21a93e13 7412 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7413
7414 if (!rdev->irq.installed) {
7415 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7416 return -EINVAL;
7417 }
7418 /* don't enable anything if the ih is disabled */
7419 if (!rdev->ih.enabled) {
7420 cik_disable_interrupts(rdev);
7421 /* force the active interrupt state to all disabled */
7422 cik_disable_interrupt_state(rdev);
7423 return 0;
7424 }
7425
4214faf6
AD		 7426 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7427 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7428 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7429
f6b355dd
AD		 7430 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7431 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7432 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7433 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7434 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7435 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7436
21a93e13
AD		 7437 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7438 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7439
2b0781a6 7440 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7441
a59781bb
AD		 7442 /* enable CP interrupts on all rings */
7443 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7444 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7445 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7446 }
2b0781a6
AD		 7447 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7448 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7449 DRM_DEBUG("si_irq_set: sw int cp1\n");
7450 if (ring->me == 1) {
7451 switch (ring->pipe) {
7452 case 0:
7453 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7454 break;
2b0781a6
AD		 7455 default:
7456 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7457 break;
7458 }
7459 } else {
7460 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7461 }
7462 }
7463 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7464 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7465 DRM_DEBUG("si_irq_set: sw int cp2\n");
7466 if (ring->me == 1) {
7467 switch (ring->pipe) {
7468 case 0:
7469 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7470 break;
2b0781a6
AD		 7471 default:
7472 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7473 break;
7474 }
7475 } else {
7476 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7477 }
7478 }
a59781bb 7479
21a93e13
AD		 7480 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7481 DRM_DEBUG("cik_irq_set: sw int dma\n");
7482 dma_cntl |= TRAP_ENABLE;
7483 }
7484
7485 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7486 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7487 dma_cntl1 |= TRAP_ENABLE;
7488 }
7489
a59781bb
AD		 7490 if (rdev->irq.crtc_vblank_int[0] ||
7491 atomic_read(&rdev->irq.pflip[0])) {
7492 DRM_DEBUG("cik_irq_set: vblank 0\n");
7493 crtc1 |= VBLANK_INTERRUPT_MASK;
7494 }
7495 if (rdev->irq.crtc_vblank_int[1] ||
7496 atomic_read(&rdev->irq.pflip[1])) {
7497 DRM_DEBUG("cik_irq_set: vblank 1\n");
7498 crtc2 |= VBLANK_INTERRUPT_MASK;
7499 }
7500 if (rdev->irq.crtc_vblank_int[2] ||
7501 atomic_read(&rdev->irq.pflip[2])) {
7502 DRM_DEBUG("cik_irq_set: vblank 2\n");
7503 crtc3 |= VBLANK_INTERRUPT_MASK;
7504 }
7505 if (rdev->irq.crtc_vblank_int[3] ||
7506 atomic_read(&rdev->irq.pflip[3])) {
7507 DRM_DEBUG("cik_irq_set: vblank 3\n");
7508 crtc4 |= VBLANK_INTERRUPT_MASK;
7509 }
7510 if (rdev->irq.crtc_vblank_int[4] ||
7511 atomic_read(&rdev->irq.pflip[4])) {
7512 DRM_DEBUG("cik_irq_set: vblank 4\n");
7513 crtc5 |= VBLANK_INTERRUPT_MASK;
7514 }
7515 if (rdev->irq.crtc_vblank_int[5] ||
7516 atomic_read(&rdev->irq.pflip[5])) {
7517 DRM_DEBUG("cik_irq_set: vblank 5\n");
7518 crtc6 |= VBLANK_INTERRUPT_MASK;
7519 }
7520 if (rdev->irq.hpd[0]) {
7521 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7522 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7523 }
7524 if (rdev->irq.hpd[1]) {
7525 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7526 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7527 }
7528 if (rdev->irq.hpd[2]) {
7529 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7530 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7531 }
7532 if (rdev->irq.hpd[3]) {
7533 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7534 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7535 }
7536 if (rdev->irq.hpd[4]) {
7537 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7538 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7539 }
7540 if (rdev->irq.hpd[5]) {
7541 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7542 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7543 }
7544
7545 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7546
21a93e13
AD		 7547 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7548 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7549
2b0781a6 7550 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7551
a59781bb
AD		 7552 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7553
7554 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7555 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7556 if (rdev->num_crtc >= 4) {
7557 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7558 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7559 }
7560 if (rdev->num_crtc >= 6) {
7561 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7562 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7563 }
7564
f5d636d2
CK		 7565 if (rdev->num_crtc >= 2) {
7566 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7567 GRPH_PFLIP_INT_MASK);
7568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7569 GRPH_PFLIP_INT_MASK);
7570 }
7571 if (rdev->num_crtc >= 4) {
7572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7573 GRPH_PFLIP_INT_MASK);
7574 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7575 GRPH_PFLIP_INT_MASK);
7576 }
7577 if (rdev->num_crtc >= 6) {
7578 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7579 GRPH_PFLIP_INT_MASK);
7580 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7581 GRPH_PFLIP_INT_MASK);
7582 }
7583
a59781bb
AD		 7584 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7585 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7586 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7587 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7588 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7589 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7590
cffefd9b
AD		 7591 /* posting read */
7592 RREG32(SRBM_STATUS);
7593
a59781bb
AD		 7594 return 0;
7595}
7596
7597/**
7598 * cik_irq_ack - ack interrupt sources
7599 *
7600 * @rdev: radeon_device pointer
7601 *
7602 * Ack interrupt sources on the GPU (vblanks, hpd,
7603 * etc.) (CIK). Certain interrupts sources are sw
7604 * generated and do not require an explicit ack.
7605 */
7606static inline void cik_irq_ack(struct radeon_device *rdev)
7607{
7608 u32 tmp;
7609
7610 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7611 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7612 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7613 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7614 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7615 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7616 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7617
f5d636d2
CK		 7618 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7619 EVERGREEN_CRTC0_REGISTER_OFFSET);
7620 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7621 EVERGREEN_CRTC1_REGISTER_OFFSET);
7622 if (rdev->num_crtc >= 4) {
7623 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7624 EVERGREEN_CRTC2_REGISTER_OFFSET);
7625 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7626 EVERGREEN_CRTC3_REGISTER_OFFSET);
7627 }
7628 if (rdev->num_crtc >= 6) {
7629 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7630 EVERGREEN_CRTC4_REGISTER_OFFSET);
7631 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7632 EVERGREEN_CRTC5_REGISTER_OFFSET);
7633 }
7634
7635 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7640 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7641 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7642 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7643 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7644 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7645 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7646 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7647 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7648 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7649
7650 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7651 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7652 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7653 GRPH_PFLIP_INT_CLEAR);
7654 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7655 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7656 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7657 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7658 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7659 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7660 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7661 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7662 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7663 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7664 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7665 }
7666
7667 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7668 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7669 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7670 GRPH_PFLIP_INT_CLEAR);
7671 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7672 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7673 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7674 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7675 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7676 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7677 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7678 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7679 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7680 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7681 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7682 }
7683
7684 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7685 tmp = RREG32(DC_HPD1_INT_CONTROL);
7686 tmp |= DC_HPDx_INT_ACK;
7687 WREG32(DC_HPD1_INT_CONTROL, tmp);
7688 }
7689 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7690 tmp = RREG32(DC_HPD2_INT_CONTROL);
7691 tmp |= DC_HPDx_INT_ACK;
7692 WREG32(DC_HPD2_INT_CONTROL, tmp);
7693 }
7694 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7695 tmp = RREG32(DC_HPD3_INT_CONTROL);
7696 tmp |= DC_HPDx_INT_ACK;
7697 WREG32(DC_HPD3_INT_CONTROL, tmp);
7698 }
7699 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7700 tmp = RREG32(DC_HPD4_INT_CONTROL);
7701 tmp |= DC_HPDx_INT_ACK;
7702 WREG32(DC_HPD4_INT_CONTROL, tmp);
7703 }
7704 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7705 tmp = RREG32(DC_HPD5_INT_CONTROL);
7706 tmp |= DC_HPDx_INT_ACK;
7707 WREG32(DC_HPD5_INT_CONTROL, tmp);
7708 }
7709 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7710 tmp = RREG32(DC_HPD5_INT_CONTROL);
7711 tmp |= DC_HPDx_INT_ACK;
7712 WREG32(DC_HPD6_INT_CONTROL, tmp);
7713 }
f6b355dd
AD		 7714 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7715 tmp = RREG32(DC_HPD1_INT_CONTROL);
7716 tmp |= DC_HPDx_RX_INT_ACK;
7717 WREG32(DC_HPD1_INT_CONTROL, tmp);
7718 }
7719 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7720 tmp = RREG32(DC_HPD2_INT_CONTROL);
7721 tmp |= DC_HPDx_RX_INT_ACK;
7722 WREG32(DC_HPD2_INT_CONTROL, tmp);
7723 }
7724 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7725 tmp = RREG32(DC_HPD3_INT_CONTROL);
7726 tmp |= DC_HPDx_RX_INT_ACK;
7727 WREG32(DC_HPD3_INT_CONTROL, tmp);
7728 }
7729 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7730 tmp = RREG32(DC_HPD4_INT_CONTROL);
7731 tmp |= DC_HPDx_RX_INT_ACK;
7732 WREG32(DC_HPD4_INT_CONTROL, tmp);
7733 }
7734 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7735 tmp = RREG32(DC_HPD5_INT_CONTROL);
7736 tmp |= DC_HPDx_RX_INT_ACK;
7737 WREG32(DC_HPD5_INT_CONTROL, tmp);
7738 }
7739 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7740 tmp = RREG32(DC_HPD5_INT_CONTROL);
7741 tmp |= DC_HPDx_RX_INT_ACK;
7742 WREG32(DC_HPD6_INT_CONTROL, tmp);
7743 }
a59781bb
AD		 7744}
7745
7746/**
7747 * cik_irq_disable - disable interrupts
7748 *
7749 * @rdev: radeon_device pointer
7750 *
7751 * Disable interrupts on the hw (CIK).
7752 */
7753static void cik_irq_disable(struct radeon_device *rdev)
7754{
7755 cik_disable_interrupts(rdev);
7756 /* Wait and acknowledge irq */
7757 mdelay(1);
7758 cik_irq_ack(rdev);
7759 cik_disable_interrupt_state(rdev);
7760}
7761
7762/**
7763 * cik_irq_disable - disable interrupts for suspend
7764 *
7765 * @rdev: radeon_device pointer
7766 *
7767 * Disable interrupts and stop the RLC (CIK).
7768 * Used for suspend.
7769 */
7770static void cik_irq_suspend(struct radeon_device *rdev)
7771{
7772 cik_irq_disable(rdev);
7773 cik_rlc_stop(rdev);
7774}
7775
7776/**
7777 * cik_irq_fini - tear down interrupt support
7778 *
7779 * @rdev: radeon_device pointer
7780 *
7781 * Disable interrupts on the hw and free the IH ring
7782 * buffer (CIK).
7783 * Used for driver unload.
7784 */
7785static void cik_irq_fini(struct radeon_device *rdev)
7786{
7787 cik_irq_suspend(rdev);
7788 r600_ih_ring_fini(rdev);
7789}
7790
7791/**
7792 * cik_get_ih_wptr - get the IH ring buffer wptr
7793 *
7794 * @rdev: radeon_device pointer
7795 *
7796 * Get the IH ring buffer wptr from either the register
7797 * or the writeback memory buffer (CIK). Also check for
7798 * ring buffer overflow and deal with it.
7799 * Used by cik_irq_process().
7800 * Returns the value of the wptr.
7801 */
7802static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7803{
7804 u32 wptr, tmp;
7805
7806 if (rdev->wb.enabled)
7807 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7808 else
7809 wptr = RREG32(IH_RB_WPTR);
7810
7811 if (wptr & RB_OVERFLOW) {
11bab0ae 7812 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7813 /* When a ring buffer overflow happen start parsing interrupt
7814 * from the last not overwritten vector (wptr + 16). Hopefully
7815 * this should allow us to catchup.
7816 */
6cc2fda2
MD		 7817 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7818 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7819 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7820 tmp = RREG32(IH_RB_CNTL);
7821 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7822 WREG32(IH_RB_CNTL, tmp);
7823 }
7824 return (wptr & rdev->ih.ptr_mask);
7825}
7826
7827/* CIK IV Ring
7828 * Each IV ring entry is 128 bits:
7829 * [7:0] - interrupt source id
7830 * [31:8] - reserved
7831 * [59:32] - interrupt source data
7832 * [63:60] - reserved
21a93e13
AD		 7833 * [71:64] - RINGID
7834 * CP:
7835 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7836 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7837 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7838 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7839 * PIPE_ID - ME0 0=3D
7840 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7841 * SDMA:
7842 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7843 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7844 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7845 * [79:72] - VMID
7846 * [95:80] - PASID
7847 * [127:96] - reserved
7848 */
7849/**
7850 * cik_irq_process - interrupt handler
7851 *
7852 * @rdev: radeon_device pointer
7853 *
7854 * Interrupt hander (CIK). Walk the IH ring,
7855 * ack interrupts and schedule work to handle
7856 * interrupt events.
7857 * Returns irq process return code.
7858 */
7859int cik_irq_process(struct radeon_device *rdev)
7860{
2b0781a6
AD		 7861 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7862 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7863 u32 wptr;
7864 u32 rptr;
7865 u32 src_id, src_data, ring_id;
7866 u8 me_id, pipe_id, queue_id;
7867 u32 ring_index;
7868 bool queue_hotplug = false;
f6b355dd 7869 bool queue_dp = false;
a59781bb 7870 bool queue_reset = false;
3ec7d11b 7871 u32 addr, status, mc_client;
41a524ab 7872 bool queue_thermal = false;
a59781bb
AD		 7873
7874 if (!rdev->ih.enabled || rdev->shutdown)
7875 return IRQ_NONE;
7876
7877 wptr = cik_get_ih_wptr(rdev);
7878
7879restart_ih:
7880 /* is somebody else already processing irqs? */
7881 if (atomic_xchg(&rdev->ih.lock, 1))
7882 return IRQ_NONE;
7883
7884 rptr = rdev->ih.rptr;
7885 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7886
7887 /* Order reading of wptr vs. reading of IH ring data */
7888 rmb();
7889
7890 /* display interrupts */
7891 cik_irq_ack(rdev);
7892
7893 while (rptr != wptr) {
7894 /* wptr/rptr are in bytes! */
7895 ring_index = rptr / 4;
e28740ec
OG		 7896
7897 radeon_kfd_interrupt(rdev,
7898 (const void *) &rdev->ih.ring[ring_index]);
7899
a59781bb
AD		 7900 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7901 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7902 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7903
7904 switch (src_id) {
7905 case 1: /* D1 vblank/vline */
7906 switch (src_data) {
7907 case 0: /* D1 vblank */
7908 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7909 if (rdev->irq.crtc_vblank_int[0]) {
7910 drm_handle_vblank(rdev->ddev, 0);
7911 rdev->pm.vblank_sync = true;
7912 wake_up(&rdev->irq.vblank_queue);
7913 }
7914 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 7915 radeon_crtc_handle_vblank(rdev, 0);
a59781bb
AD		 7916 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7917 DRM_DEBUG("IH: D1 vblank\n");
7918 }
7919 break;
7920 case 1: /* D1 vline */
7921 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7922 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7923 DRM_DEBUG("IH: D1 vline\n");
7924 }
7925 break;
7926 default:
7927 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7928 break;
7929 }
7930 break;
7931 case 2: /* D2 vblank/vline */
7932 switch (src_data) {
7933 case 0: /* D2 vblank */
7934 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7935 if (rdev->irq.crtc_vblank_int[1]) {
7936 drm_handle_vblank(rdev->ddev, 1);
7937 rdev->pm.vblank_sync = true;
7938 wake_up(&rdev->irq.vblank_queue);
7939 }
7940 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 7941 radeon_crtc_handle_vblank(rdev, 1);
a59781bb
AD		 7942 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7943 DRM_DEBUG("IH: D2 vblank\n");
7944 }
7945 break;
7946 case 1: /* D2 vline */
7947 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7948 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7949 DRM_DEBUG("IH: D2 vline\n");
7950 }
7951 break;
7952 default:
7953 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7954 break;
7955 }
7956 break;
7957 case 3: /* D3 vblank/vline */
7958 switch (src_data) {
7959 case 0: /* D3 vblank */
7960 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7961 if (rdev->irq.crtc_vblank_int[2]) {
7962 drm_handle_vblank(rdev->ddev, 2);
7963 rdev->pm.vblank_sync = true;
7964 wake_up(&rdev->irq.vblank_queue);
7965 }
7966 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 7967 radeon_crtc_handle_vblank(rdev, 2);
a59781bb
AD		 7968 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7969 DRM_DEBUG("IH: D3 vblank\n");
7970 }
7971 break;
7972 case 1: /* D3 vline */
7973 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7974 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7975 DRM_DEBUG("IH: D3 vline\n");
7976 }
7977 break;
7978 default:
7979 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7980 break;
7981 }
7982 break;
7983 case 4: /* D4 vblank/vline */
7984 switch (src_data) {
7985 case 0: /* D4 vblank */
7986 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7987 if (rdev->irq.crtc_vblank_int[3]) {
7988 drm_handle_vblank(rdev->ddev, 3);
7989 rdev->pm.vblank_sync = true;
7990 wake_up(&rdev->irq.vblank_queue);
7991 }
7992 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 7993 radeon_crtc_handle_vblank(rdev, 3);
a59781bb
AD		 7994 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7995 DRM_DEBUG("IH: D4 vblank\n");
7996 }
7997 break;
7998 case 1: /* D4 vline */
7999 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
8000 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
8001 DRM_DEBUG("IH: D4 vline\n");
8002 }
8003 break;
8004 default:
8005 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8006 break;
8007 }
8008 break;
8009 case 5: /* D5 vblank/vline */
8010 switch (src_data) {
8011 case 0: /* D5 vblank */
8012 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
8013 if (rdev->irq.crtc_vblank_int[4]) {
8014 drm_handle_vblank(rdev->ddev, 4);
8015 rdev->pm.vblank_sync = true;
8016 wake_up(&rdev->irq.vblank_queue);
8017 }
8018 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 8019 radeon_crtc_handle_vblank(rdev, 4);
a59781bb
AD		 8020 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
8021 DRM_DEBUG("IH: D5 vblank\n");
8022 }
8023 break;
8024 case 1: /* D5 vline */
8025 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
8026 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
8027 DRM_DEBUG("IH: D5 vline\n");
8028 }
8029 break;
8030 default:
8031 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8032 break;
8033 }
8034 break;
8035 case 6: /* D6 vblank/vline */
8036 switch (src_data) {
8037 case 0: /* D6 vblank */
8038 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
8039 if (rdev->irq.crtc_vblank_int[5]) {
8040 drm_handle_vblank(rdev->ddev, 5);
8041 rdev->pm.vblank_sync = true;
8042 wake_up(&rdev->irq.vblank_queue);
8043 }
8044 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 8045 radeon_crtc_handle_vblank(rdev, 5);
a59781bb
AD		 8046 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
8047 DRM_DEBUG("IH: D6 vblank\n");
8048 }
8049 break;
8050 case 1: /* D6 vline */
8051 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
8052 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
8053 DRM_DEBUG("IH: D6 vline\n");
8054 }
8055 break;
8056 default:
8057 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8058 break;
8059 }
8060 break;
f5d636d2
CK		 8061 case 8: /* D1 page flip */
8062 case 10: /* D2 page flip */
8063 case 12: /* D3 page flip */
8064 case 14: /* D4 page flip */
8065 case 16: /* D5 page flip */
8066 case 18: /* D6 page flip */
8067 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 8068 if (radeon_use_pflipirq > 0)
8069 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 8070 break;
a59781bb
AD		 8071 case 42: /* HPD hotplug */
8072 switch (src_data) {
8073 case 0:
8074 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8075 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8076 queue_hotplug = true;
8077 DRM_DEBUG("IH: HPD1\n");
8078 }
8079 break;
8080 case 1:
8081 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8082 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8083 queue_hotplug = true;
8084 DRM_DEBUG("IH: HPD2\n");
8085 }
8086 break;
8087 case 2:
8088 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8089 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8090 queue_hotplug = true;
8091 DRM_DEBUG("IH: HPD3\n");
8092 }
8093 break;
8094 case 3:
8095 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8096 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8097 queue_hotplug = true;
8098 DRM_DEBUG("IH: HPD4\n");
8099 }
8100 break;
8101 case 4:
8102 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8103 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8104 queue_hotplug = true;
8105 DRM_DEBUG("IH: HPD5\n");
8106 }
8107 break;
8108 case 5:
8109 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8110 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8111 queue_hotplug = true;
8112 DRM_DEBUG("IH: HPD6\n");
8113 }
8114 break;
f6b355dd
AD		 8115 case 6:
8116 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
8117 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
8118 queue_dp = true;
8119 DRM_DEBUG("IH: HPD_RX 1\n");
8120 }
8121 break;
8122 case 7:
8123 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
8124 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
8125 queue_dp = true;
8126 DRM_DEBUG("IH: HPD_RX 2\n");
8127 }
8128 break;
8129 case 8:
8130 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
8131 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
8132 queue_dp = true;
8133 DRM_DEBUG("IH: HPD_RX 3\n");
8134 }
8135 break;
8136 case 9:
8137 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
8138 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
8139 queue_dp = true;
8140 DRM_DEBUG("IH: HPD_RX 4\n");
8141 }
8142 break;
8143 case 10:
8144 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
8145 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
8146 queue_dp = true;
8147 DRM_DEBUG("IH: HPD_RX 5\n");
8148 }
8149 break;
8150 case 11:
8151 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
8152 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
8153 queue_dp = true;
8154 DRM_DEBUG("IH: HPD_RX 6\n");
8155 }
8156 break;
a59781bb
AD		 8157 default:
8158 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8159 break;
8160 }
8161 break;
dc12a3ec
LL		 8162 case 96:
8163 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
8164 WREG32(SRBM_INT_ACK, 0x1);
8165 break;
6a3808b8
CK		 8166 case 124: /* UVD */
8167 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8168 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8169 break;
9d97c99b
AD		 8170 case 146:
8171 case 147:
3ec7d11b
AD		 8172 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8173 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8174 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8175 /* reset addr and status */
8176 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8177 if (addr == 0x0 && status == 0x0)
8178 break;
9d97c99b
AD		 8179 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8180 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8181 addr);
9d97c99b 8182 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8183 status);
8184 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8185 break;
d93f7937
CK		 8186 case 167: /* VCE */
8187 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8188 switch (src_data) {
8189 case 0:
8190 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8191 break;
8192 case 1:
8193 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8194 break;
8195 default:
8196 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8197 break;
8198 }
8199 break;
a59781bb
AD		 8200 case 176: /* GFX RB CP_INT */
8201 case 177: /* GFX IB CP_INT */
8202 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8203 break;
8204 case 181: /* CP EOP event */
8205 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8206 /* XXX check the bitfield order! */
8207 me_id = (ring_id & 0x60) >> 5;
8208 pipe_id = (ring_id & 0x18) >> 3;
8209 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8210 switch (me_id) {
8211 case 0:
8212 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8213 break;
8214 case 1:
a59781bb 8215 case 2:
2b0781a6
AD		 8216 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8217 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8218 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8219 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8220 break;
8221 }
8222 break;
8223 case 184: /* CP Privileged reg access */
8224 DRM_ERROR("Illegal register access in command stream\n");
8225 /* XXX check the bitfield order! */
8226 me_id = (ring_id & 0x60) >> 5;
8227 pipe_id = (ring_id & 0x18) >> 3;
8228 queue_id = (ring_id & 0x7) >> 0;
8229 switch (me_id) {
8230 case 0:
8231 /* This results in a full GPU reset, but all we need to do is soft
8232 * reset the CP for gfx
8233 */
8234 queue_reset = true;
8235 break;
8236 case 1:
8237 /* XXX compute */
2b0781a6 8238 queue_reset = true;
a59781bb
AD		 8239 break;
8240 case 2:
8241 /* XXX compute */
2b0781a6 8242 queue_reset = true;
a59781bb
AD		 8243 break;
8244 }
8245 break;
8246 case 185: /* CP Privileged inst */
8247 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8248 /* XXX check the bitfield order! */
8249 me_id = (ring_id & 0x60) >> 5;
8250 pipe_id = (ring_id & 0x18) >> 3;
8251 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8252 switch (me_id) {
8253 case 0:
8254 /* This results in a full GPU reset, but all we need to do is soft
8255 * reset the CP for gfx
8256 */
8257 queue_reset = true;
8258 break;
8259 case 1:
8260 /* XXX compute */
2b0781a6 8261 queue_reset = true;
a59781bb
AD		 8262 break;
8263 case 2:
8264 /* XXX compute */
2b0781a6 8265 queue_reset = true;
a59781bb
AD		 8266 break;
8267 }
8268 break;
21a93e13
AD		 8269 case 224: /* SDMA trap event */
8270 /* XXX check the bitfield order! */
8271 me_id = (ring_id & 0x3) >> 0;
8272 queue_id = (ring_id & 0xc) >> 2;
8273 DRM_DEBUG("IH: SDMA trap\n");
8274 switch (me_id) {
8275 case 0:
8276 switch (queue_id) {
8277 case 0:
8278 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8279 break;
8280 case 1:
8281 /* XXX compute */
8282 break;
8283 case 2:
8284 /* XXX compute */
8285 break;
8286 }
8287 break;
8288 case 1:
8289 switch (queue_id) {
8290 case 0:
8291 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8292 break;
8293 case 1:
8294 /* XXX compute */
8295 break;
8296 case 2:
8297 /* XXX compute */
8298 break;
8299 }
8300 break;
8301 }
8302 break;
41a524ab
AD		 8303 case 230: /* thermal low to high */
8304 DRM_DEBUG("IH: thermal low to high\n");
8305 rdev->pm.dpm.thermal.high_to_low = false;
8306 queue_thermal = true;
8307 break;
8308 case 231: /* thermal high to low */
8309 DRM_DEBUG("IH: thermal high to low\n");
8310 rdev->pm.dpm.thermal.high_to_low = true;
8311 queue_thermal = true;
8312 break;
8313 case 233: /* GUI IDLE */
8314 DRM_DEBUG("IH: GUI idle\n");
8315 break;
21a93e13
AD		 8316 case 241: /* SDMA Privileged inst */
8317 case 247: /* SDMA Privileged inst */
8318 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8319 /* XXX check the bitfield order! */
8320 me_id = (ring_id & 0x3) >> 0;
8321 queue_id = (ring_id & 0xc) >> 2;
8322 switch (me_id) {
8323 case 0:
8324 switch (queue_id) {
8325 case 0:
8326 queue_reset = true;
8327 break;
8328 case 1:
8329 /* XXX compute */
8330 queue_reset = true;
8331 break;
8332 case 2:
8333 /* XXX compute */
8334 queue_reset = true;
8335 break;
8336 }
8337 break;
8338 case 1:
8339 switch (queue_id) {
8340 case 0:
8341 queue_reset = true;
8342 break;
8343 case 1:
8344 /* XXX compute */
8345 queue_reset = true;
8346 break;
8347 case 2:
8348 /* XXX compute */
8349 queue_reset = true;
8350 break;
8351 }
8352 break;
8353 }
8354 break;
a59781bb
AD		 8355 default:
8356 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8357 break;
8358 }
8359
8360 /* wptr/rptr are in bytes! */
8361 rptr += 16;
8362 rptr &= rdev->ih.ptr_mask;
f55e03b9 8363 WREG32(IH_RB_RPTR, rptr);
a59781bb 8364 }
f6b355dd
AD		 8365 if (queue_dp)
8366 schedule_work(&rdev->dp_work);
a59781bb
AD		 8367 if (queue_hotplug)
8368 schedule_work(&rdev->hotplug_work);
3c036389
CK		 8369 if (queue_reset) {
8370 rdev->needs_reset = true;
8371 wake_up_all(&rdev->fence_queue);
8372 }
41a524ab
AD		 8373 if (queue_thermal)
8374 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8375 rdev->ih.rptr = rptr;
a59781bb
AD		 8376 atomic_set(&rdev->ih.lock, 0);
8377
8378 /* make sure wptr hasn't changed while processing */
8379 wptr = cik_get_ih_wptr(rdev);
8380 if (wptr != rptr)
8381 goto restart_ih;
8382
8383 return IRQ_HANDLED;
8384}
7bf94a2c
AD		 8385
8386/*
8387 * startup/shutdown callbacks
8388 */
8389/**
8390 * cik_startup - program the asic to a functional state
8391 *
8392 * @rdev: radeon_device pointer
8393 *
8394 * Programs the asic to a functional state (CIK).
8395 * Called by cik_init() and cik_resume().
8396 * Returns 0 for success, error for failure.
8397 */
8398static int cik_startup(struct radeon_device *rdev)
8399{
8400 struct radeon_ring *ring;
0e16e4cf 8401 u32 nop;
7bf94a2c
AD		 8402 int r;
8403
8a7cd276
AD		 8404 /* enable pcie gen2/3 link */
8405 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8406 /* enable aspm */
8407 cik_program_aspm(rdev);
8a7cd276 8408
e5903d39
AD		 8409 /* scratch needs to be initialized before MC */
8410 r = r600_vram_scratch_init(rdev);
8411 if (r)
8412 return r;
8413
6fab3feb
AD		 8414 cik_mc_program(rdev);
8415
6c7bccea 8416 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8417 r = ci_mc_load_microcode(rdev);
8418 if (r) {
8419 DRM_ERROR("Failed to load MC firmware!\n");
8420 return r;
8421 }
8422 }
8423
7bf94a2c
AD		 8424 r = cik_pcie_gart_enable(rdev);
8425 if (r)
8426 return r;
8427 cik_gpu_init(rdev);
8428
8429 /* allocate rlc buffers */
22c775ce
AD		 8430 if (rdev->flags & RADEON_IS_IGP) {
8431 if (rdev->family == CHIP_KAVERI) {
8432 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8433 rdev->rlc.reg_list_size =
8434 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8435 } else {
8436 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8437 rdev->rlc.reg_list_size =
8438 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8439 }
8440 }
8441 rdev->rlc.cs_data = ci_cs_data;
8442 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8443 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8444 if (r) {
8445 DRM_ERROR("Failed to init rlc BOs!\n");
8446 return r;
8447 }
8448
8449 /* allocate wb buffer */
8450 r = radeon_wb_init(rdev);
8451 if (r)
8452 return r;
8453
963e81f9
AD		 8454 /* allocate mec buffers */
8455 r = cik_mec_init(rdev);
8456 if (r) {
8457 DRM_ERROR("Failed to init MEC BOs!\n");
8458 return r;
8459 }
8460
7bf94a2c
AD		 8461 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8462 if (r) {
8463 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8464 return r;
8465 }
8466
963e81f9
AD		 8467 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8468 if (r) {
8469 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8470 return r;
8471 }
8472
8473 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8474 if (r) {
8475 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8476 return r;
8477 }
8478
7bf94a2c
AD		 8479 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8480 if (r) {
8481 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8482 return r;
8483 }
8484
8485 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8486 if (r) {
8487 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8488 return r;
8489 }
8490
2ce529da 8491 r = radeon_uvd_resume(rdev);
87167bb1 8492 if (!r) {
2ce529da
AD		 8493 r = uvd_v4_2_resume(rdev);
8494 if (!r) {
8495 r = radeon_fence_driver_start_ring(rdev,
8496 R600_RING_TYPE_UVD_INDEX);
8497 if (r)
8498 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8499 }
87167bb1
CK		 8500 }
8501 if (r)
8502 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8503
d93f7937
CK		 8504 r = radeon_vce_resume(rdev);
8505 if (!r) {
8506 r = vce_v2_0_resume(rdev);
8507 if (!r)
8508 r = radeon_fence_driver_start_ring(rdev,
8509 TN_RING_TYPE_VCE1_INDEX);
8510 if (!r)
8511 r = radeon_fence_driver_start_ring(rdev,
8512 TN_RING_TYPE_VCE2_INDEX);
8513 }
8514 if (r) {
8515 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8516 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8517 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8518 }
8519
7bf94a2c
AD		 8520 /* Enable IRQ */
8521 if (!rdev->irq.installed) {
8522 r = radeon_irq_kms_init(rdev);
8523 if (r)
8524 return r;
8525 }
8526
8527 r = cik_irq_init(rdev);
8528 if (r) {
8529 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8530 radeon_irq_kms_fini(rdev);
8531 return r;
8532 }
8533 cik_irq_set(rdev);
8534
0e16e4cf 8535 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8536 if (rdev->new_fw)
8537 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8538 else
8539 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8540 } else {
8541 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8542 }
8543
7bf94a2c
AD		 8544 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8545 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8546 nop);
7bf94a2c
AD		 8547 if (r)
8548 return r;
8549
963e81f9 8550 /* set up the compute queues */
2615b53a 8551 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8552 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8553 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8554 nop);
963e81f9
AD		 8555 if (r)
8556 return r;
8557 ring->me = 1; /* first MEC */
8558 ring->pipe = 0; /* first pipe */
8559 ring->queue = 0; /* first queue */
8560 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8561
2615b53a 8562 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8563 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8564 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8565 nop);
963e81f9
AD		 8566 if (r)
8567 return r;
8568 /* dGPU only have 1 MEC */
8569 ring->me = 1; /* first MEC */
8570 ring->pipe = 0; /* first pipe */
8571 ring->queue = 1; /* second queue */
8572 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8573
7bf94a2c
AD		 8574 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8575 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8576 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8577 if (r)
8578 return r;
8579
8580 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8581 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8582 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8583 if (r)
8584 return r;
8585
8586 r = cik_cp_resume(rdev);
8587 if (r)
8588 return r;
8589
8590 r = cik_sdma_resume(rdev);
8591 if (r)
8592 return r;
8593
87167bb1
CK		 8594 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8595 if (ring->ring_size) {
02c9f7fa 8596 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8597 RADEON_CP_PACKET2);
87167bb1 8598 if (!r)
e409b128 8599 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8600 if (r)
8601 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8602 }
8603
d93f7937
CK		 8604 r = -ENOENT;
8605
8606 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8607 if (ring->ring_size)
8608 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8609 VCE_CMD_NO_OP);
8610
8611 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8612 if (ring->ring_size)
8613 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8614 VCE_CMD_NO_OP);
8615
8616 if (!r)
8617 r = vce_v1_0_init(rdev);
8618 else if (r != -ENOENT)
8619 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8620
7bf94a2c
AD		 8621 r = radeon_ib_pool_init(rdev);
8622 if (r) {
8623 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8624 return r;
8625 }
8626
8627 r = radeon_vm_manager_init(rdev);
8628 if (r) {
8629 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8630 return r;
8631 }
8632
bfc1f97d 8633 r = radeon_audio_init(rdev);
b530602f
AD		 8634 if (r)
8635 return r;
8636
e28740ec
OG		 8637 r = radeon_kfd_resume(rdev);
8638 if (r)
8639 return r;
8640
7bf94a2c
AD		 8641 return 0;
8642}
8643
8644/**
8645 * cik_resume - resume the asic to a functional state
8646 *
8647 * @rdev: radeon_device pointer
8648 *
8649 * Programs the asic to a functional state (CIK).
8650 * Called at resume.
8651 * Returns 0 for success, error for failure.
8652 */
8653int cik_resume(struct radeon_device *rdev)
8654{
8655 int r;
8656
8657 /* post card */
8658 atom_asic_init(rdev->mode_info.atom_context);
8659
0aafd313
AD		 8660 /* init golden registers */
8661 cik_init_golden_registers(rdev);
8662
bc6a6295
AD		 8663 if (rdev->pm.pm_method == PM_METHOD_DPM)
8664 radeon_pm_resume(rdev);
6c7bccea 8665
7bf94a2c
AD		 8666 rdev->accel_working = true;
8667 r = cik_startup(rdev);
8668 if (r) {
8669 DRM_ERROR("cik startup failed on resume\n");
8670 rdev->accel_working = false;
8671 return r;
8672 }
8673
8674 return r;
8675
8676}
8677
8678/**
8679 * cik_suspend - suspend the asic
8680 *
8681 * @rdev: radeon_device pointer
8682 *
8683 * Bring the chip into a state suitable for suspend (CIK).
8684 * Called at suspend.
8685 * Returns 0 for success.
8686 */
8687int cik_suspend(struct radeon_device *rdev)
8688{
e28740ec 8689 radeon_kfd_suspend(rdev);
6c7bccea 8690 radeon_pm_suspend(rdev);
7991d665 8691 radeon_audio_fini(rdev);
7bf94a2c
AD		 8692 radeon_vm_manager_fini(rdev);
8693 cik_cp_enable(rdev, false);
8694 cik_sdma_enable(rdev, false);
e409b128 8695 uvd_v1_0_fini(rdev);
87167bb1 8696 radeon_uvd_suspend(rdev);
d93f7937 8697 radeon_vce_suspend(rdev);
473359bc
AD		 8698 cik_fini_pg(rdev);
8699 cik_fini_cg(rdev);
7bf94a2c
AD		 8700 cik_irq_suspend(rdev);
8701 radeon_wb_disable(rdev);
8702 cik_pcie_gart_disable(rdev);
8703 return 0;
8704}
8705
8706/* Plan is to move initialization in that function and use
8707 * helper function so that radeon_device_init pretty much
8708 * do nothing more than calling asic specific function. This
8709 * should also allow to remove a bunch of callback function
8710 * like vram_info.
8711 */
8712/**
8713 * cik_init - asic specific driver and hw init
8714 *
8715 * @rdev: radeon_device pointer
8716 *
8717 * Setup asic specific driver variables and program the hw
8718 * to a functional state (CIK).
8719 * Called at driver startup.
8720 * Returns 0 for success, errors for failure.
8721 */
8722int cik_init(struct radeon_device *rdev)
8723{
8724 struct radeon_ring *ring;
8725 int r;
8726
8727 /* Read BIOS */
8728 if (!radeon_get_bios(rdev)) {
8729 if (ASIC_IS_AVIVO(rdev))
8730 return -EINVAL;
8731 }
8732 /* Must be an ATOMBIOS */
8733 if (!rdev->is_atom_bios) {
8734 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8735 return -EINVAL;
8736 }
8737 r = radeon_atombios_init(rdev);
8738 if (r)
8739 return r;
8740
8741 /* Post card if necessary */
8742 if (!radeon_card_posted(rdev)) {
8743 if (!rdev->bios) {
8744 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8745 return -EINVAL;
8746 }
8747 DRM_INFO("GPU not posted. posting now...\n");
8748 atom_asic_init(rdev->mode_info.atom_context);
8749 }
0aafd313
AD		 8750 /* init golden registers */
8751 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8752 /* Initialize scratch registers */
8753 cik_scratch_init(rdev);
8754 /* Initialize surface registers */
8755 radeon_surface_init(rdev);
8756 /* Initialize clocks */
8757 radeon_get_clock_info(rdev->ddev);
8758
8759 /* Fence driver */
8760 r = radeon_fence_driver_init(rdev);
8761 if (r)
8762 return r;
8763
8764 /* initialize memory controller */
8765 r = cik_mc_init(rdev);
8766 if (r)
8767 return r;
8768 /* Memory manager */
8769 r = radeon_bo_init(rdev);
8770 if (r)
8771 return r;
8772
01ac8794
AD		 8773 if (rdev->flags & RADEON_IS_IGP) {
8774 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8775 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8776 r = cik_init_microcode(rdev);
8777 if (r) {
8778 DRM_ERROR("Failed to load firmware!\n");
8779 return r;
8780 }
8781 }
8782 } else {
8783 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8784 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8785 !rdev->mc_fw) {
8786 r = cik_init_microcode(rdev);
8787 if (r) {
8788 DRM_ERROR("Failed to load firmware!\n");
8789 return r;
8790 }
8791 }
8792 }
8793
6c7bccea
AD		 8794 /* Initialize power management */
8795 radeon_pm_init(rdev);
8796
7bf94a2c
AD		 8797 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8798 ring->ring_obj = NULL;
8799 r600_ring_init(rdev, ring, 1024 * 1024);
8800
963e81f9
AD		 8801 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8802 ring->ring_obj = NULL;
8803 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8804 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8805 if (r)
8806 return r;
8807
8808 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8809 ring->ring_obj = NULL;
8810 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8811 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8812 if (r)
8813 return r;
8814
7bf94a2c
AD		 8815 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8816 ring->ring_obj = NULL;
8817 r600_ring_init(rdev, ring, 256 * 1024);
8818
8819 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8820 ring->ring_obj = NULL;
8821 r600_ring_init(rdev, ring, 256 * 1024);
8822
87167bb1
CK		 8823 r = radeon_uvd_init(rdev);
8824 if (!r) {
8825 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8826 ring->ring_obj = NULL;
8827 r600_ring_init(rdev, ring, 4096);
8828 }
8829
d93f7937
CK		 8830 r = radeon_vce_init(rdev);
8831 if (!r) {
8832 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8833 ring->ring_obj = NULL;
8834 r600_ring_init(rdev, ring, 4096);
8835
8836 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8837 ring->ring_obj = NULL;
8838 r600_ring_init(rdev, ring, 4096);
8839 }
8840
7bf94a2c
AD		 8841 rdev->ih.ring_obj = NULL;
8842 r600_ih_ring_init(rdev, 64 * 1024);
8843
8844 r = r600_pcie_gart_init(rdev);
8845 if (r)
8846 return r;
8847
8848 rdev->accel_working = true;
8849 r = cik_startup(rdev);
8850 if (r) {
8851 dev_err(rdev->dev, "disabling GPU acceleration\n");
8852 cik_cp_fini(rdev);
8853 cik_sdma_fini(rdev);
8854 cik_irq_fini(rdev);
1fd11777 8855 sumo_rlc_fini(rdev);
963e81f9 8856 cik_mec_fini(rdev);
7bf94a2c
AD		 8857 radeon_wb_fini(rdev);
8858 radeon_ib_pool_fini(rdev);
8859 radeon_vm_manager_fini(rdev);
8860 radeon_irq_kms_fini(rdev);
8861 cik_pcie_gart_fini(rdev);
8862 rdev->accel_working = false;
8863 }
8864
8865 /* Don't start up if the MC ucode is missing.
8866 * The default clocks and voltages before the MC ucode
8867 * is loaded are not suffient for advanced operations.
8868 */
8869 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8870 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8871 return -EINVAL;
8872 }
8873
8874 return 0;
8875}
8876
8877/**
8878 * cik_fini - asic specific driver and hw fini
8879 *
8880 * @rdev: radeon_device pointer
8881 *
8882 * Tear down the asic specific driver variables and program the hw
8883 * to an idle state (CIK).
8884 * Called at driver unload.
8885 */
8886void cik_fini(struct radeon_device *rdev)
8887{
6c7bccea 8888 radeon_pm_fini(rdev);
7bf94a2c
AD		 8889 cik_cp_fini(rdev);
8890 cik_sdma_fini(rdev);
473359bc
AD		 8891 cik_fini_pg(rdev);
8892 cik_fini_cg(rdev);
7bf94a2c 8893 cik_irq_fini(rdev);
1fd11777 8894 sumo_rlc_fini(rdev);
963e81f9 8895 cik_mec_fini(rdev);
7bf94a2c
AD		 8896 radeon_wb_fini(rdev);
8897 radeon_vm_manager_fini(rdev);
8898 radeon_ib_pool_fini(rdev);
8899 radeon_irq_kms_fini(rdev);
e409b128 8900 uvd_v1_0_fini(rdev);
87167bb1 8901 radeon_uvd_fini(rdev);
d93f7937 8902 radeon_vce_fini(rdev);
7bf94a2c
AD		 8903 cik_pcie_gart_fini(rdev);
8904 r600_vram_scratch_fini(rdev);
8905 radeon_gem_fini(rdev);
8906 radeon_fence_driver_fini(rdev);
8907 radeon_bo_fini(rdev);
8908 radeon_atombios_fini(rdev);
8909 kfree(rdev->bios);
8910 rdev->bios = NULL;
8911}
cd84a27d 8912
134b480f
AD		 8913void dce8_program_fmt(struct drm_encoder *encoder)
8914{
8915 struct drm_device *dev = encoder->dev;
8916 struct radeon_device *rdev = dev->dev_private;
8917 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8918 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8919 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8920 int bpc = 0;
8921 u32 tmp = 0;
6214bb74 8922 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8923
6214bb74
AD		 8924 if (connector) {
8925 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8926 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8927 dither = radeon_connector->dither;
8928 }
134b480f
AD		 8929
8930 /* LVDS/eDP FMT is set up by atom */
8931 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8932 return;
8933
8934 /* not needed for analog */
8935 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8936 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8937 return;
8938
8939 if (bpc == 0)
8940 return;
8941
8942 switch (bpc) {
8943 case 6:
6214bb74 8944 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8945 /* XXX sort out optimal dither settings */
8946 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8947 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8948 else
8949 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8950 break;
8951 case 8:
6214bb74 8952 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8953 /* XXX sort out optimal dither settings */
8954 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8955 FMT_RGB_RANDOM_ENABLE |
8956 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8957 else
8958 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8959 break;
8960 case 10:
6214bb74 8961 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8962 /* XXX sort out optimal dither settings */
8963 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8964 FMT_RGB_RANDOM_ENABLE |
8965 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8966 else
8967 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8968 break;
8969 default:
8970 /* not needed */
8971 break;
8972 }
8973
8974 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8975}
8976
cd84a27d
AD		 8977/* display watermark setup */
8978/**
8979 * dce8_line_buffer_adjust - Set up the line buffer
8980 *
8981 * @rdev: radeon_device pointer
8982 * @radeon_crtc: the selected display controller
8983 * @mode: the current display mode on the selected display
8984 * controller
8985 *
8986 * Setup up the line buffer allocation for
8987 * the selected display controller (CIK).
8988 * Returns the line buffer size in pixels.
8989 */
8990static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8991 struct radeon_crtc *radeon_crtc,
8992 struct drm_display_mode *mode)
8993{
bc01a8c7
AD		 8994 u32 tmp, buffer_alloc, i;
8995 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8996 /*
8997 * Line Buffer Setup
8998 * There are 6 line buffers, one for each display controllers.
8999 * There are 3 partitions per LB. Select the number of partitions
9000 * to enable based on the display width. For display widths larger
9001 * than 4096, you need use to use 2 display controllers and combine
9002 * them using the stereo blender.
9003 */
9004 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 9005 if (mode->crtc_hdisplay < 1920) {
cd84a27d 9006 tmp = 1;
bc01a8c7
AD		 9007 buffer_alloc = 2;
9008 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 9009 tmp = 2;
bc01a8c7
AD		 9010 buffer_alloc = 2;
9011 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 9012 tmp = 0;
bc01a8c7
AD		 9013 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
9014 } else {
cd84a27d
AD		 9015 DRM_DEBUG_KMS("Mode too big for LB!\n");
9016 tmp = 0;
bc01a8c7 9017 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 9018 }
bc01a8c7 9019 } else {
cd84a27d 9020 tmp = 1;
bc01a8c7
AD		 9021 buffer_alloc = 0;
9022 }
cd84a27d
AD		 9023
9024 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
9025 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
9026
bc01a8c7
AD		 9027 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
9028 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
9029 for (i = 0; i < rdev->usec_timeout; i++) {
9030 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
9031 DMIF_BUFFERS_ALLOCATED_COMPLETED)
9032 break;
9033 udelay(1);
9034 }
9035
cd84a27d
AD		 9036 if (radeon_crtc->base.enabled && mode) {
9037 switch (tmp) {
9038 case 0:
9039 default:
9040 return 4096 * 2;
9041 case 1:
9042 return 1920 * 2;
9043 case 2:
9044 return 2560 * 2;
9045 }
9046 }
9047
9048 /* controller not enabled, so no lb used */
9049 return 0;
9050}
9051
9052/**
9053 * cik_get_number_of_dram_channels - get the number of dram channels
9054 *
9055 * @rdev: radeon_device pointer
9056 *
9057 * Look up the number of video ram channels (CIK).
9058 * Used for display watermark bandwidth calculations
9059 * Returns the number of dram channels
9060 */
9061static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
9062{
9063 u32 tmp = RREG32(MC_SHARED_CHMAP);
9064
9065 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
9066 case 0:
9067 default:
9068 return 1;
9069 case 1:
9070 return 2;
9071 case 2:
9072 return 4;
9073 case 3:
9074 return 8;
9075 case 4:
9076 return 3;
9077 case 5:
9078 return 6;
9079 case 6:
9080 return 10;
9081 case 7:
9082 return 12;
9083 case 8:
9084 return 16;
9085 }
9086}
9087
9088struct dce8_wm_params {
9089 u32 dram_channels; /* number of dram channels */
9090 u32 yclk; /* bandwidth per dram data pin in kHz */
9091 u32 sclk; /* engine clock in kHz */
9092 u32 disp_clk; /* display clock in kHz */
9093 u32 src_width; /* viewport width */
9094 u32 active_time; /* active display time in ns */
9095 u32 blank_time; /* blank time in ns */
9096 bool interlaced; /* mode is interlaced */
9097 fixed20_12 vsc; /* vertical scale ratio */
9098 u32 num_heads; /* number of active crtcs */
9099 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
9100 u32 lb_size; /* line buffer allocated to pipe */
9101 u32 vtaps; /* vertical scaler taps */
9102};
9103
9104/**
9105 * dce8_dram_bandwidth - get the dram bandwidth
9106 *
9107 * @wm: watermark calculation data
9108 *
9109 * Calculate the raw dram bandwidth (CIK).
9110 * Used for display watermark bandwidth calculations
9111 * Returns the dram bandwidth in MBytes/s
9112 */
9113static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9114{
9115 /* Calculate raw DRAM Bandwidth */
9116 fixed20_12 dram_efficiency; /* 0.7 */
9117 fixed20_12 yclk, dram_channels, bandwidth;
9118 fixed20_12 a;
9119
9120 a.full = dfixed_const(1000);
9121 yclk.full = dfixed_const(wm->yclk);
9122 yclk.full = dfixed_div(yclk, a);
9123 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9124 a.full = dfixed_const(10);
9125 dram_efficiency.full = dfixed_const(7);
9126 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9127 bandwidth.full = dfixed_mul(dram_channels, yclk);
9128 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9129
9130 return dfixed_trunc(bandwidth);
9131}
9132
9133/**
9134 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9135 *
9136 * @wm: watermark calculation data
9137 *
9138 * Calculate the dram bandwidth used for display (CIK).
9139 * Used for display watermark bandwidth calculations
9140 * Returns the dram bandwidth for display in MBytes/s
9141 */
9142static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9143{
9144 /* Calculate DRAM Bandwidth and the part allocated to display. */
9145 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9146 fixed20_12 yclk, dram_channels, bandwidth;
9147 fixed20_12 a;
9148
9149 a.full = dfixed_const(1000);
9150 yclk.full = dfixed_const(wm->yclk);
9151 yclk.full = dfixed_div(yclk, a);
9152 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9153 a.full = dfixed_const(10);
9154 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9155 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9156 bandwidth.full = dfixed_mul(dram_channels, yclk);
9157 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9158
9159 return dfixed_trunc(bandwidth);
9160}
9161
9162/**
9163 * dce8_data_return_bandwidth - get the data return bandwidth
9164 *
9165 * @wm: watermark calculation data
9166 *
9167 * Calculate the data return bandwidth used for display (CIK).
9168 * Used for display watermark bandwidth calculations
9169 * Returns the data return bandwidth in MBytes/s
9170 */
9171static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9172{
9173 /* Calculate the display Data return Bandwidth */
9174 fixed20_12 return_efficiency; /* 0.8 */
9175 fixed20_12 sclk, bandwidth;
9176 fixed20_12 a;
9177
9178 a.full = dfixed_const(1000);
9179 sclk.full = dfixed_const(wm->sclk);
9180 sclk.full = dfixed_div(sclk, a);
9181 a.full = dfixed_const(10);
9182 return_efficiency.full = dfixed_const(8);
9183 return_efficiency.full = dfixed_div(return_efficiency, a);
9184 a.full = dfixed_const(32);
9185 bandwidth.full = dfixed_mul(a, sclk);
9186 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9187
9188 return dfixed_trunc(bandwidth);
9189}
9190
9191/**
9192 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9193 *
9194 * @wm: watermark calculation data
9195 *
9196 * Calculate the dmif bandwidth used for display (CIK).
9197 * Used for display watermark bandwidth calculations
9198 * Returns the dmif bandwidth in MBytes/s
9199 */
9200static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9201{
9202 /* Calculate the DMIF Request Bandwidth */
9203 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9204 fixed20_12 disp_clk, bandwidth;
9205 fixed20_12 a, b;
9206
9207 a.full = dfixed_const(1000);
9208 disp_clk.full = dfixed_const(wm->disp_clk);
9209 disp_clk.full = dfixed_div(disp_clk, a);
9210 a.full = dfixed_const(32);
9211 b.full = dfixed_mul(a, disp_clk);
9212
9213 a.full = dfixed_const(10);
9214 disp_clk_request_efficiency.full = dfixed_const(8);
9215 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9216
9217 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9218
9219 return dfixed_trunc(bandwidth);
9220}
9221
9222/**
9223 * dce8_available_bandwidth - get the min available bandwidth
9224 *
9225 * @wm: watermark calculation data
9226 *
9227 * Calculate the min available bandwidth used for display (CIK).
9228 * Used for display watermark bandwidth calculations
9229 * Returns the min available bandwidth in MBytes/s
9230 */
9231static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9232{
9233 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9234 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9235 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9236 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9237
9238 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9239}
9240
9241/**
9242 * dce8_average_bandwidth - get the average available bandwidth
9243 *
9244 * @wm: watermark calculation data
9245 *
9246 * Calculate the average available bandwidth used for display (CIK).
9247 * Used for display watermark bandwidth calculations
9248 * Returns the average available bandwidth in MBytes/s
9249 */
9250static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9251{
9252 /* Calculate the display mode Average Bandwidth
9253 * DisplayMode should contain the source and destination dimensions,
9254 * timing, etc.
9255 */
9256 fixed20_12 bpp;
9257 fixed20_12 line_time;
9258 fixed20_12 src_width;
9259 fixed20_12 bandwidth;
9260 fixed20_12 a;
9261
9262 a.full = dfixed_const(1000);
9263 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9264 line_time.full = dfixed_div(line_time, a);
9265 bpp.full = dfixed_const(wm->bytes_per_pixel);
9266 src_width.full = dfixed_const(wm->src_width);
9267 bandwidth.full = dfixed_mul(src_width, bpp);
9268 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9269 bandwidth.full = dfixed_div(bandwidth, line_time);
9270
9271 return dfixed_trunc(bandwidth);
9272}
9273
9274/**
9275 * dce8_latency_watermark - get the latency watermark
9276 *
9277 * @wm: watermark calculation data
9278 *
9279 * Calculate the latency watermark (CIK).
9280 * Used for display watermark bandwidth calculations
9281 * Returns the latency watermark in ns
9282 */
9283static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9284{
9285 /* First calculate the latency in ns */
9286 u32 mc_latency = 2000; /* 2000 ns. */
9287 u32 available_bandwidth = dce8_available_bandwidth(wm);
9288 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9289 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9290 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9291 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9292 (wm->num_heads * cursor_line_pair_return_time);
9293 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9294 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9295 u32 tmp, dmif_size = 12288;
9296 fixed20_12 a, b, c;
9297
9298 if (wm->num_heads == 0)
9299 return 0;
9300
9301 a.full = dfixed_const(2);
9302 b.full = dfixed_const(1);
9303 if ((wm->vsc.full > a.full) ||
9304 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9305 (wm->vtaps >= 5) ||
9306 ((wm->vsc.full >= a.full) && wm->interlaced))
9307 max_src_lines_per_dst_line = 4;
9308 else
9309 max_src_lines_per_dst_line = 2;
9310
9311 a.full = dfixed_const(available_bandwidth);
9312 b.full = dfixed_const(wm->num_heads);
9313 a.full = dfixed_div(a, b);
9314
9315 b.full = dfixed_const(mc_latency + 512);
9316 c.full = dfixed_const(wm->disp_clk);
9317 b.full = dfixed_div(b, c);
9318
9319 c.full = dfixed_const(dmif_size);
9320 b.full = dfixed_div(c, b);
9321
9322 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9323
9324 b.full = dfixed_const(1000);
9325 c.full = dfixed_const(wm->disp_clk);
9326 b.full = dfixed_div(c, b);
9327 c.full = dfixed_const(wm->bytes_per_pixel);
9328 b.full = dfixed_mul(b, c);
9329
9330 lb_fill_bw = min(tmp, dfixed_trunc(b));
9331
9332 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9333 b.full = dfixed_const(1000);
9334 c.full = dfixed_const(lb_fill_bw);
9335 b.full = dfixed_div(c, b);
9336 a.full = dfixed_div(a, b);
9337 line_fill_time = dfixed_trunc(a);
9338
9339 if (line_fill_time < wm->active_time)
9340 return latency;
9341 else
9342 return latency + (line_fill_time - wm->active_time);
9343
9344}
9345
9346/**
9347 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9348 * average and available dram bandwidth
9349 *
9350 * @wm: watermark calculation data
9351 *
9352 * Check if the display average bandwidth fits in the display
9353 * dram bandwidth (CIK).
9354 * Used for display watermark bandwidth calculations
9355 * Returns true if the display fits, false if not.
9356 */
9357static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9358{
9359 if (dce8_average_bandwidth(wm) <=
9360 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9361 return true;
9362 else
9363 return false;
9364}
9365
9366/**
9367 * dce8_average_bandwidth_vs_available_bandwidth - check
9368 * average and available bandwidth
9369 *
9370 * @wm: watermark calculation data
9371 *
9372 * Check if the display average bandwidth fits in the display
9373 * available bandwidth (CIK).
9374 * Used for display watermark bandwidth calculations
9375 * Returns true if the display fits, false if not.
9376 */
9377static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9378{
9379 if (dce8_average_bandwidth(wm) <=
9380 (dce8_available_bandwidth(wm) / wm->num_heads))
9381 return true;
9382 else
9383 return false;
9384}
9385
9386/**
9387 * dce8_check_latency_hiding - check latency hiding
9388 *
9389 * @wm: watermark calculation data
9390 *
9391 * Check latency hiding (CIK).
9392 * Used for display watermark bandwidth calculations
9393 * Returns true if the display fits, false if not.
9394 */
9395static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9396{
9397 u32 lb_partitions = wm->lb_size / wm->src_width;
9398 u32 line_time = wm->active_time + wm->blank_time;
9399 u32 latency_tolerant_lines;
9400 u32 latency_hiding;
9401 fixed20_12 a;
9402
9403 a.full = dfixed_const(1);
9404 if (wm->vsc.full > a.full)
9405 latency_tolerant_lines = 1;
9406 else {
9407 if (lb_partitions <= (wm->vtaps + 1))
9408 latency_tolerant_lines = 1;
9409 else
9410 latency_tolerant_lines = 2;
9411 }
9412
9413 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9414
9415 if (dce8_latency_watermark(wm) <= latency_hiding)
9416 return true;
9417 else
9418 return false;
9419}
9420
9421/**
9422 * dce8_program_watermarks - program display watermarks
9423 *
9424 * @rdev: radeon_device pointer
9425 * @radeon_crtc: the selected display controller
9426 * @lb_size: line buffer size
9427 * @num_heads: number of display controllers in use
9428 *
9429 * Calculate and program the display watermarks for the
9430 * selected display controller (CIK).
9431 */
9432static void dce8_program_watermarks(struct radeon_device *rdev,
9433 struct radeon_crtc *radeon_crtc,
9434 u32 lb_size, u32 num_heads)
9435{
9436 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9437 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9438 u32 pixel_period;
9439 u32 line_time = 0;
9440 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9441 u32 tmp, wm_mask;
9442
9443 if (radeon_crtc->base.enabled && num_heads && mode) {
9444 pixel_period = 1000000 / (u32)mode->clock;
9445 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9446
58ea2dea
AD		 9447 /* watermark for high clocks */
9448 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9449 rdev->pm.dpm_enabled) {
9450 wm_high.yclk =
9451 radeon_dpm_get_mclk(rdev, false) * 10;
9452 wm_high.sclk =
9453 radeon_dpm_get_sclk(rdev, false) * 10;
9454 } else {
9455 wm_high.yclk = rdev->pm.current_mclk * 10;
9456 wm_high.sclk = rdev->pm.current_sclk * 10;
9457 }
9458
9459 wm_high.disp_clk = mode->clock;
9460 wm_high.src_width = mode->crtc_hdisplay;
9461 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9462 wm_high.blank_time = line_time - wm_high.active_time;
9463 wm_high.interlaced = false;
cd84a27d 9464 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9465 wm_high.interlaced = true;
9466 wm_high.vsc = radeon_crtc->vsc;
9467 wm_high.vtaps = 1;
cd84a27d 9468 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9469 wm_high.vtaps = 2;
9470 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9471 wm_high.lb_size = lb_size;
9472 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9473 wm_high.num_heads = num_heads;
cd84a27d
AD		 9474
9475 /* set for high clocks */
58ea2dea
AD		 9476 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9477
9478 /* possibly force display priority to high */
9479 /* should really do this at mode validation time... */
9480 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9481 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9482 !dce8_check_latency_hiding(&wm_high) ||
9483 (rdev->disp_priority == 2)) {
9484 DRM_DEBUG_KMS("force priority to high\n");
9485 }
9486
9487 /* watermark for low clocks */
9488 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9489 rdev->pm.dpm_enabled) {
9490 wm_low.yclk =
9491 radeon_dpm_get_mclk(rdev, true) * 10;
9492 wm_low.sclk =
9493 radeon_dpm_get_sclk(rdev, true) * 10;
9494 } else {
9495 wm_low.yclk = rdev->pm.current_mclk * 10;
9496 wm_low.sclk = rdev->pm.current_sclk * 10;
9497 }
9498
9499 wm_low.disp_clk = mode->clock;
9500 wm_low.src_width = mode->crtc_hdisplay;
9501 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9502 wm_low.blank_time = line_time - wm_low.active_time;
9503 wm_low.interlaced = false;
9504 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9505 wm_low.interlaced = true;
9506 wm_low.vsc = radeon_crtc->vsc;
9507 wm_low.vtaps = 1;
9508 if (radeon_crtc->rmx_type != RMX_OFF)
9509 wm_low.vtaps = 2;
9510 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9511 wm_low.lb_size = lb_size;
9512 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9513 wm_low.num_heads = num_heads;
9514
cd84a27d 9515 /* set for low clocks */
58ea2dea 9516 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9517
9518 /* possibly force display priority to high */
9519 /* should really do this at mode validation time... */
58ea2dea
AD		 9520 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9521 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9522 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9523 (rdev->disp_priority == 2)) {
9524 DRM_DEBUG_KMS("force priority to high\n");
9525 }
9526 }
9527
9528 /* select wm A */
9529 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9530 tmp = wm_mask;
9531 tmp &= ~LATENCY_WATERMARK_MASK(3);
9532 tmp |= LATENCY_WATERMARK_MASK(1);
9533 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9534 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9535 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9536 LATENCY_HIGH_WATERMARK(line_time)));
9537 /* select wm B */
9538 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9539 tmp &= ~LATENCY_WATERMARK_MASK(3);
9540 tmp |= LATENCY_WATERMARK_MASK(2);
9541 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9542 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9543 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9544 LATENCY_HIGH_WATERMARK(line_time)));
9545 /* restore original selection */
9546 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9547
9548 /* save values for DPM */
9549 radeon_crtc->line_time = line_time;
9550 radeon_crtc->wm_high = latency_watermark_a;
9551 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9552}
9553
9554/**
9555 * dce8_bandwidth_update - program display watermarks
9556 *
9557 * @rdev: radeon_device pointer
9558 *
9559 * Calculate and program the display watermarks and line
9560 * buffer allocation (CIK).
9561 */
9562void dce8_bandwidth_update(struct radeon_device *rdev)
9563{
9564 struct drm_display_mode *mode = NULL;
9565 u32 num_heads = 0, lb_size;
9566 int i;
9567
8efe82ca
AD		 9568 if (!rdev->mode_info.mode_config_initialized)
9569 return;
9570
cd84a27d
AD		 9571 radeon_update_display_priority(rdev);
9572
9573 for (i = 0; i < rdev->num_crtc; i++) {
9574 if (rdev->mode_info.crtcs[i]->base.enabled)
9575 num_heads++;
9576 }
9577 for (i = 0; i < rdev->num_crtc; i++) {
9578 mode = &rdev->mode_info.crtcs[i]->base.mode;
9579 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9580 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9581 }
9582}
44fa346f
AD		 9583
9584/**
9585 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9586 *
9587 * @rdev: radeon_device pointer
9588 *
9589 * Fetches a GPU clock counter snapshot (SI).
9590 * Returns the 64 bit clock counter snapshot.
9591 */
9592uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9593{
9594 uint64_t clock;
9595
9596 mutex_lock(&rdev->gpu_clock_mutex);
9597 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9598 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9599 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9600 mutex_unlock(&rdev->gpu_clock_mutex);
9601 return clock;
9602}
9603
87167bb1
CK		 9604static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9605 u32 cntl_reg, u32 status_reg)
9606{
9607 int r, i;
9608 struct atom_clock_dividers dividers;
9609 uint32_t tmp;
9610
9611 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9612 clock, false, &dividers);
9613 if (r)
9614 return r;
9615
9616 tmp = RREG32_SMC(cntl_reg);
9617 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9618 tmp |= dividers.post_divider;
9619 WREG32_SMC(cntl_reg, tmp);
9620
9621 for (i = 0; i < 100; i++) {
9622 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9623 break;
9624 mdelay(10);
9625 }
9626 if (i == 100)
9627 return -ETIMEDOUT;
9628
9629 return 0;
9630}
9631
9632int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9633{
9634 int r = 0;
9635
9636 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9637 if (r)
9638 return r;
9639
9640 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9641 return r;
9642}
9643
5ad6bf91
AD		 9644int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9645{
9646 int r, i;
9647 struct atom_clock_dividers dividers;
9648 u32 tmp;
9649
9650 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9651 ecclk, false, &dividers);
9652 if (r)
9653 return r;
9654
9655 for (i = 0; i < 100; i++) {
9656 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9657 break;
9658 mdelay(10);
9659 }
9660 if (i == 100)
9661 return -ETIMEDOUT;
9662
9663 tmp = RREG32_SMC(CG_ECLK_CNTL);
9664 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9665 tmp |= dividers.post_divider;
9666 WREG32_SMC(CG_ECLK_CNTL, tmp);
9667
9668 for (i = 0; i < 100; i++) {
9669 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9670 break;
9671 mdelay(10);
9672 }
9673 if (i == 100)
9674 return -ETIMEDOUT;
9675
9676 return 0;
9677}
9678
8a7cd276 9679static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9680{
8a7cd276
AD		 9681 struct pci_dev *root = rdev->pdev->bus->self;
9682 int bridge_pos, gpu_pos;
9683 u32 speed_cntl, mask, current_data_rate;
9684 int ret, i;
9685 u16 tmp16;
87167bb1 9686
0bd252de
AW		 9687 if (pci_is_root_bus(rdev->pdev->bus))
9688 return;
9689
8a7cd276
AD		 9690 if (radeon_pcie_gen2 == 0)
9691 return;
87167bb1 9692
8a7cd276
AD		 9693 if (rdev->flags & RADEON_IS_IGP)
9694 return;
87167bb1 9695
8a7cd276
AD		 9696 if (!(rdev->flags & RADEON_IS_PCIE))
9697 return;
87167bb1 9698
8a7cd276
AD		 9699 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9700 if (ret != 0)
9701 return;
87167bb1 9702
8a7cd276
AD		 9703 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9704 return;
87167bb1 9705
8a7cd276
AD		 9706 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9707 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9708 LC_CURRENT_DATA_RATE_SHIFT;
9709 if (mask & DRM_PCIE_SPEED_80) {
9710 if (current_data_rate == 2) {
9711 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9712 return;
9713 }
9714 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9715 } else if (mask & DRM_PCIE_SPEED_50) {
9716 if (current_data_rate == 1) {
9717 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9718 return;
9719 }
9720 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9721 }
87167bb1 9722
8a7cd276
AD		 9723 bridge_pos = pci_pcie_cap(root);
9724 if (!bridge_pos)
9725 return;
9726
9727 gpu_pos = pci_pcie_cap(rdev->pdev);
9728 if (!gpu_pos)
9729 return;
9730
9731 if (mask & DRM_PCIE_SPEED_80) {
9732 /* re-try equalization if gen3 is not already enabled */
9733 if (current_data_rate != 2) {
9734 u16 bridge_cfg, gpu_cfg;
9735 u16 bridge_cfg2, gpu_cfg2;
9736 u32 max_lw, current_lw, tmp;
9737
9738 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9739 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9740
9741 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9742 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9743
9744 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9745 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9746
9747 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9748 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9749 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9750
9751 if (current_lw < max_lw) {
9752 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9753 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9754 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9755 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9756 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9757 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9758 }
9759 }
9760
9761 for (i = 0; i < 10; i++) {
9762 /* check status */
9763 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9764 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9765 break;
9766
9767 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9768 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9769
9770 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9771 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9772
9773 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9774 tmp |= LC_SET_QUIESCE;
9775 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9776
9777 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9778 tmp |= LC_REDO_EQ;
9779 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9780
9781 mdelay(100);
9782
9783 /* linkctl */
9784 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9785 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9786 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9787 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9788
9789 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9790 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9791 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9792 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9793
9794 /* linkctl2 */
9795 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9796 tmp16 &= ~((1 << 4) | (7 << 9));
9797 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9798 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9799
9800 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9801 tmp16 &= ~((1 << 4) | (7 << 9));
9802 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9803 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9804
9805 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9806 tmp &= ~LC_SET_QUIESCE;
9807 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9808 }
9809 }
9810 }
9811
9812 /* set the link speed */
9813 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9814 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9815 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9816
9817 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9818 tmp16 &= ~0xf;
9819 if (mask & DRM_PCIE_SPEED_80)
9820 tmp16 |= 3; /* gen3 */
9821 else if (mask & DRM_PCIE_SPEED_50)
9822 tmp16 |= 2; /* gen2 */
9823 else
9824 tmp16 |= 1; /* gen1 */
9825 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9826
9827 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9828 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9829 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9830
9831 for (i = 0; i < rdev->usec_timeout; i++) {
9832 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9833 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9834 break;
9835 udelay(1);
9836 }
9837}
7235711a
AD		 9838
9839static void cik_program_aspm(struct radeon_device *rdev)
9840{
9841 u32 data, orig;
9842 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9843 bool disable_clkreq = false;
9844
9845 if (radeon_aspm == 0)
9846 return;
9847
9848 /* XXX double check IGPs */
9849 if (rdev->flags & RADEON_IS_IGP)
9850 return;
9851
9852 if (!(rdev->flags & RADEON_IS_PCIE))
9853 return;
9854
9855 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9856 data &= ~LC_XMIT_N_FTS_MASK;
9857 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9858 if (orig != data)
9859 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9860
9861 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9862 data |= LC_GO_TO_RECOVERY;
9863 if (orig != data)
9864 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9865
9866 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9867 data |= P_IGNORE_EDB_ERR;
9868 if (orig != data)
9869 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9870
9871 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9872 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9873 data |= LC_PMI_TO_L1_DIS;
9874 if (!disable_l0s)
9875 data |= LC_L0S_INACTIVITY(7);
9876
9877 if (!disable_l1) {
9878 data |= LC_L1_INACTIVITY(7);
9879 data &= ~LC_PMI_TO_L1_DIS;
9880 if (orig != data)
9881 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9882
9883 if (!disable_plloff_in_l1) {
9884 bool clk_req_support;
9885
9886 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9887 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9888 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9889 if (orig != data)
9890 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9891
9892 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9893 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9894 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9895 if (orig != data)
9896 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9897
9898 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9899 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9900 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9901 if (orig != data)
9902 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9903
9904 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9905 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9906 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9907 if (orig != data)
9908 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9909
9910 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9911 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9912 data |= LC_DYN_LANES_PWR_STATE(3);
9913 if (orig != data)
9914 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9915
0bd252de
AW		 9916 if (!disable_clkreq &&
9917 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9918 struct pci_dev *root = rdev->pdev->bus->self;
9919 u32 lnkcap;
9920
9921 clk_req_support = false;
9922 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9923 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9924 clk_req_support = true;
9925 } else {
9926 clk_req_support = false;
9927 }
9928
9929 if (clk_req_support) {
9930 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9931 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9932 if (orig != data)
9933 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9934
9935 orig = data = RREG32_SMC(THM_CLK_CNTL);
9936 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9937 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9938 if (orig != data)
9939 WREG32_SMC(THM_CLK_CNTL, data);
9940
9941 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9942 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9943 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9944 if (orig != data)
9945 WREG32_SMC(MISC_CLK_CTRL, data);
9946
9947 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9948 data &= ~BCLK_AS_XCLK;
9949 if (orig != data)
9950 WREG32_SMC(CG_CLKPIN_CNTL, data);
9951
9952 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9953 data &= ~FORCE_BIF_REFCLK_EN;
9954 if (orig != data)
9955 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9956
9957 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9958 data &= ~MPLL_CLKOUT_SEL_MASK;
9959 data |= MPLL_CLKOUT_SEL(4);
9960 if (orig != data)
9961 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9962 }
9963 }
9964 } else {
9965 if (orig != data)
9966 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9967 }
9968
9969 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9970 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9971 if (orig != data)
9972 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9973
9974 if (!disable_l0s) {
9975 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9976 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9977 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9978 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9979 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9980 data &= ~LC_L0S_INACTIVITY_MASK;
9981 if (orig != data)
9982 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9983 }
9984 }
9985 }
87167bb1 9986}
Linux kernel - Deliverables
This page took 0.672319 seconds and 5 git commands to generate.