projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/cik: use POLL_REG_MEM special op for sDMA HDP flush
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
d4775655
AD		 44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
02c81327
AD		 52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 64
a59781bb
AD		 65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 73extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
cc066715 78static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 80static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
6f2043ce 87
286d9cc6
AD		 88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
6f2043ce 124
6e2c3c0a
AD		 125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
0a5b7b0b 130 unsigned long flags;
6e2c3c0a
AD		 131 u32 r;
132
0a5b7b0b 133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
0a5b7b0b 137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
0a5b7b0b
AD		 143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
0a5b7b0b 150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 151}
152
22c775ce
AD		 153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
0aafd313
AD		 925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
8efff337
AD		 1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
0aafd313
AD		 1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
8efff337
AD		 1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
0aafd313
AD		 1532 default:
1533 break;
1534 }
1535}
1536
2c67912c
AD		 1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
75efdee1
AD		 1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
d5754ab8 1563 * @index: doorbell index
75efdee1
AD		 1564 *
1565 * Returns the value in the doorbell aperture at the
d5754ab8 1566 * requested doorbell index (CIK).
75efdee1 1567 */
d5754ab8 1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1569{
d5754ab8
AL		 1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
75efdee1 1572 } else {
d5754ab8 1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
d5754ab8 1582 * @index: doorbell index
75efdee1
AD		 1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
d5754ab8 1586 * requested doorbell index (CIK).
75efdee1 1587 */
d5754ab8 1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1589{
d5754ab8
AL		 1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
75efdee1 1592 } else {
d5754ab8 1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1594 }
1595}
1596
bc8273fe
AD		 1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
d4775655
AD		 1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
b556b12e
AD		 1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
bc8273fe
AD		 1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
6c7bccea 1700int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe
AD		 1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
bc8273fe
AD		 1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
d4775655
AD		 1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
bc8273fe
AD		 1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
02c81327
AD		 1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
02c81327
AD		 1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655
AD		 1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
02c81327
AD		 1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
02c81327
AD		 1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327 1802 break;
d4775655
AD		 1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
02c81327
AD		 1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
21a93e13 1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
cc8dbbb4 1904 /* No SMC, MC ucode on APUs */
02c81327
AD		 1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
cc8dbbb4
AD		 1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
d30d71e8 1925 err = 0;
cc8dbbb4
AD		 1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
02c81327
AD		 1932 }
1933
1934out:
02c81327
AD		 1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
02c81327
AD		 1952 }
1953 return err;
1954}
1955
8cc1a532
AD		 1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
21e438af 1994 num_pipe_configs = 16;
8cc1a532 1995
21e438af
AD		 1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
39aee490 2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
32f79a8a 2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2432 }
2433 } else if (num_pipe_configs == 4) {
2434 if (num_rbs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2437 case 0:
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2442 break;
2443 case 1:
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2448 break;
2449 case 2:
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2454 break;
2455 case 3:
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2460 break;
2461 case 4:
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2466 break;
2467 case 5:
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2470 break;
2471 case 6:
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2476 break;
2477 case 7:
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2482 break;
2483 case 8:
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2486 break;
2487 case 9:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2490 break;
2491 case 10:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2496 break;
2497 case 11:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2502 break;
2503 case 12:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2508 break;
2509 case 13:
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2512 break;
2513 case 14:
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518 break;
2519 case 16:
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2524 break;
2525 case 17:
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 break;
2531 case 27:
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2534 break;
2535 case 28:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 29:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 30:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 default:
2554 gb_tile_moden = 0;
2555 break;
2556 }
39aee490 2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2559 }
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2563 case 0:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2568 break;
2569 case 1:
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2574 break;
2575 case 2:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 break;
2581 case 3:
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2586 break;
2587 case 4:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2592 break;
2593 case 5:
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2596 break;
2597 case 6:
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2602 break;
2603 case 7:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2608 break;
2609 case 8:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2612 break;
2613 case 9:
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2616 break;
2617 case 10:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 11:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 12:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 13:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2638 break;
2639 case 14:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 16:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2650 break;
2651 case 17:
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2656 break;
2657 case 27:
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2660 break;
2661 case 28:
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2666 break;
2667 case 29:
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 break;
2673 case 30:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 default:
2680 gb_tile_moden = 0;
2681 break;
2682 }
39aee490 2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2685 }
2686 }
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2689 case 0:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2694 break;
2695 case 1:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 2:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 3:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 4:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 5:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 6:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 8:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 9:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 10:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 11:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2754 break;
2755 case 12:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 break;
2761 case 13:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2766 break;
2767 case 14:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2772 break;
2773 default:
2774 gb_tile_moden = 0;
2775 break;
2776 }
32f79a8a 2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779 }
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2783 case 0:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2788 break;
2789 case 1:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2794 break;
2795 case 2:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 3:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2806 break;
2807 case 4:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2812 break;
2813 case 5:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2816 break;
2817 case 6:
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2822 break;
2823 case 7:
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2828 break;
2829 case 8:
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2831 break;
2832 case 9:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2835 break;
2836 case 10:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 11:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 12:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 13:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2857 break;
2858 case 14:
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 break;
2864 case 16:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 17:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 27:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2879 break;
2880 case 28:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 29:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 30:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 break;
2898 default:
2899 gb_tile_moden = 0;
2900 break;
2901 }
39aee490 2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2904 }
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2907 case 0:
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2912 break;
2913 case 1:
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2918 break;
2919 case 2:
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2924 break;
2925 case 3:
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2930 break;
2931 case 4:
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2936 break;
2937 case 5:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 6:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2948 break;
2949 case 8:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 9:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 10:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 11:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 12:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2978 break;
2979 case 13:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 14:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2990 break;
2991 default:
2992 gb_tile_moden = 0;
2993 break;
2994 }
32f79a8a 2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2997 }
2998 } else
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3000}
3001
3002/**
3003 * cik_select_se_sh - select which SE, SH to address
3004 *
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3008 *
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3012 */
3013static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3015{
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3017
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3024 else
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3027}
3028
3029/**
3030 * cik_create_bitmask - create a bitmask
3031 *
3032 * @bit_width: length of the mask
3033 *
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3036 */
3037static u32 cik_create_bitmask(u32 bit_width)
3038{
3039 u32 i, mask = 0;
3040
3041 for (i = 0; i < bit_width; i++) {
3042 mask <<= 1;
3043 mask |= 1;
3044 }
3045 return mask;
3046}
3047
3048/**
3049 * cik_select_se_sh - select which SE, SH to address
3050 *
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3055 *
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3058 */
3059static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3060 u32 max_rb_num_per_se,
8cc1a532
AD		 3061 u32 sh_per_se)
3062{
3063 u32 data, mask;
3064
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3066 if (data & 1)
3067 data &= BACKEND_DISABLE_MASK;
3068 else
3069 data = 0;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3071
3072 data >>= BACKEND_DISABLE_SHIFT;
3073
9fadb352 3074 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3075
3076 return data & mask;
3077}
3078
3079/**
3080 * cik_setup_rb - setup the RBs on the asic
3081 *
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3086 *
3087 * Configures per-SE/SH RB registers (CIK).
3088 */
3089static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
9fadb352 3091 u32 max_rb_num_per_se)
8cc1a532
AD		 3092{
3093 int i, j;
3094 u32 data, mask;
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3097
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
9fadb352 3101 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3104 else
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3106 }
3107 }
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3109
3110 mask = 1;
9fadb352 3111 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3114 mask <<= 1;
3115 }
3116
439a1cff
MO		 3117 rdev->config.cik.backend_enable_mask = enabled_rbs;
3118
8cc1a532
AD		 3119 for (i = 0; i < se_num; i++) {
3120 cik_select_se_sh(rdev, i, 0xffffffff);
3121 data = 0;
3122 for (j = 0; j < sh_per_se; j++) {
3123 switch (enabled_rbs & 3) {
fc821b70
AD		 3124 case 0:
3125 if (j == 0)
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3127 else
3128 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3129 break;
8cc1a532
AD		 3130 case 1:
3131 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3132 break;
3133 case 2:
3134 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3135 break;
3136 case 3:
3137 default:
3138 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3139 break;
3140 }
3141 enabled_rbs >>= 2;
3142 }
3143 WREG32(PA_SC_RASTER_CONFIG, data);
3144 }
3145 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3146}
3147
3148/**
3149 * cik_gpu_init - setup the 3D engine
3150 *
3151 * @rdev: radeon_device pointer
3152 *
3153 * Configures the 3D engine and tiling configuration
3154 * registers so that the 3D engine is usable.
3155 */
3156static void cik_gpu_init(struct radeon_device *rdev)
3157{
3158 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3159 u32 mc_shared_chmap, mc_arb_ramcfg;
3160 u32 hdp_host_path_cntl;
3161 u32 tmp;
3162 int i, j;
3163
3164 switch (rdev->family) {
3165 case CHIP_BONAIRE:
3166 rdev->config.cik.max_shader_engines = 2;
3167 rdev->config.cik.max_tile_pipes = 4;
3168 rdev->config.cik.max_cu_per_sh = 7;
3169 rdev->config.cik.max_sh_per_se = 1;
3170 rdev->config.cik.max_backends_per_se = 2;
3171 rdev->config.cik.max_texture_channel_caches = 4;
3172 rdev->config.cik.max_gprs = 256;
3173 rdev->config.cik.max_gs_threads = 32;
3174 rdev->config.cik.max_hw_contexts = 8;
3175
3176 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3177 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3178 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3181 break;
b496038b
AD		 3182 case CHIP_HAWAII:
3183 rdev->config.cik.max_shader_engines = 4;
3184 rdev->config.cik.max_tile_pipes = 16;
3185 rdev->config.cik.max_cu_per_sh = 11;
3186 rdev->config.cik.max_sh_per_se = 1;
3187 rdev->config.cik.max_backends_per_se = 4;
3188 rdev->config.cik.max_texture_channel_caches = 16;
3189 rdev->config.cik.max_gprs = 256;
3190 rdev->config.cik.max_gs_threads = 32;
3191 rdev->config.cik.max_hw_contexts = 8;
3192
3193 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3194 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3195 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3198 break;
8cc1a532 3199 case CHIP_KAVERI:
b2e4c70a
AD		 3200 rdev->config.cik.max_shader_engines = 1;
3201 rdev->config.cik.max_tile_pipes = 4;
3202 if ((rdev->pdev->device == 0x1304) ||
3203 (rdev->pdev->device == 0x1305) ||
3204 (rdev->pdev->device == 0x130C) ||
3205 (rdev->pdev->device == 0x130F) ||
3206 (rdev->pdev->device == 0x1310) ||
3207 (rdev->pdev->device == 0x1311) ||
3208 (rdev->pdev->device == 0x131C)) {
3209 rdev->config.cik.max_cu_per_sh = 8;
3210 rdev->config.cik.max_backends_per_se = 2;
3211 } else if ((rdev->pdev->device == 0x1309) ||
3212 (rdev->pdev->device == 0x130A) ||
3213 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3214 (rdev->pdev->device == 0x1313) ||
3215 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3216 rdev->config.cik.max_cu_per_sh = 6;
3217 rdev->config.cik.max_backends_per_se = 2;
3218 } else if ((rdev->pdev->device == 0x1306) ||
3219 (rdev->pdev->device == 0x1307) ||
3220 (rdev->pdev->device == 0x130B) ||
3221 (rdev->pdev->device == 0x130E) ||
3222 (rdev->pdev->device == 0x1315) ||
3223 (rdev->pdev->device == 0x131B)) {
3224 rdev->config.cik.max_cu_per_sh = 4;
3225 rdev->config.cik.max_backends_per_se = 1;
3226 } else {
3227 rdev->config.cik.max_cu_per_sh = 3;
3228 rdev->config.cik.max_backends_per_se = 1;
3229 }
3230 rdev->config.cik.max_sh_per_se = 1;
3231 rdev->config.cik.max_texture_channel_caches = 4;
3232 rdev->config.cik.max_gprs = 256;
3233 rdev->config.cik.max_gs_threads = 16;
3234 rdev->config.cik.max_hw_contexts = 8;
3235
3236 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3237 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3238 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3240 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3241 break;
3242 case CHIP_KABINI:
3243 default:
3244 rdev->config.cik.max_shader_engines = 1;
3245 rdev->config.cik.max_tile_pipes = 2;
3246 rdev->config.cik.max_cu_per_sh = 2;
3247 rdev->config.cik.max_sh_per_se = 1;
3248 rdev->config.cik.max_backends_per_se = 1;
3249 rdev->config.cik.max_texture_channel_caches = 2;
3250 rdev->config.cik.max_gprs = 256;
3251 rdev->config.cik.max_gs_threads = 16;
3252 rdev->config.cik.max_hw_contexts = 8;
3253
3254 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3255 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3256 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3257 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3258 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3259 break;
3260 }
3261
3262 /* Initialize HDP */
3263 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3264 WREG32((0x2c14 + j), 0x00000000);
3265 WREG32((0x2c18 + j), 0x00000000);
3266 WREG32((0x2c1c + j), 0x00000000);
3267 WREG32((0x2c20 + j), 0x00000000);
3268 WREG32((0x2c24 + j), 0x00000000);
3269 }
3270
3271 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3272
3273 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3274
3275 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3276 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3277
3278 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3279 rdev->config.cik.mem_max_burst_length_bytes = 256;
3280 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3281 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3282 if (rdev->config.cik.mem_row_size_in_kb > 4)
3283 rdev->config.cik.mem_row_size_in_kb = 4;
3284 /* XXX use MC settings? */
3285 rdev->config.cik.shader_engine_tile_size = 32;
3286 rdev->config.cik.num_gpus = 1;
3287 rdev->config.cik.multi_gpu_tile_size = 64;
3288
3289 /* fix up row size */
3290 gb_addr_config &= ~ROW_SIZE_MASK;
3291 switch (rdev->config.cik.mem_row_size_in_kb) {
3292 case 1:
3293 default:
3294 gb_addr_config |= ROW_SIZE(0);
3295 break;
3296 case 2:
3297 gb_addr_config |= ROW_SIZE(1);
3298 break;
3299 case 4:
3300 gb_addr_config |= ROW_SIZE(2);
3301 break;
3302 }
3303
3304 /* setup tiling info dword. gb_addr_config is not adequate since it does
3305 * not have bank info, so create a custom tiling dword.
3306 * bits 3:0 num_pipes
3307 * bits 7:4 num_banks
3308 * bits 11:8 group_size
3309 * bits 15:12 row_size
3310 */
3311 rdev->config.cik.tile_config = 0;
3312 switch (rdev->config.cik.num_tile_pipes) {
3313 case 1:
3314 rdev->config.cik.tile_config |= (0 << 0);
3315 break;
3316 case 2:
3317 rdev->config.cik.tile_config |= (1 << 0);
3318 break;
3319 case 4:
3320 rdev->config.cik.tile_config |= (2 << 0);
3321 break;
3322 case 8:
3323 default:
3324 /* XXX what about 12? */
3325 rdev->config.cik.tile_config |= (3 << 0);
3326 break;
3327 }
a537314e
MD		 3328 rdev->config.cik.tile_config |=
3329 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3332 rdev->config.cik.tile_config |=
3333 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3334
3335 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3336 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3337 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3338 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3339 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3340 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3341 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3342 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3343
3344 cik_tiling_mode_table_init(rdev);
3345
3346 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3347 rdev->config.cik.max_sh_per_se,
3348 rdev->config.cik.max_backends_per_se);
3349
3350 /* set HW defaults for 3D engine */
3351 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3352
3353 WREG32(SX_DEBUG_1, 0x20);
3354
3355 WREG32(TA_CNTL_AUX, 0x00010000);
3356
3357 tmp = RREG32(SPI_CONFIG_CNTL);
3358 tmp |= 0x03000000;
3359 WREG32(SPI_CONFIG_CNTL, tmp);
3360
3361 WREG32(SQ_CONFIG, 1);
3362
3363 WREG32(DB_DEBUG, 0);
3364
3365 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3366 tmp |= 0x00000400;
3367 WREG32(DB_DEBUG2, tmp);
3368
3369 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3370 tmp |= 0x00020200;
3371 WREG32(DB_DEBUG3, tmp);
3372
3373 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3374 tmp |= 0x00018208;
3375 WREG32(CB_HW_CONTROL, tmp);
3376
3377 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3378
3379 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3380 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3381 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3382 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3383
3384 WREG32(VGT_NUM_INSTANCES, 1);
3385
3386 WREG32(CP_PERFMON_CNTL, 0);
3387
3388 WREG32(SQ_CONFIG, 0);
3389
3390 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3391 FORCE_EOV_MAX_REZ_CNT(255)));
3392
3393 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3394 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3395
3396 WREG32(VGT_GS_VERTEX_REUSE, 16);
3397 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3398
3399 tmp = RREG32(HDP_MISC_CNTL);
3400 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3401 WREG32(HDP_MISC_CNTL, tmp);
3402
3403 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3404 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3405
3406 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3407 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3408
3409 udelay(50);
3410}
3411
2cae3bc3
AD		 3412/*
3413 * GPU scratch registers helpers function.
3414 */
3415/**
3416 * cik_scratch_init - setup driver info for CP scratch regs
3417 *
3418 * @rdev: radeon_device pointer
3419 *
3420 * Set up the number and offset of the CP scratch registers.
3421 * NOTE: use of CP scratch registers is a legacy inferface and
3422 * is not used by default on newer asics (r6xx+). On newer asics,
3423 * memory buffers are used for fences rather than scratch regs.
3424 */
3425static void cik_scratch_init(struct radeon_device *rdev)
3426{
3427 int i;
3428
3429 rdev->scratch.num_reg = 7;
3430 rdev->scratch.reg_base = SCRATCH_REG0;
3431 for (i = 0; i < rdev->scratch.num_reg; i++) {
3432 rdev->scratch.free[i] = true;
3433 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3434 }
3435}
3436
fbc832c7
AD		 3437/**
3438 * cik_ring_test - basic gfx ring test
3439 *
3440 * @rdev: radeon_device pointer
3441 * @ring: radeon_ring structure holding ring information
3442 *
3443 * Allocate a scratch register and write to it using the gfx ring (CIK).
3444 * Provides a basic gfx ring test to verify that the ring is working.
3445 * Used by cik_cp_gfx_resume();
3446 * Returns 0 on success, error on failure.
3447 */
3448int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3449{
3450 uint32_t scratch;
3451 uint32_t tmp = 0;
3452 unsigned i;
3453 int r;
3454
3455 r = radeon_scratch_get(rdev, &scratch);
3456 if (r) {
3457 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3458 return r;
3459 }
3460 WREG32(scratch, 0xCAFEDEAD);
3461 r = radeon_ring_lock(rdev, ring, 3);
3462 if (r) {
3463 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3464 radeon_scratch_free(rdev, scratch);
3465 return r;
3466 }
3467 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3468 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3469 radeon_ring_write(ring, 0xDEADBEEF);
3470 radeon_ring_unlock_commit(rdev, ring);
963e81f9 3471
fbc832c7
AD		 3472 for (i = 0; i < rdev->usec_timeout; i++) {
3473 tmp = RREG32(scratch);
3474 if (tmp == 0xDEADBEEF)
3475 break;
3476 DRM_UDELAY(1);
3477 }
3478 if (i < rdev->usec_timeout) {
3479 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3480 } else {
3481 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3482 ring->idx, scratch, tmp);
3483 r = -EINVAL;
3484 }
3485 radeon_scratch_free(rdev, scratch);
3486 return r;
3487}
3488
780f5ddd
AD		 3489/**
3490 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3491 *
3492 * @rdev: radeon_device pointer
3493 * @ridx: radeon ring index
3494 *
3495 * Emits an hdp flush on the cp.
3496 */
3497static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3498 int ridx)
3499{
3500 struct radeon_ring *ring = &rdev->ring[ridx];
3501
3502 /* We should be using the new WAIT_REG_MEM special op packet here
3503 * but it causes the CP to hang
3504 */
3505 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3506 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3507 WRITE_DATA_DST_SEL(0)));
3508 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3509 radeon_ring_write(ring, 0);
3510 radeon_ring_write(ring, 0);
3511}
3512
2cae3bc3 3513/**
b07fdd38 3514 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3515 *
3516 * @rdev: radeon_device pointer
3517 * @fence: radeon fence object
3518 *
3519 * Emits a fence sequnce number on the gfx ring and flushes
3520 * GPU caches.
3521 */
b07fdd38
AD		 3522void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3523 struct radeon_fence *fence)
2cae3bc3
AD		 3524{
3525 struct radeon_ring *ring = &rdev->ring[fence->ring];
3526 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3527
3528 /* EVENT_WRITE_EOP - flush caches, send int */
3529 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3530 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3531 EOP_TC_ACTION_EN |
3532 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3533 EVENT_INDEX(5)));
3534 radeon_ring_write(ring, addr & 0xfffffffc);
3535 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3536 radeon_ring_write(ring, fence->seq);
3537 radeon_ring_write(ring, 0);
3538 /* HDP flush */
780f5ddd 3539 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
2cae3bc3
AD		 3540}
3541
b07fdd38
AD		 3542/**
3543 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3544 *
3545 * @rdev: radeon_device pointer
3546 * @fence: radeon fence object
3547 *
3548 * Emits a fence sequnce number on the compute ring and flushes
3549 * GPU caches.
3550 */
3551void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3552 struct radeon_fence *fence)
3553{
3554 struct radeon_ring *ring = &rdev->ring[fence->ring];
3555 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3556
3557 /* RELEASE_MEM - flush caches, send int */
3558 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3559 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3560 EOP_TC_ACTION_EN |
3561 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3562 EVENT_INDEX(5)));
3563 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3564 radeon_ring_write(ring, addr & 0xfffffffc);
3565 radeon_ring_write(ring, upper_32_bits(addr));
3566 radeon_ring_write(ring, fence->seq);
3567 radeon_ring_write(ring, 0);
3568 /* HDP flush */
780f5ddd 3569 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
b07fdd38
AD		 3570}
3571
1654b817 3572bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3573 struct radeon_ring *ring,
3574 struct radeon_semaphore *semaphore,
3575 bool emit_wait)
3576{
3577 uint64_t addr = semaphore->gpu_addr;
3578 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3579
3580 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3581 radeon_ring_write(ring, addr & 0xffffffff);
3582 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817
CK		 3583
3584 return true;
2cae3bc3
AD		 3585}
3586
c9dbd705
AD		 3587/**
3588 * cik_copy_cpdma - copy pages using the CP DMA engine
3589 *
3590 * @rdev: radeon_device pointer
3591 * @src_offset: src GPU address
3592 * @dst_offset: dst GPU address
3593 * @num_gpu_pages: number of GPU pages to xfer
3594 * @fence: radeon fence object
3595 *
3596 * Copy GPU paging using the CP DMA engine (CIK+).
3597 * Used by the radeon ttm implementation to move pages if
3598 * registered as the asic copy callback.
3599 */
3600int cik_copy_cpdma(struct radeon_device *rdev,
3601 uint64_t src_offset, uint64_t dst_offset,
3602 unsigned num_gpu_pages,
3603 struct radeon_fence **fence)
3604{
3605 struct radeon_semaphore *sem = NULL;
3606 int ring_index = rdev->asic->copy.blit_ring_index;
3607 struct radeon_ring *ring = &rdev->ring[ring_index];
3608 u32 size_in_bytes, cur_size_in_bytes, control;
3609 int i, num_loops;
3610 int r = 0;
3611
3612 r = radeon_semaphore_create(rdev, &sem);
3613 if (r) {
3614 DRM_ERROR("radeon: moving bo (%d).\n", r);
3615 return r;
3616 }
3617
3618 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3619 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3620 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3621 if (r) {
3622 DRM_ERROR("radeon: moving bo (%d).\n", r);
3623 radeon_semaphore_free(rdev, &sem, NULL);
3624 return r;
3625 }
3626
1654b817
CK		 3627 radeon_semaphore_sync_to(sem, *fence);
3628 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 3629
3630 for (i = 0; i < num_loops; i++) {
3631 cur_size_in_bytes = size_in_bytes;
3632 if (cur_size_in_bytes > 0x1fffff)
3633 cur_size_in_bytes = 0x1fffff;
3634 size_in_bytes -= cur_size_in_bytes;
3635 control = 0;
3636 if (size_in_bytes == 0)
3637 control |= PACKET3_DMA_DATA_CP_SYNC;
3638 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3639 radeon_ring_write(ring, control);
3640 radeon_ring_write(ring, lower_32_bits(src_offset));
3641 radeon_ring_write(ring, upper_32_bits(src_offset));
3642 radeon_ring_write(ring, lower_32_bits(dst_offset));
3643 radeon_ring_write(ring, upper_32_bits(dst_offset));
3644 radeon_ring_write(ring, cur_size_in_bytes);
3645 src_offset += cur_size_in_bytes;
3646 dst_offset += cur_size_in_bytes;
3647 }
3648
3649 r = radeon_fence_emit(rdev, fence, ring->idx);
3650 if (r) {
3651 radeon_ring_unlock_undo(rdev, ring);
3652 return r;
3653 }
3654
3655 radeon_ring_unlock_commit(rdev, ring);
3656 radeon_semaphore_free(rdev, &sem, *fence);
3657
3658 return r;
3659}
3660
2cae3bc3
AD		 3661/*
3662 * IB stuff
3663 */
3664/**
3665 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3666 *
3667 * @rdev: radeon_device pointer
3668 * @ib: radeon indirect buffer object
3669 *
3670 * Emits an DE (drawing engine) or CE (constant engine) IB
3671 * on the gfx ring. IBs are usually generated by userspace
3672 * acceleration drivers and submitted to the kernel for
3673 * sheduling on the ring. This function schedules the IB
3674 * on the gfx ring for execution by the GPU.
3675 */
3676void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3677{
3678 struct radeon_ring *ring = &rdev->ring[ib->ring];
3679 u32 header, control = INDIRECT_BUFFER_VALID;
3680
3681 if (ib->is_const_ib) {
3682 /* set switch buffer packet before const IB */
3683 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3684 radeon_ring_write(ring, 0);
3685
3686 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3687 } else {
3688 u32 next_rptr;
3689 if (ring->rptr_save_reg) {
3690 next_rptr = ring->wptr + 3 + 4;
3691 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3692 radeon_ring_write(ring, ((ring->rptr_save_reg -
3693 PACKET3_SET_UCONFIG_REG_START) >> 2));
3694 radeon_ring_write(ring, next_rptr);
3695 } else if (rdev->wb.enabled) {
3696 next_rptr = ring->wptr + 5 + 4;
3697 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3698 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3699 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3700 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3701 radeon_ring_write(ring, next_rptr);
3702 }
3703
3704 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3705 }
3706
3707 control |= ib->length_dw |
3708 (ib->vm ? (ib->vm->id << 24) : 0);
3709
3710 radeon_ring_write(ring, header);
3711 radeon_ring_write(ring,
3712#ifdef __BIG_ENDIAN
3713 (2 << 0) |
3714#endif
3715 (ib->gpu_addr & 0xFFFFFFFC));
3716 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3717 radeon_ring_write(ring, control);
3718}
3719
fbc832c7
AD		 3720/**
3721 * cik_ib_test - basic gfx ring IB test
3722 *
3723 * @rdev: radeon_device pointer
3724 * @ring: radeon_ring structure holding ring information
3725 *
3726 * Allocate an IB and execute it on the gfx ring (CIK).
3727 * Provides a basic gfx ring test to verify that IBs are working.
3728 * Returns 0 on success, error on failure.
3729 */
3730int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3731{
3732 struct radeon_ib ib;
3733 uint32_t scratch;
3734 uint32_t tmp = 0;
3735 unsigned i;
3736 int r;
3737
3738 r = radeon_scratch_get(rdev, &scratch);
3739 if (r) {
3740 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3741 return r;
3742 }
3743 WREG32(scratch, 0xCAFEDEAD);
3744 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3745 if (r) {
3746 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3747 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3748 return r;
3749 }
3750 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3751 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3752 ib.ptr[2] = 0xDEADBEEF;
3753 ib.length_dw = 3;
3754 r = radeon_ib_schedule(rdev, &ib, NULL);
3755 if (r) {
3756 radeon_scratch_free(rdev, scratch);
3757 radeon_ib_free(rdev, &ib);
3758 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3759 return r;
3760 }
3761 r = radeon_fence_wait(ib.fence, false);
3762 if (r) {
3763 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3764 radeon_scratch_free(rdev, scratch);
3765 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 3766 return r;
3767 }
3768 for (i = 0; i < rdev->usec_timeout; i++) {
3769 tmp = RREG32(scratch);
3770 if (tmp == 0xDEADBEEF)
3771 break;
3772 DRM_UDELAY(1);
3773 }
3774 if (i < rdev->usec_timeout) {
3775 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3776 } else {
3777 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3778 scratch, tmp);
3779 r = -EINVAL;
3780 }
3781 radeon_scratch_free(rdev, scratch);
3782 radeon_ib_free(rdev, &ib);
3783 return r;
3784}
3785
841cf442
AD		 3786/*
3787 * CP.
3788 * On CIK, gfx and compute now have independant command processors.
3789 *
3790 * GFX
3791 * Gfx consists of a single ring and can process both gfx jobs and
3792 * compute jobs. The gfx CP consists of three microengines (ME):
3793 * PFP - Pre-Fetch Parser
3794 * ME - Micro Engine
3795 * CE - Constant Engine
3796 * The PFP and ME make up what is considered the Drawing Engine (DE).
3797 * The CE is an asynchronous engine used for updating buffer desciptors
3798 * used by the DE so that they can be loaded into cache in parallel
3799 * while the DE is processing state update packets.
3800 *
3801 * Compute
3802 * The compute CP consists of two microengines (ME):
3803 * MEC1 - Compute MicroEngine 1
3804 * MEC2 - Compute MicroEngine 2
3805 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3806 * The queues are exposed to userspace and are programmed directly
3807 * by the compute runtime.
3808 */
3809/**
3810 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3811 *
3812 * @rdev: radeon_device pointer
3813 * @enable: enable or disable the MEs
3814 *
3815 * Halts or unhalts the gfx MEs.
3816 */
3817static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3818{
3819 if (enable)
3820 WREG32(CP_ME_CNTL, 0);
3821 else {
3822 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3823 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3824 }
3825 udelay(50);
3826}
3827
3828/**
3829 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3830 *
3831 * @rdev: radeon_device pointer
3832 *
3833 * Loads the gfx PFP, ME, and CE ucode.
3834 * Returns 0 for success, -EINVAL if the ucode is not available.
3835 */
3836static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3837{
3838 const __be32 *fw_data;
3839 int i;
3840
3841 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3842 return -EINVAL;
3843
3844 cik_cp_gfx_enable(rdev, false);
3845
3846 /* PFP */
3847 fw_data = (const __be32 *)rdev->pfp_fw->data;
3848 WREG32(CP_PFP_UCODE_ADDR, 0);
3849 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3850 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3851 WREG32(CP_PFP_UCODE_ADDR, 0);
3852
3853 /* CE */
3854 fw_data = (const __be32 *)rdev->ce_fw->data;
3855 WREG32(CP_CE_UCODE_ADDR, 0);
3856 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3857 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3858 WREG32(CP_CE_UCODE_ADDR, 0);
3859
3860 /* ME */
3861 fw_data = (const __be32 *)rdev->me_fw->data;
3862 WREG32(CP_ME_RAM_WADDR, 0);
3863 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3864 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3865 WREG32(CP_ME_RAM_WADDR, 0);
3866
3867 WREG32(CP_PFP_UCODE_ADDR, 0);
3868 WREG32(CP_CE_UCODE_ADDR, 0);
3869 WREG32(CP_ME_RAM_WADDR, 0);
3870 WREG32(CP_ME_RAM_RADDR, 0);
3871 return 0;
3872}
3873
3874/**
3875 * cik_cp_gfx_start - start the gfx ring
3876 *
3877 * @rdev: radeon_device pointer
3878 *
3879 * Enables the ring and loads the clear state context and other
3880 * packets required to init the ring.
3881 * Returns 0 for success, error for failure.
3882 */
3883static int cik_cp_gfx_start(struct radeon_device *rdev)
3884{
3885 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3886 int r, i;
3887
3888 /* init the CP */
3889 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3890 WREG32(CP_ENDIAN_SWAP, 0);
3891 WREG32(CP_DEVICE_ID, 1);
3892
3893 cik_cp_gfx_enable(rdev, true);
3894
3895 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3896 if (r) {
3897 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3898 return r;
3899 }
3900
3901 /* init the CE partitions. CE only used for gfx on CIK */
3902 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3903 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3904 radeon_ring_write(ring, 0xc000);
3905 radeon_ring_write(ring, 0xc000);
3906
3907 /* setup clear context state */
3908 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3909 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3910
3911 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3912 radeon_ring_write(ring, 0x80000000);
3913 radeon_ring_write(ring, 0x80000000);
3914
3915 for (i = 0; i < cik_default_size; i++)
3916 radeon_ring_write(ring, cik_default_state[i]);
3917
3918 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3919 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3920
3921 /* set clear context state */
3922 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3923 radeon_ring_write(ring, 0);
3924
3925 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3926 radeon_ring_write(ring, 0x00000316);
3927 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3928 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3929
3930 radeon_ring_unlock_commit(rdev, ring);
3931
3932 return 0;
3933}
3934
3935/**
3936 * cik_cp_gfx_fini - stop the gfx ring
3937 *
3938 * @rdev: radeon_device pointer
3939 *
3940 * Stop the gfx ring and tear down the driver ring
3941 * info.
3942 */
3943static void cik_cp_gfx_fini(struct radeon_device *rdev)
3944{
3945 cik_cp_gfx_enable(rdev, false);
3946 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3947}
3948
3949/**
3950 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3951 *
3952 * @rdev: radeon_device pointer
3953 *
3954 * Program the location and size of the gfx ring buffer
3955 * and test it to make sure it's working.
3956 * Returns 0 for success, error for failure.
3957 */
3958static int cik_cp_gfx_resume(struct radeon_device *rdev)
3959{
3960 struct radeon_ring *ring;
3961 u32 tmp;
3962 u32 rb_bufsz;
3963 u64 rb_addr;
3964 int r;
3965
3966 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 3967 if (rdev->family != CHIP_HAWAII)
3968 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 3969
3970 /* Set the write pointer delay */
3971 WREG32(CP_RB_WPTR_DELAY, 0);
3972
3973 /* set the RB to use vmid 0 */
3974 WREG32(CP_RB_VMID, 0);
3975
3976 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3977
3978 /* ring 0 - compute and gfx */
3979 /* Set ring buffer size */
3980 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3981 rb_bufsz = order_base_2(ring->ring_size / 8);
3982 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 3983#ifdef __BIG_ENDIAN
3984 tmp |= BUF_SWAP_32BIT;
3985#endif
3986 WREG32(CP_RB0_CNTL, tmp);
3987
3988 /* Initialize the ring buffer's read and write pointers */
3989 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3990 ring->wptr = 0;
3991 WREG32(CP_RB0_WPTR, ring->wptr);
3992
3993 /* set the wb address wether it's enabled or not */
3994 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3995 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3996
3997 /* scratch register shadowing is no longer supported */
3998 WREG32(SCRATCH_UMSK, 0);
3999
4000 if (!rdev->wb.enabled)
4001 tmp |= RB_NO_UPDATE;
4002
4003 mdelay(1);
4004 WREG32(CP_RB0_CNTL, tmp);
4005
4006 rb_addr = ring->gpu_addr >> 8;
4007 WREG32(CP_RB0_BASE, rb_addr);
4008 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4009
4010 ring->rptr = RREG32(CP_RB0_RPTR);
4011
4012 /* start the ring */
4013 cik_cp_gfx_start(rdev);
4014 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4015 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4016 if (r) {
4017 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4018 return r;
4019 }
4020 return 0;
4021}
4022
ea31bf69
AD		 4023u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4024 struct radeon_ring *ring)
963e81f9
AD		 4025{
4026 u32 rptr;
4027
ea31bf69
AD		 4028 if (rdev->wb.enabled)
4029 rptr = rdev->wb.wb[ring->rptr_offs/4];
4030 else
4031 rptr = RREG32(CP_RB0_RPTR);
4032
4033 return rptr;
4034}
4035
4036u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4037 struct radeon_ring *ring)
4038{
4039 u32 wptr;
4040
4041 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4042
ea31bf69
AD		 4043 return wptr;
4044}
4045
4046void cik_gfx_set_wptr(struct radeon_device *rdev,
4047 struct radeon_ring *ring)
4048{
4049 WREG32(CP_RB0_WPTR, ring->wptr);
4050 (void)RREG32(CP_RB0_WPTR);
4051}
4052
4053u32 cik_compute_get_rptr(struct radeon_device *rdev,
4054 struct radeon_ring *ring)
4055{
4056 u32 rptr;
963e81f9
AD		 4057
4058 if (rdev->wb.enabled) {
ea31bf69 4059 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4060 } else {
f61d5b46 4061 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4062 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4063 rptr = RREG32(CP_HQD_PQ_RPTR);
4064 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4065 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4066 }
963e81f9
AD		 4067
4068 return rptr;
4069}
4070
ea31bf69
AD		 4071u32 cik_compute_get_wptr(struct radeon_device *rdev,
4072 struct radeon_ring *ring)
963e81f9
AD		 4073{
4074 u32 wptr;
4075
4076 if (rdev->wb.enabled) {
ea31bf69
AD		 4077 /* XXX check if swapping is necessary on BE */
4078 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4079 } else {
f61d5b46 4080 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4081 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4082 wptr = RREG32(CP_HQD_PQ_WPTR);
4083 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4084 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4085 }
963e81f9
AD		 4086
4087 return wptr;
4088}
4089
ea31bf69
AD		 4090void cik_compute_set_wptr(struct radeon_device *rdev,
4091 struct radeon_ring *ring)
963e81f9 4092{
ea31bf69
AD		 4093 /* XXX check if swapping is necessary on BE */
4094 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4095 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4096}
4097
841cf442
AD		 4098/**
4099 * cik_cp_compute_enable - enable/disable the compute CP MEs
4100 *
4101 * @rdev: radeon_device pointer
4102 * @enable: enable or disable the MEs
4103 *
4104 * Halts or unhalts the compute MEs.
4105 */
4106static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4107{
4108 if (enable)
4109 WREG32(CP_MEC_CNTL, 0);
4110 else
4111 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4112 udelay(50);
4113}
4114
4115/**
4116 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4117 *
4118 * @rdev: radeon_device pointer
4119 *
4120 * Loads the compute MEC1&2 ucode.
4121 * Returns 0 for success, -EINVAL if the ucode is not available.
4122 */
4123static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4124{
4125 const __be32 *fw_data;
4126 int i;
4127
4128 if (!rdev->mec_fw)
4129 return -EINVAL;
4130
4131 cik_cp_compute_enable(rdev, false);
4132
4133 /* MEC1 */
4134 fw_data = (const __be32 *)rdev->mec_fw->data;
4135 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4136 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4137 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4138 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4139
4140 if (rdev->family == CHIP_KAVERI) {
4141 /* MEC2 */
4142 fw_data = (const __be32 *)rdev->mec_fw->data;
4143 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4144 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4145 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4146 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4147 }
4148
4149 return 0;
4150}
4151
4152/**
4153 * cik_cp_compute_start - start the compute queues
4154 *
4155 * @rdev: radeon_device pointer
4156 *
4157 * Enable the compute queues.
4158 * Returns 0 for success, error for failure.
4159 */
4160static int cik_cp_compute_start(struct radeon_device *rdev)
4161{
963e81f9
AD		 4162 cik_cp_compute_enable(rdev, true);
4163
841cf442
AD		 4164 return 0;
4165}
4166
4167/**
4168 * cik_cp_compute_fini - stop the compute queues
4169 *
4170 * @rdev: radeon_device pointer
4171 *
4172 * Stop the compute queues and tear down the driver queue
4173 * info.
4174 */
4175static void cik_cp_compute_fini(struct radeon_device *rdev)
4176{
963e81f9
AD		 4177 int i, idx, r;
4178
841cf442 4179 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4180
4181 for (i = 0; i < 2; i++) {
4182 if (i == 0)
4183 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4184 else
4185 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4186
4187 if (rdev->ring[idx].mqd_obj) {
4188 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4189 if (unlikely(r != 0))
4190 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4191
4192 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4193 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4194
4195 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4196 rdev->ring[idx].mqd_obj = NULL;
4197 }
4198 }
841cf442
AD		 4199}
4200
963e81f9
AD		 4201static void cik_mec_fini(struct radeon_device *rdev)
4202{
4203 int r;
4204
4205 if (rdev->mec.hpd_eop_obj) {
4206 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4207 if (unlikely(r != 0))
4208 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4209 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4210 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4211
4212 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4213 rdev->mec.hpd_eop_obj = NULL;
4214 }
4215}
4216
4217#define MEC_HPD_SIZE 2048
4218
4219static int cik_mec_init(struct radeon_device *rdev)
4220{
4221 int r;
4222 u32 *hpd;
4223
4224 /*
4225 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4226 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4227 */
4228 if (rdev->family == CHIP_KAVERI)
4229 rdev->mec.num_mec = 2;
4230 else
4231 rdev->mec.num_mec = 1;
4232 rdev->mec.num_pipe = 4;
4233 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4234
4235 if (rdev->mec.hpd_eop_obj == NULL) {
4236 r = radeon_bo_create(rdev,
4237 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4238 PAGE_SIZE, true,
4239 RADEON_GEM_DOMAIN_GTT, NULL,
4240 &rdev->mec.hpd_eop_obj);
4241 if (r) {
4242 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4243 return r;
4244 }
4245 }
4246
4247 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4248 if (unlikely(r != 0)) {
4249 cik_mec_fini(rdev);
4250 return r;
4251 }
4252 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4253 &rdev->mec.hpd_eop_gpu_addr);
4254 if (r) {
4255 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4256 cik_mec_fini(rdev);
4257 return r;
4258 }
4259 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4260 if (r) {
4261 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4262 cik_mec_fini(rdev);
4263 return r;
4264 }
4265
4266 /* clear memory. Not sure if this is required or not */
4267 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4268
4269 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4270 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4271
4272 return 0;
4273}
4274
4275struct hqd_registers
4276{
4277 u32 cp_mqd_base_addr;
4278 u32 cp_mqd_base_addr_hi;
4279 u32 cp_hqd_active;
4280 u32 cp_hqd_vmid;
4281 u32 cp_hqd_persistent_state;
4282 u32 cp_hqd_pipe_priority;
4283 u32 cp_hqd_queue_priority;
4284 u32 cp_hqd_quantum;
4285 u32 cp_hqd_pq_base;
4286 u32 cp_hqd_pq_base_hi;
4287 u32 cp_hqd_pq_rptr;
4288 u32 cp_hqd_pq_rptr_report_addr;
4289 u32 cp_hqd_pq_rptr_report_addr_hi;
4290 u32 cp_hqd_pq_wptr_poll_addr;
4291 u32 cp_hqd_pq_wptr_poll_addr_hi;
4292 u32 cp_hqd_pq_doorbell_control;
4293 u32 cp_hqd_pq_wptr;
4294 u32 cp_hqd_pq_control;
4295 u32 cp_hqd_ib_base_addr;
4296 u32 cp_hqd_ib_base_addr_hi;
4297 u32 cp_hqd_ib_rptr;
4298 u32 cp_hqd_ib_control;
4299 u32 cp_hqd_iq_timer;
4300 u32 cp_hqd_iq_rptr;
4301 u32 cp_hqd_dequeue_request;
4302 u32 cp_hqd_dma_offload;
4303 u32 cp_hqd_sema_cmd;
4304 u32 cp_hqd_msg_type;
4305 u32 cp_hqd_atomic0_preop_lo;
4306 u32 cp_hqd_atomic0_preop_hi;
4307 u32 cp_hqd_atomic1_preop_lo;
4308 u32 cp_hqd_atomic1_preop_hi;
4309 u32 cp_hqd_hq_scheduler0;
4310 u32 cp_hqd_hq_scheduler1;
4311 u32 cp_mqd_control;
4312};
4313
4314struct bonaire_mqd
4315{
4316 u32 header;
4317 u32 dispatch_initiator;
4318 u32 dimensions[3];
4319 u32 start_idx[3];
4320 u32 num_threads[3];
4321 u32 pipeline_stat_enable;
4322 u32 perf_counter_enable;
4323 u32 pgm[2];
4324 u32 tba[2];
4325 u32 tma[2];
4326 u32 pgm_rsrc[2];
4327 u32 vmid;
4328 u32 resource_limits;
4329 u32 static_thread_mgmt01[2];
4330 u32 tmp_ring_size;
4331 u32 static_thread_mgmt23[2];
4332 u32 restart[3];
4333 u32 thread_trace_enable;
4334 u32 reserved1;
4335 u32 user_data[16];
4336 u32 vgtcs_invoke_count[2];
4337 struct hqd_registers queue_state;
4338 u32 dequeue_cntr;
4339 u32 interrupt_queue[64];
4340};
4341
841cf442
AD		 4342/**
4343 * cik_cp_compute_resume - setup the compute queue registers
4344 *
4345 * @rdev: radeon_device pointer
4346 *
4347 * Program the compute queues and test them to make sure they
4348 * are working.
4349 * Returns 0 for success, error for failure.
4350 */
4351static int cik_cp_compute_resume(struct radeon_device *rdev)
4352{
963e81f9
AD		 4353 int r, i, idx;
4354 u32 tmp;
4355 bool use_doorbell = true;
4356 u64 hqd_gpu_addr;
4357 u64 mqd_gpu_addr;
4358 u64 eop_gpu_addr;
4359 u64 wb_gpu_addr;
4360 u32 *buf;
4361 struct bonaire_mqd *mqd;
841cf442 4362
841cf442
AD		 4363 r = cik_cp_compute_start(rdev);
4364 if (r)
4365 return r;
963e81f9
AD		 4366
4367 /* fix up chicken bits */
4368 tmp = RREG32(CP_CPF_DEBUG);
4369 tmp |= (1 << 23);
4370 WREG32(CP_CPF_DEBUG, tmp);
4371
4372 /* init the pipes */
f61d5b46 4373 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4374 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4375 int me = (i < 4) ? 1 : 2;
4376 int pipe = (i < 4) ? i : (i - 4);
4377
4378 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4379
4380 cik_srbm_select(rdev, me, pipe, 0, 0);
4381
4382 /* write the EOP addr */
4383 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4384 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4385
4386 /* set the VMID assigned */
4387 WREG32(CP_HPD_EOP_VMID, 0);
4388
4389 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4390 tmp = RREG32(CP_HPD_EOP_CONTROL);
4391 tmp &= ~EOP_SIZE_MASK;
b72a8925 4392 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 4393 WREG32(CP_HPD_EOP_CONTROL, tmp);
4394 }
4395 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4396 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4397
4398 /* init the queues. Just two for now. */
4399 for (i = 0; i < 2; i++) {
4400 if (i == 0)
4401 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4402 else
4403 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4404
4405 if (rdev->ring[idx].mqd_obj == NULL) {
4406 r = radeon_bo_create(rdev,
4407 sizeof(struct bonaire_mqd),
4408 PAGE_SIZE, true,
4409 RADEON_GEM_DOMAIN_GTT, NULL,
4410 &rdev->ring[idx].mqd_obj);
4411 if (r) {
4412 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4413 return r;
4414 }
4415 }
4416
4417 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4418 if (unlikely(r != 0)) {
4419 cik_cp_compute_fini(rdev);
4420 return r;
4421 }
4422 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4423 &mqd_gpu_addr);
4424 if (r) {
4425 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4426 cik_cp_compute_fini(rdev);
4427 return r;
4428 }
4429 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4430 if (r) {
4431 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4432 cik_cp_compute_fini(rdev);
4433 return r;
4434 }
4435
963e81f9
AD		 4436 /* init the mqd struct */
4437 memset(buf, 0, sizeof(struct bonaire_mqd));
4438
4439 mqd = (struct bonaire_mqd *)buf;
4440 mqd->header = 0xC0310800;
4441 mqd->static_thread_mgmt01[0] = 0xffffffff;
4442 mqd->static_thread_mgmt01[1] = 0xffffffff;
4443 mqd->static_thread_mgmt23[0] = 0xffffffff;
4444 mqd->static_thread_mgmt23[1] = 0xffffffff;
4445
f61d5b46 4446 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4447 cik_srbm_select(rdev, rdev->ring[idx].me,
4448 rdev->ring[idx].pipe,
4449 rdev->ring[idx].queue, 0);
4450
4451 /* disable wptr polling */
4452 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4453 tmp &= ~WPTR_POLL_EN;
4454 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4455
4456 /* enable doorbell? */
4457 mqd->queue_state.cp_hqd_pq_doorbell_control =
4458 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4459 if (use_doorbell)
4460 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4461 else
4462 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4463 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4464 mqd->queue_state.cp_hqd_pq_doorbell_control);
4465
4466 /* disable the queue if it's active */
4467 mqd->queue_state.cp_hqd_dequeue_request = 0;
4468 mqd->queue_state.cp_hqd_pq_rptr = 0;
4469 mqd->queue_state.cp_hqd_pq_wptr= 0;
4470 if (RREG32(CP_HQD_ACTIVE) & 1) {
4471 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4472 for (i = 0; i < rdev->usec_timeout; i++) {
4473 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4474 break;
4475 udelay(1);
4476 }
4477 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4478 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4479 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4480 }
4481
4482 /* set the pointer to the MQD */
4483 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4484 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4485 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4486 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4487 /* set MQD vmid to 0 */
4488 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4489 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4490 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4491
4492 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4493 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4494 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4495 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4496 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4497 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4498
4499 /* set up the HQD, this is similar to CP_RB0_CNTL */
4500 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4501 mqd->queue_state.cp_hqd_pq_control &=
4502 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4503
4504 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4505 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4506 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4507 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4508#ifdef __BIG_ENDIAN
4509 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4510#endif
4511 mqd->queue_state.cp_hqd_pq_control &=
4512 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4513 mqd->queue_state.cp_hqd_pq_control |=
4514 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4515 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4516
4517 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4518 if (i == 0)
4519 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4520 else
4521 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4522 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4523 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4524 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4525 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4526 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4527
4528 /* set the wb address wether it's enabled or not */
4529 if (i == 0)
4530 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4531 else
4532 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4533 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4534 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4535 upper_32_bits(wb_gpu_addr) & 0xffff;
4536 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4537 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4538 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4539 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4540
4541 /* enable the doorbell if requested */
4542 if (use_doorbell) {
4543 mqd->queue_state.cp_hqd_pq_doorbell_control =
4544 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4545 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4546 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4547 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4548 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4549 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4550 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4551
4552 } else {
4553 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4554 }
4555 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4556 mqd->queue_state.cp_hqd_pq_doorbell_control);
4557
4558 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4559 rdev->ring[idx].wptr = 0;
4560 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4561 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4562 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4563 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4564
4565 /* set the vmid for the queue */
4566 mqd->queue_state.cp_hqd_vmid = 0;
4567 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4568
4569 /* activate the queue */
4570 mqd->queue_state.cp_hqd_active = 1;
4571 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4572
4573 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4574 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4575
4576 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4577 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4578
4579 rdev->ring[idx].ready = true;
4580 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4581 if (r)
4582 rdev->ring[idx].ready = false;
4583 }
4584
841cf442
AD		 4585 return 0;
4586}
4587
841cf442
AD		 4588static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4589{
4590 cik_cp_gfx_enable(rdev, enable);
4591 cik_cp_compute_enable(rdev, enable);
4592}
4593
841cf442
AD		 4594static int cik_cp_load_microcode(struct radeon_device *rdev)
4595{
4596 int r;
4597
4598 r = cik_cp_gfx_load_microcode(rdev);
4599 if (r)
4600 return r;
4601 r = cik_cp_compute_load_microcode(rdev);
4602 if (r)
4603 return r;
4604
4605 return 0;
4606}
4607
841cf442
AD		 4608static void cik_cp_fini(struct radeon_device *rdev)
4609{
4610 cik_cp_gfx_fini(rdev);
4611 cik_cp_compute_fini(rdev);
4612}
4613
841cf442
AD		 4614static int cik_cp_resume(struct radeon_device *rdev)
4615{
4616 int r;
4617
4214faf6
AD		 4618 cik_enable_gui_idle_interrupt(rdev, false);
4619
841cf442
AD		 4620 r = cik_cp_load_microcode(rdev);
4621 if (r)
4622 return r;
4623
4624 r = cik_cp_gfx_resume(rdev);
4625 if (r)
4626 return r;
4627 r = cik_cp_compute_resume(rdev);
4628 if (r)
4629 return r;
4630
4214faf6
AD		 4631 cik_enable_gui_idle_interrupt(rdev, true);
4632
841cf442
AD		 4633 return 0;
4634}
4635
cc066715 4636static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4637{
6f2043ce
AD		 4638 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4639 RREG32(GRBM_STATUS));
4640 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4641 RREG32(GRBM_STATUS2));
4642 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4643 RREG32(GRBM_STATUS_SE0));
4644 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4645 RREG32(GRBM_STATUS_SE1));
4646 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4647 RREG32(GRBM_STATUS_SE2));
4648 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4649 RREG32(GRBM_STATUS_SE3));
4650 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4651 RREG32(SRBM_STATUS));
4652 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4653 RREG32(SRBM_STATUS2));
cc066715
AD		 4654 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4655 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4656 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4657 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4658 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4659 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4660 RREG32(CP_STALLED_STAT1));
4661 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4662 RREG32(CP_STALLED_STAT2));
4663 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4664 RREG32(CP_STALLED_STAT3));
4665 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4666 RREG32(CP_CPF_BUSY_STAT));
4667 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4668 RREG32(CP_CPF_STALLED_STAT1));
4669 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4670 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4671 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4672 RREG32(CP_CPC_STALLED_STAT1));
4673 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4674}
6f2043ce 4675
21a93e13 4676/**
cc066715 4677 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4678 *
4679 * @rdev: radeon_device pointer
21a93e13 4680 *
cc066715
AD		 4681 * Check which blocks are busy and return the relevant reset
4682 * mask to be used by cik_gpu_soft_reset().
4683 * Returns a mask of the blocks to be reset.
21a93e13 4684 */
2483b4ea 4685u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4686{
cc066715
AD		 4687 u32 reset_mask = 0;
4688 u32 tmp;
21a93e13 4689
cc066715
AD		 4690 /* GRBM_STATUS */
4691 tmp = RREG32(GRBM_STATUS);
4692 if (tmp & (PA_BUSY | SC_BUSY |
4693 BCI_BUSY | SX_BUSY |
4694 TA_BUSY | VGT_BUSY |
4695 DB_BUSY | CB_BUSY |
4696 GDS_BUSY | SPI_BUSY |
4697 IA_BUSY | IA_BUSY_NO_DMA))
4698 reset_mask |= RADEON_RESET_GFX;
21a93e13 4699
cc066715
AD		 4700 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4701 reset_mask |= RADEON_RESET_CP;
21a93e13 4702
cc066715
AD		 4703 /* GRBM_STATUS2 */
4704 tmp = RREG32(GRBM_STATUS2);
4705 if (tmp & RLC_BUSY)
4706 reset_mask |= RADEON_RESET_RLC;
21a93e13 4707
cc066715
AD		 4708 /* SDMA0_STATUS_REG */
4709 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4710 if (!(tmp & SDMA_IDLE))
4711 reset_mask |= RADEON_RESET_DMA;
21a93e13 4712
cc066715
AD		 4713 /* SDMA1_STATUS_REG */
4714 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4715 if (!(tmp & SDMA_IDLE))
4716 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4717
cc066715
AD		 4718 /* SRBM_STATUS2 */
4719 tmp = RREG32(SRBM_STATUS2);
4720 if (tmp & SDMA_BUSY)
4721 reset_mask |= RADEON_RESET_DMA;
21a93e13 4722
cc066715
AD		 4723 if (tmp & SDMA1_BUSY)
4724 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4725
cc066715
AD		 4726 /* SRBM_STATUS */
4727 tmp = RREG32(SRBM_STATUS);
21a93e13 4728
cc066715
AD		 4729 if (tmp & IH_BUSY)
4730 reset_mask |= RADEON_RESET_IH;
21a93e13 4731
cc066715
AD		 4732 if (tmp & SEM_BUSY)
4733 reset_mask |= RADEON_RESET_SEM;
21a93e13 4734
cc066715
AD		 4735 if (tmp & GRBM_RQ_PENDING)
4736 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4737
cc066715
AD		 4738 if (tmp & VMC_BUSY)
4739 reset_mask |= RADEON_RESET_VMC;
21a93e13 4740
cc066715
AD		 4741 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4742 MCC_BUSY | MCD_BUSY))
4743 reset_mask |= RADEON_RESET_MC;
21a93e13 4744
cc066715
AD		 4745 if (evergreen_is_display_hung(rdev))
4746 reset_mask |= RADEON_RESET_DISPLAY;
4747
4748 /* Skip MC reset as it's mostly likely not hung, just busy */
4749 if (reset_mask & RADEON_RESET_MC) {
4750 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4751 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4752 }
cc066715
AD		 4753
4754 return reset_mask;
21a93e13
AD		 4755}
4756
4757/**
cc066715 4758 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4759 *
4760 * @rdev: radeon_device pointer
cc066715 4761 * @reset_mask: mask of which blocks to reset
21a93e13 4762 *
cc066715 4763 * Soft reset the blocks specified in @reset_mask.
21a93e13 4764 */
cc066715 4765static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4766{
6f2043ce 4767 struct evergreen_mc_save save;
cc066715
AD		 4768 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4769 u32 tmp;
21a93e13 4770
cc066715
AD		 4771 if (reset_mask == 0)
4772 return;
21a93e13 4773
cc066715 4774 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4775
cc066715
AD		 4776 cik_print_gpu_status_regs(rdev);
4777 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4778 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4779 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4780 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4781
fb2c7f4d
AD		 4782 /* disable CG/PG */
4783 cik_fini_pg(rdev);
4784 cik_fini_cg(rdev);
4785
cc066715
AD		 4786 /* stop the rlc */
4787 cik_rlc_stop(rdev);
21a93e13 4788
cc066715
AD		 4789 /* Disable GFX parsing/prefetching */
4790 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4791
cc066715
AD		 4792 /* Disable MEC parsing/prefetching */
4793 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4794
cc066715
AD		 4795 if (reset_mask & RADEON_RESET_DMA) {
4796 /* sdma0 */
4797 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4798 tmp |= SDMA_HALT;
4799 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4800 }
4801 if (reset_mask & RADEON_RESET_DMA1) {
4802 /* sdma1 */
4803 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4804 tmp |= SDMA_HALT;
4805 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4806 }
21a93e13 4807
6f2043ce 4808 evergreen_mc_stop(rdev, &save);
cc066715 4809 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4810 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4811 }
21a93e13 4812
cc066715
AD		 4813 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4814 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4815
cc066715
AD		 4816 if (reset_mask & RADEON_RESET_CP) {
4817 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4818
cc066715
AD		 4819 srbm_soft_reset |= SOFT_RESET_GRBM;
4820 }
21a93e13 4821
cc066715
AD		 4822 if (reset_mask & RADEON_RESET_DMA)
4823 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4824
cc066715
AD		 4825 if (reset_mask & RADEON_RESET_DMA1)
4826 srbm_soft_reset |= SOFT_RESET_SDMA1;
4827
4828 if (reset_mask & RADEON_RESET_DISPLAY)
4829 srbm_soft_reset |= SOFT_RESET_DC;
4830
4831 if (reset_mask & RADEON_RESET_RLC)
4832 grbm_soft_reset |= SOFT_RESET_RLC;
4833
4834 if (reset_mask & RADEON_RESET_SEM)
4835 srbm_soft_reset |= SOFT_RESET_SEM;
4836
4837 if (reset_mask & RADEON_RESET_IH)
4838 srbm_soft_reset |= SOFT_RESET_IH;
4839
4840 if (reset_mask & RADEON_RESET_GRBM)
4841 srbm_soft_reset |= SOFT_RESET_GRBM;
4842
4843 if (reset_mask & RADEON_RESET_VMC)
4844 srbm_soft_reset |= SOFT_RESET_VMC;
4845
4846 if (!(rdev->flags & RADEON_IS_IGP)) {
4847 if (reset_mask & RADEON_RESET_MC)
4848 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 4849 }
4850
cc066715
AD		 4851 if (grbm_soft_reset) {
4852 tmp = RREG32(GRBM_SOFT_RESET);
4853 tmp |= grbm_soft_reset;
4854 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4855 WREG32(GRBM_SOFT_RESET, tmp);
4856 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 4857
cc066715 4858 udelay(50);
21a93e13 4859
cc066715
AD		 4860 tmp &= ~grbm_soft_reset;
4861 WREG32(GRBM_SOFT_RESET, tmp);
4862 tmp = RREG32(GRBM_SOFT_RESET);
4863 }
21a93e13 4864
cc066715
AD		 4865 if (srbm_soft_reset) {
4866 tmp = RREG32(SRBM_SOFT_RESET);
4867 tmp |= srbm_soft_reset;
4868 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4869 WREG32(SRBM_SOFT_RESET, tmp);
4870 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 4871
cc066715 4872 udelay(50);
21a93e13 4873
cc066715
AD		 4874 tmp &= ~srbm_soft_reset;
4875 WREG32(SRBM_SOFT_RESET, tmp);
4876 tmp = RREG32(SRBM_SOFT_RESET);
4877 }
21a93e13 4878
6f2043ce
AD		 4879 /* Wait a little for things to settle down */
4880 udelay(50);
21a93e13 4881
6f2043ce 4882 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4883 udelay(50);
4884
4885 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 4886}
4887
0279ed19
AD		 4888struct kv_reset_save_regs {
4889 u32 gmcon_reng_execute;
4890 u32 gmcon_misc;
4891 u32 gmcon_misc3;
4892};
4893
4894static void kv_save_regs_for_reset(struct radeon_device *rdev,
4895 struct kv_reset_save_regs *save)
4896{
4897 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4898 save->gmcon_misc = RREG32(GMCON_MISC);
4899 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4900
4901 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4902 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4903 STCTRL_STUTTER_EN));
4904}
4905
4906static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4907 struct kv_reset_save_regs *save)
4908{
4909 int i;
4910
4911 WREG32(GMCON_PGFSM_WRITE, 0);
4912 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4913
4914 for (i = 0; i < 5; i++)
4915 WREG32(GMCON_PGFSM_WRITE, 0);
4916
4917 WREG32(GMCON_PGFSM_WRITE, 0);
4918 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4919
4920 for (i = 0; i < 5; i++)
4921 WREG32(GMCON_PGFSM_WRITE, 0);
4922
4923 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4924 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4925
4926 for (i = 0; i < 5; i++)
4927 WREG32(GMCON_PGFSM_WRITE, 0);
4928
4929 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4930 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4931
4932 for (i = 0; i < 5; i++)
4933 WREG32(GMCON_PGFSM_WRITE, 0);
4934
4935 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4936 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4937
4938 for (i = 0; i < 5; i++)
4939 WREG32(GMCON_PGFSM_WRITE, 0);
4940
4941 WREG32(GMCON_PGFSM_WRITE, 0);
4942 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
4943
4944 for (i = 0; i < 5; i++)
4945 WREG32(GMCON_PGFSM_WRITE, 0);
4946
4947 WREG32(GMCON_PGFSM_WRITE, 0x420000);
4948 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
4949
4950 for (i = 0; i < 5; i++)
4951 WREG32(GMCON_PGFSM_WRITE, 0);
4952
4953 WREG32(GMCON_PGFSM_WRITE, 0x120202);
4954 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
4955
4956 for (i = 0; i < 5; i++)
4957 WREG32(GMCON_PGFSM_WRITE, 0);
4958
4959 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
4960 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
4961
4962 for (i = 0; i < 5; i++)
4963 WREG32(GMCON_PGFSM_WRITE, 0);
4964
4965 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
4966 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
4967
4968 for (i = 0; i < 5; i++)
4969 WREG32(GMCON_PGFSM_WRITE, 0);
4970
4971 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
4972 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
4973
4974 WREG32(GMCON_MISC3, save->gmcon_misc3);
4975 WREG32(GMCON_MISC, save->gmcon_misc);
4976 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
4977}
4978
4979static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
4980{
4981 struct evergreen_mc_save save;
4982 struct kv_reset_save_regs kv_save = { 0 };
4983 u32 tmp, i;
4984
4985 dev_info(rdev->dev, "GPU pci config reset\n");
4986
4987 /* disable dpm? */
4988
4989 /* disable cg/pg */
4990 cik_fini_pg(rdev);
4991 cik_fini_cg(rdev);
4992
4993 /* Disable GFX parsing/prefetching */
4994 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4995
4996 /* Disable MEC parsing/prefetching */
4997 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4998
4999 /* sdma0 */
5000 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5001 tmp |= SDMA_HALT;
5002 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5003 /* sdma1 */
5004 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5005 tmp |= SDMA_HALT;
5006 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5007 /* XXX other engines? */
5008
5009 /* halt the rlc, disable cp internal ints */
5010 cik_rlc_stop(rdev);
5011
5012 udelay(50);
5013
5014 /* disable mem access */
5015 evergreen_mc_stop(rdev, &save);
5016 if (evergreen_mc_wait_for_idle(rdev)) {
5017 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5018 }
5019
5020 if (rdev->flags & RADEON_IS_IGP)
5021 kv_save_regs_for_reset(rdev, &kv_save);
5022
5023 /* disable BM */
5024 pci_clear_master(rdev->pdev);
5025 /* reset */
5026 radeon_pci_config_reset(rdev);
5027
5028 udelay(100);
5029
5030 /* wait for asic to come out of reset */
5031 for (i = 0; i < rdev->usec_timeout; i++) {
5032 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5033 break;
5034 udelay(1);
5035 }
5036
5037 /* does asic init need to be run first??? */
5038 if (rdev->flags & RADEON_IS_IGP)
5039 kv_restore_regs_for_reset(rdev, &kv_save);
5040}
5041
21a93e13 5042/**
cc066715 5043 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5044 *
5045 * @rdev: radeon_device pointer
5046 *
cc066715
AD		 5047 * Look up which blocks are hung and attempt
5048 * to reset them.
6f2043ce 5049 * Returns 0 for success.
21a93e13 5050 */
6f2043ce 5051int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5052{
cc066715 5053 u32 reset_mask;
21a93e13 5054
cc066715 5055 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5056
cc066715
AD		 5057 if (reset_mask)
5058 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5059
0279ed19 5060 /* try soft reset */
cc066715 5061 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5062
cc066715
AD		 5063 reset_mask = cik_gpu_check_soft_reset(rdev);
5064
0279ed19
AD		 5065 /* try pci config reset */
5066 if (reset_mask && radeon_hard_reset)
5067 cik_gpu_pci_config_reset(rdev);
5068
5069 reset_mask = cik_gpu_check_soft_reset(rdev);
5070
cc066715
AD		 5071 if (!reset_mask)
5072 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5073
5074 return 0;
5075}
5076
5077/**
cc066715 5078 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5079 *
5080 * @rdev: radeon_device pointer
cc066715 5081 * @ring: radeon_ring structure holding ring information
21a93e13 5082 *
cc066715
AD		 5083 * Check if the 3D engine is locked up (CIK).
5084 * Returns true if the engine is locked, false if not.
21a93e13 5085 */
cc066715 5086bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5087{
cc066715 5088 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5089
cc066715
AD		 5090 if (!(reset_mask & (RADEON_RESET_GFX |
5091 RADEON_RESET_COMPUTE |
5092 RADEON_RESET_CP))) {
5093 radeon_ring_lockup_update(ring);
5094 return false;
21a93e13 5095 }
cc066715
AD		 5096 /* force CP activities */
5097 radeon_ring_force_activity(rdev, ring);
5098 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5099}
5100
1c49165d 5101/* MC */
21a93e13 5102/**
1c49165d 5103 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5104 *
5105 * @rdev: radeon_device pointer
21a93e13 5106 *
1c49165d
AD		 5107 * Set the location of vram, gart, and AGP in the GPU's
5108 * physical address space (CIK).
21a93e13 5109 */
1c49165d 5110static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5111{
1c49165d 5112 struct evergreen_mc_save save;
21a93e13 5113 u32 tmp;
1c49165d 5114 int i, j;
21a93e13 5115
1c49165d
AD		 5116 /* Initialize HDP */
5117 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5118 WREG32((0x2c14 + j), 0x00000000);
5119 WREG32((0x2c18 + j), 0x00000000);
5120 WREG32((0x2c1c + j), 0x00000000);
5121 WREG32((0x2c20 + j), 0x00000000);
5122 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5123 }
1c49165d 5124 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5125
1c49165d
AD		 5126 evergreen_mc_stop(rdev, &save);
5127 if (radeon_mc_wait_for_idle(rdev)) {
5128 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5129 }
1c49165d
AD		 5130 /* Lockout access through VGA aperture*/
5131 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5132 /* Update configuration */
5133 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5134 rdev->mc.vram_start >> 12);
5135 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5136 rdev->mc.vram_end >> 12);
5137 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5138 rdev->vram_scratch.gpu_addr >> 12);
5139 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5140 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5141 WREG32(MC_VM_FB_LOCATION, tmp);
5142 /* XXX double check these! */
5143 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5144 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5145 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5146 WREG32(MC_VM_AGP_BASE, 0);
5147 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5148 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5149 if (radeon_mc_wait_for_idle(rdev)) {
5150 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5151 }
1c49165d
AD		 5152 evergreen_mc_resume(rdev, &save);
5153 /* we need to own VRAM, so turn off the VGA renderer here
5154 * to stop it overwriting our objects */
5155 rv515_vga_render_disable(rdev);
21a93e13
AD		 5156}
5157
5158/**
1c49165d 5159 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5160 *
5161 * @rdev: radeon_device pointer
21a93e13 5162 *
1c49165d
AD		 5163 * Look up the amount of vram, vram width, and decide how to place
5164 * vram and gart within the GPU's physical address space (CIK).
5165 * Returns 0 for success.
21a93e13 5166 */
1c49165d 5167static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5168{
1c49165d
AD		 5169 u32 tmp;
5170 int chansize, numchan;
21a93e13 5171
1c49165d
AD		 5172 /* Get VRAM informations */
5173 rdev->mc.vram_is_ddr = true;
5174 tmp = RREG32(MC_ARB_RAMCFG);
5175 if (tmp & CHANSIZE_MASK) {
5176 chansize = 64;
21a93e13 5177 } else {
1c49165d 5178 chansize = 32;
21a93e13 5179 }
1c49165d
AD		 5180 tmp = RREG32(MC_SHARED_CHMAP);
5181 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5182 case 0:
5183 default:
5184 numchan = 1;
5185 break;
5186 case 1:
5187 numchan = 2;
5188 break;
5189 case 2:
5190 numchan = 4;
5191 break;
5192 case 3:
5193 numchan = 8;
5194 break;
5195 case 4:
5196 numchan = 3;
5197 break;
5198 case 5:
5199 numchan = 6;
5200 break;
5201 case 6:
5202 numchan = 10;
5203 break;
5204 case 7:
5205 numchan = 12;
5206 break;
5207 case 8:
5208 numchan = 16;
5209 break;
5210 }
5211 rdev->mc.vram_width = numchan * chansize;
5212 /* Could aper size report 0 ? */
5213 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5214 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5215 /* size in MB on si */
13c5bfda
AD		 5216 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5217 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5218 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5219 si_vram_gtt_location(rdev, &rdev->mc);
5220 radeon_update_bandwidth_info(rdev);
5221
5222 return 0;
5223}
5224
5225/*
5226 * GART
5227 * VMID 0 is the physical GPU addresses as used by the kernel.
5228 * VMIDs 1-15 are used for userspace clients and are handled
5229 * by the radeon vm/hsa code.
5230 */
5231/**
5232 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5233 *
5234 * @rdev: radeon_device pointer
5235 *
5236 * Flush the TLB for the VMID 0 page table (CIK).
5237 */
5238void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5239{
5240 /* flush hdp cache */
5241 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5242
5243 /* bits 0-15 are the VM contexts0-15 */
5244 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5245}
5246
5247/**
5248 * cik_pcie_gart_enable - gart enable
5249 *
5250 * @rdev: radeon_device pointer
5251 *
5252 * This sets up the TLBs, programs the page tables for VMID0,
5253 * sets up the hw for VMIDs 1-15 which are allocated on
5254 * demand, and sets up the global locations for the LDS, GDS,
5255 * and GPUVM for FSA64 clients (CIK).
5256 * Returns 0 for success, errors for failure.
5257 */
5258static int cik_pcie_gart_enable(struct radeon_device *rdev)
5259{
5260 int r, i;
5261
5262 if (rdev->gart.robj == NULL) {
5263 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5264 return -EINVAL;
5265 }
5266 r = radeon_gart_table_vram_pin(rdev);
5267 if (r)
5268 return r;
5269 radeon_gart_restore(rdev);
5270 /* Setup TLB control */
5271 WREG32(MC_VM_MX_L1_TLB_CNTL,
5272 (0xA << 7) |
5273 ENABLE_L1_TLB |
5274 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5275 ENABLE_ADVANCED_DRIVER_MODEL |
5276 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5277 /* Setup L2 cache */
5278 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5279 ENABLE_L2_FRAGMENT_PROCESSING |
5280 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5281 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5282 EFFECTIVE_L2_QUEUE_SIZE(7) |
5283 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5284 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5285 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5286 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5287 /* setup context0 */
5288 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5289 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5290 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5291 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5292 (u32)(rdev->dummy_page.addr >> 12));
5293 WREG32(VM_CONTEXT0_CNTL2, 0);
5294 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5295 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5296
5297 WREG32(0x15D4, 0);
5298 WREG32(0x15D8, 0);
5299 WREG32(0x15DC, 0);
5300
5301 /* empty context1-15 */
5302 /* FIXME start with 4G, once using 2 level pt switch to full
5303 * vm size space
5304 */
5305 /* set vm size, must be a multiple of 4 */
5306 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5307 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5308 for (i = 1; i < 16; i++) {
5309 if (i < 8)
5310 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5311 rdev->gart.table_addr >> 12);
5312 else
5313 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5314 rdev->gart.table_addr >> 12);
5315 }
5316
5317 /* enable context1-15 */
5318 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5319 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5320 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5321 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 5322 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5323 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5324 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5325 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5326 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5327 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5328 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5329 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5330 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5331 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5332 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5333 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 5334
5335 /* TC cache setup ??? */
5336 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5337 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5338 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5339
5340 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5341 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5342 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5343 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5344 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5345
5346 WREG32(TC_CFG_L1_VOLATILE, 0);
5347 WREG32(TC_CFG_L2_VOLATILE, 0);
5348
5349 if (rdev->family == CHIP_KAVERI) {
5350 u32 tmp = RREG32(CHUB_CONTROL);
5351 tmp &= ~BYPASS_VM;
5352 WREG32(CHUB_CONTROL, tmp);
5353 }
5354
5355 /* XXX SH_MEM regs */
5356 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5357 mutex_lock(&rdev->srbm_mutex);
1c49165d 5358 for (i = 0; i < 16; i++) {
b556b12e 5359 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5360 /* CP and shaders */
1c49165d
AD		 5361 WREG32(SH_MEM_CONFIG, 0);
5362 WREG32(SH_MEM_APE1_BASE, 1);
5363 WREG32(SH_MEM_APE1_LIMIT, 0);
5364 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5365 /* SDMA GFX */
5366 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5367 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5368 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5369 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5370 /* XXX SDMA RLC - todo */
1c49165d 5371 }
b556b12e 5372 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5373 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5374
5375 cik_pcie_gart_tlb_flush(rdev);
5376 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5377 (unsigned)(rdev->mc.gtt_size >> 20),
5378 (unsigned long long)rdev->gart.table_addr);
5379 rdev->gart.ready = true;
5380 return 0;
5381}
5382
5383/**
5384 * cik_pcie_gart_disable - gart disable
5385 *
5386 * @rdev: radeon_device pointer
5387 *
5388 * This disables all VM page table (CIK).
5389 */
5390static void cik_pcie_gart_disable(struct radeon_device *rdev)
5391{
5392 /* Disable all tables */
5393 WREG32(VM_CONTEXT0_CNTL, 0);
5394 WREG32(VM_CONTEXT1_CNTL, 0);
5395 /* Setup TLB control */
5396 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5397 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5398 /* Setup L2 cache */
5399 WREG32(VM_L2_CNTL,
5400 ENABLE_L2_FRAGMENT_PROCESSING |
5401 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5402 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5403 EFFECTIVE_L2_QUEUE_SIZE(7) |
5404 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5405 WREG32(VM_L2_CNTL2, 0);
5406 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5407 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5408 radeon_gart_table_vram_unpin(rdev);
5409}
5410
5411/**
5412 * cik_pcie_gart_fini - vm fini callback
5413 *
5414 * @rdev: radeon_device pointer
5415 *
5416 * Tears down the driver GART/VM setup (CIK).
5417 */
5418static void cik_pcie_gart_fini(struct radeon_device *rdev)
5419{
5420 cik_pcie_gart_disable(rdev);
5421 radeon_gart_table_vram_free(rdev);
5422 radeon_gart_fini(rdev);
5423}
5424
5425/* vm parser */
5426/**
5427 * cik_ib_parse - vm ib_parse callback
5428 *
5429 * @rdev: radeon_device pointer
5430 * @ib: indirect buffer pointer
5431 *
5432 * CIK uses hw IB checking so this is a nop (CIK).
5433 */
5434int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5435{
5436 return 0;
5437}
5438
5439/*
5440 * vm
5441 * VMID 0 is the physical GPU addresses as used by the kernel.
5442 * VMIDs 1-15 are used for userspace clients and are handled
5443 * by the radeon vm/hsa code.
5444 */
5445/**
5446 * cik_vm_init - cik vm init callback
5447 *
5448 * @rdev: radeon_device pointer
5449 *
5450 * Inits cik specific vm parameters (number of VMs, base of vram for
5451 * VMIDs 1-15) (CIK).
5452 * Returns 0 for success.
5453 */
5454int cik_vm_init(struct radeon_device *rdev)
5455{
5456 /* number of VMs */
5457 rdev->vm_manager.nvm = 16;
5458 /* base offset of vram pages */
5459 if (rdev->flags & RADEON_IS_IGP) {
5460 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5461 tmp <<= 22;
5462 rdev->vm_manager.vram_base_offset = tmp;
5463 } else
5464 rdev->vm_manager.vram_base_offset = 0;
5465
5466 return 0;
5467}
5468
5469/**
5470 * cik_vm_fini - cik vm fini callback
5471 *
5472 * @rdev: radeon_device pointer
5473 *
5474 * Tear down any asic specific VM setup (CIK).
5475 */
5476void cik_vm_fini(struct radeon_device *rdev)
5477{
5478}
5479
3ec7d11b
AD		 5480/**
5481 * cik_vm_decode_fault - print human readable fault info
5482 *
5483 * @rdev: radeon_device pointer
5484 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5485 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5486 *
5487 * Print human readable fault information (CIK).
5488 */
5489static void cik_vm_decode_fault(struct radeon_device *rdev,
5490 u32 status, u32 addr, u32 mc_client)
5491{
939c0d3c 5492 u32 mc_id;
3ec7d11b
AD		 5493 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5494 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5495 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5496 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5497
939c0d3c
AD		 5498 if (rdev->family == CHIP_HAWAII)
5499 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5500 else
5501 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5502
328a50c7 5503 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5504 protections, vmid, addr,
5505 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5506 block, mc_client, mc_id);
3ec7d11b
AD		 5507}
5508
f96ab484
AD		 5509/**
5510 * cik_vm_flush - cik vm flush using the CP
5511 *
5512 * @rdev: radeon_device pointer
5513 *
5514 * Update the page table base and flush the VM TLB
5515 * using the CP (CIK).
5516 */
5517void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5518{
5519 struct radeon_ring *ring = &rdev->ring[ridx];
5520
5521 if (vm == NULL)
5522 return;
5523
5524 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5525 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5526 WRITE_DATA_DST_SEL(0)));
5527 if (vm->id < 8) {
5528 radeon_ring_write(ring,
5529 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5530 } else {
5531 radeon_ring_write(ring,
5532 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5533 }
5534 radeon_ring_write(ring, 0);
5535 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5536
5537 /* update SH_MEM_* regs */
5538 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5539 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5540 WRITE_DATA_DST_SEL(0)));
5541 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5542 radeon_ring_write(ring, 0);
5543 radeon_ring_write(ring, VMID(vm->id));
5544
5545 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5546 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5547 WRITE_DATA_DST_SEL(0)));
5548 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5549 radeon_ring_write(ring, 0);
5550
5551 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5552 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5553 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5554 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5555
f96ab484
AD		 5556 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5557 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5558 WRITE_DATA_DST_SEL(0)));
5559 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5560 radeon_ring_write(ring, 0);
5561 radeon_ring_write(ring, VMID(0));
6f2043ce 5562
f96ab484 5563 /* HDP flush */
780f5ddd 5564 cik_hdp_flush_cp_ring_emit(rdev, ridx);
f96ab484
AD		 5565
5566 /* bits 0-15 are the VM contexts0-15 */
5567 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5568 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5569 WRITE_DATA_DST_SEL(0)));
5570 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5571 radeon_ring_write(ring, 0);
5572 radeon_ring_write(ring, 1 << vm->id);
5573
b07fdd38
AD		 5574 /* compute doesn't have PFP */
5575 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5576 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5577 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5578 radeon_ring_write(ring, 0x0);
5579 }
cc066715 5580}
6f2043ce 5581
f6796cae
AD		 5582/*
5583 * RLC
5584 * The RLC is a multi-purpose microengine that handles a
5585 * variety of functions, the most important of which is
5586 * the interrupt controller.
5587 */
866d83de
AD		 5588static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5589 bool enable)
f6796cae 5590{
866d83de 5591 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5592
866d83de
AD		 5593 if (enable)
5594 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5595 else
5596 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5597 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5598}
f6796cae 5599
866d83de 5600static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5601{
cc066715 5602 u32 tmp;
6f2043ce 5603
866d83de
AD		 5604 tmp = RREG32(RLC_LB_CNTL);
5605 if (enable)
5606 tmp |= LOAD_BALANCE_ENABLE;
5607 else
5608 tmp &= ~LOAD_BALANCE_ENABLE;
5609 WREG32(RLC_LB_CNTL, tmp);
5610}
cc066715 5611
866d83de
AD		 5612static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5613{
5614 u32 i, j, k;
5615 u32 mask;
cc066715 5616
f6796cae
AD		 5617 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5618 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5619 cik_select_se_sh(rdev, i, j);
5620 for (k = 0; k < rdev->usec_timeout; k++) {
5621 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5622 break;
5623 udelay(1);
5624 }
5625 }
5626 }
5627 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 5628
f6796cae
AD		 5629 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5630 for (k = 0; k < rdev->usec_timeout; k++) {
5631 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5632 break;
5633 udelay(1);
5634 }
5635}
cc066715 5636
22c775ce
AD		 5637static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5638{
5639 u32 tmp;
cc066715 5640
22c775ce
AD		 5641 tmp = RREG32(RLC_CNTL);
5642 if (tmp != rlc)
5643 WREG32(RLC_CNTL, rlc);
5644}
cc066715 5645
22c775ce
AD		 5646static u32 cik_halt_rlc(struct radeon_device *rdev)
5647{
5648 u32 data, orig;
cc066715 5649
22c775ce 5650 orig = data = RREG32(RLC_CNTL);
cc066715 5651
22c775ce
AD		 5652 if (data & RLC_ENABLE) {
5653 u32 i;
cc066715 5654
22c775ce
AD		 5655 data &= ~RLC_ENABLE;
5656 WREG32(RLC_CNTL, data);
cc066715 5657
22c775ce
AD		 5658 for (i = 0; i < rdev->usec_timeout; i++) {
5659 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5660 break;
5661 udelay(1);
5662 }
cc066715 5663
22c775ce
AD		 5664 cik_wait_for_rlc_serdes(rdev);
5665 }
cc066715 5666
22c775ce
AD		 5667 return orig;
5668}
cc066715 5669
a412fce0
AD		 5670void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5671{
5672 u32 tmp, i, mask;
5673
5674 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5675 WREG32(RLC_GPR_REG2, tmp);
5676
5677 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5678 for (i = 0; i < rdev->usec_timeout; i++) {
5679 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5680 break;
5681 udelay(1);
5682 }
5683
5684 for (i = 0; i < rdev->usec_timeout; i++) {
5685 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5686 break;
5687 udelay(1);
5688 }
5689}
5690
5691void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5692{
5693 u32 tmp;
5694
5695 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5696 WREG32(RLC_GPR_REG2, tmp);
5697}
5698
866d83de
AD		 5699/**
5700 * cik_rlc_stop - stop the RLC ME
5701 *
5702 * @rdev: radeon_device pointer
5703 *
5704 * Halt the RLC ME (MicroEngine) (CIK).
5705 */
5706static void cik_rlc_stop(struct radeon_device *rdev)
5707{
22c775ce 5708 WREG32(RLC_CNTL, 0);
866d83de
AD		 5709
5710 cik_enable_gui_idle_interrupt(rdev, false);
5711
866d83de
AD		 5712 cik_wait_for_rlc_serdes(rdev);
5713}
5714
f6796cae
AD		 5715/**
5716 * cik_rlc_start - start the RLC ME
5717 *
5718 * @rdev: radeon_device pointer
5719 *
5720 * Unhalt the RLC ME (MicroEngine) (CIK).
5721 */
5722static void cik_rlc_start(struct radeon_device *rdev)
5723{
f6796cae 5724 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5725
866d83de 5726 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5727
f6796cae 5728 udelay(50);
6f2043ce
AD		 5729}
5730
5731/**
f6796cae 5732 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5733 *
5734 * @rdev: radeon_device pointer
5735 *
f6796cae
AD		 5736 * Initialize the RLC registers, load the ucode,
5737 * and start the RLC (CIK).
5738 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5739 */
f6796cae 5740static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5741{
22c775ce 5742 u32 i, size, tmp;
f6796cae 5743 const __be32 *fw_data;
cc066715 5744
f6796cae
AD		 5745 if (!rdev->rlc_fw)
5746 return -EINVAL;
cc066715 5747
f6796cae
AD		 5748 switch (rdev->family) {
5749 case CHIP_BONAIRE:
d4775655 5750 case CHIP_HAWAII:
f6796cae
AD		 5751 default:
5752 size = BONAIRE_RLC_UCODE_SIZE;
5753 break;
5754 case CHIP_KAVERI:
5755 size = KV_RLC_UCODE_SIZE;
5756 break;
5757 case CHIP_KABINI:
5758 size = KB_RLC_UCODE_SIZE;
5759 break;
5760 }
cc066715 5761
cc066715
AD		 5762 cik_rlc_stop(rdev);
5763
22c775ce
AD		 5764 /* disable CG */
5765 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5766 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5767
866d83de 5768 si_rlc_reset(rdev);
6f2043ce 5769
22c775ce 5770 cik_init_pg(rdev);
6f2043ce 5771
22c775ce 5772 cik_init_cg(rdev);
cc066715 5773
f6796cae
AD		 5774 WREG32(RLC_LB_CNTR_INIT, 0);
5775 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5776
f6796cae
AD		 5777 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5778 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5779 WREG32(RLC_LB_PARAMS, 0x00600408);
5780 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5781
f6796cae
AD		 5782 WREG32(RLC_MC_CNTL, 0);
5783 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5784
f6796cae
AD		 5785 fw_data = (const __be32 *)rdev->rlc_fw->data;
5786 WREG32(RLC_GPM_UCODE_ADDR, 0);
5787 for (i = 0; i < size; i++)
5788 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5789 WREG32(RLC_GPM_UCODE_ADDR, 0);
cc066715 5790
866d83de
AD		 5791 /* XXX - find out what chips support lbpw */
5792 cik_enable_lbpw(rdev, false);
cc066715 5793
22c775ce
AD		 5794 if (rdev->family == CHIP_BONAIRE)
5795 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5796
f6796cae 5797 cik_rlc_start(rdev);
cc066715 5798
f6796cae
AD		 5799 return 0;
5800}
cc066715 5801
22c775ce
AD		 5802static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5803{
5804 u32 data, orig, tmp, tmp2;
cc066715 5805
22c775ce 5806 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 5807
473359bc 5808 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 5809 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5810
22c775ce 5811 tmp = cik_halt_rlc(rdev);
cc066715 5812
22c775ce
AD		 5813 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5814 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5815 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5816 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5817 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 5818
22c775ce 5819 cik_update_rlc(rdev, tmp);
cc066715 5820
22c775ce
AD		 5821 data |= CGCG_EN | CGLS_EN;
5822 } else {
ddc76ff6 5823 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 5824
22c775ce
AD		 5825 RREG32(CB_CGTT_SCLK_CTRL);
5826 RREG32(CB_CGTT_SCLK_CTRL);
5827 RREG32(CB_CGTT_SCLK_CTRL);
5828 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 5829
22c775ce 5830 data &= ~(CGCG_EN | CGLS_EN);
cc066715 5831 }
6f2043ce 5832
22c775ce
AD		 5833 if (orig != data)
5834 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 5835
6f2043ce
AD		 5836}
5837
22c775ce 5838static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 5839{
22c775ce
AD		 5840 u32 data, orig, tmp = 0;
5841
473359bc
AD		 5842 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5843 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5844 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5845 orig = data = RREG32(CP_MEM_SLP_CNTL);
5846 data |= CP_MEM_LS_EN;
5847 if (orig != data)
5848 WREG32(CP_MEM_SLP_CNTL, data);
5849 }
5850 }
cc066715 5851
22c775ce
AD		 5852 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5853 data &= 0xfffffffd;
5854 if (orig != data)
5855 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5856
5857 tmp = cik_halt_rlc(rdev);
5858
5859 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5860 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5861 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5862 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5863 WREG32(RLC_SERDES_WR_CTRL, data);
5864
5865 cik_update_rlc(rdev, tmp);
5866
473359bc
AD		 5867 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5868 orig = data = RREG32(CGTS_SM_CTRL_REG);
5869 data &= ~SM_MODE_MASK;
5870 data |= SM_MODE(0x2);
5871 data |= SM_MODE_ENABLE;
5872 data &= ~CGTS_OVERRIDE;
5873 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5874 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5875 data &= ~CGTS_LS_OVERRIDE;
5876 data &= ~ON_MONITOR_ADD_MASK;
5877 data |= ON_MONITOR_ADD_EN;
5878 data |= ON_MONITOR_ADD(0x96);
5879 if (orig != data)
5880 WREG32(CGTS_SM_CTRL_REG, data);
5881 }
22c775ce
AD		 5882 } else {
5883 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5884 data |= 0x00000002;
5885 if (orig != data)
5886 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5887
5888 data = RREG32(RLC_MEM_SLP_CNTL);
5889 if (data & RLC_MEM_LS_EN) {
5890 data &= ~RLC_MEM_LS_EN;
5891 WREG32(RLC_MEM_SLP_CNTL, data);
5892 }
6f2043ce 5893
22c775ce
AD		 5894 data = RREG32(CP_MEM_SLP_CNTL);
5895 if (data & CP_MEM_LS_EN) {
5896 data &= ~CP_MEM_LS_EN;
5897 WREG32(CP_MEM_SLP_CNTL, data);
5898 }
cc066715 5899
22c775ce
AD		 5900 orig = data = RREG32(CGTS_SM_CTRL_REG);
5901 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5902 if (orig != data)
5903 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 5904
22c775ce 5905 tmp = cik_halt_rlc(rdev);
cc066715 5906
22c775ce
AD		 5907 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5908 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5909 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5910 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5911 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 5912
22c775ce 5913 cik_update_rlc(rdev, tmp);
cc066715 5914 }
6f2043ce 5915}
1c49165d 5916
22c775ce 5917static const u32 mc_cg_registers[] =
21a93e13 5918{
22c775ce
AD		 5919 MC_HUB_MISC_HUB_CG,
5920 MC_HUB_MISC_SIP_CG,
5921 MC_HUB_MISC_VM_CG,
5922 MC_XPB_CLK_GAT,
5923 ATC_MISC_CG,
5924 MC_CITF_MISC_WR_CG,
5925 MC_CITF_MISC_RD_CG,
5926 MC_CITF_MISC_VM_CG,
5927 VM_L2_CG,
5928};
21a93e13 5929
22c775ce
AD		 5930static void cik_enable_mc_ls(struct radeon_device *rdev,
5931 bool enable)
1c49165d 5932{
22c775ce
AD		 5933 int i;
5934 u32 orig, data;
1c49165d 5935
22c775ce
AD		 5936 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5937 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5938 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5939 data |= MC_LS_ENABLE;
5940 else
5941 data &= ~MC_LS_ENABLE;
5942 if (data != orig)
5943 WREG32(mc_cg_registers[i], data);
1c49165d 5944 }
22c775ce 5945}
1c49165d 5946
22c775ce
AD		 5947static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5948 bool enable)
5949{
5950 int i;
5951 u32 orig, data;
5952
5953 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5954 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5955 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5956 data |= MC_CG_ENABLE;
5957 else
5958 data &= ~MC_CG_ENABLE;
5959 if (data != orig)
5960 WREG32(mc_cg_registers[i], data);
1c49165d 5961 }
1c49165d
AD		 5962}
5963
22c775ce
AD		 5964static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5965 bool enable)
1c49165d 5966{
22c775ce 5967 u32 orig, data;
1c49165d 5968
473359bc 5969 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5970 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5971 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 5972 } else {
22c775ce
AD		 5973 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5974 data |= 0xff000000;
5975 if (data != orig)
5976 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5977
22c775ce
AD		 5978 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5979 data |= 0xff000000;
5980 if (data != orig)
5981 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5982 }
1c49165d
AD		 5983}
5984
22c775ce
AD		 5985static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5986 bool enable)
1c49165d 5987{
22c775ce
AD		 5988 u32 orig, data;
5989
473359bc 5990 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5991 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5992 data |= 0x100;
5993 if (orig != data)
5994 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5995
5996 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5997 data |= 0x100;
5998 if (orig != data)
5999 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6000 } else {
6001 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6002 data &= ~0x100;
6003 if (orig != data)
6004 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6005
22c775ce
AD		 6006 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6007 data &= ~0x100;
6008 if (orig != data)
6009 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6010 }
1c49165d
AD		 6011}
6012
22c775ce
AD		 6013static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6014 bool enable)
1c49165d 6015{
22c775ce 6016 u32 orig, data;
1c49165d 6017
473359bc 6018 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6019 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6020 data = 0xfff;
6021 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6022
22c775ce
AD		 6023 orig = data = RREG32(UVD_CGC_CTRL);
6024 data |= DCM;
6025 if (orig != data)
6026 WREG32(UVD_CGC_CTRL, data);
6027 } else {
6028 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6029 data &= ~0xfff;
6030 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6031
22c775ce
AD		 6032 orig = data = RREG32(UVD_CGC_CTRL);
6033 data &= ~DCM;
6034 if (orig != data)
6035 WREG32(UVD_CGC_CTRL, data);
1c49165d 6036 }
22c775ce 6037}
1c49165d 6038
473359bc
AD		 6039static void cik_enable_bif_mgls(struct radeon_device *rdev,
6040 bool enable)
6041{
6042 u32 orig, data;
1c49165d 6043
473359bc 6044 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6045
473359bc
AD		 6046 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6047 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6048 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6049 else
6050 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6051 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6052
473359bc
AD		 6053 if (orig != data)
6054 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6055}
1c49165d 6056
22c775ce
AD		 6057static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6058 bool enable)
6059{
6060 u32 orig, data;
1c49165d 6061
22c775ce 6062 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6063
473359bc 6064 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6065 data &= ~CLOCK_GATING_DIS;
6066 else
6067 data |= CLOCK_GATING_DIS;
6068
6069 if (orig != data)
6070 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6071}
6072
22c775ce
AD		 6073static void cik_enable_hdp_ls(struct radeon_device *rdev,
6074 bool enable)
1c49165d 6075{
22c775ce
AD		 6076 u32 orig, data;
6077
6078 orig = data = RREG32(HDP_MEM_POWER_LS);
6079
473359bc 6080 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6081 data |= HDP_LS_ENABLE;
6082 else
6083 data &= ~HDP_LS_ENABLE;
6084
6085 if (orig != data)
6086 WREG32(HDP_MEM_POWER_LS, data);
6087}
6088
6089void cik_update_cg(struct radeon_device *rdev,
6090 u32 block, bool enable)
6091{
4214faf6 6092
22c775ce 6093 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6094 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6095 /* order matters! */
6096 if (enable) {
6097 cik_enable_mgcg(rdev, true);
6098 cik_enable_cgcg(rdev, true);
6099 } else {
6100 cik_enable_cgcg(rdev, false);
6101 cik_enable_mgcg(rdev, false);
6102 }
4214faf6 6103 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6104 }
6105
6106 if (block & RADEON_CG_BLOCK_MC) {
6107 if (!(rdev->flags & RADEON_IS_IGP)) {
6108 cik_enable_mc_mgcg(rdev, enable);
6109 cik_enable_mc_ls(rdev, enable);
6110 }
6111 }
6112
6113 if (block & RADEON_CG_BLOCK_SDMA) {
6114 cik_enable_sdma_mgcg(rdev, enable);
6115 cik_enable_sdma_mgls(rdev, enable);
6116 }
6117
473359bc
AD		 6118 if (block & RADEON_CG_BLOCK_BIF) {
6119 cik_enable_bif_mgls(rdev, enable);
6120 }
6121
22c775ce
AD		 6122 if (block & RADEON_CG_BLOCK_UVD) {
6123 if (rdev->has_uvd)
6124 cik_enable_uvd_mgcg(rdev, enable);
6125 }
6126
6127 if (block & RADEON_CG_BLOCK_HDP) {
6128 cik_enable_hdp_mgcg(rdev, enable);
6129 cik_enable_hdp_ls(rdev, enable);
6130 }
1c49165d
AD		 6131}
6132
22c775ce 6133static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6134{
22c775ce 6135
ddc76ff6 6136 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6137
6138 if (rdev->has_uvd)
6139 si_init_uvd_internal_cg(rdev);
6140
6141 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6142 RADEON_CG_BLOCK_SDMA |
473359bc 6143 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6144 RADEON_CG_BLOCK_UVD |
6145 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6146}
6147
473359bc 6148static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6149{
473359bc
AD		 6150 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6151 RADEON_CG_BLOCK_SDMA |
6152 RADEON_CG_BLOCK_BIF |
6153 RADEON_CG_BLOCK_UVD |
6154 RADEON_CG_BLOCK_HDP), false);
6155
6156 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6157}
6158
22c775ce
AD		 6159static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6160 bool enable)
1c49165d 6161{
22c775ce 6162 u32 data, orig;
1c49165d 6163
22c775ce 6164 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6165 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6166 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6167 else
6168 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6169 if (orig != data)
6170 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6171}
6172
22c775ce
AD		 6173static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6174 bool enable)
1c49165d 6175{
22c775ce
AD		 6176 u32 data, orig;
6177
6178 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6179 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6180 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6181 else
6182 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6183 if (orig != data)
6184 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6185}
6186
22c775ce 6187static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6188{
22c775ce 6189 u32 data, orig;
3ec7d11b 6190
22c775ce 6191 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6192 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6193 data &= ~DISABLE_CP_PG;
6194 else
6195 data |= DISABLE_CP_PG;
6196 if (orig != data)
6197 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6198}
6199
22c775ce 6200static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6201{
22c775ce 6202 u32 data, orig;
f96ab484 6203
22c775ce 6204 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6205 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6206 data &= ~DISABLE_GDS_PG;
6207 else
6208 data |= DISABLE_GDS_PG;
6209 if (orig != data)
6210 WREG32(RLC_PG_CNTL, data);
6211}
6212
6213#define CP_ME_TABLE_SIZE 96
6214#define CP_ME_TABLE_OFFSET 2048
6215#define CP_MEC_TABLE_OFFSET 4096
6216
6217void cik_init_cp_pg_table(struct radeon_device *rdev)
6218{
6219 const __be32 *fw_data;
6220 volatile u32 *dst_ptr;
6221 int me, i, max_me = 4;
6222 u32 bo_offset = 0;
6223 u32 table_offset;
6224
6225 if (rdev->family == CHIP_KAVERI)
6226 max_me = 5;
6227
6228 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6229 return;
6230
22c775ce
AD		 6231 /* write the cp table buffer */
6232 dst_ptr = rdev->rlc.cp_table_ptr;
6233 for (me = 0; me < max_me; me++) {
6234 if (me == 0) {
6235 fw_data = (const __be32 *)rdev->ce_fw->data;
6236 table_offset = CP_ME_TABLE_OFFSET;
6237 } else if (me == 1) {
6238 fw_data = (const __be32 *)rdev->pfp_fw->data;
6239 table_offset = CP_ME_TABLE_OFFSET;
6240 } else if (me == 2) {
6241 fw_data = (const __be32 *)rdev->me_fw->data;
6242 table_offset = CP_ME_TABLE_OFFSET;
6243 } else {
6244 fw_data = (const __be32 *)rdev->mec_fw->data;
6245 table_offset = CP_MEC_TABLE_OFFSET;
6246 }
6247
6248 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6ba81e53 6249 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
22c775ce
AD		 6250 }
6251 bo_offset += CP_ME_TABLE_SIZE;
f96ab484 6252 }
22c775ce 6253}
f96ab484 6254
22c775ce
AD		 6255static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6256 bool enable)
6257{
6258 u32 data, orig;
6259
2b19d17f 6260 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6261 orig = data = RREG32(RLC_PG_CNTL);
6262 data |= GFX_PG_ENABLE;
6263 if (orig != data)
6264 WREG32(RLC_PG_CNTL, data);
6265
6266 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6267 data |= AUTO_PG_EN;
6268 if (orig != data)
6269 WREG32(RLC_AUTO_PG_CTRL, data);
6270 } else {
6271 orig = data = RREG32(RLC_PG_CNTL);
6272 data &= ~GFX_PG_ENABLE;
6273 if (orig != data)
6274 WREG32(RLC_PG_CNTL, data);
f96ab484 6275
22c775ce
AD		 6276 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6277 data &= ~AUTO_PG_EN;
6278 if (orig != data)
6279 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6280
22c775ce
AD		 6281 data = RREG32(DB_RENDER_CONTROL);
6282 }
6283}
f96ab484 6284
22c775ce
AD		 6285static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6286{
6287 u32 mask = 0, tmp, tmp1;
6288 int i;
f96ab484 6289
22c775ce
AD		 6290 cik_select_se_sh(rdev, se, sh);
6291 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6292 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6293 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6294
22c775ce 6295 tmp &= 0xffff0000;
f96ab484 6296
22c775ce
AD		 6297 tmp |= tmp1;
6298 tmp >>= 16;
6299
6300 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6301 mask <<= 1;
6302 mask |= 1;
b07fdd38 6303 }
22c775ce
AD		 6304
6305 return (~tmp) & mask;
f96ab484
AD		 6306}
6307
22c775ce 6308static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6309{
22c775ce
AD		 6310 u32 i, j, k, active_cu_number = 0;
6311 u32 mask, counter, cu_bitmap;
6312 u32 tmp = 0;
d0e092d9 6313
22c775ce
AD		 6314 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6315 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6316 mask = 1;
6317 cu_bitmap = 0;
6318 counter = 0;
6319 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6320 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6321 if (counter < 2)
6322 cu_bitmap |= mask;
6323 counter ++;
d0e092d9 6324 }
22c775ce 6325 mask <<= 1;
d0e092d9 6326 }
d0e092d9 6327
22c775ce
AD		 6328 active_cu_number += counter;
6329 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6330 }
d0e092d9 6331 }
22c775ce
AD		 6332
6333 WREG32(RLC_PG_AO_CU_MASK, tmp);
6334
6335 tmp = RREG32(RLC_MAX_PG_CU);
6336 tmp &= ~MAX_PU_CU_MASK;
6337 tmp |= MAX_PU_CU(active_cu_number);
6338 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6339}
6340
22c775ce
AD		 6341static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6342 bool enable)
605de6b9 6343{
22c775ce 6344 u32 data, orig;
605de6b9 6345
22c775ce 6346 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6347 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6348 data |= STATIC_PER_CU_PG_ENABLE;
6349 else
6350 data &= ~STATIC_PER_CU_PG_ENABLE;
6351 if (orig != data)
6352 WREG32(RLC_PG_CNTL, data);
6353}
6354
6355static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6356 bool enable)
6357{
6358 u32 data, orig;
605de6b9 6359
22c775ce 6360 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6361 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6362 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6363 else
22c775ce
AD		 6364 data &= ~DYN_PER_CU_PG_ENABLE;
6365 if (orig != data)
6366 WREG32(RLC_PG_CNTL, data);
6367}
605de6b9 6368
22c775ce
AD		 6369#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6370#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6371
6372static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6373{
6374 u32 data, orig;
6375 u32 i;
6376
6377 if (rdev->rlc.cs_data) {
6378 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6379 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6380 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6381 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6382 } else {
22c775ce
AD		 6383 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6384 for (i = 0; i < 3; i++)
6385 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6386 }
6387 if (rdev->rlc.reg_list) {
6388 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6389 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6390 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6391 }
605de6b9 6392
22c775ce
AD		 6393 orig = data = RREG32(RLC_PG_CNTL);
6394 data |= GFX_PG_SRC;
6395 if (orig != data)
6396 WREG32(RLC_PG_CNTL, data);
605de6b9 6397
22c775ce
AD		 6398 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6399 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6400
22c775ce
AD		 6401 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6402 data &= ~IDLE_POLL_COUNT_MASK;
6403 data |= IDLE_POLL_COUNT(0x60);
6404 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6405
22c775ce
AD		 6406 data = 0x10101010;
6407 WREG32(RLC_PG_DELAY, data);
605de6b9 6408
22c775ce
AD		 6409 data = RREG32(RLC_PG_DELAY_2);
6410 data &= ~0xff;
6411 data |= 0x3;
6412 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6413
22c775ce
AD		 6414 data = RREG32(RLC_AUTO_PG_CTRL);
6415 data &= ~GRBM_REG_SGIT_MASK;
6416 data |= GRBM_REG_SGIT(0x700);
6417 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6418
605de6b9
AD		 6419}
6420
22c775ce 6421static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6422{
473359bc
AD		 6423 cik_enable_gfx_cgpg(rdev, enable);
6424 cik_enable_gfx_static_mgpg(rdev, enable);
6425 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6426}
f6796cae 6427
a0f38609
AD		 6428u32 cik_get_csb_size(struct radeon_device *rdev)
6429{
6430 u32 count = 0;
6431 const struct cs_section_def *sect = NULL;
6432 const struct cs_extent_def *ext = NULL;
f6796cae 6433
a0f38609
AD		 6434 if (rdev->rlc.cs_data == NULL)
6435 return 0;
f6796cae 6436
a0f38609
AD		 6437 /* begin clear state */
6438 count += 2;
6439 /* context control state */
6440 count += 3;
6441
6442 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6443 for (ext = sect->section; ext->extent != NULL; ++ext) {
6444 if (sect->id == SECT_CONTEXT)
6445 count += 2 + ext->reg_count;
6446 else
6447 return 0;
f6796cae
AD		 6448 }
6449 }
a0f38609
AD		 6450 /* pa_sc_raster_config/pa_sc_raster_config1 */
6451 count += 4;
6452 /* end clear state */
6453 count += 2;
6454 /* clear state */
6455 count += 2;
f6796cae 6456
a0f38609 6457 return count;
f6796cae
AD		 6458}
6459
a0f38609 6460void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6461{
a0f38609
AD		 6462 u32 count = 0, i;
6463 const struct cs_section_def *sect = NULL;
6464 const struct cs_extent_def *ext = NULL;
f6796cae 6465
a0f38609
AD		 6466 if (rdev->rlc.cs_data == NULL)
6467 return;
6468 if (buffer == NULL)
6469 return;
f6796cae 6470
6ba81e53
AD		 6471 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6472 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6473
6ba81e53
AD		 6474 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6475 buffer[count++] = cpu_to_le32(0x80000000);
6476 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6477
6478 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6479 for (ext = sect->section; ext->extent != NULL; ++ext) {
6480 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6481 buffer[count++] =
6482 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6483 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6484 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6485 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6486 } else {
6487 return;
6488 }
6489 }
6490 }
f6796cae 6491
6ba81e53
AD		 6492 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6493 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6494 switch (rdev->family) {
6495 case CHIP_BONAIRE:
6ba81e53
AD		 6496 buffer[count++] = cpu_to_le32(0x16000012);
6497 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6498 break;
6499 case CHIP_KAVERI:
6ba81e53
AD		 6500 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6501 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6502 break;
6503 case CHIP_KABINI:
6ba81e53
AD		 6504 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6505 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6506 break;
bbfe90bd
AD		 6507 case CHIP_HAWAII:
6508 buffer[count++] = 0x3a00161a;
6509 buffer[count++] = 0x0000002e;
6510 break;
a0f38609 6511 default:
6ba81e53
AD		 6512 buffer[count++] = cpu_to_le32(0x00000000);
6513 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6514 break;
6515 }
6516
6ba81e53
AD		 6517 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6518 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6519
6ba81e53
AD		 6520 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6521 buffer[count++] = cpu_to_le32(0);
a0f38609 6522}
f6796cae 6523
473359bc 6524static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6525{
473359bc 6526 if (rdev->pg_flags) {
22c775ce
AD		 6527 cik_enable_sck_slowdown_on_pu(rdev, true);
6528 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6529 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6530 cik_init_gfx_cgpg(rdev);
6531 cik_enable_cp_pg(rdev, true);
6532 cik_enable_gds_pg(rdev, true);
6533 }
22c775ce
AD		 6534 cik_init_ao_cu_mask(rdev);
6535 cik_update_gfx_pg(rdev, true);
6536 }
6537}
f6796cae 6538
473359bc
AD		 6539static void cik_fini_pg(struct radeon_device *rdev)
6540{
6541 if (rdev->pg_flags) {
6542 cik_update_gfx_pg(rdev, false);
2b19d17f 6543 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6544 cik_enable_cp_pg(rdev, false);
6545 cik_enable_gds_pg(rdev, false);
6546 }
6547 }
f6796cae 6548}
a59781bb
AD		 6549
6550/*
6551 * Interrupts
6552 * Starting with r6xx, interrupts are handled via a ring buffer.
6553 * Ring buffers are areas of GPU accessible memory that the GPU
6554 * writes interrupt vectors into and the host reads vectors out of.
6555 * There is a rptr (read pointer) that determines where the
6556 * host is currently reading, and a wptr (write pointer)
6557 * which determines where the GPU has written. When the
6558 * pointers are equal, the ring is idle. When the GPU
6559 * writes vectors to the ring buffer, it increments the
6560 * wptr. When there is an interrupt, the host then starts
6561 * fetching commands and processing them until the pointers are
6562 * equal again at which point it updates the rptr.
6563 */
6564
6565/**
6566 * cik_enable_interrupts - Enable the interrupt ring buffer
6567 *
6568 * @rdev: radeon_device pointer
6569 *
6570 * Enable the interrupt ring buffer (CIK).
6571 */
6572static void cik_enable_interrupts(struct radeon_device *rdev)
6573{
6574 u32 ih_cntl = RREG32(IH_CNTL);
6575 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6576
6577 ih_cntl |= ENABLE_INTR;
6578 ih_rb_cntl |= IH_RB_ENABLE;
6579 WREG32(IH_CNTL, ih_cntl);
6580 WREG32(IH_RB_CNTL, ih_rb_cntl);
6581 rdev->ih.enabled = true;
6582}
6583
6584/**
6585 * cik_disable_interrupts - Disable the interrupt ring buffer
6586 *
6587 * @rdev: radeon_device pointer
6588 *
6589 * Disable the interrupt ring buffer (CIK).
6590 */
6591static void cik_disable_interrupts(struct radeon_device *rdev)
6592{
6593 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6594 u32 ih_cntl = RREG32(IH_CNTL);
6595
6596 ih_rb_cntl &= ~IH_RB_ENABLE;
6597 ih_cntl &= ~ENABLE_INTR;
6598 WREG32(IH_RB_CNTL, ih_rb_cntl);
6599 WREG32(IH_CNTL, ih_cntl);
6600 /* set rptr, wptr to 0 */
6601 WREG32(IH_RB_RPTR, 0);
6602 WREG32(IH_RB_WPTR, 0);
6603 rdev->ih.enabled = false;
6604 rdev->ih.rptr = 0;
6605}
6606
6607/**
6608 * cik_disable_interrupt_state - Disable all interrupt sources
6609 *
6610 * @rdev: radeon_device pointer
6611 *
6612 * Clear all interrupt enable bits used by the driver (CIK).
6613 */
6614static void cik_disable_interrupt_state(struct radeon_device *rdev)
6615{
6616 u32 tmp;
6617
6618 /* gfx ring */
4214faf6
AD		 6619 tmp = RREG32(CP_INT_CNTL_RING0) &
6620 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6621 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6622 /* sdma */
6623 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6624 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6625 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6626 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6627 /* compute queues */
6628 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6629 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6630 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6631 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6632 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6633 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6634 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6635 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6636 /* grbm */
6637 WREG32(GRBM_INT_CNTL, 0);
6638 /* vline/vblank, etc. */
6639 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6640 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6641 if (rdev->num_crtc >= 4) {
6642 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6643 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6644 }
6645 if (rdev->num_crtc >= 6) {
6646 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6647 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6648 }
6649
6650 /* dac hotplug */
6651 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6652
6653 /* digital hotplug */
6654 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6655 WREG32(DC_HPD1_INT_CONTROL, tmp);
6656 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6657 WREG32(DC_HPD2_INT_CONTROL, tmp);
6658 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6659 WREG32(DC_HPD3_INT_CONTROL, tmp);
6660 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6661 WREG32(DC_HPD4_INT_CONTROL, tmp);
6662 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6663 WREG32(DC_HPD5_INT_CONTROL, tmp);
6664 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6665 WREG32(DC_HPD6_INT_CONTROL, tmp);
6666
6667}
6668
6669/**
6670 * cik_irq_init - init and enable the interrupt ring
6671 *
6672 * @rdev: radeon_device pointer
6673 *
6674 * Allocate a ring buffer for the interrupt controller,
6675 * enable the RLC, disable interrupts, enable the IH
6676 * ring buffer and enable it (CIK).
6677 * Called at device load and reume.
6678 * Returns 0 for success, errors for failure.
6679 */
6680static int cik_irq_init(struct radeon_device *rdev)
6681{
6682 int ret = 0;
6683 int rb_bufsz;
6684 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6685
6686 /* allocate ring */
6687 ret = r600_ih_ring_alloc(rdev);
6688 if (ret)
6689 return ret;
6690
6691 /* disable irqs */
6692 cik_disable_interrupts(rdev);
6693
6694 /* init rlc */
6695 ret = cik_rlc_resume(rdev);
6696 if (ret) {
6697 r600_ih_ring_fini(rdev);
6698 return ret;
6699 }
6700
6701 /* setup interrupt control */
6702 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6703 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6704 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6705 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6706 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6707 */
6708 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6709 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6710 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6711 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6712
6713 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6714 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6715
6716 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6717 IH_WPTR_OVERFLOW_CLEAR |
6718 (rb_bufsz << 1));
6719
6720 if (rdev->wb.enabled)
6721 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6722
6723 /* set the writeback address whether it's enabled or not */
6724 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6725 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6726
6727 WREG32(IH_RB_CNTL, ih_rb_cntl);
6728
6729 /* set rptr, wptr to 0 */
6730 WREG32(IH_RB_RPTR, 0);
6731 WREG32(IH_RB_WPTR, 0);
6732
6733 /* Default settings for IH_CNTL (disabled at first) */
6734 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6735 /* RPTR_REARM only works if msi's are enabled */
6736 if (rdev->msi_enabled)
6737 ih_cntl |= RPTR_REARM;
6738 WREG32(IH_CNTL, ih_cntl);
6739
6740 /* force the active interrupt state to all disabled */
6741 cik_disable_interrupt_state(rdev);
6742
6743 pci_set_master(rdev->pdev);
6744
6745 /* enable irqs */
6746 cik_enable_interrupts(rdev);
6747
6748 return ret;
6749}
6750
6751/**
6752 * cik_irq_set - enable/disable interrupt sources
6753 *
6754 * @rdev: radeon_device pointer
6755 *
6756 * Enable interrupt sources on the GPU (vblanks, hpd,
6757 * etc.) (CIK).
6758 * Returns 0 for success, errors for failure.
6759 */
6760int cik_irq_set(struct radeon_device *rdev)
6761{
4214faf6 6762 u32 cp_int_cntl;
2b0781a6
AD		 6763 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6764 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6765 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6766 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6767 u32 grbm_int_cntl = 0;
21a93e13 6768 u32 dma_cntl, dma_cntl1;
41a524ab 6769 u32 thermal_int;
a59781bb
AD		 6770
6771 if (!rdev->irq.installed) {
6772 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6773 return -EINVAL;
6774 }
6775 /* don't enable anything if the ih is disabled */
6776 if (!rdev->ih.enabled) {
6777 cik_disable_interrupts(rdev);
6778 /* force the active interrupt state to all disabled */
6779 cik_disable_interrupt_state(rdev);
6780 return 0;
6781 }
6782
4214faf6
AD		 6783 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6784 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6785 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6786
a59781bb
AD		 6787 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6788 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6789 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6790 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6791 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6792 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6793
21a93e13
AD		 6794 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6795 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6796
2b0781a6
AD		 6797 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6798 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6799 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6800 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6801 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6802 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6803 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6804 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6805
cc8dbbb4
AD		 6806 if (rdev->flags & RADEON_IS_IGP)
6807 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6808 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6809 else
6810 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6811 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6812
a59781bb
AD		 6813 /* enable CP interrupts on all rings */
6814 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6815 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6816 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6817 }
2b0781a6
AD		 6818 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6819 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6820 DRM_DEBUG("si_irq_set: sw int cp1\n");
6821 if (ring->me == 1) {
6822 switch (ring->pipe) {
6823 case 0:
6824 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6825 break;
6826 case 1:
6827 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6828 break;
6829 case 2:
6830 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6831 break;
6832 case 3:
6833 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6834 break;
6835 default:
6836 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6837 break;
6838 }
6839 } else if (ring->me == 2) {
6840 switch (ring->pipe) {
6841 case 0:
6842 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6843 break;
6844 case 1:
6845 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6846 break;
6847 case 2:
6848 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6849 break;
6850 case 3:
6851 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6852 break;
6853 default:
6854 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6855 break;
6856 }
6857 } else {
6858 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6859 }
6860 }
6861 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6862 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6863 DRM_DEBUG("si_irq_set: sw int cp2\n");
6864 if (ring->me == 1) {
6865 switch (ring->pipe) {
6866 case 0:
6867 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6868 break;
6869 case 1:
6870 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6871 break;
6872 case 2:
6873 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6874 break;
6875 case 3:
6876 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6877 break;
6878 default:
6879 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6880 break;
6881 }
6882 } else if (ring->me == 2) {
6883 switch (ring->pipe) {
6884 case 0:
6885 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6886 break;
6887 case 1:
6888 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6889 break;
6890 case 2:
6891 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6892 break;
6893 case 3:
6894 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6895 break;
6896 default:
6897 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6898 break;
6899 }
6900 } else {
6901 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6902 }
6903 }
a59781bb 6904
21a93e13
AD		 6905 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6906 DRM_DEBUG("cik_irq_set: sw int dma\n");
6907 dma_cntl |= TRAP_ENABLE;
6908 }
6909
6910 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6911 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6912 dma_cntl1 |= TRAP_ENABLE;
6913 }
6914
a59781bb
AD		 6915 if (rdev->irq.crtc_vblank_int[0] ||
6916 atomic_read(&rdev->irq.pflip[0])) {
6917 DRM_DEBUG("cik_irq_set: vblank 0\n");
6918 crtc1 |= VBLANK_INTERRUPT_MASK;
6919 }
6920 if (rdev->irq.crtc_vblank_int[1] ||
6921 atomic_read(&rdev->irq.pflip[1])) {
6922 DRM_DEBUG("cik_irq_set: vblank 1\n");
6923 crtc2 |= VBLANK_INTERRUPT_MASK;
6924 }
6925 if (rdev->irq.crtc_vblank_int[2] ||
6926 atomic_read(&rdev->irq.pflip[2])) {
6927 DRM_DEBUG("cik_irq_set: vblank 2\n");
6928 crtc3 |= VBLANK_INTERRUPT_MASK;
6929 }
6930 if (rdev->irq.crtc_vblank_int[3] ||
6931 atomic_read(&rdev->irq.pflip[3])) {
6932 DRM_DEBUG("cik_irq_set: vblank 3\n");
6933 crtc4 |= VBLANK_INTERRUPT_MASK;
6934 }
6935 if (rdev->irq.crtc_vblank_int[4] ||
6936 atomic_read(&rdev->irq.pflip[4])) {
6937 DRM_DEBUG("cik_irq_set: vblank 4\n");
6938 crtc5 |= VBLANK_INTERRUPT_MASK;
6939 }
6940 if (rdev->irq.crtc_vblank_int[5] ||
6941 atomic_read(&rdev->irq.pflip[5])) {
6942 DRM_DEBUG("cik_irq_set: vblank 5\n");
6943 crtc6 |= VBLANK_INTERRUPT_MASK;
6944 }
6945 if (rdev->irq.hpd[0]) {
6946 DRM_DEBUG("cik_irq_set: hpd 1\n");
6947 hpd1 |= DC_HPDx_INT_EN;
6948 }
6949 if (rdev->irq.hpd[1]) {
6950 DRM_DEBUG("cik_irq_set: hpd 2\n");
6951 hpd2 |= DC_HPDx_INT_EN;
6952 }
6953 if (rdev->irq.hpd[2]) {
6954 DRM_DEBUG("cik_irq_set: hpd 3\n");
6955 hpd3 |= DC_HPDx_INT_EN;
6956 }
6957 if (rdev->irq.hpd[3]) {
6958 DRM_DEBUG("cik_irq_set: hpd 4\n");
6959 hpd4 |= DC_HPDx_INT_EN;
6960 }
6961 if (rdev->irq.hpd[4]) {
6962 DRM_DEBUG("cik_irq_set: hpd 5\n");
6963 hpd5 |= DC_HPDx_INT_EN;
6964 }
6965 if (rdev->irq.hpd[5]) {
6966 DRM_DEBUG("cik_irq_set: hpd 6\n");
6967 hpd6 |= DC_HPDx_INT_EN;
6968 }
6969
41a524ab
AD		 6970 if (rdev->irq.dpm_thermal) {
6971 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6972 if (rdev->flags & RADEON_IS_IGP)
6973 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6974 else
6975 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6976 }
6977
a59781bb
AD		 6978 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6979
21a93e13
AD		 6980 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6981 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6982
2b0781a6
AD		 6983 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6984 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6985 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6986 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6987 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6988 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6989 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6990 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6991
a59781bb
AD		 6992 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6993
6994 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6995 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6996 if (rdev->num_crtc >= 4) {
6997 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6998 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6999 }
7000 if (rdev->num_crtc >= 6) {
7001 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7002 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7003 }
7004
7005 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7006 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7007 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7008 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7009 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7010 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7011
cc8dbbb4
AD		 7012 if (rdev->flags & RADEON_IS_IGP)
7013 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7014 else
7015 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 7016
a59781bb
AD		 7017 return 0;
7018}
7019
7020/**
7021 * cik_irq_ack - ack interrupt sources
7022 *
7023 * @rdev: radeon_device pointer
7024 *
7025 * Ack interrupt sources on the GPU (vblanks, hpd,
7026 * etc.) (CIK). Certain interrupts sources are sw
7027 * generated and do not require an explicit ack.
7028 */
7029static inline void cik_irq_ack(struct radeon_device *rdev)
7030{
7031 u32 tmp;
7032
7033 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7034 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7035 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7036 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7037 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7038 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7039 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7040
7041 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7042 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7043 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7044 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7045 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7046 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7047 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7048 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7049
7050 if (rdev->num_crtc >= 4) {
7051 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7052 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7053 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7054 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7055 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7056 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7057 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7058 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7059 }
7060
7061 if (rdev->num_crtc >= 6) {
7062 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7063 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7064 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7065 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7066 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7067 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7068 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7069 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7070 }
7071
7072 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7073 tmp = RREG32(DC_HPD1_INT_CONTROL);
7074 tmp |= DC_HPDx_INT_ACK;
7075 WREG32(DC_HPD1_INT_CONTROL, tmp);
7076 }
7077 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7078 tmp = RREG32(DC_HPD2_INT_CONTROL);
7079 tmp |= DC_HPDx_INT_ACK;
7080 WREG32(DC_HPD2_INT_CONTROL, tmp);
7081 }
7082 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7083 tmp = RREG32(DC_HPD3_INT_CONTROL);
7084 tmp |= DC_HPDx_INT_ACK;
7085 WREG32(DC_HPD3_INT_CONTROL, tmp);
7086 }
7087 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7088 tmp = RREG32(DC_HPD4_INT_CONTROL);
7089 tmp |= DC_HPDx_INT_ACK;
7090 WREG32(DC_HPD4_INT_CONTROL, tmp);
7091 }
7092 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7093 tmp = RREG32(DC_HPD5_INT_CONTROL);
7094 tmp |= DC_HPDx_INT_ACK;
7095 WREG32(DC_HPD5_INT_CONTROL, tmp);
7096 }
7097 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7098 tmp = RREG32(DC_HPD5_INT_CONTROL);
7099 tmp |= DC_HPDx_INT_ACK;
7100 WREG32(DC_HPD6_INT_CONTROL, tmp);
7101 }
7102}
7103
7104/**
7105 * cik_irq_disable - disable interrupts
7106 *
7107 * @rdev: radeon_device pointer
7108 *
7109 * Disable interrupts on the hw (CIK).
7110 */
7111static void cik_irq_disable(struct radeon_device *rdev)
7112{
7113 cik_disable_interrupts(rdev);
7114 /* Wait and acknowledge irq */
7115 mdelay(1);
7116 cik_irq_ack(rdev);
7117 cik_disable_interrupt_state(rdev);
7118}
7119
7120/**
7121 * cik_irq_disable - disable interrupts for suspend
7122 *
7123 * @rdev: radeon_device pointer
7124 *
7125 * Disable interrupts and stop the RLC (CIK).
7126 * Used for suspend.
7127 */
7128static void cik_irq_suspend(struct radeon_device *rdev)
7129{
7130 cik_irq_disable(rdev);
7131 cik_rlc_stop(rdev);
7132}
7133
7134/**
7135 * cik_irq_fini - tear down interrupt support
7136 *
7137 * @rdev: radeon_device pointer
7138 *
7139 * Disable interrupts on the hw and free the IH ring
7140 * buffer (CIK).
7141 * Used for driver unload.
7142 */
7143static void cik_irq_fini(struct radeon_device *rdev)
7144{
7145 cik_irq_suspend(rdev);
7146 r600_ih_ring_fini(rdev);
7147}
7148
7149/**
7150 * cik_get_ih_wptr - get the IH ring buffer wptr
7151 *
7152 * @rdev: radeon_device pointer
7153 *
7154 * Get the IH ring buffer wptr from either the register
7155 * or the writeback memory buffer (CIK). Also check for
7156 * ring buffer overflow and deal with it.
7157 * Used by cik_irq_process().
7158 * Returns the value of the wptr.
7159 */
7160static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7161{
7162 u32 wptr, tmp;
7163
7164 if (rdev->wb.enabled)
7165 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7166 else
7167 wptr = RREG32(IH_RB_WPTR);
7168
7169 if (wptr & RB_OVERFLOW) {
7170 /* When a ring buffer overflow happen start parsing interrupt
7171 * from the last not overwritten vector (wptr + 16). Hopefully
7172 * this should allow us to catchup.
7173 */
7174 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7175 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7176 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7177 tmp = RREG32(IH_RB_CNTL);
7178 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7179 WREG32(IH_RB_CNTL, tmp);
7180 }
7181 return (wptr & rdev->ih.ptr_mask);
7182}
7183
7184/* CIK IV Ring
7185 * Each IV ring entry is 128 bits:
7186 * [7:0] - interrupt source id
7187 * [31:8] - reserved
7188 * [59:32] - interrupt source data
7189 * [63:60] - reserved
21a93e13
AD		 7190 * [71:64] - RINGID
7191 * CP:
7192 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7193 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7194 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7195 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7196 * PIPE_ID - ME0 0=3D
7197 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7198 * SDMA:
7199 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7200 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7201 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7202 * [79:72] - VMID
7203 * [95:80] - PASID
7204 * [127:96] - reserved
7205 */
7206/**
7207 * cik_irq_process - interrupt handler
7208 *
7209 * @rdev: radeon_device pointer
7210 *
7211 * Interrupt hander (CIK). Walk the IH ring,
7212 * ack interrupts and schedule work to handle
7213 * interrupt events.
7214 * Returns irq process return code.
7215 */
7216int cik_irq_process(struct radeon_device *rdev)
7217{
2b0781a6
AD		 7218 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7219 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7220 u32 wptr;
7221 u32 rptr;
7222 u32 src_id, src_data, ring_id;
7223 u8 me_id, pipe_id, queue_id;
7224 u32 ring_index;
7225 bool queue_hotplug = false;
7226 bool queue_reset = false;
3ec7d11b 7227 u32 addr, status, mc_client;
41a524ab 7228 bool queue_thermal = false;
a59781bb
AD		 7229
7230 if (!rdev->ih.enabled || rdev->shutdown)
7231 return IRQ_NONE;
7232
7233 wptr = cik_get_ih_wptr(rdev);
7234
7235restart_ih:
7236 /* is somebody else already processing irqs? */
7237 if (atomic_xchg(&rdev->ih.lock, 1))
7238 return IRQ_NONE;
7239
7240 rptr = rdev->ih.rptr;
7241 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7242
7243 /* Order reading of wptr vs. reading of IH ring data */
7244 rmb();
7245
7246 /* display interrupts */
7247 cik_irq_ack(rdev);
7248
7249 while (rptr != wptr) {
7250 /* wptr/rptr are in bytes! */
7251 ring_index = rptr / 4;
7252 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7253 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7254 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7255
7256 switch (src_id) {
7257 case 1: /* D1 vblank/vline */
7258 switch (src_data) {
7259 case 0: /* D1 vblank */
7260 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7261 if (rdev->irq.crtc_vblank_int[0]) {
7262 drm_handle_vblank(rdev->ddev, 0);
7263 rdev->pm.vblank_sync = true;
7264 wake_up(&rdev->irq.vblank_queue);
7265 }
7266 if (atomic_read(&rdev->irq.pflip[0]))
7267 radeon_crtc_handle_flip(rdev, 0);
7268 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7269 DRM_DEBUG("IH: D1 vblank\n");
7270 }
7271 break;
7272 case 1: /* D1 vline */
7273 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7274 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7275 DRM_DEBUG("IH: D1 vline\n");
7276 }
7277 break;
7278 default:
7279 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7280 break;
7281 }
7282 break;
7283 case 2: /* D2 vblank/vline */
7284 switch (src_data) {
7285 case 0: /* D2 vblank */
7286 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7287 if (rdev->irq.crtc_vblank_int[1]) {
7288 drm_handle_vblank(rdev->ddev, 1);
7289 rdev->pm.vblank_sync = true;
7290 wake_up(&rdev->irq.vblank_queue);
7291 }
7292 if (atomic_read(&rdev->irq.pflip[1]))
7293 radeon_crtc_handle_flip(rdev, 1);
7294 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7295 DRM_DEBUG("IH: D2 vblank\n");
7296 }
7297 break;
7298 case 1: /* D2 vline */
7299 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7300 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7301 DRM_DEBUG("IH: D2 vline\n");
7302 }
7303 break;
7304 default:
7305 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7306 break;
7307 }
7308 break;
7309 case 3: /* D3 vblank/vline */
7310 switch (src_data) {
7311 case 0: /* D3 vblank */
7312 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7313 if (rdev->irq.crtc_vblank_int[2]) {
7314 drm_handle_vblank(rdev->ddev, 2);
7315 rdev->pm.vblank_sync = true;
7316 wake_up(&rdev->irq.vblank_queue);
7317 }
7318 if (atomic_read(&rdev->irq.pflip[2]))
7319 radeon_crtc_handle_flip(rdev, 2);
7320 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7321 DRM_DEBUG("IH: D3 vblank\n");
7322 }
7323 break;
7324 case 1: /* D3 vline */
7325 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7326 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7327 DRM_DEBUG("IH: D3 vline\n");
7328 }
7329 break;
7330 default:
7331 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7332 break;
7333 }
7334 break;
7335 case 4: /* D4 vblank/vline */
7336 switch (src_data) {
7337 case 0: /* D4 vblank */
7338 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7339 if (rdev->irq.crtc_vblank_int[3]) {
7340 drm_handle_vblank(rdev->ddev, 3);
7341 rdev->pm.vblank_sync = true;
7342 wake_up(&rdev->irq.vblank_queue);
7343 }
7344 if (atomic_read(&rdev->irq.pflip[3]))
7345 radeon_crtc_handle_flip(rdev, 3);
7346 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7347 DRM_DEBUG("IH: D4 vblank\n");
7348 }
7349 break;
7350 case 1: /* D4 vline */
7351 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7352 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7353 DRM_DEBUG("IH: D4 vline\n");
7354 }
7355 break;
7356 default:
7357 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7358 break;
7359 }
7360 break;
7361 case 5: /* D5 vblank/vline */
7362 switch (src_data) {
7363 case 0: /* D5 vblank */
7364 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7365 if (rdev->irq.crtc_vblank_int[4]) {
7366 drm_handle_vblank(rdev->ddev, 4);
7367 rdev->pm.vblank_sync = true;
7368 wake_up(&rdev->irq.vblank_queue);
7369 }
7370 if (atomic_read(&rdev->irq.pflip[4]))
7371 radeon_crtc_handle_flip(rdev, 4);
7372 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7373 DRM_DEBUG("IH: D5 vblank\n");
7374 }
7375 break;
7376 case 1: /* D5 vline */
7377 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7378 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7379 DRM_DEBUG("IH: D5 vline\n");
7380 }
7381 break;
7382 default:
7383 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7384 break;
7385 }
7386 break;
7387 case 6: /* D6 vblank/vline */
7388 switch (src_data) {
7389 case 0: /* D6 vblank */
7390 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7391 if (rdev->irq.crtc_vblank_int[5]) {
7392 drm_handle_vblank(rdev->ddev, 5);
7393 rdev->pm.vblank_sync = true;
7394 wake_up(&rdev->irq.vblank_queue);
7395 }
7396 if (atomic_read(&rdev->irq.pflip[5]))
7397 radeon_crtc_handle_flip(rdev, 5);
7398 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7399 DRM_DEBUG("IH: D6 vblank\n");
7400 }
7401 break;
7402 case 1: /* D6 vline */
7403 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7404 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7405 DRM_DEBUG("IH: D6 vline\n");
7406 }
7407 break;
7408 default:
7409 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7410 break;
7411 }
7412 break;
7413 case 42: /* HPD hotplug */
7414 switch (src_data) {
7415 case 0:
7416 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7417 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7418 queue_hotplug = true;
7419 DRM_DEBUG("IH: HPD1\n");
7420 }
7421 break;
7422 case 1:
7423 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7424 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7425 queue_hotplug = true;
7426 DRM_DEBUG("IH: HPD2\n");
7427 }
7428 break;
7429 case 2:
7430 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7431 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7432 queue_hotplug = true;
7433 DRM_DEBUG("IH: HPD3\n");
7434 }
7435 break;
7436 case 3:
7437 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7438 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7439 queue_hotplug = true;
7440 DRM_DEBUG("IH: HPD4\n");
7441 }
7442 break;
7443 case 4:
7444 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7445 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7446 queue_hotplug = true;
7447 DRM_DEBUG("IH: HPD5\n");
7448 }
7449 break;
7450 case 5:
7451 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7452 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7453 queue_hotplug = true;
7454 DRM_DEBUG("IH: HPD6\n");
7455 }
7456 break;
7457 default:
7458 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7459 break;
7460 }
7461 break;
6a3808b8
CK		 7462 case 124: /* UVD */
7463 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7464 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7465 break;
9d97c99b
AD		 7466 case 146:
7467 case 147:
3ec7d11b
AD		 7468 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7469 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7470 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 7471 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7472 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7473 addr);
9d97c99b 7474 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7475 status);
7476 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 7477 /* reset addr and status */
7478 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7479 break;
a59781bb
AD		 7480 case 176: /* GFX RB CP_INT */
7481 case 177: /* GFX IB CP_INT */
7482 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7483 break;
7484 case 181: /* CP EOP event */
7485 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7486 /* XXX check the bitfield order! */
7487 me_id = (ring_id & 0x60) >> 5;
7488 pipe_id = (ring_id & 0x18) >> 3;
7489 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7490 switch (me_id) {
7491 case 0:
7492 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7493 break;
7494 case 1:
a59781bb 7495 case 2:
2b0781a6
AD		 7496 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7497 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7498 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7499 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7500 break;
7501 }
7502 break;
7503 case 184: /* CP Privileged reg access */
7504 DRM_ERROR("Illegal register access in command stream\n");
7505 /* XXX check the bitfield order! */
7506 me_id = (ring_id & 0x60) >> 5;
7507 pipe_id = (ring_id & 0x18) >> 3;
7508 queue_id = (ring_id & 0x7) >> 0;
7509 switch (me_id) {
7510 case 0:
7511 /* This results in a full GPU reset, but all we need to do is soft
7512 * reset the CP for gfx
7513 */
7514 queue_reset = true;
7515 break;
7516 case 1:
7517 /* XXX compute */
2b0781a6 7518 queue_reset = true;
a59781bb
AD		 7519 break;
7520 case 2:
7521 /* XXX compute */
2b0781a6 7522 queue_reset = true;
a59781bb
AD		 7523 break;
7524 }
7525 break;
7526 case 185: /* CP Privileged inst */
7527 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 7528 /* XXX check the bitfield order! */
7529 me_id = (ring_id & 0x60) >> 5;
7530 pipe_id = (ring_id & 0x18) >> 3;
7531 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7532 switch (me_id) {
7533 case 0:
7534 /* This results in a full GPU reset, but all we need to do is soft
7535 * reset the CP for gfx
7536 */
7537 queue_reset = true;
7538 break;
7539 case 1:
7540 /* XXX compute */
2b0781a6 7541 queue_reset = true;
a59781bb
AD		 7542 break;
7543 case 2:
7544 /* XXX compute */
2b0781a6 7545 queue_reset = true;
a59781bb
AD		 7546 break;
7547 }
7548 break;
21a93e13
AD		 7549 case 224: /* SDMA trap event */
7550 /* XXX check the bitfield order! */
7551 me_id = (ring_id & 0x3) >> 0;
7552 queue_id = (ring_id & 0xc) >> 2;
7553 DRM_DEBUG("IH: SDMA trap\n");
7554 switch (me_id) {
7555 case 0:
7556 switch (queue_id) {
7557 case 0:
7558 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7559 break;
7560 case 1:
7561 /* XXX compute */
7562 break;
7563 case 2:
7564 /* XXX compute */
7565 break;
7566 }
7567 break;
7568 case 1:
7569 switch (queue_id) {
7570 case 0:
7571 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7572 break;
7573 case 1:
7574 /* XXX compute */
7575 break;
7576 case 2:
7577 /* XXX compute */
7578 break;
7579 }
7580 break;
7581 }
7582 break;
41a524ab
AD		 7583 case 230: /* thermal low to high */
7584 DRM_DEBUG("IH: thermal low to high\n");
7585 rdev->pm.dpm.thermal.high_to_low = false;
7586 queue_thermal = true;
7587 break;
7588 case 231: /* thermal high to low */
7589 DRM_DEBUG("IH: thermal high to low\n");
7590 rdev->pm.dpm.thermal.high_to_low = true;
7591 queue_thermal = true;
7592 break;
7593 case 233: /* GUI IDLE */
7594 DRM_DEBUG("IH: GUI idle\n");
7595 break;
21a93e13
AD		 7596 case 241: /* SDMA Privileged inst */
7597 case 247: /* SDMA Privileged inst */
7598 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7599 /* XXX check the bitfield order! */
7600 me_id = (ring_id & 0x3) >> 0;
7601 queue_id = (ring_id & 0xc) >> 2;
7602 switch (me_id) {
7603 case 0:
7604 switch (queue_id) {
7605 case 0:
7606 queue_reset = true;
7607 break;
7608 case 1:
7609 /* XXX compute */
7610 queue_reset = true;
7611 break;
7612 case 2:
7613 /* XXX compute */
7614 queue_reset = true;
7615 break;
7616 }
7617 break;
7618 case 1:
7619 switch (queue_id) {
7620 case 0:
7621 queue_reset = true;
7622 break;
7623 case 1:
7624 /* XXX compute */
7625 queue_reset = true;
7626 break;
7627 case 2:
7628 /* XXX compute */
7629 queue_reset = true;
7630 break;
7631 }
7632 break;
7633 }
7634 break;
a59781bb
AD		 7635 default:
7636 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7637 break;
7638 }
7639
7640 /* wptr/rptr are in bytes! */
7641 rptr += 16;
7642 rptr &= rdev->ih.ptr_mask;
7643 }
7644 if (queue_hotplug)
7645 schedule_work(&rdev->hotplug_work);
7646 if (queue_reset)
7647 schedule_work(&rdev->reset_work);
41a524ab
AD		 7648 if (queue_thermal)
7649 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 7650 rdev->ih.rptr = rptr;
7651 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7652 atomic_set(&rdev->ih.lock, 0);
7653
7654 /* make sure wptr hasn't changed while processing */
7655 wptr = cik_get_ih_wptr(rdev);
7656 if (wptr != rptr)
7657 goto restart_ih;
7658
7659 return IRQ_HANDLED;
7660}
7bf94a2c
AD		 7661
7662/*
7663 * startup/shutdown callbacks
7664 */
7665/**
7666 * cik_startup - program the asic to a functional state
7667 *
7668 * @rdev: radeon_device pointer
7669 *
7670 * Programs the asic to a functional state (CIK).
7671 * Called by cik_init() and cik_resume().
7672 * Returns 0 for success, error for failure.
7673 */
7674static int cik_startup(struct radeon_device *rdev)
7675{
7676 struct radeon_ring *ring;
7677 int r;
7678
8a7cd276
AD		 7679 /* enable pcie gen2/3 link */
7680 cik_pcie_gen3_enable(rdev);
7235711a
AD		 7681 /* enable aspm */
7682 cik_program_aspm(rdev);
8a7cd276 7683
e5903d39
AD		 7684 /* scratch needs to be initialized before MC */
7685 r = r600_vram_scratch_init(rdev);
7686 if (r)
7687 return r;
7688
6fab3feb
AD		 7689 cik_mc_program(rdev);
7690
6c7bccea 7691 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 7692 r = ci_mc_load_microcode(rdev);
7693 if (r) {
7694 DRM_ERROR("Failed to load MC firmware!\n");
7695 return r;
7696 }
7697 }
7698
7bf94a2c
AD		 7699 r = cik_pcie_gart_enable(rdev);
7700 if (r)
7701 return r;
7702 cik_gpu_init(rdev);
7703
7704 /* allocate rlc buffers */
22c775ce
AD		 7705 if (rdev->flags & RADEON_IS_IGP) {
7706 if (rdev->family == CHIP_KAVERI) {
7707 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7708 rdev->rlc.reg_list_size =
7709 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7710 } else {
7711 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7712 rdev->rlc.reg_list_size =
7713 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7714 }
7715 }
7716 rdev->rlc.cs_data = ci_cs_data;
7717 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7718 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7719 if (r) {
7720 DRM_ERROR("Failed to init rlc BOs!\n");
7721 return r;
7722 }
7723
7724 /* allocate wb buffer */
7725 r = radeon_wb_init(rdev);
7726 if (r)
7727 return r;
7728
963e81f9
AD		 7729 /* allocate mec buffers */
7730 r = cik_mec_init(rdev);
7731 if (r) {
7732 DRM_ERROR("Failed to init MEC BOs!\n");
7733 return r;
7734 }
7735
7bf94a2c
AD		 7736 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7737 if (r) {
7738 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7739 return r;
7740 }
7741
963e81f9
AD		 7742 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7743 if (r) {
7744 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7745 return r;
7746 }
7747
7748 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7749 if (r) {
7750 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7751 return r;
7752 }
7753
7bf94a2c
AD		 7754 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7755 if (r) {
7756 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7757 return r;
7758 }
7759
7760 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7761 if (r) {
7762 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7763 return r;
7764 }
7765
2ce529da 7766 r = radeon_uvd_resume(rdev);
87167bb1 7767 if (!r) {
2ce529da
AD		 7768 r = uvd_v4_2_resume(rdev);
7769 if (!r) {
7770 r = radeon_fence_driver_start_ring(rdev,
7771 R600_RING_TYPE_UVD_INDEX);
7772 if (r)
7773 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7774 }
87167bb1
CK		 7775 }
7776 if (r)
7777 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7778
7bf94a2c
AD		 7779 /* Enable IRQ */
7780 if (!rdev->irq.installed) {
7781 r = radeon_irq_kms_init(rdev);
7782 if (r)
7783 return r;
7784 }
7785
7786 r = cik_irq_init(rdev);
7787 if (r) {
7788 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7789 radeon_irq_kms_fini(rdev);
7790 return r;
7791 }
7792 cik_irq_set(rdev);
7793
7794 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7795 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1dac28eb 7796 PACKET3(PACKET3_NOP, 0x3FFF));
7bf94a2c
AD		 7797 if (r)
7798 return r;
7799
963e81f9 7800 /* set up the compute queues */
2615b53a 7801 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7802 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7803 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 7804 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7805 if (r)
7806 return r;
7807 ring->me = 1; /* first MEC */
7808 ring->pipe = 0; /* first pipe */
7809 ring->queue = 0; /* first queue */
7810 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7811
2615b53a 7812 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7813 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7814 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 7815 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7816 if (r)
7817 return r;
7818 /* dGPU only have 1 MEC */
7819 ring->me = 1; /* first MEC */
7820 ring->pipe = 0; /* first pipe */
7821 ring->queue = 1; /* second queue */
7822 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7823
7bf94a2c
AD		 7824 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7825 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 7826 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7827 if (r)
7828 return r;
7829
7830 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7831 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 7832 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7833 if (r)
7834 return r;
7835
7836 r = cik_cp_resume(rdev);
7837 if (r)
7838 return r;
7839
7840 r = cik_sdma_resume(rdev);
7841 if (r)
7842 return r;
7843
87167bb1
CK		 7844 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7845 if (ring->ring_size) {
02c9f7fa 7846 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 7847 RADEON_CP_PACKET2);
87167bb1 7848 if (!r)
e409b128 7849 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7850 if (r)
7851 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7852 }
7853
7bf94a2c
AD		 7854 r = radeon_ib_pool_init(rdev);
7855 if (r) {
7856 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7857 return r;
7858 }
7859
7860 r = radeon_vm_manager_init(rdev);
7861 if (r) {
7862 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7863 return r;
7864 }
7865
b530602f
AD		 7866 r = dce6_audio_init(rdev);
7867 if (r)
7868 return r;
7869
7bf94a2c
AD		 7870 return 0;
7871}
7872
7873/**
7874 * cik_resume - resume the asic to a functional state
7875 *
7876 * @rdev: radeon_device pointer
7877 *
7878 * Programs the asic to a functional state (CIK).
7879 * Called at resume.
7880 * Returns 0 for success, error for failure.
7881 */
7882int cik_resume(struct radeon_device *rdev)
7883{
7884 int r;
7885
7886 /* post card */
7887 atom_asic_init(rdev->mode_info.atom_context);
7888
0aafd313
AD		 7889 /* init golden registers */
7890 cik_init_golden_registers(rdev);
7891
6c7bccea
AD		 7892 radeon_pm_resume(rdev);
7893
7bf94a2c
AD		 7894 rdev->accel_working = true;
7895 r = cik_startup(rdev);
7896 if (r) {
7897 DRM_ERROR("cik startup failed on resume\n");
7898 rdev->accel_working = false;
7899 return r;
7900 }
7901
7902 return r;
7903
7904}
7905
7906/**
7907 * cik_suspend - suspend the asic
7908 *
7909 * @rdev: radeon_device pointer
7910 *
7911 * Bring the chip into a state suitable for suspend (CIK).
7912 * Called at suspend.
7913 * Returns 0 for success.
7914 */
7915int cik_suspend(struct radeon_device *rdev)
7916{
6c7bccea 7917 radeon_pm_suspend(rdev);
b530602f 7918 dce6_audio_fini(rdev);
7bf94a2c
AD		 7919 radeon_vm_manager_fini(rdev);
7920 cik_cp_enable(rdev, false);
7921 cik_sdma_enable(rdev, false);
e409b128 7922 uvd_v1_0_fini(rdev);
87167bb1 7923 radeon_uvd_suspend(rdev);
473359bc
AD		 7924 cik_fini_pg(rdev);
7925 cik_fini_cg(rdev);
7bf94a2c
AD		 7926 cik_irq_suspend(rdev);
7927 radeon_wb_disable(rdev);
7928 cik_pcie_gart_disable(rdev);
7929 return 0;
7930}
7931
7932/* Plan is to move initialization in that function and use
7933 * helper function so that radeon_device_init pretty much
7934 * do nothing more than calling asic specific function. This
7935 * should also allow to remove a bunch of callback function
7936 * like vram_info.
7937 */
7938/**
7939 * cik_init - asic specific driver and hw init
7940 *
7941 * @rdev: radeon_device pointer
7942 *
7943 * Setup asic specific driver variables and program the hw
7944 * to a functional state (CIK).
7945 * Called at driver startup.
7946 * Returns 0 for success, errors for failure.
7947 */
7948int cik_init(struct radeon_device *rdev)
7949{
7950 struct radeon_ring *ring;
7951 int r;
7952
7953 /* Read BIOS */
7954 if (!radeon_get_bios(rdev)) {
7955 if (ASIC_IS_AVIVO(rdev))
7956 return -EINVAL;
7957 }
7958 /* Must be an ATOMBIOS */
7959 if (!rdev->is_atom_bios) {
7960 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7961 return -EINVAL;
7962 }
7963 r = radeon_atombios_init(rdev);
7964 if (r)
7965 return r;
7966
7967 /* Post card if necessary */
7968 if (!radeon_card_posted(rdev)) {
7969 if (!rdev->bios) {
7970 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7971 return -EINVAL;
7972 }
7973 DRM_INFO("GPU not posted. posting now...\n");
7974 atom_asic_init(rdev->mode_info.atom_context);
7975 }
0aafd313
AD		 7976 /* init golden registers */
7977 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7978 /* Initialize scratch registers */
7979 cik_scratch_init(rdev);
7980 /* Initialize surface registers */
7981 radeon_surface_init(rdev);
7982 /* Initialize clocks */
7983 radeon_get_clock_info(rdev->ddev);
7984
7985 /* Fence driver */
7986 r = radeon_fence_driver_init(rdev);
7987 if (r)
7988 return r;
7989
7990 /* initialize memory controller */
7991 r = cik_mc_init(rdev);
7992 if (r)
7993 return r;
7994 /* Memory manager */
7995 r = radeon_bo_init(rdev);
7996 if (r)
7997 return r;
7998
01ac8794
AD		 7999 if (rdev->flags & RADEON_IS_IGP) {
8000 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8001 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8002 r = cik_init_microcode(rdev);
8003 if (r) {
8004 DRM_ERROR("Failed to load firmware!\n");
8005 return r;
8006 }
8007 }
8008 } else {
8009 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8010 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8011 !rdev->mc_fw) {
8012 r = cik_init_microcode(rdev);
8013 if (r) {
8014 DRM_ERROR("Failed to load firmware!\n");
8015 return r;
8016 }
8017 }
8018 }
8019
6c7bccea
AD		 8020 /* Initialize power management */
8021 radeon_pm_init(rdev);
8022
7bf94a2c
AD		 8023 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8024 ring->ring_obj = NULL;
8025 r600_ring_init(rdev, ring, 1024 * 1024);
8026
963e81f9
AD		 8027 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8028 ring->ring_obj = NULL;
8029 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8030 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8031 if (r)
8032 return r;
8033
8034 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8035 ring->ring_obj = NULL;
8036 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8037 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8038 if (r)
8039 return r;
8040
7bf94a2c
AD		 8041 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8042 ring->ring_obj = NULL;
8043 r600_ring_init(rdev, ring, 256 * 1024);
8044
8045 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8046 ring->ring_obj = NULL;
8047 r600_ring_init(rdev, ring, 256 * 1024);
8048
87167bb1
CK		 8049 r = radeon_uvd_init(rdev);
8050 if (!r) {
8051 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8052 ring->ring_obj = NULL;
8053 r600_ring_init(rdev, ring, 4096);
8054 }
8055
7bf94a2c
AD		 8056 rdev->ih.ring_obj = NULL;
8057 r600_ih_ring_init(rdev, 64 * 1024);
8058
8059 r = r600_pcie_gart_init(rdev);
8060 if (r)
8061 return r;
8062
8063 rdev->accel_working = true;
8064 r = cik_startup(rdev);
8065 if (r) {
8066 dev_err(rdev->dev, "disabling GPU acceleration\n");
8067 cik_cp_fini(rdev);
8068 cik_sdma_fini(rdev);
8069 cik_irq_fini(rdev);
1fd11777 8070 sumo_rlc_fini(rdev);
963e81f9 8071 cik_mec_fini(rdev);
7bf94a2c
AD		 8072 radeon_wb_fini(rdev);
8073 radeon_ib_pool_fini(rdev);
8074 radeon_vm_manager_fini(rdev);
8075 radeon_irq_kms_fini(rdev);
8076 cik_pcie_gart_fini(rdev);
8077 rdev->accel_working = false;
8078 }
8079
8080 /* Don't start up if the MC ucode is missing.
8081 * The default clocks and voltages before the MC ucode
8082 * is loaded are not suffient for advanced operations.
8083 */
8084 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8085 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8086 return -EINVAL;
8087 }
8088
8089 return 0;
8090}
8091
8092/**
8093 * cik_fini - asic specific driver and hw fini
8094 *
8095 * @rdev: radeon_device pointer
8096 *
8097 * Tear down the asic specific driver variables and program the hw
8098 * to an idle state (CIK).
8099 * Called at driver unload.
8100 */
8101void cik_fini(struct radeon_device *rdev)
8102{
6c7bccea 8103 radeon_pm_fini(rdev);
7bf94a2c
AD		 8104 cik_cp_fini(rdev);
8105 cik_sdma_fini(rdev);
473359bc
AD		 8106 cik_fini_pg(rdev);
8107 cik_fini_cg(rdev);
7bf94a2c 8108 cik_irq_fini(rdev);
1fd11777 8109 sumo_rlc_fini(rdev);
963e81f9 8110 cik_mec_fini(rdev);
7bf94a2c
AD		 8111 radeon_wb_fini(rdev);
8112 radeon_vm_manager_fini(rdev);
8113 radeon_ib_pool_fini(rdev);
8114 radeon_irq_kms_fini(rdev);
e409b128 8115 uvd_v1_0_fini(rdev);
87167bb1 8116 radeon_uvd_fini(rdev);
7bf94a2c
AD		 8117 cik_pcie_gart_fini(rdev);
8118 r600_vram_scratch_fini(rdev);
8119 radeon_gem_fini(rdev);
8120 radeon_fence_driver_fini(rdev);
8121 radeon_bo_fini(rdev);
8122 radeon_atombios_fini(rdev);
8123 kfree(rdev->bios);
8124 rdev->bios = NULL;
8125}
cd84a27d 8126
134b480f
AD		 8127void dce8_program_fmt(struct drm_encoder *encoder)
8128{
8129 struct drm_device *dev = encoder->dev;
8130 struct radeon_device *rdev = dev->dev_private;
8131 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8132 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8133 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8134 int bpc = 0;
8135 u32 tmp = 0;
6214bb74 8136 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8137
6214bb74
AD		 8138 if (connector) {
8139 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8140 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8141 dither = radeon_connector->dither;
8142 }
134b480f
AD		 8143
8144 /* LVDS/eDP FMT is set up by atom */
8145 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8146 return;
8147
8148 /* not needed for analog */
8149 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8150 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8151 return;
8152
8153 if (bpc == 0)
8154 return;
8155
8156 switch (bpc) {
8157 case 6:
6214bb74 8158 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8159 /* XXX sort out optimal dither settings */
8160 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8161 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8162 else
8163 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8164 break;
8165 case 8:
6214bb74 8166 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8167 /* XXX sort out optimal dither settings */
8168 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8169 FMT_RGB_RANDOM_ENABLE |
8170 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8171 else
8172 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8173 break;
8174 case 10:
6214bb74 8175 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8176 /* XXX sort out optimal dither settings */
8177 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8178 FMT_RGB_RANDOM_ENABLE |
8179 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8180 else
8181 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8182 break;
8183 default:
8184 /* not needed */
8185 break;
8186 }
8187
8188 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8189}
8190
cd84a27d
AD		 8191/* display watermark setup */
8192/**
8193 * dce8_line_buffer_adjust - Set up the line buffer
8194 *
8195 * @rdev: radeon_device pointer
8196 * @radeon_crtc: the selected display controller
8197 * @mode: the current display mode on the selected display
8198 * controller
8199 *
8200 * Setup up the line buffer allocation for
8201 * the selected display controller (CIK).
8202 * Returns the line buffer size in pixels.
8203 */
8204static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8205 struct radeon_crtc *radeon_crtc,
8206 struct drm_display_mode *mode)
8207{
bc01a8c7
AD		 8208 u32 tmp, buffer_alloc, i;
8209 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8210 /*
8211 * Line Buffer Setup
8212 * There are 6 line buffers, one for each display controllers.
8213 * There are 3 partitions per LB. Select the number of partitions
8214 * to enable based on the display width. For display widths larger
8215 * than 4096, you need use to use 2 display controllers and combine
8216 * them using the stereo blender.
8217 */
8218 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8219 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8220 tmp = 1;
bc01a8c7
AD		 8221 buffer_alloc = 2;
8222 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8223 tmp = 2;
bc01a8c7
AD		 8224 buffer_alloc = 2;
8225 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8226 tmp = 0;
bc01a8c7
AD		 8227 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8228 } else {
cd84a27d
AD		 8229 DRM_DEBUG_KMS("Mode too big for LB!\n");
8230 tmp = 0;
bc01a8c7 8231 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8232 }
bc01a8c7 8233 } else {
cd84a27d 8234 tmp = 1;
bc01a8c7
AD		 8235 buffer_alloc = 0;
8236 }
cd84a27d
AD		 8237
8238 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8239 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8240
bc01a8c7
AD		 8241 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8242 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8243 for (i = 0; i < rdev->usec_timeout; i++) {
8244 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8245 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8246 break;
8247 udelay(1);
8248 }
8249
cd84a27d
AD		 8250 if (radeon_crtc->base.enabled && mode) {
8251 switch (tmp) {
8252 case 0:
8253 default:
8254 return 4096 * 2;
8255 case 1:
8256 return 1920 * 2;
8257 case 2:
8258 return 2560 * 2;
8259 }
8260 }
8261
8262 /* controller not enabled, so no lb used */
8263 return 0;
8264}
8265
8266/**
8267 * cik_get_number_of_dram_channels - get the number of dram channels
8268 *
8269 * @rdev: radeon_device pointer
8270 *
8271 * Look up the number of video ram channels (CIK).
8272 * Used for display watermark bandwidth calculations
8273 * Returns the number of dram channels
8274 */
8275static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8276{
8277 u32 tmp = RREG32(MC_SHARED_CHMAP);
8278
8279 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8280 case 0:
8281 default:
8282 return 1;
8283 case 1:
8284 return 2;
8285 case 2:
8286 return 4;
8287 case 3:
8288 return 8;
8289 case 4:
8290 return 3;
8291 case 5:
8292 return 6;
8293 case 6:
8294 return 10;
8295 case 7:
8296 return 12;
8297 case 8:
8298 return 16;
8299 }
8300}
8301
8302struct dce8_wm_params {
8303 u32 dram_channels; /* number of dram channels */
8304 u32 yclk; /* bandwidth per dram data pin in kHz */
8305 u32 sclk; /* engine clock in kHz */
8306 u32 disp_clk; /* display clock in kHz */
8307 u32 src_width; /* viewport width */
8308 u32 active_time; /* active display time in ns */
8309 u32 blank_time; /* blank time in ns */
8310 bool interlaced; /* mode is interlaced */
8311 fixed20_12 vsc; /* vertical scale ratio */
8312 u32 num_heads; /* number of active crtcs */
8313 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8314 u32 lb_size; /* line buffer allocated to pipe */
8315 u32 vtaps; /* vertical scaler taps */
8316};
8317
8318/**
8319 * dce8_dram_bandwidth - get the dram bandwidth
8320 *
8321 * @wm: watermark calculation data
8322 *
8323 * Calculate the raw dram bandwidth (CIK).
8324 * Used for display watermark bandwidth calculations
8325 * Returns the dram bandwidth in MBytes/s
8326 */
8327static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8328{
8329 /* Calculate raw DRAM Bandwidth */
8330 fixed20_12 dram_efficiency; /* 0.7 */
8331 fixed20_12 yclk, dram_channels, bandwidth;
8332 fixed20_12 a;
8333
8334 a.full = dfixed_const(1000);
8335 yclk.full = dfixed_const(wm->yclk);
8336 yclk.full = dfixed_div(yclk, a);
8337 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8338 a.full = dfixed_const(10);
8339 dram_efficiency.full = dfixed_const(7);
8340 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8341 bandwidth.full = dfixed_mul(dram_channels, yclk);
8342 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8343
8344 return dfixed_trunc(bandwidth);
8345}
8346
8347/**
8348 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8349 *
8350 * @wm: watermark calculation data
8351 *
8352 * Calculate the dram bandwidth used for display (CIK).
8353 * Used for display watermark bandwidth calculations
8354 * Returns the dram bandwidth for display in MBytes/s
8355 */
8356static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8357{
8358 /* Calculate DRAM Bandwidth and the part allocated to display. */
8359 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8360 fixed20_12 yclk, dram_channels, bandwidth;
8361 fixed20_12 a;
8362
8363 a.full = dfixed_const(1000);
8364 yclk.full = dfixed_const(wm->yclk);
8365 yclk.full = dfixed_div(yclk, a);
8366 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8367 a.full = dfixed_const(10);
8368 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8369 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8370 bandwidth.full = dfixed_mul(dram_channels, yclk);
8371 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8372
8373 return dfixed_trunc(bandwidth);
8374}
8375
8376/**
8377 * dce8_data_return_bandwidth - get the data return bandwidth
8378 *
8379 * @wm: watermark calculation data
8380 *
8381 * Calculate the data return bandwidth used for display (CIK).
8382 * Used for display watermark bandwidth calculations
8383 * Returns the data return bandwidth in MBytes/s
8384 */
8385static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8386{
8387 /* Calculate the display Data return Bandwidth */
8388 fixed20_12 return_efficiency; /* 0.8 */
8389 fixed20_12 sclk, bandwidth;
8390 fixed20_12 a;
8391
8392 a.full = dfixed_const(1000);
8393 sclk.full = dfixed_const(wm->sclk);
8394 sclk.full = dfixed_div(sclk, a);
8395 a.full = dfixed_const(10);
8396 return_efficiency.full = dfixed_const(8);
8397 return_efficiency.full = dfixed_div(return_efficiency, a);
8398 a.full = dfixed_const(32);
8399 bandwidth.full = dfixed_mul(a, sclk);
8400 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8401
8402 return dfixed_trunc(bandwidth);
8403}
8404
8405/**
8406 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8407 *
8408 * @wm: watermark calculation data
8409 *
8410 * Calculate the dmif bandwidth used for display (CIK).
8411 * Used for display watermark bandwidth calculations
8412 * Returns the dmif bandwidth in MBytes/s
8413 */
8414static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8415{
8416 /* Calculate the DMIF Request Bandwidth */
8417 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8418 fixed20_12 disp_clk, bandwidth;
8419 fixed20_12 a, b;
8420
8421 a.full = dfixed_const(1000);
8422 disp_clk.full = dfixed_const(wm->disp_clk);
8423 disp_clk.full = dfixed_div(disp_clk, a);
8424 a.full = dfixed_const(32);
8425 b.full = dfixed_mul(a, disp_clk);
8426
8427 a.full = dfixed_const(10);
8428 disp_clk_request_efficiency.full = dfixed_const(8);
8429 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8430
8431 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8432
8433 return dfixed_trunc(bandwidth);
8434}
8435
8436/**
8437 * dce8_available_bandwidth - get the min available bandwidth
8438 *
8439 * @wm: watermark calculation data
8440 *
8441 * Calculate the min available bandwidth used for display (CIK).
8442 * Used for display watermark bandwidth calculations
8443 * Returns the min available bandwidth in MBytes/s
8444 */
8445static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8446{
8447 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8448 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8449 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8450 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8451
8452 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8453}
8454
8455/**
8456 * dce8_average_bandwidth - get the average available bandwidth
8457 *
8458 * @wm: watermark calculation data
8459 *
8460 * Calculate the average available bandwidth used for display (CIK).
8461 * Used for display watermark bandwidth calculations
8462 * Returns the average available bandwidth in MBytes/s
8463 */
8464static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8465{
8466 /* Calculate the display mode Average Bandwidth
8467 * DisplayMode should contain the source and destination dimensions,
8468 * timing, etc.
8469 */
8470 fixed20_12 bpp;
8471 fixed20_12 line_time;
8472 fixed20_12 src_width;
8473 fixed20_12 bandwidth;
8474 fixed20_12 a;
8475
8476 a.full = dfixed_const(1000);
8477 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8478 line_time.full = dfixed_div(line_time, a);
8479 bpp.full = dfixed_const(wm->bytes_per_pixel);
8480 src_width.full = dfixed_const(wm->src_width);
8481 bandwidth.full = dfixed_mul(src_width, bpp);
8482 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8483 bandwidth.full = dfixed_div(bandwidth, line_time);
8484
8485 return dfixed_trunc(bandwidth);
8486}
8487
8488/**
8489 * dce8_latency_watermark - get the latency watermark
8490 *
8491 * @wm: watermark calculation data
8492 *
8493 * Calculate the latency watermark (CIK).
8494 * Used for display watermark bandwidth calculations
8495 * Returns the latency watermark in ns
8496 */
8497static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8498{
8499 /* First calculate the latency in ns */
8500 u32 mc_latency = 2000; /* 2000 ns. */
8501 u32 available_bandwidth = dce8_available_bandwidth(wm);
8502 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8503 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8504 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8505 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8506 (wm->num_heads * cursor_line_pair_return_time);
8507 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8508 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8509 u32 tmp, dmif_size = 12288;
8510 fixed20_12 a, b, c;
8511
8512 if (wm->num_heads == 0)
8513 return 0;
8514
8515 a.full = dfixed_const(2);
8516 b.full = dfixed_const(1);
8517 if ((wm->vsc.full > a.full) ||
8518 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8519 (wm->vtaps >= 5) ||
8520 ((wm->vsc.full >= a.full) && wm->interlaced))
8521 max_src_lines_per_dst_line = 4;
8522 else
8523 max_src_lines_per_dst_line = 2;
8524
8525 a.full = dfixed_const(available_bandwidth);
8526 b.full = dfixed_const(wm->num_heads);
8527 a.full = dfixed_div(a, b);
8528
8529 b.full = dfixed_const(mc_latency + 512);
8530 c.full = dfixed_const(wm->disp_clk);
8531 b.full = dfixed_div(b, c);
8532
8533 c.full = dfixed_const(dmif_size);
8534 b.full = dfixed_div(c, b);
8535
8536 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8537
8538 b.full = dfixed_const(1000);
8539 c.full = dfixed_const(wm->disp_clk);
8540 b.full = dfixed_div(c, b);
8541 c.full = dfixed_const(wm->bytes_per_pixel);
8542 b.full = dfixed_mul(b, c);
8543
8544 lb_fill_bw = min(tmp, dfixed_trunc(b));
8545
8546 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8547 b.full = dfixed_const(1000);
8548 c.full = dfixed_const(lb_fill_bw);
8549 b.full = dfixed_div(c, b);
8550 a.full = dfixed_div(a, b);
8551 line_fill_time = dfixed_trunc(a);
8552
8553 if (line_fill_time < wm->active_time)
8554 return latency;
8555 else
8556 return latency + (line_fill_time - wm->active_time);
8557
8558}
8559
8560/**
8561 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8562 * average and available dram bandwidth
8563 *
8564 * @wm: watermark calculation data
8565 *
8566 * Check if the display average bandwidth fits in the display
8567 * dram bandwidth (CIK).
8568 * Used for display watermark bandwidth calculations
8569 * Returns true if the display fits, false if not.
8570 */
8571static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8572{
8573 if (dce8_average_bandwidth(wm) <=
8574 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8575 return true;
8576 else
8577 return false;
8578}
8579
8580/**
8581 * dce8_average_bandwidth_vs_available_bandwidth - check
8582 * average and available bandwidth
8583 *
8584 * @wm: watermark calculation data
8585 *
8586 * Check if the display average bandwidth fits in the display
8587 * available bandwidth (CIK).
8588 * Used for display watermark bandwidth calculations
8589 * Returns true if the display fits, false if not.
8590 */
8591static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8592{
8593 if (dce8_average_bandwidth(wm) <=
8594 (dce8_available_bandwidth(wm) / wm->num_heads))
8595 return true;
8596 else
8597 return false;
8598}
8599
8600/**
8601 * dce8_check_latency_hiding - check latency hiding
8602 *
8603 * @wm: watermark calculation data
8604 *
8605 * Check latency hiding (CIK).
8606 * Used for display watermark bandwidth calculations
8607 * Returns true if the display fits, false if not.
8608 */
8609static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8610{
8611 u32 lb_partitions = wm->lb_size / wm->src_width;
8612 u32 line_time = wm->active_time + wm->blank_time;
8613 u32 latency_tolerant_lines;
8614 u32 latency_hiding;
8615 fixed20_12 a;
8616
8617 a.full = dfixed_const(1);
8618 if (wm->vsc.full > a.full)
8619 latency_tolerant_lines = 1;
8620 else {
8621 if (lb_partitions <= (wm->vtaps + 1))
8622 latency_tolerant_lines = 1;
8623 else
8624 latency_tolerant_lines = 2;
8625 }
8626
8627 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8628
8629 if (dce8_latency_watermark(wm) <= latency_hiding)
8630 return true;
8631 else
8632 return false;
8633}
8634
8635/**
8636 * dce8_program_watermarks - program display watermarks
8637 *
8638 * @rdev: radeon_device pointer
8639 * @radeon_crtc: the selected display controller
8640 * @lb_size: line buffer size
8641 * @num_heads: number of display controllers in use
8642 *
8643 * Calculate and program the display watermarks for the
8644 * selected display controller (CIK).
8645 */
8646static void dce8_program_watermarks(struct radeon_device *rdev,
8647 struct radeon_crtc *radeon_crtc,
8648 u32 lb_size, u32 num_heads)
8649{
8650 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 8651 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 8652 u32 pixel_period;
8653 u32 line_time = 0;
8654 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8655 u32 tmp, wm_mask;
8656
8657 if (radeon_crtc->base.enabled && num_heads && mode) {
8658 pixel_period = 1000000 / (u32)mode->clock;
8659 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8660
58ea2dea
AD		 8661 /* watermark for high clocks */
8662 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8663 rdev->pm.dpm_enabled) {
8664 wm_high.yclk =
8665 radeon_dpm_get_mclk(rdev, false) * 10;
8666 wm_high.sclk =
8667 radeon_dpm_get_sclk(rdev, false) * 10;
8668 } else {
8669 wm_high.yclk = rdev->pm.current_mclk * 10;
8670 wm_high.sclk = rdev->pm.current_sclk * 10;
8671 }
8672
8673 wm_high.disp_clk = mode->clock;
8674 wm_high.src_width = mode->crtc_hdisplay;
8675 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8676 wm_high.blank_time = line_time - wm_high.active_time;
8677 wm_high.interlaced = false;
cd84a27d 8678 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 8679 wm_high.interlaced = true;
8680 wm_high.vsc = radeon_crtc->vsc;
8681 wm_high.vtaps = 1;
cd84a27d 8682 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 8683 wm_high.vtaps = 2;
8684 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8685 wm_high.lb_size = lb_size;
8686 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8687 wm_high.num_heads = num_heads;
cd84a27d
AD		 8688
8689 /* set for high clocks */
58ea2dea
AD		 8690 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8691
8692 /* possibly force display priority to high */
8693 /* should really do this at mode validation time... */
8694 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8695 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8696 !dce8_check_latency_hiding(&wm_high) ||
8697 (rdev->disp_priority == 2)) {
8698 DRM_DEBUG_KMS("force priority to high\n");
8699 }
8700
8701 /* watermark for low clocks */
8702 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8703 rdev->pm.dpm_enabled) {
8704 wm_low.yclk =
8705 radeon_dpm_get_mclk(rdev, true) * 10;
8706 wm_low.sclk =
8707 radeon_dpm_get_sclk(rdev, true) * 10;
8708 } else {
8709 wm_low.yclk = rdev->pm.current_mclk * 10;
8710 wm_low.sclk = rdev->pm.current_sclk * 10;
8711 }
8712
8713 wm_low.disp_clk = mode->clock;
8714 wm_low.src_width = mode->crtc_hdisplay;
8715 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8716 wm_low.blank_time = line_time - wm_low.active_time;
8717 wm_low.interlaced = false;
8718 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8719 wm_low.interlaced = true;
8720 wm_low.vsc = radeon_crtc->vsc;
8721 wm_low.vtaps = 1;
8722 if (radeon_crtc->rmx_type != RMX_OFF)
8723 wm_low.vtaps = 2;
8724 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8725 wm_low.lb_size = lb_size;
8726 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8727 wm_low.num_heads = num_heads;
8728
cd84a27d 8729 /* set for low clocks */
58ea2dea 8730 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 8731
8732 /* possibly force display priority to high */
8733 /* should really do this at mode validation time... */
58ea2dea
AD		 8734 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8735 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8736 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 8737 (rdev->disp_priority == 2)) {
8738 DRM_DEBUG_KMS("force priority to high\n");
8739 }
8740 }
8741
8742 /* select wm A */
8743 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8744 tmp = wm_mask;
8745 tmp &= ~LATENCY_WATERMARK_MASK(3);
8746 tmp |= LATENCY_WATERMARK_MASK(1);
8747 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8748 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8749 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8750 LATENCY_HIGH_WATERMARK(line_time)));
8751 /* select wm B */
8752 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8753 tmp &= ~LATENCY_WATERMARK_MASK(3);
8754 tmp |= LATENCY_WATERMARK_MASK(2);
8755 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8756 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8757 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8758 LATENCY_HIGH_WATERMARK(line_time)));
8759 /* restore original selection */
8760 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 8761
8762 /* save values for DPM */
8763 radeon_crtc->line_time = line_time;
8764 radeon_crtc->wm_high = latency_watermark_a;
8765 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 8766}
8767
8768/**
8769 * dce8_bandwidth_update - program display watermarks
8770 *
8771 * @rdev: radeon_device pointer
8772 *
8773 * Calculate and program the display watermarks and line
8774 * buffer allocation (CIK).
8775 */
8776void dce8_bandwidth_update(struct radeon_device *rdev)
8777{
8778 struct drm_display_mode *mode = NULL;
8779 u32 num_heads = 0, lb_size;
8780 int i;
8781
8782 radeon_update_display_priority(rdev);
8783
8784 for (i = 0; i < rdev->num_crtc; i++) {
8785 if (rdev->mode_info.crtcs[i]->base.enabled)
8786 num_heads++;
8787 }
8788 for (i = 0; i < rdev->num_crtc; i++) {
8789 mode = &rdev->mode_info.crtcs[i]->base.mode;
8790 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8791 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8792 }
8793}
44fa346f
AD		 8794
8795/**
8796 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8797 *
8798 * @rdev: radeon_device pointer
8799 *
8800 * Fetches a GPU clock counter snapshot (SI).
8801 * Returns the 64 bit clock counter snapshot.
8802 */
8803uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8804{
8805 uint64_t clock;
8806
8807 mutex_lock(&rdev->gpu_clock_mutex);
8808 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8809 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8810 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8811 mutex_unlock(&rdev->gpu_clock_mutex);
8812 return clock;
8813}
8814
87167bb1
CK		 8815static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8816 u32 cntl_reg, u32 status_reg)
8817{
8818 int r, i;
8819 struct atom_clock_dividers dividers;
8820 uint32_t tmp;
8821
8822 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8823 clock, false, &dividers);
8824 if (r)
8825 return r;
8826
8827 tmp = RREG32_SMC(cntl_reg);
8828 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8829 tmp |= dividers.post_divider;
8830 WREG32_SMC(cntl_reg, tmp);
8831
8832 for (i = 0; i < 100; i++) {
8833 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8834 break;
8835 mdelay(10);
8836 }
8837 if (i == 100)
8838 return -ETIMEDOUT;
8839
8840 return 0;
8841}
8842
8843int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8844{
8845 int r = 0;
8846
8847 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8848 if (r)
8849 return r;
8850
8851 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8852 return r;
8853}
8854
8a7cd276 8855static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 8856{
8a7cd276
AD		 8857 struct pci_dev *root = rdev->pdev->bus->self;
8858 int bridge_pos, gpu_pos;
8859 u32 speed_cntl, mask, current_data_rate;
8860 int ret, i;
8861 u16 tmp16;
87167bb1 8862
8a7cd276
AD		 8863 if (radeon_pcie_gen2 == 0)
8864 return;
87167bb1 8865
8a7cd276
AD		 8866 if (rdev->flags & RADEON_IS_IGP)
8867 return;
87167bb1 8868
8a7cd276
AD		 8869 if (!(rdev->flags & RADEON_IS_PCIE))
8870 return;
87167bb1 8871
8a7cd276
AD		 8872 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8873 if (ret != 0)
8874 return;
87167bb1 8875
8a7cd276
AD		 8876 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8877 return;
87167bb1 8878
8a7cd276
AD		 8879 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8880 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8881 LC_CURRENT_DATA_RATE_SHIFT;
8882 if (mask & DRM_PCIE_SPEED_80) {
8883 if (current_data_rate == 2) {
8884 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8885 return;
8886 }
8887 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8888 } else if (mask & DRM_PCIE_SPEED_50) {
8889 if (current_data_rate == 1) {
8890 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8891 return;
8892 }
8893 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8894 }
87167bb1 8895
8a7cd276
AD		 8896 bridge_pos = pci_pcie_cap(root);
8897 if (!bridge_pos)
8898 return;
8899
8900 gpu_pos = pci_pcie_cap(rdev->pdev);
8901 if (!gpu_pos)
8902 return;
8903
8904 if (mask & DRM_PCIE_SPEED_80) {
8905 /* re-try equalization if gen3 is not already enabled */
8906 if (current_data_rate != 2) {
8907 u16 bridge_cfg, gpu_cfg;
8908 u16 bridge_cfg2, gpu_cfg2;
8909 u32 max_lw, current_lw, tmp;
8910
8911 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8912 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8913
8914 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8915 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8916
8917 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8918 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8919
8920 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8921 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8922 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8923
8924 if (current_lw < max_lw) {
8925 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8926 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8927 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8928 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8929 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8930 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8931 }
8932 }
8933
8934 for (i = 0; i < 10; i++) {
8935 /* check status */
8936 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8937 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8938 break;
8939
8940 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8941 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8942
8943 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8944 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8945
8946 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8947 tmp |= LC_SET_QUIESCE;
8948 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8949
8950 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8951 tmp |= LC_REDO_EQ;
8952 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8953
8954 mdelay(100);
8955
8956 /* linkctl */
8957 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8958 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8959 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8960 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8961
8962 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8963 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8964 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8965 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8966
8967 /* linkctl2 */
8968 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8969 tmp16 &= ~((1 << 4) | (7 << 9));
8970 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8971 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8972
8973 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8974 tmp16 &= ~((1 << 4) | (7 << 9));
8975 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8976 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8977
8978 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8979 tmp &= ~LC_SET_QUIESCE;
8980 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8981 }
8982 }
8983 }
8984
8985 /* set the link speed */
8986 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8987 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8988 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8989
8990 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8991 tmp16 &= ~0xf;
8992 if (mask & DRM_PCIE_SPEED_80)
8993 tmp16 |= 3; /* gen3 */
8994 else if (mask & DRM_PCIE_SPEED_50)
8995 tmp16 |= 2; /* gen2 */
8996 else
8997 tmp16 |= 1; /* gen1 */
8998 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8999
9000 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9001 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9002 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9003
9004 for (i = 0; i < rdev->usec_timeout; i++) {
9005 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9006 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9007 break;
9008 udelay(1);
9009 }
9010}
7235711a
AD		 9011
9012static void cik_program_aspm(struct radeon_device *rdev)
9013{
9014 u32 data, orig;
9015 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9016 bool disable_clkreq = false;
9017
9018 if (radeon_aspm == 0)
9019 return;
9020
9021 /* XXX double check IGPs */
9022 if (rdev->flags & RADEON_IS_IGP)
9023 return;
9024
9025 if (!(rdev->flags & RADEON_IS_PCIE))
9026 return;
9027
9028 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9029 data &= ~LC_XMIT_N_FTS_MASK;
9030 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9031 if (orig != data)
9032 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9033
9034 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9035 data |= LC_GO_TO_RECOVERY;
9036 if (orig != data)
9037 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9038
9039 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9040 data |= P_IGNORE_EDB_ERR;
9041 if (orig != data)
9042 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9043
9044 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9045 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9046 data |= LC_PMI_TO_L1_DIS;
9047 if (!disable_l0s)
9048 data |= LC_L0S_INACTIVITY(7);
9049
9050 if (!disable_l1) {
9051 data |= LC_L1_INACTIVITY(7);
9052 data &= ~LC_PMI_TO_L1_DIS;
9053 if (orig != data)
9054 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9055
9056 if (!disable_plloff_in_l1) {
9057 bool clk_req_support;
9058
9059 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9060 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9061 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9062 if (orig != data)
9063 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9064
9065 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9066 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9067 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9068 if (orig != data)
9069 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9070
9071 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9072 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9073 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9074 if (orig != data)
9075 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9076
9077 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9078 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9079 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9080 if (orig != data)
9081 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9082
9083 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9084 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9085 data |= LC_DYN_LANES_PWR_STATE(3);
9086 if (orig != data)
9087 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9088
9089 if (!disable_clkreq) {
9090 struct pci_dev *root = rdev->pdev->bus->self;
9091 u32 lnkcap;
9092
9093 clk_req_support = false;
9094 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9095 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9096 clk_req_support = true;
9097 } else {
9098 clk_req_support = false;
9099 }
9100
9101 if (clk_req_support) {
9102 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9103 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9104 if (orig != data)
9105 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9106
9107 orig = data = RREG32_SMC(THM_CLK_CNTL);
9108 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9109 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9110 if (orig != data)
9111 WREG32_SMC(THM_CLK_CNTL, data);
9112
9113 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9114 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9115 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9116 if (orig != data)
9117 WREG32_SMC(MISC_CLK_CTRL, data);
9118
9119 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9120 data &= ~BCLK_AS_XCLK;
9121 if (orig != data)
9122 WREG32_SMC(CG_CLKPIN_CNTL, data);
9123
9124 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9125 data &= ~FORCE_BIF_REFCLK_EN;
9126 if (orig != data)
9127 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9128
9129 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9130 data &= ~MPLL_CLKOUT_SEL_MASK;
9131 data |= MPLL_CLKOUT_SEL(4);
9132 if (orig != data)
9133 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9134 }
9135 }
9136 } else {
9137 if (orig != data)
9138 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9139 }
9140
9141 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9142 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9143 if (orig != data)
9144 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9145
9146 if (!disable_l0s) {
9147 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9148 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9149 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9150 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9151 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9152 data &= ~LC_L0S_INACTIVITY_MASK;
9153 if (orig != data)
9154 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9155 }
9156 }
9157 }
87167bb1 9158}
Linux kernel - Deliverables
This page took 0.562625 seconds and 5 git commands to generate.