projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add get_allowed_info_register for r6xx/r7xx
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
bfc1f97d 29#include "radeon_audio.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 149extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 150
d4788db3
AD		 151static const u32 evergreen_golden_registers[] =
152{
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
6abafb78 192 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
195};
196
197static const u32 evergreen_golden_registers2[] =
198{
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
213};
214
215static const u32 cypress_mgcg_init[] =
216{
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
366};
367
368static const u32 redwood_mgcg_init[] =
369{
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
438};
439
440static const u32 cedar_golden_registers[] =
441{
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
6abafb78 479 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 480 0x9508, 0xffffffff, 0x00000002
481};
482
483static const u32 cedar_mgcg_init[] =
484{
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
535};
536
537static const u32 juniper_mgcg_init[] =
538{
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
633};
634
635static const u32 supersumo_golden_registers[] =
636{
637 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 638 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
708};
709
710static const u32 sumo_golden_registers[] =
711{
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
717};
718
719static const u32 wrestler_golden_registers[] =
720{
721 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 722 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
770};
771
772static const u32 barts_golden_registers[] =
773{
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
819};
820
821static const u32 turks_golden_registers[] =
822{
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
870};
871
872static const u32 caicos_golden_registers[] =
873{
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
921};
922
923static void evergreen_init_golden_registers(struct radeon_device *rdev)
924{
925 switch (rdev->family) {
926 case CHIP_CYPRESS:
927 case CHIP_HEMLOCK:
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
935 cypress_mgcg_init,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
937 break;
938 case CHIP_JUNIPER:
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
946 juniper_mgcg_init,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
948 break;
949 case CHIP_REDWOOD:
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
957 redwood_mgcg_init,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
959 break;
960 case CHIP_CEDAR:
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
968 cedar_mgcg_init,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
970 break;
971 case CHIP_PALM:
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
975 break;
976 case CHIP_SUMO:
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
980 break;
981 case CHIP_SUMO2:
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
988 break;
989 case CHIP_BARTS:
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
993 break;
994 case CHIP_TURKS:
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
998 break;
999 case CHIP_CAICOS:
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1003 break;
1004 default:
1005 break;
1006 }
1007}
1008
285484e2
JG		 1009void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1010 unsigned *bankh, unsigned *mtaspect,
1011 unsigned *tile_split)
1012{
1013 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1014 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1015 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1016 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1017 switch (*bankw) {
1018 default:
1019 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1020 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1021 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1022 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1023 }
1024 switch (*bankh) {
1025 default:
1026 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1027 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1028 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1029 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1030 }
1031 switch (*mtaspect) {
1032 default:
1033 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1034 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1035 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1036 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1037 }
1038}
1039
23d33ba3
AD		 1040static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1041 u32 cntl_reg, u32 status_reg)
1042{
1043 int r, i;
1044 struct atom_clock_dividers dividers;
1045
1046 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1047 clock, false, &dividers);
1048 if (r)
1049 return r;
1050
1051 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1052
1053 for (i = 0; i < 100; i++) {
1054 if (RREG32(status_reg) & DCLK_STATUS)
1055 break;
1056 mdelay(10);
1057 }
1058 if (i == 100)
1059 return -ETIMEDOUT;
1060
1061 return 0;
1062}
1063
1064int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1065{
1066 int r = 0;
1067 u32 cg_scratch = RREG32(CG_SCRATCH1);
1068
1069 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1070 if (r)
1071 goto done;
1072 cg_scratch &= 0xffff0000;
1073 cg_scratch |= vclk / 100; /* Mhz */
1074
1075 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1076 if (r)
1077 goto done;
1078 cg_scratch &= 0x0000ffff;
1079 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1080
1081done:
1082 WREG32(CG_SCRATCH1, cg_scratch);
1083
1084 return r;
1085}
1086
a8b4925c
AD		 1087int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1088{
1089 /* start off with something large */
facd112d 1090 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1091 int r;
1092
4ed10835
CK		 1093 /* bypass vclk and dclk with bclk */
1094 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1095 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1096 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1097
1098 /* put PLL in bypass mode */
1099 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1100
1101 if (!vclk || !dclk) {
1102 /* keep the Bypass mode, put PLL to sleep */
1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1104 return 0;
1105 }
1106
facd112d
CK		 1107 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1108 16384, 0x03FFFFFF, 0, 128, 5,
1109 &fb_div, &vclk_div, &dclk_div);
1110 if (r)
1111 return r;
a8b4925c
AD		 1112
1113 /* set VCO_MODE to 1 */
1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1115
1116 /* toggle UPLL_SLEEP to 1 then back to 0 */
1117 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1119
1120 /* deassert UPLL_RESET */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1122
1123 mdelay(1);
1124
facd112d 1125 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1126 if (r)
1127 return r;
1128
1129 /* assert UPLL_RESET again */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1131
1132 /* disable spread spectrum. */
1133 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1134
1135 /* set feedback divider */
facd112d 1136 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1137
1138 /* set ref divider to 0 */
1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1140
facd112d 1141 if (fb_div < 307200)
a8b4925c
AD		 1142 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1143 else
1144 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1145
1146 /* set PDIV_A and PDIV_B */
1147 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1148 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1149 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1150
1151 /* give the PLL some time to settle */
1152 mdelay(15);
1153
1154 /* deassert PLL_RESET */
1155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1156
1157 mdelay(15);
1158
1159 /* switch from bypass mode to normal mode */
1160 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1161
facd112d 1162 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1163 if (r)
1164 return r;
1165
1166 /* switch VCLK and DCLK selection */
1167 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1168 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1169 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1170
1171 mdelay(100);
1172
1173 return 0;
1174}
1175
d054ac16
AD		 1176void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1177{
c11592fe
YW		 1178 int readrq;
1179 u16 v;
d054ac16 1180
c11592fe
YW		 1181 readrq = pcie_get_readrq(rdev->pdev);
1182 v = ffs(readrq) - 8;
d054ac16
AD		 1183 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1184 * to avoid hangs or perfomance issues
1185 */
c11592fe
YW		 1186 if ((v == 0) || (v == 6) || (v == 7))
1187 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1188}
1189
134b480f
AD		 1190void dce4_program_fmt(struct drm_encoder *encoder)
1191{
1192 struct drm_device *dev = encoder->dev;
1193 struct radeon_device *rdev = dev->dev_private;
1194 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1195 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1196 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1197 int bpc = 0;
1198 u32 tmp = 0;
6214bb74 1199 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1200
6214bb74
AD		 1201 if (connector) {
1202 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1203 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1204 dither = radeon_connector->dither;
1205 }
134b480f
AD		 1206
1207 /* LVDS/eDP FMT is set up by atom */
1208 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1209 return;
1210
1211 /* not needed for analog */
1212 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1213 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1214 return;
1215
1216 if (bpc == 0)
1217 return;
1218
1219 switch (bpc) {
1220 case 6:
6214bb74 1221 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1222 /* XXX sort out optimal dither settings */
1223 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1224 FMT_SPATIAL_DITHER_EN);
1225 else
1226 tmp |= FMT_TRUNCATE_EN;
1227 break;
1228 case 8:
6214bb74 1229 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1230 /* XXX sort out optimal dither settings */
1231 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1232 FMT_RGB_RANDOM_ENABLE |
1233 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1234 else
1235 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1236 break;
1237 case 10:
1238 default:
1239 /* not needed */
1240 break;
1241 }
1242
1243 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1244}
1245
10257a6d
AD		 1246static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1247{
1248 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1249 return true;
1250 else
1251 return false;
1252}
1253
1254static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1255{
1256 u32 pos1, pos2;
1257
1258 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1260
1261 if (pos1 != pos2)
1262 return true;
1263 else
1264 return false;
1265}
1266
377edc8b
AD		 1267/**
1268 * dce4_wait_for_vblank - vblank wait asic callback.
1269 *
1270 * @rdev: radeon_device pointer
1271 * @crtc: crtc to wait for vblank on
1272 *
1273 * Wait for vblank on the requested crtc (evergreen+).
1274 */
3ae19b75
AD		 1275void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1276{
10257a6d 1277 unsigned i = 0;
3ae19b75 1278
4a15903d
AD		 1279 if (crtc >= rdev->num_crtc)
1280 return;
1281
10257a6d
AD		 1282 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1283 return;
1284
1285 /* depending on when we hit vblank, we may be close to active; if so,
1286 * wait for another frame.
1287 */
1288 while (dce4_is_in_vblank(rdev, crtc)) {
1289 if (i++ % 100 == 0) {
1290 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1291 break;
3ae19b75 1292 }
10257a6d
AD		 1293 }
1294
1295 while (!dce4_is_in_vblank(rdev, crtc)) {
1296 if (i++ % 100 == 0) {
1297 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1298 break;
3ae19b75
AD		 1299 }
1300 }
1301}
1302
377edc8b
AD		 1303/**
1304 * evergreen_page_flip - pageflip callback.
1305 *
1306 * @rdev: radeon_device pointer
1307 * @crtc_id: crtc to cleanup pageflip on
1308 * @crtc_base: new address of the crtc (GPU MC address)
1309 *
1310 * Does the actual pageflip (evergreen+).
1311 * During vblank we take the crtc lock and wait for the update_pending
1312 * bit to go high, when it does, we release the lock, and allow the
1313 * double buffered update to take place.
1314 * Returns the current update pending status.
1315 */
157fa14d 1316void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
6f34be50
AD		 1317{
1318 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1319 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1320 int i;
6f34be50
AD		 1321
1322 /* Lock the graphics update lock */
1323 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1324 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1325
1326 /* update the scanout addresses */
1327 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1328 upper_32_bits(crtc_base));
1329 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1330 (u32)crtc_base);
1331
1332 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1333 upper_32_bits(crtc_base));
1334 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1335 (u32)crtc_base);
1336
1337 /* Wait for update_pending to go high. */
f6496479
AD		 1338 for (i = 0; i < rdev->usec_timeout; i++) {
1339 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1340 break;
1341 udelay(1);
1342 }
6f34be50
AD		 1343 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1344
1345 /* Unlock the lock, so double-buffering can take place inside vblank */
1346 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1347 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
157fa14d
CK		 1348}
1349
1350/**
1351 * evergreen_page_flip_pending - check if page flip is still pending
1352 *
1353 * @rdev: radeon_device pointer
1354 * @crtc_id: crtc to check
1355 *
1356 * Returns the current update pending status.
1357 */
1358bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1359{
1360 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1361
1362 /* Return current update_pending status: */
157fa14d
CK		 1363 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1364 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1365}
1366
21a8122a 1367/* get temperature in millidegrees */
20d391d7 1368int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1369{
1c88d74f
AD		 1370 u32 temp, toffset;
1371 int actual_temp = 0;
67b3f823
AD		 1372
1373 if (rdev->family == CHIP_JUNIPER) {
1374 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1375 TOFFSET_SHIFT;
1376 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1377 TS0_ADC_DOUT_SHIFT;
1378
1379 if (toffset & 0x100)
1380 actual_temp = temp / 2 - (0x200 - toffset);
1381 else
1382 actual_temp = temp / 2 + toffset;
1383
1384 actual_temp = actual_temp * 1000;
1385
1386 } else {
1387 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1388 ASIC_T_SHIFT;
1389
1390 if (temp & 0x400)
1391 actual_temp = -256;
1392 else if (temp & 0x200)
1393 actual_temp = 255;
1394 else if (temp & 0x100) {
1395 actual_temp = temp & 0x1ff;
1396 actual_temp |= ~0x1ff;
1397 } else
1398 actual_temp = temp & 0xff;
1399
1400 actual_temp = (actual_temp * 1000) / 2;
1401 }
21a8122a 1402
67b3f823 1403 return actual_temp;
21a8122a
AD		 1404}
1405
20d391d7 1406int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1407{
1408 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1409 int actual_temp = temp - 49;
e33df25f
AD		 1410
1411 return actual_temp * 1000;
1412}
1413
377edc8b
AD		 1414/**
1415 * sumo_pm_init_profile - Initialize power profiles callback.
1416 *
1417 * @rdev: radeon_device pointer
1418 *
1419 * Initialize the power states used in profile mode
1420 * (sumo, trinity, SI).
1421 * Used for profile mode only.
1422 */
a4c9e2ee
AD		 1423void sumo_pm_init_profile(struct radeon_device *rdev)
1424{
1425 int idx;
1426
1427 /* default */
1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1431 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1432
1433 /* low,mid sh/mh */
1434 if (rdev->flags & RADEON_IS_MOBILITY)
1435 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1436 else
1437 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1438
1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1442 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1443
1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1447 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1448
1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1452 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1453
1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1457 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1458
1459 /* high sh/mh */
1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1464 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1465 rdev->pm.power_state[idx].num_clock_modes - 1;
1466
1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1471 rdev->pm.power_state[idx].num_clock_modes - 1;
1472}
1473
27810fb2
AD		 1474/**
1475 * btc_pm_init_profile - Initialize power profiles callback.
1476 *
1477 * @rdev: radeon_device pointer
1478 *
1479 * Initialize the power states used in profile mode
1480 * (BTC, cayman).
1481 * Used for profile mode only.
1482 */
1483void btc_pm_init_profile(struct radeon_device *rdev)
1484{
1485 int idx;
1486
1487 /* default */
1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1492 /* starting with BTC, there is one state that is used for both
1493 * MH and SH. Difference is that we always use the high clock index for
1494 * mclk.
1495 */
1496 if (rdev->flags & RADEON_IS_MOBILITY)
1497 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1498 else
1499 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1500 /* low sh */
1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1504 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1505 /* mid sh */
1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1509 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1510 /* high sh */
1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1514 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1515 /* low mh */
1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1520 /* mid mh */
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1524 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1525 /* high mh */
1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1530}
1531
377edc8b
AD		 1532/**
1533 * evergreen_pm_misc - set additional pm hw parameters callback.
1534 *
1535 * @rdev: radeon_device pointer
1536 *
1537 * Set non-clock parameters associated with a power state
1538 * (voltage, etc.) (evergreen+).
1539 */
49e02b73
AD		 1540void evergreen_pm_misc(struct radeon_device *rdev)
1541{
a081a9d6
RM		 1542 int req_ps_idx = rdev->pm.requested_power_state_index;
1543 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1544 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1545 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1546
2feea49a 1547 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1548 /* 0xff0x are flags rather then an actual voltage */
1549 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1550 return;
2feea49a 1551 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1552 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1553 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1554 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1555 }
7ae764b1
AD		 1556
1557 /* starting with BTC, there is one state that is used for both
1558 * MH and SH. Difference is that we always use the high clock index for
1559 * mclk and vddci.
1560 */
1561 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1562 (rdev->family >= CHIP_BARTS) &&
1563 rdev->pm.active_crtc_count &&
1564 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1565 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1566 voltage = &rdev->pm.power_state[req_ps_idx].
1567 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1568
c6cf7777
AD		 1569 /* 0xff0x are flags rather then an actual voltage */
1570 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1571 return;
2feea49a
AD		 1572 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1573 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1574 rdev->pm.current_vddci = voltage->vddci;
1575 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1576 }
1577 }
49e02b73
AD		 1578}
1579
377edc8b
AD		 1580/**
1581 * evergreen_pm_prepare - pre-power state change callback.
1582 *
1583 * @rdev: radeon_device pointer
1584 *
1585 * Prepare for a power state change (evergreen+).
1586 */
49e02b73
AD		 1587void evergreen_pm_prepare(struct radeon_device *rdev)
1588{
1589 struct drm_device *ddev = rdev->ddev;
1590 struct drm_crtc *crtc;
1591 struct radeon_crtc *radeon_crtc;
1592 u32 tmp;
1593
1594 /* disable any active CRTCs */
1595 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1596 radeon_crtc = to_radeon_crtc(crtc);
1597 if (radeon_crtc->enabled) {
1598 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1599 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1600 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1601 }
1602 }
1603}
1604
377edc8b
AD		 1605/**
1606 * evergreen_pm_finish - post-power state change callback.
1607 *
1608 * @rdev: radeon_device pointer
1609 *
1610 * Clean up after a power state change (evergreen+).
1611 */
49e02b73
AD		 1612void evergreen_pm_finish(struct radeon_device *rdev)
1613{
1614 struct drm_device *ddev = rdev->ddev;
1615 struct drm_crtc *crtc;
1616 struct radeon_crtc *radeon_crtc;
1617 u32 tmp;
1618
1619 /* enable any active CRTCs */
1620 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1621 radeon_crtc = to_radeon_crtc(crtc);
1622 if (radeon_crtc->enabled) {
1623 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1624 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1625 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1626 }
1627 }
1628}
1629
377edc8b
AD		 1630/**
1631 * evergreen_hpd_sense - hpd sense callback.
1632 *
1633 * @rdev: radeon_device pointer
1634 * @hpd: hpd (hotplug detect) pin
1635 *
1636 * Checks if a digital monitor is connected (evergreen+).
1637 * Returns true if connected, false if not connected.
1638 */
bcc1c2a1
AD		 1639bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1640{
1641 bool connected = false;
0ca2ab52
AD		 1642
1643 switch (hpd) {
1644 case RADEON_HPD_1:
1645 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1646 connected = true;
1647 break;
1648 case RADEON_HPD_2:
1649 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1650 connected = true;
1651 break;
1652 case RADEON_HPD_3:
1653 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1654 connected = true;
1655 break;
1656 case RADEON_HPD_4:
1657 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1658 connected = true;
1659 break;
1660 case RADEON_HPD_5:
1661 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1662 connected = true;
1663 break;
1664 case RADEON_HPD_6:
1665 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1666 connected = true;
ebc54ffe 1667 break;
0ca2ab52
AD		 1668 default:
1669 break;
1670 }
1671
bcc1c2a1
AD		 1672 return connected;
1673}
1674
377edc8b
AD		 1675/**
1676 * evergreen_hpd_set_polarity - hpd set polarity callback.
1677 *
1678 * @rdev: radeon_device pointer
1679 * @hpd: hpd (hotplug detect) pin
1680 *
1681 * Set the polarity of the hpd pin (evergreen+).
1682 */
bcc1c2a1
AD		 1683void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1684 enum radeon_hpd_id hpd)
1685{
0ca2ab52
AD		 1686 u32 tmp;
1687 bool connected = evergreen_hpd_sense(rdev, hpd);
1688
1689 switch (hpd) {
1690 case RADEON_HPD_1:
1691 tmp = RREG32(DC_HPD1_INT_CONTROL);
1692 if (connected)
1693 tmp &= ~DC_HPDx_INT_POLARITY;
1694 else
1695 tmp |= DC_HPDx_INT_POLARITY;
1696 WREG32(DC_HPD1_INT_CONTROL, tmp);
1697 break;
1698 case RADEON_HPD_2:
1699 tmp = RREG32(DC_HPD2_INT_CONTROL);
1700 if (connected)
1701 tmp &= ~DC_HPDx_INT_POLARITY;
1702 else
1703 tmp |= DC_HPDx_INT_POLARITY;
1704 WREG32(DC_HPD2_INT_CONTROL, tmp);
1705 break;
1706 case RADEON_HPD_3:
1707 tmp = RREG32(DC_HPD3_INT_CONTROL);
1708 if (connected)
1709 tmp &= ~DC_HPDx_INT_POLARITY;
1710 else
1711 tmp |= DC_HPDx_INT_POLARITY;
1712 WREG32(DC_HPD3_INT_CONTROL, tmp);
1713 break;
1714 case RADEON_HPD_4:
1715 tmp = RREG32(DC_HPD4_INT_CONTROL);
1716 if (connected)
1717 tmp &= ~DC_HPDx_INT_POLARITY;
1718 else
1719 tmp |= DC_HPDx_INT_POLARITY;
1720 WREG32(DC_HPD4_INT_CONTROL, tmp);
1721 break;
1722 case RADEON_HPD_5:
1723 tmp = RREG32(DC_HPD5_INT_CONTROL);
1724 if (connected)
1725 tmp &= ~DC_HPDx_INT_POLARITY;
1726 else
1727 tmp |= DC_HPDx_INT_POLARITY;
1728 WREG32(DC_HPD5_INT_CONTROL, tmp);
1729 break;
1730 case RADEON_HPD_6:
1731 tmp = RREG32(DC_HPD6_INT_CONTROL);
1732 if (connected)
1733 tmp &= ~DC_HPDx_INT_POLARITY;
1734 else
1735 tmp |= DC_HPDx_INT_POLARITY;
1736 WREG32(DC_HPD6_INT_CONTROL, tmp);
1737 break;
1738 default:
1739 break;
1740 }
bcc1c2a1
AD		 1741}
1742
377edc8b
AD		 1743/**
1744 * evergreen_hpd_init - hpd setup callback.
1745 *
1746 * @rdev: radeon_device pointer
1747 *
1748 * Setup the hpd pins used by the card (evergreen+).
1749 * Enable the pin, set the polarity, and enable the hpd interrupts.
1750 */
bcc1c2a1
AD		 1751void evergreen_hpd_init(struct radeon_device *rdev)
1752{
0ca2ab52
AD		 1753 struct drm_device *dev = rdev->ddev;
1754 struct drm_connector *connector;
fb98257a 1755 unsigned enabled = 0;
0ca2ab52
AD		 1756 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1757 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1758
0ca2ab52
AD		 1759 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1760 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1761
1762 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1763 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1764 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1765 * aux dp channel on imac and help (but not completely fix)
1766 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1767 * also avoid interrupt storms during dpms.
1768 */
1769 continue;
1770 }
0ca2ab52
AD		 1771 switch (radeon_connector->hpd.hpd) {
1772 case RADEON_HPD_1:
1773 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1774 break;
1775 case RADEON_HPD_2:
1776 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1777 break;
1778 case RADEON_HPD_3:
1779 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1780 break;
1781 case RADEON_HPD_4:
1782 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1783 break;
1784 case RADEON_HPD_5:
1785 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1786 break;
1787 case RADEON_HPD_6:
1788 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1789 break;
1790 default:
1791 break;
1792 }
64912e99 1793 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1794 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1795 }
fb98257a 1796 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1797}
1798
377edc8b
AD		 1799/**
1800 * evergreen_hpd_fini - hpd tear down callback.
1801 *
1802 * @rdev: radeon_device pointer
1803 *
1804 * Tear down the hpd pins used by the card (evergreen+).
1805 * Disable the hpd interrupts.
1806 */
0ca2ab52 1807void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1808{
0ca2ab52
AD		 1809 struct drm_device *dev = rdev->ddev;
1810 struct drm_connector *connector;
fb98257a 1811 unsigned disabled = 0;
0ca2ab52
AD		 1812
1813 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1814 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1815 switch (radeon_connector->hpd.hpd) {
1816 case RADEON_HPD_1:
1817 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1818 break;
1819 case RADEON_HPD_2:
1820 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1821 break;
1822 case RADEON_HPD_3:
1823 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1824 break;
1825 case RADEON_HPD_4:
1826 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1827 break;
1828 case RADEON_HPD_5:
1829 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1830 break;
1831 case RADEON_HPD_6:
1832 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1833 break;
1834 default:
1835 break;
1836 }
fb98257a 1837 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1838 }
fb98257a 1839 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1840}
1841
f9d9c362
AD		 1842/* watermark setup */
1843
1844static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1845 struct radeon_crtc *radeon_crtc,
1846 struct drm_display_mode *mode,
1847 struct drm_display_mode *other_mode)
1848{
0b31e023
AD		 1849 u32 tmp, buffer_alloc, i;
1850 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1851 /*
1852 * Line Buffer Setup
1853 * There are 3 line buffers, each one shared by 2 display controllers.
1854 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1855 * the display controllers. The paritioning is done via one of four
1856 * preset allocations specified in bits 2:0:
1857 * first display controller
1858 * 0 - first half of lb (3840 * 2)
1859 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1860 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1861 * 3 - first 1/4 of lb (1920 * 2)
1862 * second display controller
1863 * 4 - second half of lb (3840 * 2)
1864 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1865 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1866 * 7 - last 1/4 of lb (1920 * 2)
1867 */
12dfc843
AD		 1868 /* this can get tricky if we have two large displays on a paired group
1869 * of crtcs. Ideally for multiple large displays we'd assign them to
1870 * non-linked crtcs for maximum line buffer allocation.
1871 */
1872 if (radeon_crtc->base.enabled && mode) {
0b31e023 1873 if (other_mode) {
f9d9c362 1874 tmp = 0; /* 1/2 */
0b31e023
AD		 1875 buffer_alloc = 1;
1876 } else {
12dfc843 1877 tmp = 2; /* whole */
0b31e023
AD		 1878 buffer_alloc = 2;
1879 }
1880 } else {
12dfc843 1881 tmp = 0;
0b31e023
AD		 1882 buffer_alloc = 0;
1883 }
f9d9c362
AD		 1884
1885 /* second controller of the pair uses second half of the lb */
1886 if (radeon_crtc->crtc_id % 2)
1887 tmp += 4;
1888 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1889
0b31e023
AD		 1890 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1891 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1892 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1893 for (i = 0; i < rdev->usec_timeout; i++) {
1894 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1895 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1896 break;
1897 udelay(1);
1898 }
1899 }
1900
12dfc843
AD		 1901 if (radeon_crtc->base.enabled && mode) {
1902 switch (tmp) {
1903 case 0:
1904 case 4:
1905 default:
1906 if (ASIC_IS_DCE5(rdev))
1907 return 4096 * 2;
1908 else
1909 return 3840 * 2;
1910 case 1:
1911 case 5:
1912 if (ASIC_IS_DCE5(rdev))
1913 return 6144 * 2;
1914 else
1915 return 5760 * 2;
1916 case 2:
1917 case 6:
1918 if (ASIC_IS_DCE5(rdev))
1919 return 8192 * 2;
1920 else
1921 return 7680 * 2;
1922 case 3:
1923 case 7:
1924 if (ASIC_IS_DCE5(rdev))
1925 return 2048 * 2;
1926 else
1927 return 1920 * 2;
1928 }
f9d9c362 1929 }
12dfc843
AD		 1930
1931 /* controller not enabled, so no lb used */
1932 return 0;
f9d9c362
AD		 1933}
1934
ca7db22b 1935u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1936{
1937 u32 tmp = RREG32(MC_SHARED_CHMAP);
1938
1939 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1940 case 0:
1941 default:
1942 return 1;
1943 case 1:
1944 return 2;
1945 case 2:
1946 return 4;
1947 case 3:
1948 return 8;
1949 }
1950}
1951
1952struct evergreen_wm_params {
1953 u32 dram_channels; /* number of dram channels */
1954 u32 yclk; /* bandwidth per dram data pin in kHz */
1955 u32 sclk; /* engine clock in kHz */
1956 u32 disp_clk; /* display clock in kHz */
1957 u32 src_width; /* viewport width */
1958 u32 active_time; /* active display time in ns */
1959 u32 blank_time; /* blank time in ns */
1960 bool interlaced; /* mode is interlaced */
1961 fixed20_12 vsc; /* vertical scale ratio */
1962 u32 num_heads; /* number of active crtcs */
1963 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1964 u32 lb_size; /* line buffer allocated to pipe */
1965 u32 vtaps; /* vertical scaler taps */
1966};
1967
1968static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1969{
1970 /* Calculate DRAM Bandwidth and the part allocated to display. */
1971 fixed20_12 dram_efficiency; /* 0.7 */
1972 fixed20_12 yclk, dram_channels, bandwidth;
1973 fixed20_12 a;
1974
1975 a.full = dfixed_const(1000);
1976 yclk.full = dfixed_const(wm->yclk);
1977 yclk.full = dfixed_div(yclk, a);
1978 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1979 a.full = dfixed_const(10);
1980 dram_efficiency.full = dfixed_const(7);
1981 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1982 bandwidth.full = dfixed_mul(dram_channels, yclk);
1983 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1984
1985 return dfixed_trunc(bandwidth);
1986}
1987
1988static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1989{
1990 /* Calculate DRAM Bandwidth and the part allocated to display. */
1991 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1992 fixed20_12 yclk, dram_channels, bandwidth;
1993 fixed20_12 a;
1994
1995 a.full = dfixed_const(1000);
1996 yclk.full = dfixed_const(wm->yclk);
1997 yclk.full = dfixed_div(yclk, a);
1998 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1999 a.full = dfixed_const(10);
2000 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2001 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2002 bandwidth.full = dfixed_mul(dram_channels, yclk);
2003 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2004
2005 return dfixed_trunc(bandwidth);
2006}
2007
2008static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2009{
2010 /* Calculate the display Data return Bandwidth */
2011 fixed20_12 return_efficiency; /* 0.8 */
2012 fixed20_12 sclk, bandwidth;
2013 fixed20_12 a;
2014
2015 a.full = dfixed_const(1000);
2016 sclk.full = dfixed_const(wm->sclk);
2017 sclk.full = dfixed_div(sclk, a);
2018 a.full = dfixed_const(10);
2019 return_efficiency.full = dfixed_const(8);
2020 return_efficiency.full = dfixed_div(return_efficiency, a);
2021 a.full = dfixed_const(32);
2022 bandwidth.full = dfixed_mul(a, sclk);
2023 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2024
2025 return dfixed_trunc(bandwidth);
2026}
2027
2028static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2029{
2030 /* Calculate the DMIF Request Bandwidth */
2031 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2032 fixed20_12 disp_clk, bandwidth;
2033 fixed20_12 a;
2034
2035 a.full = dfixed_const(1000);
2036 disp_clk.full = dfixed_const(wm->disp_clk);
2037 disp_clk.full = dfixed_div(disp_clk, a);
2038 a.full = dfixed_const(10);
2039 disp_clk_request_efficiency.full = dfixed_const(8);
2040 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2041 a.full = dfixed_const(32);
2042 bandwidth.full = dfixed_mul(a, disp_clk);
2043 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2044
2045 return dfixed_trunc(bandwidth);
2046}
2047
2048static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2049{
2050 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2051 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2052 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2053 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2054
2055 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2056}
2057
2058static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2059{
2060 /* Calculate the display mode Average Bandwidth
2061 * DisplayMode should contain the source and destination dimensions,
2062 * timing, etc.
2063 */
2064 fixed20_12 bpp;
2065 fixed20_12 line_time;
2066 fixed20_12 src_width;
2067 fixed20_12 bandwidth;
2068 fixed20_12 a;
2069
2070 a.full = dfixed_const(1000);
2071 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2072 line_time.full = dfixed_div(line_time, a);
2073 bpp.full = dfixed_const(wm->bytes_per_pixel);
2074 src_width.full = dfixed_const(wm->src_width);
2075 bandwidth.full = dfixed_mul(src_width, bpp);
2076 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2077 bandwidth.full = dfixed_div(bandwidth, line_time);
2078
2079 return dfixed_trunc(bandwidth);
2080}
2081
2082static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2083{
2084 /* First calcualte the latency in ns */
2085 u32 mc_latency = 2000; /* 2000 ns. */
2086 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2087 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2088 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2089 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2090 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2091 (wm->num_heads * cursor_line_pair_return_time);
2092 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2093 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2094 fixed20_12 a, b, c;
2095
2096 if (wm->num_heads == 0)
2097 return 0;
2098
2099 a.full = dfixed_const(2);
2100 b.full = dfixed_const(1);
2101 if ((wm->vsc.full > a.full) ||
2102 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2103 (wm->vtaps >= 5) ||
2104 ((wm->vsc.full >= a.full) && wm->interlaced))
2105 max_src_lines_per_dst_line = 4;
2106 else
2107 max_src_lines_per_dst_line = 2;
2108
2109 a.full = dfixed_const(available_bandwidth);
2110 b.full = dfixed_const(wm->num_heads);
2111 a.full = dfixed_div(a, b);
2112
2113 b.full = dfixed_const(1000);
2114 c.full = dfixed_const(wm->disp_clk);
2115 b.full = dfixed_div(c, b);
2116 c.full = dfixed_const(wm->bytes_per_pixel);
2117 b.full = dfixed_mul(b, c);
2118
2119 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2120
2121 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2122 b.full = dfixed_const(1000);
2123 c.full = dfixed_const(lb_fill_bw);
2124 b.full = dfixed_div(c, b);
2125 a.full = dfixed_div(a, b);
2126 line_fill_time = dfixed_trunc(a);
2127
2128 if (line_fill_time < wm->active_time)
2129 return latency;
2130 else
2131 return latency + (line_fill_time - wm->active_time);
2132
2133}
2134
2135static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2136{
2137 if (evergreen_average_bandwidth(wm) <=
2138 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2139 return true;
2140 else
2141 return false;
2142};
2143
2144static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2145{
2146 if (evergreen_average_bandwidth(wm) <=
2147 (evergreen_available_bandwidth(wm) / wm->num_heads))
2148 return true;
2149 else
2150 return false;
2151};
2152
2153static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2154{
2155 u32 lb_partitions = wm->lb_size / wm->src_width;
2156 u32 line_time = wm->active_time + wm->blank_time;
2157 u32 latency_tolerant_lines;
2158 u32 latency_hiding;
2159 fixed20_12 a;
2160
2161 a.full = dfixed_const(1);
2162 if (wm->vsc.full > a.full)
2163 latency_tolerant_lines = 1;
2164 else {
2165 if (lb_partitions <= (wm->vtaps + 1))
2166 latency_tolerant_lines = 1;
2167 else
2168 latency_tolerant_lines = 2;
2169 }
2170
2171 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2172
2173 if (evergreen_latency_watermark(wm) <= latency_hiding)
2174 return true;
2175 else
2176 return false;
2177}
2178
2179static void evergreen_program_watermarks(struct radeon_device *rdev,
2180 struct radeon_crtc *radeon_crtc,
2181 u32 lb_size, u32 num_heads)
2182{
2183 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2184 struct evergreen_wm_params wm_low, wm_high;
2185 u32 dram_channels;
f9d9c362
AD		 2186 u32 pixel_period;
2187 u32 line_time = 0;
2188 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2189 u32 priority_a_mark = 0, priority_b_mark = 0;
2190 u32 priority_a_cnt = PRIORITY_OFF;
2191 u32 priority_b_cnt = PRIORITY_OFF;
2192 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2193 u32 tmp, arb_control3;
2194 fixed20_12 a, b, c;
2195
2196 if (radeon_crtc->base.enabled && num_heads && mode) {
2197 pixel_period = 1000000 / (u32)mode->clock;
2198 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2199 priority_a_cnt = 0;
2200 priority_b_cnt = 0;
cf0cfdd7
AD		 2201 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2202
2203 /* watermark for high clocks */
2204 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2205 wm_high.yclk =
2206 radeon_dpm_get_mclk(rdev, false) * 10;
2207 wm_high.sclk =
2208 radeon_dpm_get_sclk(rdev, false) * 10;
2209 } else {
2210 wm_high.yclk = rdev->pm.current_mclk * 10;
2211 wm_high.sclk = rdev->pm.current_sclk * 10;
2212 }
f9d9c362 2213
cf0cfdd7
AD		 2214 wm_high.disp_clk = mode->clock;
2215 wm_high.src_width = mode->crtc_hdisplay;
2216 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2217 wm_high.blank_time = line_time - wm_high.active_time;
2218 wm_high.interlaced = false;
f9d9c362 2219 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2220 wm_high.interlaced = true;
2221 wm_high.vsc = radeon_crtc->vsc;
2222 wm_high.vtaps = 1;
f9d9c362 2223 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2224 wm_high.vtaps = 2;
2225 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2226 wm_high.lb_size = lb_size;
2227 wm_high.dram_channels = dram_channels;
2228 wm_high.num_heads = num_heads;
2229
2230 /* watermark for low clocks */
2231 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2232 wm_low.yclk =
2233 radeon_dpm_get_mclk(rdev, true) * 10;
2234 wm_low.sclk =
2235 radeon_dpm_get_sclk(rdev, true) * 10;
2236 } else {
2237 wm_low.yclk = rdev->pm.current_mclk * 10;
2238 wm_low.sclk = rdev->pm.current_sclk * 10;
2239 }
2240
2241 wm_low.disp_clk = mode->clock;
2242 wm_low.src_width = mode->crtc_hdisplay;
2243 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2244 wm_low.blank_time = line_time - wm_low.active_time;
2245 wm_low.interlaced = false;
2246 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2247 wm_low.interlaced = true;
2248 wm_low.vsc = radeon_crtc->vsc;
2249 wm_low.vtaps = 1;
2250 if (radeon_crtc->rmx_type != RMX_OFF)
2251 wm_low.vtaps = 2;
2252 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2253 wm_low.lb_size = lb_size;
2254 wm_low.dram_channels = dram_channels;
2255 wm_low.num_heads = num_heads;
f9d9c362
AD		 2256
2257 /* set for high clocks */
cf0cfdd7 2258 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2259 /* set for low clocks */
cf0cfdd7 2260 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2261
2262 /* possibly force display priority to high */
2263 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2264 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2265 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2266 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2267 (rdev->disp_priority == 2)) {
cf0cfdd7 2268 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2269 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2270 }
2271 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2272 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2273 !evergreen_check_latency_hiding(&wm_low) ||
2274 (rdev->disp_priority == 2)) {
2275 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2276 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2277 }
2278
2279 a.full = dfixed_const(1000);
2280 b.full = dfixed_const(mode->clock);
2281 b.full = dfixed_div(b, a);
2282 c.full = dfixed_const(latency_watermark_a);
2283 c.full = dfixed_mul(c, b);
2284 c.full = dfixed_mul(c, radeon_crtc->hsc);
2285 c.full = dfixed_div(c, a);
2286 a.full = dfixed_const(16);
2287 c.full = dfixed_div(c, a);
2288 priority_a_mark = dfixed_trunc(c);
2289 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2290
2291 a.full = dfixed_const(1000);
2292 b.full = dfixed_const(mode->clock);
2293 b.full = dfixed_div(b, a);
2294 c.full = dfixed_const(latency_watermark_b);
2295 c.full = dfixed_mul(c, b);
2296 c.full = dfixed_mul(c, radeon_crtc->hsc);
2297 c.full = dfixed_div(c, a);
2298 a.full = dfixed_const(16);
2299 c.full = dfixed_div(c, a);
2300 priority_b_mark = dfixed_trunc(c);
2301 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2302 }
2303
2304 /* select wm A */
2305 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2306 tmp = arb_control3;
2307 tmp &= ~LATENCY_WATERMARK_MASK(3);
2308 tmp |= LATENCY_WATERMARK_MASK(1);
2309 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2310 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2311 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2312 LATENCY_HIGH_WATERMARK(line_time)));
2313 /* select wm B */
2314 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2315 tmp &= ~LATENCY_WATERMARK_MASK(3);
2316 tmp |= LATENCY_WATERMARK_MASK(2);
2317 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2318 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2319 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2320 LATENCY_HIGH_WATERMARK(line_time)));
2321 /* restore original selection */
2322 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2323
2324 /* write the priority marks */
2325 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2326 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2327
7178d2a6
AD		 2328 /* save values for DPM */
2329 radeon_crtc->line_time = line_time;
2330 radeon_crtc->wm_high = latency_watermark_a;
2331 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2332}
2333
377edc8b
AD		 2334/**
2335 * evergreen_bandwidth_update - update display watermarks callback.
2336 *
2337 * @rdev: radeon_device pointer
2338 *
2339 * Update the display watermarks based on the requested mode(s)
2340 * (evergreen+).
2341 */
0ca2ab52 2342void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2343{
f9d9c362
AD		 2344 struct drm_display_mode *mode0 = NULL;
2345 struct drm_display_mode *mode1 = NULL;
2346 u32 num_heads = 0, lb_size;
2347 int i;
2348
8efe82ca
AD		 2349 if (!rdev->mode_info.mode_config_initialized)
2350 return;
2351
f9d9c362
AD		 2352 radeon_update_display_priority(rdev);
2353
2354 for (i = 0; i < rdev->num_crtc; i++) {
2355 if (rdev->mode_info.crtcs[i]->base.enabled)
2356 num_heads++;
2357 }
2358 for (i = 0; i < rdev->num_crtc; i += 2) {
2359 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2360 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2361 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2362 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2363 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2364 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2365 }
bcc1c2a1
AD		 2366}
2367
377edc8b
AD		 2368/**
2369 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2370 *
2371 * @rdev: radeon_device pointer
2372 *
2373 * Wait for the MC (memory controller) to be idle.
2374 * (evergreen+).
2375 * Returns 0 if the MC is idle, -1 if not.
2376 */
b9952a8a 2377int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2378{
2379 unsigned i;
2380 u32 tmp;
2381
2382 for (i = 0; i < rdev->usec_timeout; i++) {
2383 /* read MC_STATUS */
2384 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2385 if (!tmp)
2386 return 0;
2387 udelay(1);
2388 }
2389 return -1;
2390}
2391
2392/*
2393 * GART
2394 */
0fcdb61e
AD		 2395void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2396{
2397 unsigned i;
2398 u32 tmp;
2399
6f2f48a9
AD		 2400 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2401
0fcdb61e
AD		 2402 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2403 for (i = 0; i < rdev->usec_timeout; i++) {
2404 /* read MC_STATUS */
2405 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2406 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2407 if (tmp == 2) {
2408 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2409 return;
2410 }
2411 if (tmp) {
2412 return;
2413 }
2414 udelay(1);
2415 }
2416}
2417
1109ca09 2418static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2419{
2420 u32 tmp;
0fcdb61e 2421 int r;
bcc1c2a1 2422
c9a1be96 2423 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2424 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2425 return -EINVAL;
2426 }
2427 r = radeon_gart_table_vram_pin(rdev);
2428 if (r)
2429 return r;
2430 /* Setup L2 cache */
2431 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2432 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2433 EFFECTIVE_L2_QUEUE_SIZE(7));
2434 WREG32(VM_L2_CNTL2, 0);
2435 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2436 /* Setup TLB control */
2437 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2438 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2439 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2440 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2441 if (rdev->flags & RADEON_IS_IGP) {
2442 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2443 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2444 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2445 } else {
2446 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2447 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2448 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2449 if ((rdev->family == CHIP_JUNIPER) ||
2450 (rdev->family == CHIP_CYPRESS) ||
2451 (rdev->family == CHIP_HEMLOCK) ||
2452 (rdev->family == CHIP_BARTS))
2453 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2454 }
bcc1c2a1
AD		 2455 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2456 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2457 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2458 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2459 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2460 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2461 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2462 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2463 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2464 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2465 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2466 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2467
0fcdb61e 2468 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2469 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2470 (unsigned)(rdev->mc.gtt_size >> 20),
2471 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2472 rdev->gart.ready = true;
2473 return 0;
2474}
2475
1109ca09 2476static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2477{
2478 u32 tmp;
bcc1c2a1
AD		 2479
2480 /* Disable all tables */
0fcdb61e
AD		 2481 WREG32(VM_CONTEXT0_CNTL, 0);
2482 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2483
2484 /* Setup L2 cache */
2485 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2486 EFFECTIVE_L2_QUEUE_SIZE(7));
2487 WREG32(VM_L2_CNTL2, 0);
2488 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2489 /* Setup TLB control */
2490 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2491 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2492 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2493 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2494 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2495 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2496 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2497 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2498 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2499}
2500
1109ca09 2501static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2502{
2503 evergreen_pcie_gart_disable(rdev);
2504 radeon_gart_table_vram_free(rdev);
2505 radeon_gart_fini(rdev);
2506}
2507
2508
1109ca09 2509static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2510{
2511 u32 tmp;
bcc1c2a1
AD		 2512
2513 /* Setup L2 cache */
2514 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2515 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2516 EFFECTIVE_L2_QUEUE_SIZE(7));
2517 WREG32(VM_L2_CNTL2, 0);
2518 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2519 /* Setup TLB control */
2520 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2521 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2522 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2523 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2524 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2525 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2526 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2527 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2528 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2529 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2530 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2531 WREG32(VM_CONTEXT0_CNTL, 0);
2532 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2533}
2534
b9952a8a 2535void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2536{
62444b74
AD		 2537 u32 crtc_enabled, tmp, frame_count, blackout;
2538 int i, j;
2539
5153550a
AD		 2540 if (!ASIC_IS_NODCE(rdev)) {
2541 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2542 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2543
5153550a
AD		 2544 /* disable VGA render */
2545 WREG32(VGA_RENDER_CONTROL, 0);
2546 }
62444b74
AD		 2547 /* blank the display controllers */
2548 for (i = 0; i < rdev->num_crtc; i++) {
2549 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2550 if (crtc_enabled) {
2551 save->crtc_enabled[i] = true;
2552 if (ASIC_IS_DCE6(rdev)) {
2553 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2554 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2555 radeon_wait_for_vblank(rdev, i);
abf1457b 2556 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2557 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2558 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
f0d7bfb9 2559 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2560 }
2561 } else {
2562 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2563 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2564 radeon_wait_for_vblank(rdev, i);
abf1457b 2565 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2566 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2567 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2568 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2569 }
2570 }
2571 /* wait for the next frame */
2572 frame_count = radeon_get_vblank_counter(rdev, i);
2573 for (j = 0; j < rdev->usec_timeout; j++) {
2574 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2575 break;
2576 udelay(1);
2577 }
abf1457b
AD		 2578
2579 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2580 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2582 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2583 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2585 save->crtc_enabled[i] = false;
2586 /* ***** */
804cc4a0
AD		 2587 } else {
2588 save->crtc_enabled[i] = false;
62444b74 2589 }
18007401 2590 }
bcc1c2a1 2591
62444b74
AD		 2592 radeon_mc_wait_for_idle(rdev);
2593
2594 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2595 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2596 /* Block CPU access */
2597 WREG32(BIF_FB_EN, 0);
2598 /* blackout the MC */
2599 blackout &= ~BLACKOUT_MODE_MASK;
2600 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2601 }
ed39fadd
AD		 2602 /* wait for the MC to settle */
2603 udelay(100);
968c0166
AD		 2604
2605 /* lock double buffered regs */
2606 for (i = 0; i < rdev->num_crtc; i++) {
2607 if (save->crtc_enabled[i]) {
2608 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2609 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2610 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2611 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2612 }
2613 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2614 if (!(tmp & 1)) {
2615 tmp |= 1;
2616 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2617 }
2618 }
2619 }
bcc1c2a1
AD		 2620}
2621
b9952a8a 2622void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2623{
62444b74
AD		 2624 u32 tmp, frame_count;
2625 int i, j;
18007401 2626
62444b74
AD		 2627 /* update crtc base addresses */
2628 for (i = 0; i < rdev->num_crtc; i++) {
2629 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2630 upper_32_bits(rdev->mc.vram_start));
62444b74 2631 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2632 upper_32_bits(rdev->mc.vram_start));
62444b74 2633 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2634 (u32)rdev->mc.vram_start);
62444b74 2635 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2636 (u32)rdev->mc.vram_start);
2637 }
5153550a
AD		 2638
2639 if (!ASIC_IS_NODCE(rdev)) {
2640 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2641 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2642 }
62444b74 2643
968c0166
AD		 2644 /* unlock regs and wait for update */
2645 for (i = 0; i < rdev->num_crtc; i++) {
2646 if (save->crtc_enabled[i]) {
2647 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2648 if ((tmp & 0x7) != 3) {
2649 tmp &= ~0x7;
2650 tmp |= 0x3;
968c0166
AD		 2651 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2652 }
2653 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2654 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2655 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2656 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2657 }
2658 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2659 if (tmp & 1) {
2660 tmp &= ~1;
2661 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2662 }
2663 for (j = 0; j < rdev->usec_timeout; j++) {
2664 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2665 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2666 break;
2667 udelay(1);
2668 }
2669 }
2670 }
2671
62444b74
AD		 2672 /* unblackout the MC */
2673 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2674 tmp &= ~BLACKOUT_MODE_MASK;
2675 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2676 /* allow CPU access */
2677 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2678
2679 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2680 if (save->crtc_enabled[i]) {
62444b74
AD		 2681 if (ASIC_IS_DCE6(rdev)) {
2682 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2683 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2684 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2685 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2686 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2687 } else {
2688 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2689 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2690 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2691 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2692 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2693 }
2694 /* wait for the next frame */
2695 frame_count = radeon_get_vblank_counter(rdev, i);
2696 for (j = 0; j < rdev->usec_timeout; j++) {
2697 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2698 break;
2699 udelay(1);
2700 }
2701 }
2702 }
5153550a
AD		 2703 if (!ASIC_IS_NODCE(rdev)) {
2704 /* Unlock vga access */
2705 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2706 mdelay(1);
2707 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2708 }
bcc1c2a1
AD		 2709}
2710
755d819e 2711void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2712{
2713 struct evergreen_mc_save save;
2714 u32 tmp;
2715 int i, j;
2716
2717 /* Initialize HDP */
2718 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2719 WREG32((0x2c14 + j), 0x00000000);
2720 WREG32((0x2c18 + j), 0x00000000);
2721 WREG32((0x2c1c + j), 0x00000000);
2722 WREG32((0x2c20 + j), 0x00000000);
2723 WREG32((0x2c24 + j), 0x00000000);
2724 }
2725 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2726
2727 evergreen_mc_stop(rdev, &save);
2728 if (evergreen_mc_wait_for_idle(rdev)) {
2729 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2730 }
2731 /* Lockout access through VGA aperture*/
2732 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2733 /* Update configuration */
2734 if (rdev->flags & RADEON_IS_AGP) {
2735 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2736 /* VRAM before AGP */
2737 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2738 rdev->mc.vram_start >> 12);
2739 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2740 rdev->mc.gtt_end >> 12);
2741 } else {
2742 /* VRAM after AGP */
2743 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2744 rdev->mc.gtt_start >> 12);
2745 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2746 rdev->mc.vram_end >> 12);
2747 }
2748 } else {
2749 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2750 rdev->mc.vram_start >> 12);
2751 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2752 rdev->mc.vram_end >> 12);
2753 }
3b9832f6 2754 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2755 /* llano/ontario only */
2756 if ((rdev->family == CHIP_PALM) ||
2757 (rdev->family == CHIP_SUMO) ||
2758 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2759 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2760 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2761 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2762 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2763 }
bcc1c2a1
AD		 2764 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2765 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2766 WREG32(MC_VM_FB_LOCATION, tmp);
2767 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2768 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2769 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2770 if (rdev->flags & RADEON_IS_AGP) {
2771 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2772 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2773 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2774 } else {
2775 WREG32(MC_VM_AGP_BASE, 0);
2776 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2777 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2778 }
2779 if (evergreen_mc_wait_for_idle(rdev)) {
2780 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2781 }
2782 evergreen_mc_resume(rdev, &save);
2783 /* we need to own VRAM, so turn off the VGA renderer here
2784 * to stop it overwriting our objects */
2785 rv515_vga_render_disable(rdev);
2786}
2787
bcc1c2a1
AD		 2788/*
2789 * CP.
2790 */
12920591
AD		 2791void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2792{
876dc9f3 2793 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2794 u32 next_rptr;
7b1f2485 2795
12920591 2796 /* set to DX10/11 mode */
e32eb50d
CK		 2797 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2798 radeon_ring_write(ring, 1);
45df6803
CK		 2799
2800 if (ring->rptr_save_reg) {
89d35807 2801 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2802 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2803 radeon_ring_write(ring, ((ring->rptr_save_reg -
2804 PACKET3_SET_CONFIG_REG_START) >> 2));
2805 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2806 } else if (rdev->wb.enabled) {
2807 next_rptr = ring->wptr + 5 + 4;
2808 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2809 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2810 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2811 radeon_ring_write(ring, next_rptr);
2812 radeon_ring_write(ring, 0);
45df6803
CK		 2813 }
2814
e32eb50d
CK		 2815 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2816 radeon_ring_write(ring,
0f234f5f
AD		 2817#ifdef __BIG_ENDIAN
2818 (2 << 0) |
2819#endif
2820 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2821 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2822 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2823}
2824
bcc1c2a1
AD		 2825
2826static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2827{
fe251e2f
AD		 2828 const __be32 *fw_data;
2829 int i;
2830
2831 if (!rdev->me_fw || !rdev->pfp_fw)
2832 return -EINVAL;
bcc1c2a1 2833
fe251e2f 2834 r700_cp_stop(rdev);
0f234f5f
AD		 2835 WREG32(CP_RB_CNTL,
2836#ifdef __BIG_ENDIAN
2837 BUF_SWAP_32BIT |
2838#endif
2839 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2840
2841 fw_data = (const __be32 *)rdev->pfp_fw->data;
2842 WREG32(CP_PFP_UCODE_ADDR, 0);
2843 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2844 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2845 WREG32(CP_PFP_UCODE_ADDR, 0);
2846
2847 fw_data = (const __be32 *)rdev->me_fw->data;
2848 WREG32(CP_ME_RAM_WADDR, 0);
2849 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2850 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2851
2852 WREG32(CP_PFP_UCODE_ADDR, 0);
2853 WREG32(CP_ME_RAM_WADDR, 0);
2854 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2855 return 0;
2856}
2857
7e7b41d2
AD		 2858static int evergreen_cp_start(struct radeon_device *rdev)
2859{
e32eb50d 2860 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2861 int r, i;
7e7b41d2
AD		 2862 uint32_t cp_me;
2863
e32eb50d 2864 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2865 if (r) {
2866 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2867 return r;
2868 }
e32eb50d
CK		 2869 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2870 radeon_ring_write(ring, 0x1);
2871 radeon_ring_write(ring, 0x0);
2872 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2873 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2874 radeon_ring_write(ring, 0);
2875 radeon_ring_write(ring, 0);
1538a9e0 2876 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2877
2878 cp_me = 0xff;
2879 WREG32(CP_ME_CNTL, cp_me);
2880
e32eb50d 2881 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2882 if (r) {
2883 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2884 return r;
2885 }
2281a378
AD		 2886
2887 /* setup clear context state */
e32eb50d
CK		 2888 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2889 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2890
2891 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2892 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2893
e32eb50d
CK		 2894 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2895 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2896
2897 /* set clear context state */
e32eb50d
CK		 2898 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2899 radeon_ring_write(ring, 0);
2281a378
AD		 2900
2901 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2902 radeon_ring_write(ring, 0xc0026f00);
2903 radeon_ring_write(ring, 0x00000000);
2904 radeon_ring_write(ring, 0x00000000);
2905 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2906
2907 /* Clear consts */
e32eb50d
CK		 2908 radeon_ring_write(ring, 0xc0036f00);
2909 radeon_ring_write(ring, 0x00000bc4);
2910 radeon_ring_write(ring, 0xffffffff);
2911 radeon_ring_write(ring, 0xffffffff);
2912 radeon_ring_write(ring, 0xffffffff);
2281a378 2913
e32eb50d
CK		 2914 radeon_ring_write(ring, 0xc0026900);
2915 radeon_ring_write(ring, 0x00000316);
2916 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2917 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2918
1538a9e0 2919 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2920
2921 return 0;
2922}
2923
1109ca09 2924static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2925{
e32eb50d 2926 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2927 u32 tmp;
2928 u32 rb_bufsz;
2929 int r;
2930
2931 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2932 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2933 SOFT_RESET_PA |
2934 SOFT_RESET_SH |
2935 SOFT_RESET_VGT |
a49a50da 2936 SOFT_RESET_SPI |
fe251e2f
AD		 2937 SOFT_RESET_SX));
2938 RREG32(GRBM_SOFT_RESET);
2939 mdelay(15);
2940 WREG32(GRBM_SOFT_RESET, 0);
2941 RREG32(GRBM_SOFT_RESET);
2942
2943 /* Set ring buffer size */
b72a8925
DV		 2944 rb_bufsz = order_base_2(ring->ring_size / 8);
2945 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2946#ifdef __BIG_ENDIAN
2947 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2948#endif
fe251e2f 2949 WREG32(CP_RB_CNTL, tmp);
15d3332f 2950 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2951 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2952
2953 /* Set the write pointer delay */
2954 WREG32(CP_RB_WPTR_DELAY, 0);
2955
2956 /* Initialize the ring buffer's read and write pointers */
2957 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2958 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2959 ring->wptr = 0;
2960 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2961
48fc7f7e 2962 /* set the wb address whether it's enabled or not */
0f234f5f 2963 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2964 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2965 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2966 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2967
2968 if (rdev->wb.enabled)
2969 WREG32(SCRATCH_UMSK, 0xff);
2970 else {
2971 tmp |= RB_NO_UPDATE;
2972 WREG32(SCRATCH_UMSK, 0);
2973 }
2974
fe251e2f
AD		 2975 mdelay(1);
2976 WREG32(CP_RB_CNTL, tmp);
2977
e32eb50d 2978 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2979 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2980
7e7b41d2 2981 evergreen_cp_start(rdev);
e32eb50d 2982 ring->ready = true;
f712812e 2983 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2984 if (r) {
e32eb50d 2985 ring->ready = false;
fe251e2f
AD		 2986 return r;
2987 }
2988 return 0;
2989}
bcc1c2a1
AD		 2990
2991/*
2992 * Core functions
2993 */
bcc1c2a1
AD		 2994static void evergreen_gpu_init(struct radeon_device *rdev)
2995{
416a2bd2 2996 u32 gb_addr_config;
32fcdbf4 2997 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2998 u32 sx_debug_1;
2999 u32 smx_dc_ctl0;
3000 u32 sq_config;
3001 u32 sq_lds_resource_mgmt;
3002 u32 sq_gpr_resource_mgmt_1;
3003 u32 sq_gpr_resource_mgmt_2;
3004 u32 sq_gpr_resource_mgmt_3;
3005 u32 sq_thread_resource_mgmt;
3006 u32 sq_thread_resource_mgmt_2;
3007 u32 sq_stack_resource_mgmt_1;
3008 u32 sq_stack_resource_mgmt_2;
3009 u32 sq_stack_resource_mgmt_3;
3010 u32 vgt_cache_invalidation;
f25a5c63 3011 u32 hdp_host_path_cntl, tmp;
416a2bd2 3012 u32 disabled_rb_mask;
072c44bf 3013 int i, j, ps_thread_count;
32fcdbf4
AD		 3014
3015 switch (rdev->family) {
3016 case CHIP_CYPRESS:
3017 case CHIP_HEMLOCK:
3018 rdev->config.evergreen.num_ses = 2;
3019 rdev->config.evergreen.max_pipes = 4;
3020 rdev->config.evergreen.max_tile_pipes = 8;
3021 rdev->config.evergreen.max_simds = 10;
3022 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3023 rdev->config.evergreen.max_gprs = 256;
3024 rdev->config.evergreen.max_threads = 248;
3025 rdev->config.evergreen.max_gs_threads = 32;
3026 rdev->config.evergreen.max_stack_entries = 512;
3027 rdev->config.evergreen.sx_num_of_sets = 4;
3028 rdev->config.evergreen.sx_max_export_size = 256;
3029 rdev->config.evergreen.sx_max_export_pos_size = 64;
3030 rdev->config.evergreen.sx_max_export_smx_size = 192;
3031 rdev->config.evergreen.max_hw_contexts = 8;
3032 rdev->config.evergreen.sq_num_cf_insts = 2;
3033
3034 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3035 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3036 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3037 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3038 break;
3039 case CHIP_JUNIPER:
3040 rdev->config.evergreen.num_ses = 1;
3041 rdev->config.evergreen.max_pipes = 4;
3042 rdev->config.evergreen.max_tile_pipes = 4;
3043 rdev->config.evergreen.max_simds = 10;
3044 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3045 rdev->config.evergreen.max_gprs = 256;
3046 rdev->config.evergreen.max_threads = 248;
3047 rdev->config.evergreen.max_gs_threads = 32;
3048 rdev->config.evergreen.max_stack_entries = 512;
3049 rdev->config.evergreen.sx_num_of_sets = 4;
3050 rdev->config.evergreen.sx_max_export_size = 256;
3051 rdev->config.evergreen.sx_max_export_pos_size = 64;
3052 rdev->config.evergreen.sx_max_export_smx_size = 192;
3053 rdev->config.evergreen.max_hw_contexts = 8;
3054 rdev->config.evergreen.sq_num_cf_insts = 2;
3055
3056 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3057 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3058 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3059 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3060 break;
3061 case CHIP_REDWOOD:
3062 rdev->config.evergreen.num_ses = 1;
3063 rdev->config.evergreen.max_pipes = 4;
3064 rdev->config.evergreen.max_tile_pipes = 4;
3065 rdev->config.evergreen.max_simds = 5;
3066 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3067 rdev->config.evergreen.max_gprs = 256;
3068 rdev->config.evergreen.max_threads = 248;
3069 rdev->config.evergreen.max_gs_threads = 32;
3070 rdev->config.evergreen.max_stack_entries = 256;
3071 rdev->config.evergreen.sx_num_of_sets = 4;
3072 rdev->config.evergreen.sx_max_export_size = 256;
3073 rdev->config.evergreen.sx_max_export_pos_size = 64;
3074 rdev->config.evergreen.sx_max_export_smx_size = 192;
3075 rdev->config.evergreen.max_hw_contexts = 8;
3076 rdev->config.evergreen.sq_num_cf_insts = 2;
3077
3078 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3079 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3080 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3081 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3082 break;
3083 case CHIP_CEDAR:
3084 default:
3085 rdev->config.evergreen.num_ses = 1;
3086 rdev->config.evergreen.max_pipes = 2;
3087 rdev->config.evergreen.max_tile_pipes = 2;
3088 rdev->config.evergreen.max_simds = 2;
3089 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3090 rdev->config.evergreen.max_gprs = 256;
3091 rdev->config.evergreen.max_threads = 192;
3092 rdev->config.evergreen.max_gs_threads = 16;
3093 rdev->config.evergreen.max_stack_entries = 256;
3094 rdev->config.evergreen.sx_num_of_sets = 4;
3095 rdev->config.evergreen.sx_max_export_size = 128;
3096 rdev->config.evergreen.sx_max_export_pos_size = 32;
3097 rdev->config.evergreen.sx_max_export_smx_size = 96;
3098 rdev->config.evergreen.max_hw_contexts = 4;
3099 rdev->config.evergreen.sq_num_cf_insts = 1;
3100
d5e455e4
AD		 3101 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3102 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3103 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3104 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3105 break;
3106 case CHIP_PALM:
3107 rdev->config.evergreen.num_ses = 1;
3108 rdev->config.evergreen.max_pipes = 2;
3109 rdev->config.evergreen.max_tile_pipes = 2;
3110 rdev->config.evergreen.max_simds = 2;
3111 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3112 rdev->config.evergreen.max_gprs = 256;
3113 rdev->config.evergreen.max_threads = 192;
3114 rdev->config.evergreen.max_gs_threads = 16;
3115 rdev->config.evergreen.max_stack_entries = 256;
3116 rdev->config.evergreen.sx_num_of_sets = 4;
3117 rdev->config.evergreen.sx_max_export_size = 128;
3118 rdev->config.evergreen.sx_max_export_pos_size = 32;
3119 rdev->config.evergreen.sx_max_export_smx_size = 96;
3120 rdev->config.evergreen.max_hw_contexts = 4;
3121 rdev->config.evergreen.sq_num_cf_insts = 1;
3122
d5c5a72f
AD		 3123 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3124 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3125 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3126 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3127 break;
3128 case CHIP_SUMO:
3129 rdev->config.evergreen.num_ses = 1;
3130 rdev->config.evergreen.max_pipes = 4;
bd25f078 3131 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3132 if (rdev->pdev->device == 0x9648)
3133 rdev->config.evergreen.max_simds = 3;
3134 else if ((rdev->pdev->device == 0x9647) ||
3135 (rdev->pdev->device == 0x964a))
3136 rdev->config.evergreen.max_simds = 4;
3137 else
3138 rdev->config.evergreen.max_simds = 5;
3139 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3140 rdev->config.evergreen.max_gprs = 256;
3141 rdev->config.evergreen.max_threads = 248;
3142 rdev->config.evergreen.max_gs_threads = 32;
3143 rdev->config.evergreen.max_stack_entries = 256;
3144 rdev->config.evergreen.sx_num_of_sets = 4;
3145 rdev->config.evergreen.sx_max_export_size = 256;
3146 rdev->config.evergreen.sx_max_export_pos_size = 64;
3147 rdev->config.evergreen.sx_max_export_smx_size = 192;
3148 rdev->config.evergreen.max_hw_contexts = 8;
3149 rdev->config.evergreen.sq_num_cf_insts = 2;
3150
3151 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3152 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3153 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3154 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3155 break;
3156 case CHIP_SUMO2:
3157 rdev->config.evergreen.num_ses = 1;
3158 rdev->config.evergreen.max_pipes = 4;
3159 rdev->config.evergreen.max_tile_pipes = 4;
3160 rdev->config.evergreen.max_simds = 2;
3161 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3162 rdev->config.evergreen.max_gprs = 256;
3163 rdev->config.evergreen.max_threads = 248;
3164 rdev->config.evergreen.max_gs_threads = 32;
3165 rdev->config.evergreen.max_stack_entries = 512;
3166 rdev->config.evergreen.sx_num_of_sets = 4;
3167 rdev->config.evergreen.sx_max_export_size = 256;
3168 rdev->config.evergreen.sx_max_export_pos_size = 64;
3169 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3170 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3171 rdev->config.evergreen.sq_num_cf_insts = 2;
3172
adb68fa2
AD		 3173 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3174 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3175 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3176 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3177 break;
3178 case CHIP_BARTS:
3179 rdev->config.evergreen.num_ses = 2;
3180 rdev->config.evergreen.max_pipes = 4;
3181 rdev->config.evergreen.max_tile_pipes = 8;
3182 rdev->config.evergreen.max_simds = 7;
3183 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3184 rdev->config.evergreen.max_gprs = 256;
3185 rdev->config.evergreen.max_threads = 248;
3186 rdev->config.evergreen.max_gs_threads = 32;
3187 rdev->config.evergreen.max_stack_entries = 512;
3188 rdev->config.evergreen.sx_num_of_sets = 4;
3189 rdev->config.evergreen.sx_max_export_size = 256;
3190 rdev->config.evergreen.sx_max_export_pos_size = 64;
3191 rdev->config.evergreen.sx_max_export_smx_size = 192;
3192 rdev->config.evergreen.max_hw_contexts = 8;
3193 rdev->config.evergreen.sq_num_cf_insts = 2;
3194
3195 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3196 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3197 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3198 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3199 break;
3200 case CHIP_TURKS:
3201 rdev->config.evergreen.num_ses = 1;
3202 rdev->config.evergreen.max_pipes = 4;
3203 rdev->config.evergreen.max_tile_pipes = 4;
3204 rdev->config.evergreen.max_simds = 6;
3205 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3206 rdev->config.evergreen.max_gprs = 256;
3207 rdev->config.evergreen.max_threads = 248;
3208 rdev->config.evergreen.max_gs_threads = 32;
3209 rdev->config.evergreen.max_stack_entries = 256;
3210 rdev->config.evergreen.sx_num_of_sets = 4;
3211 rdev->config.evergreen.sx_max_export_size = 256;
3212 rdev->config.evergreen.sx_max_export_pos_size = 64;
3213 rdev->config.evergreen.sx_max_export_smx_size = 192;
3214 rdev->config.evergreen.max_hw_contexts = 8;
3215 rdev->config.evergreen.sq_num_cf_insts = 2;
3216
3217 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3218 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3219 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3220 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3221 break;
3222 case CHIP_CAICOS:
3223 rdev->config.evergreen.num_ses = 1;
bd25f078 3224 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3225 rdev->config.evergreen.max_tile_pipes = 2;
3226 rdev->config.evergreen.max_simds = 2;
3227 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3228 rdev->config.evergreen.max_gprs = 256;
3229 rdev->config.evergreen.max_threads = 192;
3230 rdev->config.evergreen.max_gs_threads = 16;
3231 rdev->config.evergreen.max_stack_entries = 256;
3232 rdev->config.evergreen.sx_num_of_sets = 4;
3233 rdev->config.evergreen.sx_max_export_size = 128;
3234 rdev->config.evergreen.sx_max_export_pos_size = 32;
3235 rdev->config.evergreen.sx_max_export_smx_size = 96;
3236 rdev->config.evergreen.max_hw_contexts = 4;
3237 rdev->config.evergreen.sq_num_cf_insts = 1;
3238
32fcdbf4
AD		 3239 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3240 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3241 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3242 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3243 break;
3244 }
3245
3246 /* Initialize HDP */
3247 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3248 WREG32((0x2c14 + j), 0x00000000);
3249 WREG32((0x2c18 + j), 0x00000000);
3250 WREG32((0x2c1c + j), 0x00000000);
3251 WREG32((0x2c20 + j), 0x00000000);
3252 WREG32((0x2c24 + j), 0x00000000);
3253 }
3254
3255 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
acc1522a
CK		 3256 WREG32(SRBM_INT_CNTL, 0x1);
3257 WREG32(SRBM_INT_ACK, 0x1);
32fcdbf4 3258
d054ac16
AD		 3259 evergreen_fix_pci_max_read_req_size(rdev);
3260
32fcdbf4 3261 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3262 if ((rdev->family == CHIP_PALM) ||
3263 (rdev->family == CHIP_SUMO) ||
3264 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3265 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3266 else
3267 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3268
1aa52bd3
AD		 3269 /* setup tiling info dword. gb_addr_config is not adequate since it does
3270 * not have bank info, so create a custom tiling dword.
3271 * bits 3:0 num_pipes
3272 * bits 7:4 num_banks
3273 * bits 11:8 group_size
3274 * bits 15:12 row_size
3275 */
3276 rdev->config.evergreen.tile_config = 0;
3277 switch (rdev->config.evergreen.max_tile_pipes) {
3278 case 1:
3279 default:
3280 rdev->config.evergreen.tile_config |= (0 << 0);
3281 break;
3282 case 2:
3283 rdev->config.evergreen.tile_config |= (1 << 0);
3284 break;
3285 case 4:
3286 rdev->config.evergreen.tile_config |= (2 << 0);
3287 break;
3288 case 8:
3289 rdev->config.evergreen.tile_config |= (3 << 0);
3290 break;
3291 }
d698a34d 3292 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3293 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3294 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3295 else {
c8d15edc
AD		 3296 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3297 case 0: /* four banks */
29d65406 3298 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3299 break;
3300 case 1: /* eight banks */
3301 rdev->config.evergreen.tile_config |= 1 << 4;
3302 break;
3303 case 2: /* sixteen banks */
3304 default:
3305 rdev->config.evergreen.tile_config |= 2 << 4;
3306 break;
3307 }
29d65406 3308 }
416a2bd2 3309 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3310 rdev->config.evergreen.tile_config |=
3311 ((gb_addr_config & 0x30000000) >> 28) << 12;
3312
416a2bd2
AD		 3313 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3314 u32 efuse_straps_4;
3315 u32 efuse_straps_3;
32fcdbf4 3316
ff82bbc4
AD		 3317 efuse_straps_4 = RREG32_RCU(0x204);
3318 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3319 tmp = (((efuse_straps_4 & 0xf) << 4) |
3320 ((efuse_straps_3 & 0xf0000000) >> 28));
3321 } else {
3322 tmp = 0;
3323 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3324 u32 rb_disable_bitmap;
3325
3326 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3327 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3328 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3329 tmp <<= 4;
3330 tmp |= rb_disable_bitmap;
32fcdbf4 3331 }
416a2bd2
AD		 3332 }
3333 /* enabled rb are just the one not disabled :) */
3334 disabled_rb_mask = tmp;
cedb655a
AD		 3335 tmp = 0;
3336 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3337 tmp |= (1 << i);
3338 /* if all the backends are disabled, fix it up here */
3339 if ((disabled_rb_mask & tmp) == tmp) {
3340 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3341 disabled_rb_mask &= ~(1 << i);
3342 }
32fcdbf4 3343
65fcf668
AD		 3344 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3345 u32 simd_disable_bitmap;
3346
3347 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3348 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3349 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3350 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3351 tmp <<= 16;
3352 tmp |= simd_disable_bitmap;
3353 }
3354 rdev->config.evergreen.active_simds = hweight32(~tmp);
3355
416a2bd2
AD		 3356 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3357 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3358
416a2bd2
AD		 3359 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3360 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3361 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3362 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3363 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3364 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3365 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3366
f7eb9730
AD		 3367 if ((rdev->config.evergreen.max_backends == 1) &&
3368 (rdev->flags & RADEON_IS_IGP)) {
3369 if ((disabled_rb_mask & 3) == 1) {
3370 /* RB0 disabled, RB1 enabled */
3371 tmp = 0x11111111;
3372 } else {
3373 /* RB1 disabled, RB0 enabled */
3374 tmp = 0x00000000;
3375 }
3376 } else {
3377 tmp = gb_addr_config & NUM_PIPES_MASK;
3378 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3379 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3380 }
416a2bd2 3381 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3382
3383 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3384 WREG32(CGTS_TCC_DISABLE, 0);
3385 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3386 WREG32(CGTS_USER_TCC_DISABLE, 0);
3387
3388 /* set HW defaults for 3D engine */
3389 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3390 ROQ_IB2_START(0x2b)));
3391
3392 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3393
3394 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3395 SYNC_GRADIENT |
3396 SYNC_WALKER |
3397 SYNC_ALIGNER));
3398
3399 sx_debug_1 = RREG32(SX_DEBUG_1);
3400 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3401 WREG32(SX_DEBUG_1, sx_debug_1);
3402
3403
3404 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3405 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3406 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3407 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3408
b866d133
AD		 3409 if (rdev->family <= CHIP_SUMO2)
3410 WREG32(SMX_SAR_CTL0, 0x00010000);
3411
32fcdbf4
AD		 3412 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3413 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3414 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3415
3416 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3417 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3418 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3419
3420 WREG32(VGT_NUM_INSTANCES, 1);
3421 WREG32(SPI_CONFIG_CNTL, 0);
3422 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3423 WREG32(CP_PERFMON_CNTL, 0);
3424
3425 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3426 FETCH_FIFO_HIWATER(0x4) |
3427 DONE_FIFO_HIWATER(0xe0) |
3428 ALU_UPDATE_FIFO_HIWATER(0x8)));
3429
3430 sq_config = RREG32(SQ_CONFIG);
3431 sq_config &= ~(PS_PRIO(3) |
3432 VS_PRIO(3) |
3433 GS_PRIO(3) |
3434 ES_PRIO(3));
3435 sq_config |= (VC_ENABLE |
3436 EXPORT_SRC_C |
3437 PS_PRIO(0) |
3438 VS_PRIO(1) |
3439 GS_PRIO(2) |
3440 ES_PRIO(3));
3441
d5e455e4
AD		 3442 switch (rdev->family) {
3443 case CHIP_CEDAR:
3444 case CHIP_PALM:
d5c5a72f
AD		 3445 case CHIP_SUMO:
3446 case CHIP_SUMO2:
adb68fa2 3447 case CHIP_CAICOS:
32fcdbf4
AD		 3448 /* no vertex cache */
3449 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3450 break;
3451 default:
3452 break;
3453 }
32fcdbf4
AD		 3454
3455 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3456
3457 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3458 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3459 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3460 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3461 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3462 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3463 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3464
d5e455e4
AD		 3465 switch (rdev->family) {
3466 case CHIP_CEDAR:
3467 case CHIP_PALM:
d5c5a72f
AD		 3468 case CHIP_SUMO:
3469 case CHIP_SUMO2:
32fcdbf4 3470 ps_thread_count = 96;
d5e455e4
AD		 3471 break;
3472 default:
32fcdbf4 3473 ps_thread_count = 128;
d5e455e4
AD		 3474 break;
3475 }
32fcdbf4
AD		 3476
3477 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3478 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3479 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3480 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3481 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3482 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3483
3484 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3487 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3488 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3489 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3490
3491 WREG32(SQ_CONFIG, sq_config);
3492 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3493 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3494 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3495 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3496 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3497 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3498 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3499 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3500 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3501 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3502
3503 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3504 FORCE_EOV_MAX_REZ_CNT(255)));
3505
d5e455e4
AD		 3506 switch (rdev->family) {
3507 case CHIP_CEDAR:
3508 case CHIP_PALM:
d5c5a72f
AD		 3509 case CHIP_SUMO:
3510 case CHIP_SUMO2:
adb68fa2 3511 case CHIP_CAICOS:
32fcdbf4 3512 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3513 break;
3514 default:
32fcdbf4 3515 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3516 break;
3517 }
32fcdbf4
AD		 3518 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3519 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3520
3521 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3522 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3523 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3524
60a4a3e0
AD		 3525 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3526 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3527
32fcdbf4
AD		 3528 WREG32(CB_PERF_CTR0_SEL_0, 0);
3529 WREG32(CB_PERF_CTR0_SEL_1, 0);
3530 WREG32(CB_PERF_CTR1_SEL_0, 0);
3531 WREG32(CB_PERF_CTR1_SEL_1, 0);
3532 WREG32(CB_PERF_CTR2_SEL_0, 0);
3533 WREG32(CB_PERF_CTR2_SEL_1, 0);
3534 WREG32(CB_PERF_CTR3_SEL_0, 0);
3535 WREG32(CB_PERF_CTR3_SEL_1, 0);
3536
60a4a3e0
AD		 3537 /* clear render buffer base addresses */
3538 WREG32(CB_COLOR0_BASE, 0);
3539 WREG32(CB_COLOR1_BASE, 0);
3540 WREG32(CB_COLOR2_BASE, 0);
3541 WREG32(CB_COLOR3_BASE, 0);
3542 WREG32(CB_COLOR4_BASE, 0);
3543 WREG32(CB_COLOR5_BASE, 0);
3544 WREG32(CB_COLOR6_BASE, 0);
3545 WREG32(CB_COLOR7_BASE, 0);
3546 WREG32(CB_COLOR8_BASE, 0);
3547 WREG32(CB_COLOR9_BASE, 0);
3548 WREG32(CB_COLOR10_BASE, 0);
3549 WREG32(CB_COLOR11_BASE, 0);
3550
3551 /* set the shader const cache sizes to 0 */
3552 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3553 WREG32(i, 0);
3554 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3555 WREG32(i, 0);
3556
f25a5c63
AD		 3557 tmp = RREG32(HDP_MISC_CNTL);
3558 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3559 WREG32(HDP_MISC_CNTL, tmp);
3560
32fcdbf4
AD		 3561 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3562 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3563
3564 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3565
3566 udelay(50);
3567
bcc1c2a1
AD		 3568}
3569
3570int evergreen_mc_init(struct radeon_device *rdev)
3571{
bcc1c2a1
AD		 3572 u32 tmp;
3573 int chansize, numchan;
bcc1c2a1
AD		 3574
3575 /* Get VRAM informations */
3576 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3577 if ((rdev->family == CHIP_PALM) ||
3578 (rdev->family == CHIP_SUMO) ||
3579 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3580 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3581 else
3582 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3583 if (tmp & CHANSIZE_OVERRIDE) {
3584 chansize = 16;
3585 } else if (tmp & CHANSIZE_MASK) {
3586 chansize = 64;
3587 } else {
3588 chansize = 32;
3589 }
3590 tmp = RREG32(MC_SHARED_CHMAP);
3591 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3592 case 0:
3593 default:
3594 numchan = 1;
3595 break;
3596 case 1:
3597 numchan = 2;
3598 break;
3599 case 2:
3600 numchan = 4;
3601 break;
3602 case 3:
3603 numchan = 8;
3604 break;
3605 }
3606 rdev->mc.vram_width = numchan * chansize;
3607 /* Could aper size report 0 ? */
01d73a69
JC		 3608 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3609 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3610 /* Setup GPU memory space */
05b3ef69
AD		 3611 if ((rdev->family == CHIP_PALM) ||
3612 (rdev->family == CHIP_SUMO) ||
3613 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3614 /* size in bytes on fusion */
3615 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3616 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3617 } else {
05b3ef69 3618 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3619 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3620 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3621 }
51e5fcd3 3622 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3623 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3624 radeon_update_bandwidth_info(rdev);
3625
bcc1c2a1
AD		 3626 return 0;
3627}
d594e46a 3628
187e3593 3629void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3630{
64c56e8c 3631 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3632 RREG32(GRBM_STATUS));
64c56e8c 3633 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3634 RREG32(GRBM_STATUS_SE0));
64c56e8c 3635 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3636 RREG32(GRBM_STATUS_SE1));
64c56e8c 3637 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3638 RREG32(SRBM_STATUS));
a65a4369
AD		 3639 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3640 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3641 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3642 RREG32(CP_STALLED_STAT1));
3643 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3644 RREG32(CP_STALLED_STAT2));
3645 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3646 RREG32(CP_BUSY_STAT));
3647 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3648 RREG32(CP_STAT));
eaaa6983
JG		 3649 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3650 RREG32(DMA_STATUS_REG));
168757ea
AD		 3651 if (rdev->family >= CHIP_CAYMAN) {
3652 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3653 RREG32(DMA_STATUS_REG + 0x800));
3654 }
0ecebb9e
AD		 3655}
3656
168757ea 3657bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3658{
a65a4369
AD		 3659 u32 crtc_hung = 0;
3660 u32 crtc_status[6];
3661 u32 i, j, tmp;
3662
3663 for (i = 0; i < rdev->num_crtc; i++) {
3664 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3665 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3666 crtc_hung |= (1 << i);
3667 }
3668 }
3669
3670 for (j = 0; j < 10; j++) {
3671 for (i = 0; i < rdev->num_crtc; i++) {
3672 if (crtc_hung & (1 << i)) {
3673 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3674 if (tmp != crtc_status[i])
3675 crtc_hung &= ~(1 << i);
3676 }
3677 }
3678 if (crtc_hung == 0)
3679 return false;
3680 udelay(100);
3681 }
3682
3683 return true;
3684}
3685
2483b4ea 3686u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3687{
3688 u32 reset_mask = 0;
b7630473 3689 u32 tmp;
0ecebb9e 3690
a65a4369
AD		 3691 /* GRBM_STATUS */
3692 tmp = RREG32(GRBM_STATUS);
3693 if (tmp & (PA_BUSY | SC_BUSY |
3694 SH_BUSY | SX_BUSY |
3695 TA_BUSY | VGT_BUSY |
3696 DB_BUSY | CB_BUSY |
3697 SPI_BUSY | VGT_BUSY_NO_DMA))
3698 reset_mask |= RADEON_RESET_GFX;
3699
3700 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3701 CP_BUSY | CP_COHERENCY_BUSY))
3702 reset_mask |= RADEON_RESET_CP;
3703
3704 if (tmp & GRBM_EE_BUSY)
3705 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3706
a65a4369
AD		 3707 /* DMA_STATUS_REG */
3708 tmp = RREG32(DMA_STATUS_REG);
3709 if (!(tmp & DMA_IDLE))
3710 reset_mask |= RADEON_RESET_DMA;
3711
3712 /* SRBM_STATUS2 */
3713 tmp = RREG32(SRBM_STATUS2);
3714 if (tmp & DMA_BUSY)
3715 reset_mask |= RADEON_RESET_DMA;
3716
3717 /* SRBM_STATUS */
3718 tmp = RREG32(SRBM_STATUS);
3719 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3720 reset_mask |= RADEON_RESET_RLC;
3721
3722 if (tmp & IH_BUSY)
3723 reset_mask |= RADEON_RESET_IH;
3724
3725 if (tmp & SEM_BUSY)
3726 reset_mask |= RADEON_RESET_SEM;
3727
3728 if (tmp & GRBM_RQ_PENDING)
3729 reset_mask |= RADEON_RESET_GRBM;
3730
3731 if (tmp & VMC_BUSY)
3732 reset_mask |= RADEON_RESET_VMC;
3733
3734 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3735 MCC_BUSY | MCD_BUSY))
3736 reset_mask |= RADEON_RESET_MC;
3737
3738 if (evergreen_is_display_hung(rdev))
3739 reset_mask |= RADEON_RESET_DISPLAY;
3740
3741 /* VM_L2_STATUS */
3742 tmp = RREG32(VM_L2_STATUS);
3743 if (tmp & L2_BUSY)
3744 reset_mask |= RADEON_RESET_VMC;
3745
d808fc88
AD		 3746 /* Skip MC reset as it's mostly likely not hung, just busy */
3747 if (reset_mask & RADEON_RESET_MC) {
3748 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3749 reset_mask &= ~RADEON_RESET_MC;
3750 }
3751
a65a4369
AD		 3752 return reset_mask;
3753}
3754
3755static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3756{
3757 struct evergreen_mc_save save;
3758 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3759 u32 tmp;
19fc42ed 3760
0ecebb9e 3761 if (reset_mask == 0)
a65a4369 3762 return;
0ecebb9e
AD		 3763
3764 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3765
b7630473
AD		 3766 evergreen_print_gpu_status_regs(rdev);
3767
b7630473
AD		 3768 /* Disable CP parsing/prefetching */
3769 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3770
3771 if (reset_mask & RADEON_RESET_DMA) {
3772 /* Disable DMA */
3773 tmp = RREG32(DMA_RB_CNTL);
3774 tmp &= ~DMA_RB_ENABLE;
3775 WREG32(DMA_RB_CNTL, tmp);
3776 }
3777
b21b6e7a
AD		 3778 udelay(50);
3779
3780 evergreen_mc_stop(rdev, &save);
3781 if (evergreen_mc_wait_for_idle(rdev)) {
3782 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3783 }
3784
b7630473
AD		 3785 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3786 grbm_soft_reset |= SOFT_RESET_DB |
3787 SOFT_RESET_CB |
3788 SOFT_RESET_PA |
3789 SOFT_RESET_SC |
3790 SOFT_RESET_SPI |
3791 SOFT_RESET_SX |
3792 SOFT_RESET_SH |
3793 SOFT_RESET_TC |
3794 SOFT_RESET_TA |
3795 SOFT_RESET_VC |
3796 SOFT_RESET_VGT;
3797 }
3798
3799 if (reset_mask & RADEON_RESET_CP) {
3800 grbm_soft_reset |= SOFT_RESET_CP |
3801 SOFT_RESET_VGT;
3802
3803 srbm_soft_reset |= SOFT_RESET_GRBM;
3804 }
0ecebb9e
AD		 3805
3806 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3807 srbm_soft_reset |= SOFT_RESET_DMA;
3808
a65a4369
AD		 3809 if (reset_mask & RADEON_RESET_DISPLAY)
3810 srbm_soft_reset |= SOFT_RESET_DC;
3811
3812 if (reset_mask & RADEON_RESET_RLC)
3813 srbm_soft_reset |= SOFT_RESET_RLC;
3814
3815 if (reset_mask & RADEON_RESET_SEM)
3816 srbm_soft_reset |= SOFT_RESET_SEM;
3817
3818 if (reset_mask & RADEON_RESET_IH)
3819 srbm_soft_reset |= SOFT_RESET_IH;
3820
3821 if (reset_mask & RADEON_RESET_GRBM)
3822 srbm_soft_reset |= SOFT_RESET_GRBM;
3823
3824 if (reset_mask & RADEON_RESET_VMC)
3825 srbm_soft_reset |= SOFT_RESET_VMC;
3826
24178ec4
AD		 3827 if (!(rdev->flags & RADEON_IS_IGP)) {
3828 if (reset_mask & RADEON_RESET_MC)
3829 srbm_soft_reset |= SOFT_RESET_MC;
3830 }
a65a4369 3831
b7630473
AD		 3832 if (grbm_soft_reset) {
3833 tmp = RREG32(GRBM_SOFT_RESET);
3834 tmp |= grbm_soft_reset;
3835 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3836 WREG32(GRBM_SOFT_RESET, tmp);
3837 tmp = RREG32(GRBM_SOFT_RESET);
3838
3839 udelay(50);
3840
3841 tmp &= ~grbm_soft_reset;
3842 WREG32(GRBM_SOFT_RESET, tmp);
3843 tmp = RREG32(GRBM_SOFT_RESET);
3844 }
3845
3846 if (srbm_soft_reset) {
3847 tmp = RREG32(SRBM_SOFT_RESET);
3848 tmp |= srbm_soft_reset;
3849 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3850 WREG32(SRBM_SOFT_RESET, tmp);
3851 tmp = RREG32(SRBM_SOFT_RESET);
3852
3853 udelay(50);
3854
3855 tmp &= ~srbm_soft_reset;
3856 WREG32(SRBM_SOFT_RESET, tmp);
3857 tmp = RREG32(SRBM_SOFT_RESET);
3858 }
0ecebb9e
AD		 3859
3860 /* Wait a little for things to settle down */
3861 udelay(50);
3862
747943ea 3863 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3864 udelay(50);
3865
b7630473 3866 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3867}
3868
b5470b03
AD		 3869void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3870{
3871 struct evergreen_mc_save save;
3872 u32 tmp, i;
3873
3874 dev_info(rdev->dev, "GPU pci config reset\n");
3875
3876 /* disable dpm? */
3877
3878 /* Disable CP parsing/prefetching */
3879 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3880 udelay(50);
3881 /* Disable DMA */
3882 tmp = RREG32(DMA_RB_CNTL);
3883 tmp &= ~DMA_RB_ENABLE;
3884 WREG32(DMA_RB_CNTL, tmp);
3885 /* XXX other engines? */
3886
3887 /* halt the rlc */
3888 r600_rlc_stop(rdev);
3889
3890 udelay(50);
3891
3892 /* set mclk/sclk to bypass */
3893 rv770_set_clk_bypass_mode(rdev);
3894 /* disable BM */
3895 pci_clear_master(rdev->pdev);
3896 /* disable mem access */
3897 evergreen_mc_stop(rdev, &save);
3898 if (evergreen_mc_wait_for_idle(rdev)) {
3899 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3900 }
3901 /* reset */
3902 radeon_pci_config_reset(rdev);
3903 /* wait for asic to come out of reset */
3904 for (i = 0; i < rdev->usec_timeout; i++) {
3905 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3906 break;
3907 udelay(1);
3908 }
3909}
3910
a2d07b74 3911int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3912{
a65a4369
AD		 3913 u32 reset_mask;
3914
3915 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3916
3917 if (reset_mask)
3918 r600_set_bios_scratch_engine_hung(rdev, true);
3919
b5470b03 3920 /* try soft reset */
a65a4369
AD		 3921 evergreen_gpu_soft_reset(rdev, reset_mask);
3922
3923 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3924
b5470b03
AD		 3925 /* try pci config reset */
3926 if (reset_mask && radeon_hard_reset)
3927 evergreen_gpu_pci_config_reset(rdev);
3928
3929 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3930
a65a4369
AD		 3931 if (!reset_mask)
3932 r600_set_bios_scratch_engine_hung(rdev, false);
3933
3934 return 0;
747943ea
AD		 3935}
3936
123bc183
AD		 3937/**
3938 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3939 *
3940 * @rdev: radeon_device pointer
3941 * @ring: radeon_ring structure holding ring information
3942 *
3943 * Check if the GFX engine is locked up.
3944 * Returns true if the engine appears to be locked up, false if not.
3945 */
3946bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3947{
3948 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3949
3950 if (!(reset_mask & (RADEON_RESET_GFX |
3951 RADEON_RESET_COMPUTE |
3952 RADEON_RESET_CP))) {
ff212f25 3953 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 3954 return false;
3955 }
123bc183
AD		 3956 return radeon_ring_test_lockup(rdev, ring);
3957}
3958
2948f5e6
AD		 3959/*
3960 * RLC
3961 */
3962#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3963#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3964
3965void sumo_rlc_fini(struct radeon_device *rdev)
3966{
3967 int r;
3968
3969 /* save restore block */
3970 if (rdev->rlc.save_restore_obj) {
3971 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3972 if (unlikely(r != 0))
3973 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3974 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3975 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3976
3977 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3978 rdev->rlc.save_restore_obj = NULL;
3979 }
3980
3981 /* clear state block */
3982 if (rdev->rlc.clear_state_obj) {
3983 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3984 if (unlikely(r != 0))
3985 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3986 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3987 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3988
3989 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3990 rdev->rlc.clear_state_obj = NULL;
3991 }
22c775ce
AD		 3992
3993 /* clear state block */
3994 if (rdev->rlc.cp_table_obj) {
3995 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3996 if (unlikely(r != 0))
3997 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3998 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3999 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4000
4001 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4002 rdev->rlc.cp_table_obj = NULL;
4003 }
2948f5e6
AD		 4004}
4005
22c775ce
AD		 4006#define CP_ME_TABLE_SIZE 96
4007
2948f5e6
AD		 4008int sumo_rlc_init(struct radeon_device *rdev)
4009{
1fd11777 4010 const u32 *src_ptr;
2948f5e6
AD		 4011 volatile u32 *dst_ptr;
4012 u32 dws, data, i, j, k, reg_num;
59a82d0e 4013 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4014 u64 reg_list_mc_addr;
1fd11777 4015 const struct cs_section_def *cs_data;
2948f5e6
AD		 4016 int r;
4017
4018 src_ptr = rdev->rlc.reg_list;
4019 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4020 if (rdev->family >= CHIP_BONAIRE) {
4021 dws += (5 * 16) + 48 + 48 + 64;
4022 }
2948f5e6
AD		 4023 cs_data = rdev->rlc.cs_data;
4024
10b7ca7e
AD		 4025 if (src_ptr) {
4026 /* save restore block */
4027 if (rdev->rlc.save_restore_obj == NULL) {
4028 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4029 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4030 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4031 if (r) {
4032 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4033 return r;
4034 }
4035 }
4036
4037 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4038 if (unlikely(r != 0)) {
4039 sumo_rlc_fini(rdev);
4040 return r;
4041 }
4042 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4043 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4044 if (r) {
10b7ca7e
AD		 4045 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4046 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4047 sumo_rlc_fini(rdev);
2948f5e6
AD		 4048 return r;
4049 }
2948f5e6 4050
10b7ca7e
AD		 4051 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4052 if (r) {
4053 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4054 sumo_rlc_fini(rdev);
4055 return r;
4056 }
4057 /* write the sr buffer */
4058 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4059 if (rdev->family >= CHIP_TAHITI) {
4060 /* SI */
59a82d0e 4061 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4062 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4063 } else {
4064 /* ON/LN/TN */
4065 /* format:
4066 * dw0: (reg2 << 16) | reg1
4067 * dw1: reg1 save space
4068 * dw2: reg2 save space
4069 */
4070 for (i = 0; i < dws; i++) {
4071 data = src_ptr[i] >> 2;
4072 i++;
4073 if (i < dws)
4074 data |= (src_ptr[i] >> 2) << 16;
4075 j = (((i - 1) * 3) / 2);
6ba81e53 4076 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4077 }
4078 j = ((i * 3) / 2);
6ba81e53 4079 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4080 }
10b7ca7e 4081 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4082 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4083 }
2948f5e6 4084
10b7ca7e
AD		 4085 if (cs_data) {
4086 /* clear state block */
a0f38609
AD		 4087 if (rdev->family >= CHIP_BONAIRE) {
4088 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4089 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4090 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4091 dws = rdev->rlc.clear_state_size + (256 / 4);
4092 } else {
4093 reg_list_num = 0;
4094 dws = 0;
4095 for (i = 0; cs_data[i].section != NULL; i++) {
4096 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4097 reg_list_num++;
4098 dws += cs_data[i].section[j].reg_count;
4099 }
10b7ca7e 4100 }
59a82d0e
AD		 4101 reg_list_blk_index = (3 * reg_list_num + 2);
4102 dws += reg_list_blk_index;
4103 rdev->rlc.clear_state_size = dws;
2948f5e6 4104 }
2948f5e6 4105
10b7ca7e 4106 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4107 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4108 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4109 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4110 if (r) {
4111 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4112 sumo_rlc_fini(rdev);
4113 return r;
4114 }
4115 }
4116 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4117 if (unlikely(r != 0)) {
4118 sumo_rlc_fini(rdev);
4119 return r;
4120 }
4121 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4122 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4123 if (r) {
10b7ca7e
AD		 4124 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4125 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4126 sumo_rlc_fini(rdev);
4127 return r;
4128 }
2948f5e6 4129
10b7ca7e
AD		 4130 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4131 if (r) {
4132 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4133 sumo_rlc_fini(rdev);
4134 return r;
4135 }
4136 /* set up the cs buffer */
4137 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4138 if (rdev->family >= CHIP_BONAIRE) {
4139 cik_get_csb_buffer(rdev, dst_ptr);
4140 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4141 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4142 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4143 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4144 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4145 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4146 } else {
4147 reg_list_hdr_blk_index = 0;
4148 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4149 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4150 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4151 reg_list_hdr_blk_index++;
59a82d0e
AD		 4152 for (i = 0; cs_data[i].section != NULL; i++) {
4153 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4154 reg_num = cs_data[i].section[j].reg_count;
4155 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4156 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4157 reg_list_hdr_blk_index++;
4158
4159 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4160 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4161 reg_list_hdr_blk_index++;
4162
4163 data = 0x08000000 | (reg_num * 4);
6ba81e53 4164 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4165 reg_list_hdr_blk_index++;
4166
4167 for (k = 0; k < reg_num; k++) {
4168 data = cs_data[i].section[j].extent[k];
6ba81e53 4169 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4170 }
4171 reg_list_mc_addr += reg_num * 4;
4172 reg_list_blk_index += reg_num;
10b7ca7e 4173 }
2948f5e6 4174 }
6ba81e53 4175 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4176 }
10b7ca7e
AD		 4177 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4178 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4179 }
2948f5e6 4180
22c775ce
AD		 4181 if (rdev->rlc.cp_table_size) {
4182 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4183 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4184 PAGE_SIZE, true,
4185 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4186 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4187 if (r) {
4188 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4189 sumo_rlc_fini(rdev);
4190 return r;
4191 }
4192 }
4193
4194 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4195 if (unlikely(r != 0)) {
4196 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4197 sumo_rlc_fini(rdev);
4198 return r;
4199 }
4200 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4201 &rdev->rlc.cp_table_gpu_addr);
4202 if (r) {
4203 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4204 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4205 sumo_rlc_fini(rdev);
4206 return r;
4207 }
4208 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4209 if (r) {
4210 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4211 sumo_rlc_fini(rdev);
4212 return r;
4213 }
4214
4215 cik_init_cp_pg_table(rdev);
4216
4217 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4218 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4219
4220 }
2948f5e6
AD		 4221
4222 return 0;
4223}
4224
4225static void evergreen_rlc_start(struct radeon_device *rdev)
4226{
8ba10463
AD		 4227 u32 mask = RLC_ENABLE;
4228
4229 if (rdev->flags & RADEON_IS_IGP) {
4230 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4231 }
4232
4233 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4234}
4235
4236int evergreen_rlc_resume(struct radeon_device *rdev)
4237{
4238 u32 i;
4239 const __be32 *fw_data;
4240
4241 if (!rdev->rlc_fw)
4242 return -EINVAL;
4243
4244 r600_rlc_stop(rdev);
4245
4246 WREG32(RLC_HB_CNTL, 0);
4247
4248 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4249 if (rdev->family == CHIP_ARUBA) {
4250 u32 always_on_bitmap =
4251 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4252 /* find out the number of active simds */
4253 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4254 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4255 tmp = hweight32(~tmp);
4256 if (tmp == rdev->config.cayman.max_simds_per_se) {
4257 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4258 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4259 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4260 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4261 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4262 }
4263 } else {
4264 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4265 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4266 }
2948f5e6
AD		 4267 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4268 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4269 } else {
4270 WREG32(RLC_HB_BASE, 0);
4271 WREG32(RLC_HB_RPTR, 0);
4272 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4273 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4274 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4275 }
2948f5e6
AD		 4276 WREG32(RLC_MC_CNTL, 0);
4277 WREG32(RLC_UCODE_CNTL, 0);
4278
4279 fw_data = (const __be32 *)rdev->rlc_fw->data;
4280 if (rdev->family >= CHIP_ARUBA) {
4281 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4282 WREG32(RLC_UCODE_ADDR, i);
4283 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4284 }
4285 } else if (rdev->family >= CHIP_CAYMAN) {
4286 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4287 WREG32(RLC_UCODE_ADDR, i);
4288 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289 }
4290 } else {
4291 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4292 WREG32(RLC_UCODE_ADDR, i);
4293 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4294 }
4295 }
4296 WREG32(RLC_UCODE_ADDR, 0);
4297
4298 evergreen_rlc_start(rdev);
4299
4300 return 0;
4301}
4302
45f9a39b
AD		 4303/* Interrupts */
4304
4305u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4306{
46437057 4307 if (crtc >= rdev->num_crtc)
45f9a39b 4308 return 0;
46437057
AD		 4309 else
4310 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4311}
4312
4313void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4314{
4315 u32 tmp;
4316
1b37078b
AD		 4317 if (rdev->family >= CHIP_CAYMAN) {
4318 cayman_cp_int_cntl_setup(rdev, 0,
4319 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4320 cayman_cp_int_cntl_setup(rdev, 1, 0);
4321 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4322 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4323 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4324 } else
4325 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4326 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4327 WREG32(DMA_CNTL, tmp);
45f9a39b 4328 WREG32(GRBM_INT_CNTL, 0);
acc1522a 4329 WREG32(SRBM_INT_CNTL, 0);
45f9a39b
AD		 4330 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4331 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4332 if (rdev->num_crtc >= 4) {
18007401
AD		 4333 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4334 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4335 }
4336 if (rdev->num_crtc >= 6) {
18007401
AD		 4337 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4338 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4339 }
45f9a39b
AD		 4340
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4343 if (rdev->num_crtc >= 4) {
18007401
AD		 4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4346 }
4347 if (rdev->num_crtc >= 6) {
18007401
AD		 4348 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4349 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4350 }
45f9a39b 4351
e9a321c6
AD		 4352 /* only one DAC on DCE5 */
4353 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4354 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4355 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4356
4357 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358 WREG32(DC_HPD1_INT_CONTROL, tmp);
4359 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360 WREG32(DC_HPD2_INT_CONTROL, tmp);
4361 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4362 WREG32(DC_HPD3_INT_CONTROL, tmp);
4363 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4364 WREG32(DC_HPD4_INT_CONTROL, tmp);
4365 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4366 WREG32(DC_HPD5_INT_CONTROL, tmp);
4367 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4368 WREG32(DC_HPD6_INT_CONTROL, tmp);
4369
4370}
4371
4372int evergreen_irq_set(struct radeon_device *rdev)
4373{
4374 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4375 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4376 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4377 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4378 u32 grbm_int_cntl = 0;
f122c610 4379 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4380 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4381 u32 thermal_int = 0;
45f9a39b
AD		 4382
4383 if (!rdev->irq.installed) {
fce7d61b 4384 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4385 return -EINVAL;
4386 }
4387 /* don't enable anything if the ih is disabled */
4388 if (!rdev->ih.enabled) {
4389 r600_disable_interrupts(rdev);
4390 /* force the active interrupt state to all disabled */
4391 evergreen_disable_interrupt_state(rdev);
4392 return 0;
4393 }
4394
4395 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4396 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4397 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4398 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4399 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4400 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4401 if (rdev->family == CHIP_ARUBA)
4402 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4403 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4404 else
4405 thermal_int = RREG32(CG_THERMAL_INT) &
4406 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4407
f122c610
AD		 4408 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4409 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4410 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4411 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4412 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4413 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4414
233d1ad5
AD		 4415 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4416
1b37078b
AD		 4417 if (rdev->family >= CHIP_CAYMAN) {
4418 /* enable CP interrupts on all rings */
736fc37f 4419 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4420 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4421 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4422 }
736fc37f 4423 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4424 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4425 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4426 }
736fc37f 4427 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4428 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4429 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4430 }
4431 } else {
736fc37f 4432 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4433 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4434 cp_int_cntl |= RB_INT_ENABLE;
4435 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4436 }
45f9a39b 4437 }
1b37078b 4438
233d1ad5
AD		 4439 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4440 DRM_DEBUG("r600_irq_set: sw int dma\n");
4441 dma_cntl |= TRAP_ENABLE;
4442 }
4443
f60cbd11
AD		 4444 if (rdev->family >= CHIP_CAYMAN) {
4445 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4446 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4447 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4448 dma_cntl1 |= TRAP_ENABLE;
4449 }
4450 }
4451
dc50ba7f
AD		 4452 if (rdev->irq.dpm_thermal) {
4453 DRM_DEBUG("dpm thermal\n");
4454 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4455 }
4456
6f34be50 4457 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4458 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4459 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4460 crtc1 |= VBLANK_INT_MASK;
4461 }
6f34be50 4462 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4463 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4464 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4465 crtc2 |= VBLANK_INT_MASK;
4466 }
6f34be50 4467 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4468 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4469 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4470 crtc3 |= VBLANK_INT_MASK;
4471 }
6f34be50 4472 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4473 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4474 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4475 crtc4 |= VBLANK_INT_MASK;
4476 }
6f34be50 4477 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4478 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4479 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4480 crtc5 |= VBLANK_INT_MASK;
4481 }
6f34be50 4482 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4483 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4484 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4485 crtc6 |= VBLANK_INT_MASK;
4486 }
4487 if (rdev->irq.hpd[0]) {
4488 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4489 hpd1 |= DC_HPDx_INT_EN;
4490 }
4491 if (rdev->irq.hpd[1]) {
4492 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4493 hpd2 |= DC_HPDx_INT_EN;
4494 }
4495 if (rdev->irq.hpd[2]) {
4496 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4497 hpd3 |= DC_HPDx_INT_EN;
4498 }
4499 if (rdev->irq.hpd[3]) {
4500 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4501 hpd4 |= DC_HPDx_INT_EN;
4502 }
4503 if (rdev->irq.hpd[4]) {
4504 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4505 hpd5 |= DC_HPDx_INT_EN;
4506 }
4507 if (rdev->irq.hpd[5]) {
4508 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4509 hpd6 |= DC_HPDx_INT_EN;
4510 }
f122c610
AD		 4511 if (rdev->irq.afmt[0]) {
4512 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4513 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4514 }
4515 if (rdev->irq.afmt[1]) {
4516 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4517 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4518 }
4519 if (rdev->irq.afmt[2]) {
4520 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4521 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4522 }
4523 if (rdev->irq.afmt[3]) {
4524 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4525 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4526 }
4527 if (rdev->irq.afmt[4]) {
4528 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4529 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4530 }
4531 if (rdev->irq.afmt[5]) {
4532 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4533 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4534 }
45f9a39b 4535
1b37078b
AD		 4536 if (rdev->family >= CHIP_CAYMAN) {
4537 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4538 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4539 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4540 } else
4541 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4542
4543 WREG32(DMA_CNTL, dma_cntl);
4544
f60cbd11
AD		 4545 if (rdev->family >= CHIP_CAYMAN)
4546 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4547
2031f77c 4548 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4549
4550 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4551 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4552 if (rdev->num_crtc >= 4) {
18007401
AD		 4553 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4554 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4555 }
4556 if (rdev->num_crtc >= 6) {
18007401
AD		 4557 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4558 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4559 }
45f9a39b 4560
f5d636d2
CK		 4561 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4562 GRPH_PFLIP_INT_MASK);
4563 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4564 GRPH_PFLIP_INT_MASK);
b7eff394 4565 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4566 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4567 GRPH_PFLIP_INT_MASK);
4568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4569 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4570 }
4571 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4573 GRPH_PFLIP_INT_MASK);
4574 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4575 GRPH_PFLIP_INT_MASK);
b7eff394 4576 }
6f34be50 4577
45f9a39b
AD		 4578 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4579 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4580 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4581 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4582 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4583 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4584 if (rdev->family == CHIP_ARUBA)
4585 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4586 else
4587 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4588
f122c610
AD		 4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4590 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4591 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4592 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4593 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4594 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4595
c320bb5f
AD		 4596 /* posting read */
4597 RREG32(SRBM_STATUS);
4598
bcc1c2a1
AD		 4599 return 0;
4600}
4601
cbdd4501 4602static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4603{
4604 u32 tmp;
4605
6f34be50
AD		 4606 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4607 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4608 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4609 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4610 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4611 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4612 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4613 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4614 if (rdev->num_crtc >= 4) {
4615 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4617 }
4618 if (rdev->num_crtc >= 6) {
4619 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4621 }
6f34be50 4622
f122c610
AD		 4623 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4624 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4625 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4626 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4627 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4628 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4629
6f34be50
AD		 4630 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4631 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4632 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4634 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4635 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4636 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4637 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4638 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4639 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4640 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4641 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4642
b7eff394
AD		 4643 if (rdev->num_crtc >= 4) {
4644 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4645 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4646 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4647 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4649 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4650 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4651 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4652 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4653 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4655 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4656 }
4657
4658 if (rdev->num_crtc >= 6) {
4659 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4660 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4661 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4662 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4663 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4664 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4665 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4666 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4667 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4668 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4670 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4671 }
45f9a39b 4672
6f34be50 4673 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4674 tmp = RREG32(DC_HPD1_INT_CONTROL);
4675 tmp |= DC_HPDx_INT_ACK;
4676 WREG32(DC_HPD1_INT_CONTROL, tmp);
4677 }
6f34be50 4678 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4679 tmp = RREG32(DC_HPD2_INT_CONTROL);
4680 tmp |= DC_HPDx_INT_ACK;
4681 WREG32(DC_HPD2_INT_CONTROL, tmp);
4682 }
6f34be50 4683 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4684 tmp = RREG32(DC_HPD3_INT_CONTROL);
4685 tmp |= DC_HPDx_INT_ACK;
4686 WREG32(DC_HPD3_INT_CONTROL, tmp);
4687 }
6f34be50 4688 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4689 tmp = RREG32(DC_HPD4_INT_CONTROL);
4690 tmp |= DC_HPDx_INT_ACK;
4691 WREG32(DC_HPD4_INT_CONTROL, tmp);
4692 }
6f34be50 4693 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4694 tmp = RREG32(DC_HPD5_INT_CONTROL);
4695 tmp |= DC_HPDx_INT_ACK;
4696 WREG32(DC_HPD5_INT_CONTROL, tmp);
4697 }
6f34be50 4698 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4699 tmp = RREG32(DC_HPD5_INT_CONTROL);
4700 tmp |= DC_HPDx_INT_ACK;
4701 WREG32(DC_HPD6_INT_CONTROL, tmp);
4702 }
f122c610
AD		 4703 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4704 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4705 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4706 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4707 }
4708 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4709 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4710 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4711 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4712 }
4713 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4714 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4715 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4716 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4717 }
4718 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4719 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4720 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4721 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4722 }
4723 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4724 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4725 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4726 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4727 }
4728 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4729 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4730 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4731 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4732 }
45f9a39b
AD		 4733}
4734
1109ca09 4735static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4736{
45f9a39b
AD		 4737 r600_disable_interrupts(rdev);
4738 /* Wait and acknowledge irq */
4739 mdelay(1);
6f34be50 4740 evergreen_irq_ack(rdev);
45f9a39b
AD		 4741 evergreen_disable_interrupt_state(rdev);
4742}
4743
755d819e 4744void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4745{
4746 evergreen_irq_disable(rdev);
4747 r600_rlc_stop(rdev);
4748}
4749
cbdd4501 4750static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4751{
4752 u32 wptr, tmp;
4753
724c80e1 4754 if (rdev->wb.enabled)
204ae24d 4755 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4756 else
4757 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4758
4759 if (wptr & RB_OVERFLOW) {
11bab0ae 4760 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 4761 /* When a ring buffer overflow happen start parsing interrupt
4762 * from the last not overwritten vector (wptr + 16). Hopefully
4763 * this should allow us to catchup.
4764 */
6cc2fda2
MD		 4765 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4766 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 4767 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4768 tmp = RREG32(IH_RB_CNTL);
4769 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4770 WREG32(IH_RB_CNTL, tmp);
4771 }
4772 return (wptr & rdev->ih.ptr_mask);
4773}
4774
4775int evergreen_irq_process(struct radeon_device *rdev)
4776{
682f1a54
DA		 4777 u32 wptr;
4778 u32 rptr;
45f9a39b
AD		 4779 u32 src_id, src_data;
4780 u32 ring_index;
45f9a39b 4781 bool queue_hotplug = false;
f122c610 4782 bool queue_hdmi = false;
dc50ba7f 4783 bool queue_thermal = false;
54e2e49c 4784 u32 status, addr;
45f9a39b 4785
682f1a54 4786 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4787 return IRQ_NONE;
4788
682f1a54 4789 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4790
4791restart_ih:
4792 /* is somebody else already processing irqs? */
4793 if (atomic_xchg(&rdev->ih.lock, 1))
4794 return IRQ_NONE;
4795
682f1a54
DA		 4796 rptr = rdev->ih.rptr;
4797 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4798
964f6645
BH		 4799 /* Order reading of wptr vs. reading of IH ring data */
4800 rmb();
4801
45f9a39b 4802 /* display interrupts */
6f34be50 4803 evergreen_irq_ack(rdev);
45f9a39b 4804
45f9a39b
AD		 4805 while (rptr != wptr) {
4806 /* wptr/rptr are in bytes! */
4807 ring_index = rptr / 4;
0f234f5f
AD		 4808 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4809 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4810
4811 switch (src_id) {
4812 case 1: /* D1 vblank/vline */
4813 switch (src_data) {
4814 case 0: /* D1 vblank */
6f34be50 4815 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4816 if (rdev->irq.crtc_vblank_int[0]) {
4817 drm_handle_vblank(rdev->ddev, 0);
4818 rdev->pm.vblank_sync = true;
4819 wake_up(&rdev->irq.vblank_queue);
4820 }
736fc37f 4821 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 4822 radeon_crtc_handle_vblank(rdev, 0);
6f34be50 4823 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4824 DRM_DEBUG("IH: D1 vblank\n");
4825 }
4826 break;
4827 case 1: /* D1 vline */
6f34be50
AD		 4828 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4829 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4830 DRM_DEBUG("IH: D1 vline\n");
4831 }
4832 break;
4833 default:
4834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4835 break;
4836 }
4837 break;
4838 case 2: /* D2 vblank/vline */
4839 switch (src_data) {
4840 case 0: /* D2 vblank */
6f34be50 4841 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4842 if (rdev->irq.crtc_vblank_int[1]) {
4843 drm_handle_vblank(rdev->ddev, 1);
4844 rdev->pm.vblank_sync = true;
4845 wake_up(&rdev->irq.vblank_queue);
4846 }
736fc37f 4847 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 4848 radeon_crtc_handle_vblank(rdev, 1);
6f34be50 4849 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4850 DRM_DEBUG("IH: D2 vblank\n");
4851 }
4852 break;
4853 case 1: /* D2 vline */
6f34be50
AD		 4854 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4855 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4856 DRM_DEBUG("IH: D2 vline\n");
4857 }
4858 break;
4859 default:
4860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4861 break;
4862 }
4863 break;
4864 case 3: /* D3 vblank/vline */
4865 switch (src_data) {
4866 case 0: /* D3 vblank */
6f34be50
AD		 4867 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4868 if (rdev->irq.crtc_vblank_int[2]) {
4869 drm_handle_vblank(rdev->ddev, 2);
4870 rdev->pm.vblank_sync = true;
4871 wake_up(&rdev->irq.vblank_queue);
4872 }
736fc37f 4873 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 4874 radeon_crtc_handle_vblank(rdev, 2);
6f34be50 4875 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4876 DRM_DEBUG("IH: D3 vblank\n");
4877 }
4878 break;
4879 case 1: /* D3 vline */
6f34be50
AD		 4880 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4881 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4882 DRM_DEBUG("IH: D3 vline\n");
4883 }
4884 break;
4885 default:
4886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4887 break;
4888 }
4889 break;
4890 case 4: /* D4 vblank/vline */
4891 switch (src_data) {
4892 case 0: /* D4 vblank */
6f34be50
AD		 4893 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4894 if (rdev->irq.crtc_vblank_int[3]) {
4895 drm_handle_vblank(rdev->ddev, 3);
4896 rdev->pm.vblank_sync = true;
4897 wake_up(&rdev->irq.vblank_queue);
4898 }
736fc37f 4899 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 4900 radeon_crtc_handle_vblank(rdev, 3);
6f34be50 4901 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4902 DRM_DEBUG("IH: D4 vblank\n");
4903 }
4904 break;
4905 case 1: /* D4 vline */
6f34be50
AD		 4906 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4907 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4908 DRM_DEBUG("IH: D4 vline\n");
4909 }
4910 break;
4911 default:
4912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4913 break;
4914 }
4915 break;
4916 case 5: /* D5 vblank/vline */
4917 switch (src_data) {
4918 case 0: /* D5 vblank */
6f34be50
AD		 4919 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4920 if (rdev->irq.crtc_vblank_int[4]) {
4921 drm_handle_vblank(rdev->ddev, 4);
4922 rdev->pm.vblank_sync = true;
4923 wake_up(&rdev->irq.vblank_queue);
4924 }
736fc37f 4925 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 4926 radeon_crtc_handle_vblank(rdev, 4);
6f34be50 4927 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4928 DRM_DEBUG("IH: D5 vblank\n");
4929 }
4930 break;
4931 case 1: /* D5 vline */
6f34be50
AD		 4932 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4933 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4934 DRM_DEBUG("IH: D5 vline\n");
4935 }
4936 break;
4937 default:
4938 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4939 break;
4940 }
4941 break;
4942 case 6: /* D6 vblank/vline */
4943 switch (src_data) {
4944 case 0: /* D6 vblank */
6f34be50
AD		 4945 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4946 if (rdev->irq.crtc_vblank_int[5]) {
4947 drm_handle_vblank(rdev->ddev, 5);
4948 rdev->pm.vblank_sync = true;
4949 wake_up(&rdev->irq.vblank_queue);
4950 }
736fc37f 4951 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 4952 radeon_crtc_handle_vblank(rdev, 5);
6f34be50 4953 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4954 DRM_DEBUG("IH: D6 vblank\n");
4955 }
4956 break;
4957 case 1: /* D6 vline */
6f34be50
AD		 4958 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4959 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4960 DRM_DEBUG("IH: D6 vline\n");
4961 }
4962 break;
4963 default:
4964 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4965 break;
4966 }
4967 break;
f5d636d2
CK		 4968 case 8: /* D1 page flip */
4969 case 10: /* D2 page flip */
4970 case 12: /* D3 page flip */
4971 case 14: /* D4 page flip */
4972 case 16: /* D5 page flip */
4973 case 18: /* D6 page flip */
4974 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 4975 if (radeon_use_pflipirq > 0)
4976 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 4977 break;
45f9a39b
AD		 4978 case 42: /* HPD hotplug */
4979 switch (src_data) {
4980 case 0:
6f34be50
AD		 4981 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4982 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4983 queue_hotplug = true;
4984 DRM_DEBUG("IH: HPD1\n");
4985 }
4986 break;
4987 case 1:
6f34be50
AD		 4988 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4989 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4990 queue_hotplug = true;
4991 DRM_DEBUG("IH: HPD2\n");
4992 }
4993 break;
4994 case 2:
6f34be50
AD		 4995 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4996 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4997 queue_hotplug = true;
4998 DRM_DEBUG("IH: HPD3\n");
4999 }
5000 break;
5001 case 3:
6f34be50
AD		 5002 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5003 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 5004 queue_hotplug = true;
5005 DRM_DEBUG("IH: HPD4\n");
5006 }
5007 break;
5008 case 4:
6f34be50
AD		 5009 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5010 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 5011 queue_hotplug = true;
5012 DRM_DEBUG("IH: HPD5\n");
5013 }
5014 break;
5015 case 5:
6f34be50
AD		 5016 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5017 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 5018 queue_hotplug = true;
5019 DRM_DEBUG("IH: HPD6\n");
5020 }
5021 break;
5022 default:
5023 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5024 break;
5025 }
5026 break;
f122c610
AD		 5027 case 44: /* hdmi */
5028 switch (src_data) {
5029 case 0:
5030 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5031 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5032 queue_hdmi = true;
5033 DRM_DEBUG("IH: HDMI0\n");
5034 }
5035 break;
5036 case 1:
5037 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5038 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5039 queue_hdmi = true;
5040 DRM_DEBUG("IH: HDMI1\n");
5041 }
5042 break;
5043 case 2:
5044 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5045 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5046 queue_hdmi = true;
5047 DRM_DEBUG("IH: HDMI2\n");
5048 }
5049 break;
5050 case 3:
5051 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5052 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5053 queue_hdmi = true;
5054 DRM_DEBUG("IH: HDMI3\n");
5055 }
5056 break;
5057 case 4:
5058 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5059 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5060 queue_hdmi = true;
5061 DRM_DEBUG("IH: HDMI4\n");
5062 }
5063 break;
5064 case 5:
5065 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5066 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5067 queue_hdmi = true;
5068 DRM_DEBUG("IH: HDMI5\n");
5069 }
5070 break;
5071 default:
5072 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5073 break;
5074 }
acc1522a
CK		 5075 case 96:
5076 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5077 WREG32(SRBM_INT_ACK, 0x1);
5078 break;
f2ba57b5
CK		 5079 case 124: /* UVD */
5080 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5081 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5082 break;
ae133a11
CK		 5083 case 146:
5084 case 147:
54e2e49c
AD		 5085 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5086 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5087 /* reset addr and status */
5088 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5089 if (addr == 0x0 && status == 0x0)
5090 break;
ae133a11
CK		 5091 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5092 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5093 addr);
ae133a11 5094 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5095 status);
5096 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5097 break;
45f9a39b
AD		 5098 case 176: /* CP_INT in ring buffer */
5099 case 177: /* CP_INT in IB1 */
5100 case 178: /* CP_INT in IB2 */
5101 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5102 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5103 break;
5104 case 181: /* CP EOP event */
5105 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5106 if (rdev->family >= CHIP_CAYMAN) {
5107 switch (src_data) {
5108 case 0:
5109 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5110 break;
5111 case 1:
5112 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5113 break;
5114 case 2:
5115 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5116 break;
5117 }
5118 } else
5119 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5120 break;
233d1ad5
AD		 5121 case 224: /* DMA trap event */
5122 DRM_DEBUG("IH: DMA trap\n");
5123 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5124 break;
dc50ba7f
AD		 5125 case 230: /* thermal low to high */
5126 DRM_DEBUG("IH: thermal low to high\n");
5127 rdev->pm.dpm.thermal.high_to_low = false;
5128 queue_thermal = true;
5129 break;
5130 case 231: /* thermal high to low */
5131 DRM_DEBUG("IH: thermal high to low\n");
5132 rdev->pm.dpm.thermal.high_to_low = true;
5133 queue_thermal = true;
5134 break;
2031f77c 5135 case 233: /* GUI IDLE */
303c805c 5136 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5137 break;
f60cbd11
AD		 5138 case 244: /* DMA trap event */
5139 if (rdev->family >= CHIP_CAYMAN) {
5140 DRM_DEBUG("IH: DMA1 trap\n");
5141 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5142 }
5143 break;
45f9a39b
AD		 5144 default:
5145 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5146 break;
5147 }
5148
5149 /* wptr/rptr are in bytes! */
5150 rptr += 16;
5151 rptr &= rdev->ih.ptr_mask;
f55e03b9 5152 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5153 }
45f9a39b 5154 if (queue_hotplug)
32c87fca 5155 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5156 if (queue_hdmi)
5157 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5158 if (queue_thermal && rdev->pm.dpm_enabled)
5159 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5160 rdev->ih.rptr = rptr;
c20dc369
CK		 5161 atomic_set(&rdev->ih.lock, 0);
5162
5163 /* make sure wptr hasn't changed while processing */
5164 wptr = evergreen_get_ih_wptr(rdev);
5165 if (wptr != rptr)
5166 goto restart_ih;
5167
45f9a39b
AD		 5168 return IRQ_HANDLED;
5169}
5170
bcc1c2a1
AD		 5171static int evergreen_startup(struct radeon_device *rdev)
5172{
f2ba57b5 5173 struct radeon_ring *ring;
bcc1c2a1
AD		 5174 int r;
5175
9e46a48d 5176 /* enable pcie gen2 link */
cd54033a 5177 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5178 /* enable aspm */
5179 evergreen_program_aspm(rdev);
9e46a48d 5180
e5903d39
AD		 5181 /* scratch needs to be initialized before MC */
5182 r = r600_vram_scratch_init(rdev);
5183 if (r)
5184 return r;
5185
6fab3feb
AD		 5186 evergreen_mc_program(rdev);
5187
6c7bccea 5188 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5189 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5190 if (r) {
0af62b01 5191 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5192 return r;
5193 }
5194 }
fe251e2f 5195
bcc1c2a1 5196 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5197 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5198 } else {
5199 r = evergreen_pcie_gart_enable(rdev);
5200 if (r)
5201 return r;
5202 }
bcc1c2a1 5203 evergreen_gpu_init(rdev);
bcc1c2a1 5204
2948f5e6
AD		 5205 /* allocate rlc buffers */
5206 if (rdev->flags & RADEON_IS_IGP) {
5207 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5208 rdev->rlc.reg_list_size =
5209 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5210 rdev->rlc.cs_data = evergreen_cs_data;
5211 r = sumo_rlc_init(rdev);
5212 if (r) {
5213 DRM_ERROR("Failed to init rlc BOs!\n");
5214 return r;
5215 }
5216 }
5217
724c80e1
AD		 5218 /* allocate wb buffer */
5219 r = radeon_wb_init(rdev);
5220 if (r)
5221 return r;
5222
30eb77f4
JG		 5223 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5224 if (r) {
5225 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5226 return r;
5227 }
5228
233d1ad5
AD		 5229 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5230 if (r) {
5231 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5232 return r;
5233 }
5234
e409b128 5235 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5236 if (!r) {
5237 r = radeon_fence_driver_start_ring(rdev,
5238 R600_RING_TYPE_UVD_INDEX);
5239 if (r)
5240 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5241 }
5242
5243 if (r)
5244 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5245
bcc1c2a1 5246 /* Enable IRQ */
e49f3959
AH		 5247 if (!rdev->irq.installed) {
5248 r = radeon_irq_kms_init(rdev);
5249 if (r)
5250 return r;
5251 }
5252
bcc1c2a1
AD		 5253 r = r600_irq_init(rdev);
5254 if (r) {
5255 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5256 radeon_irq_kms_fini(rdev);
5257 return r;
5258 }
45f9a39b 5259 evergreen_irq_set(rdev);
bcc1c2a1 5260
f2ba57b5 5261 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5262 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5263 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5264 if (r)
5265 return r;
233d1ad5
AD		 5266
5267 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5268 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5269 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5270 if (r)
5271 return r;
5272
bcc1c2a1
AD		 5273 r = evergreen_cp_load_microcode(rdev);
5274 if (r)
5275 return r;
fe251e2f 5276 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5277 if (r)
5278 return r;
5279 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5280 if (r)
5281 return r;
fe251e2f 5282
f2ba57b5
CK		 5283 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5284 if (ring->ring_size) {
02c9f7fa 5285 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5286 RADEON_CP_PACKET2);
f2ba57b5 5287 if (!r)
e409b128 5288 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5289
5290 if (r)
5291 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5292 }
5293
2898c348
CK		 5294 r = radeon_ib_pool_init(rdev);
5295 if (r) {
5296 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5297 return r;
2898c348 5298 }
b15ba512 5299
bfc1f97d 5300 r = radeon_audio_init(rdev);
69d2ae57
RM		 5301 if (r) {
5302 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5303 return r;
5304 }
5305
bcc1c2a1
AD		 5306 return 0;
5307}
5308
5309int evergreen_resume(struct radeon_device *rdev)
5310{
5311 int r;
5312
86f5c9ed
AD		 5313 /* reset the asic, the gfx blocks are often in a bad state
5314 * after the driver is unloaded or after a resume
5315 */
5316 if (radeon_asic_reset(rdev))
5317 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5318 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5319 * posting will perform necessary task to bring back GPU into good
5320 * shape.
5321 */
5322 /* post card */
5323 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5324
d4788db3
AD		 5325 /* init golden registers */
5326 evergreen_init_golden_registers(rdev);
5327
bc6a6295
AD		 5328 if (rdev->pm.pm_method == PM_METHOD_DPM)
5329 radeon_pm_resume(rdev);
6c7bccea 5330
b15ba512 5331 rdev->accel_working = true;
bcc1c2a1
AD		 5332 r = evergreen_startup(rdev);
5333 if (r) {
755d819e 5334 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5335 rdev->accel_working = false;
bcc1c2a1
AD		 5336 return r;
5337 }
fe251e2f 5338
bcc1c2a1
AD		 5339 return r;
5340
5341}
5342
5343int evergreen_suspend(struct radeon_device *rdev)
5344{
6c7bccea 5345 radeon_pm_suspend(rdev);
7991d665 5346 radeon_audio_fini(rdev);
e409b128 5347 uvd_v1_0_fini(rdev);
f2ba57b5 5348 radeon_uvd_suspend(rdev);
bcc1c2a1 5349 r700_cp_stop(rdev);
233d1ad5 5350 r600_dma_stop(rdev);
45f9a39b 5351 evergreen_irq_suspend(rdev);
724c80e1 5352 radeon_wb_disable(rdev);
bcc1c2a1 5353 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5354
5355 return 0;
5356}
5357
bcc1c2a1
AD		 5358/* Plan is to move initialization in that function and use
5359 * helper function so that radeon_device_init pretty much
5360 * do nothing more than calling asic specific function. This
5361 * should also allow to remove a bunch of callback function
5362 * like vram_info.
5363 */
5364int evergreen_init(struct radeon_device *rdev)
5365{
5366 int r;
5367
bcc1c2a1
AD		 5368 /* Read BIOS */
5369 if (!radeon_get_bios(rdev)) {
5370 if (ASIC_IS_AVIVO(rdev))
5371 return -EINVAL;
5372 }
5373 /* Must be an ATOMBIOS */
5374 if (!rdev->is_atom_bios) {
755d819e 5375 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5376 return -EINVAL;
5377 }
5378 r = radeon_atombios_init(rdev);
5379 if (r)
5380 return r;
86f5c9ed
AD		 5381 /* reset the asic, the gfx blocks are often in a bad state
5382 * after the driver is unloaded or after a resume
5383 */
5384 if (radeon_asic_reset(rdev))
5385 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5386 /* Post card if necessary */
fd909c37 5387 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5388 if (!rdev->bios) {
5389 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5390 return -EINVAL;
5391 }
5392 DRM_INFO("GPU not posted. posting now...\n");
5393 atom_asic_init(rdev->mode_info.atom_context);
5394 }
d4788db3
AD		 5395 /* init golden registers */
5396 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5397 /* Initialize scratch registers */
5398 r600_scratch_init(rdev);
5399 /* Initialize surface registers */
5400 radeon_surface_init(rdev);
5401 /* Initialize clocks */
5402 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5403 /* Fence driver */
5404 r = radeon_fence_driver_init(rdev);
5405 if (r)
5406 return r;
d594e46a
JG		 5407 /* initialize AGP */
5408 if (rdev->flags & RADEON_IS_AGP) {
5409 r = radeon_agp_init(rdev);
5410 if (r)
5411 radeon_agp_disable(rdev);
5412 }
5413 /* initialize memory controller */
bcc1c2a1
AD		 5414 r = evergreen_mc_init(rdev);
5415 if (r)
5416 return r;
5417 /* Memory manager */
5418 r = radeon_bo_init(rdev);
5419 if (r)
5420 return r;
45f9a39b 5421
01ac8794
AD		 5422 if (ASIC_IS_DCE5(rdev)) {
5423 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5424 r = ni_init_microcode(rdev);
5425 if (r) {
5426 DRM_ERROR("Failed to load firmware!\n");
5427 return r;
5428 }
5429 }
5430 } else {
5431 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5432 r = r600_init_microcode(rdev);
5433 if (r) {
5434 DRM_ERROR("Failed to load firmware!\n");
5435 return r;
5436 }
5437 }
5438 }
5439
6c7bccea
AD		 5440 /* Initialize power management */
5441 radeon_pm_init(rdev);
5442
e32eb50d
CK		 5443 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5444 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5445
233d1ad5
AD		 5446 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5447 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5448
f2ba57b5
CK		 5449 r = radeon_uvd_init(rdev);
5450 if (!r) {
5451 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5452 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5453 4096);
5454 }
5455
bcc1c2a1
AD		 5456 rdev->ih.ring_obj = NULL;
5457 r600_ih_ring_init(rdev, 64 * 1024);
5458
5459 r = r600_pcie_gart_init(rdev);
5460 if (r)
5461 return r;
0fcdb61e 5462
148a03bc 5463 rdev->accel_working = true;
bcc1c2a1
AD		 5464 r = evergreen_startup(rdev);
5465 if (r) {
fe251e2f
AD		 5466 dev_err(rdev->dev, "disabling GPU acceleration\n");
5467 r700_cp_fini(rdev);
233d1ad5 5468 r600_dma_fini(rdev);
fe251e2f 5469 r600_irq_fini(rdev);
2948f5e6
AD		 5470 if (rdev->flags & RADEON_IS_IGP)
5471 sumo_rlc_fini(rdev);
724c80e1 5472 radeon_wb_fini(rdev);
2898c348 5473 radeon_ib_pool_fini(rdev);
fe251e2f 5474 radeon_irq_kms_fini(rdev);
0fcdb61e 5475 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5476 rdev->accel_working = false;
5477 }
77e00f2e
AD		 5478
5479 /* Don't start up if the MC ucode is missing on BTC parts.
5480 * The default clocks and voltages before the MC ucode
5481 * is loaded are not suffient for advanced operations.
5482 */
5483 if (ASIC_IS_DCE5(rdev)) {
5484 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5485 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5486 return -EINVAL;
5487 }
5488 }
5489
bcc1c2a1
AD		 5490 return 0;
5491}
5492
5493void evergreen_fini(struct radeon_device *rdev)
5494{
6c7bccea 5495 radeon_pm_fini(rdev);
7991d665 5496 radeon_audio_fini(rdev);
45f9a39b 5497 r700_cp_fini(rdev);
233d1ad5 5498 r600_dma_fini(rdev);
bcc1c2a1 5499 r600_irq_fini(rdev);
2948f5e6
AD		 5500 if (rdev->flags & RADEON_IS_IGP)
5501 sumo_rlc_fini(rdev);
724c80e1 5502 radeon_wb_fini(rdev);
2898c348 5503 radeon_ib_pool_fini(rdev);
bcc1c2a1 5504 radeon_irq_kms_fini(rdev);
e409b128 5505 uvd_v1_0_fini(rdev);
f2ba57b5 5506 radeon_uvd_fini(rdev);
d9654413 5507 evergreen_pcie_gart_fini(rdev);
16cdf04d 5508 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5509 radeon_gem_fini(rdev);
5510 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5511 radeon_agp_fini(rdev);
5512 radeon_bo_fini(rdev);
5513 radeon_atombios_fini(rdev);
5514 kfree(rdev->bios);
5515 rdev->bios = NULL;
bcc1c2a1 5516}
9e46a48d 5517
b07759bf 5518void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5519{
7e0e4196 5520 u32 link_width_cntl, speed_cntl;
9e46a48d 5521
d42dd579
AD		 5522 if (radeon_pcie_gen2 == 0)
5523 return;
5524
9e46a48d
AD		 5525 if (rdev->flags & RADEON_IS_IGP)
5526 return;
5527
5528 if (!(rdev->flags & RADEON_IS_PCIE))
5529 return;
5530
5531 /* x2 cards have a special sequence */
5532 if (ASIC_IS_X2(rdev))
5533 return;
5534
7e0e4196
KSS		 5535 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5536 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5537 return;
5538
492d2b61 5539 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5540 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5541 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5542 return;
5543 }
5544
197bbb3d
DA		 5545 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5546
9e46a48d
AD		 5547 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5548 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5549
492d2b61 5550 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5551 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5552 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5553
492d2b61 5554 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5555 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5556 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5557
492d2b61 5558 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5559 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5560 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5561
492d2b61 5562 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5563 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5564 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5565
492d2b61 5566 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5567 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5568 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5569
5570 } else {
492d2b61 5571 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5572 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5573 if (1)
5574 link_width_cntl |= LC_UPCONFIGURE_DIS;
5575 else
5576 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5577 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5578 }
5579}
f52382d7
AD		 5580
5581void evergreen_program_aspm(struct radeon_device *rdev)
5582{
5583 u32 data, orig;
5584 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5585 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5586 /* fusion_platform = true
5587 * if the system is a fusion system
5588 * (APU or DGPU in a fusion system).
5589 * todo: check if the system is a fusion platform.
5590 */
5591 bool fusion_platform = false;
5592
1294d4a3
AD		 5593 if (radeon_aspm == 0)
5594 return;
5595
f52382d7
AD		 5596 if (!(rdev->flags & RADEON_IS_PCIE))
5597 return;
5598
5599 switch (rdev->family) {
5600 case CHIP_CYPRESS:
5601 case CHIP_HEMLOCK:
5602 case CHIP_JUNIPER:
5603 case CHIP_REDWOOD:
5604 case CHIP_CEDAR:
5605 case CHIP_SUMO:
5606 case CHIP_SUMO2:
5607 case CHIP_PALM:
5608 case CHIP_ARUBA:
5609 disable_l0s = true;
5610 break;
5611 default:
5612 disable_l0s = false;
5613 break;
5614 }
5615
5616 if (rdev->flags & RADEON_IS_IGP)
5617 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5618
5619 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5620 if (fusion_platform)
5621 data &= ~MULTI_PIF;
5622 else
5623 data |= MULTI_PIF;
5624 if (data != orig)
5625 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5626
5627 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5628 if (fusion_platform)
5629 data &= ~MULTI_PIF;
5630 else
5631 data |= MULTI_PIF;
5632 if (data != orig)
5633 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5634
5635 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5636 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5637 if (!disable_l0s) {
5638 if (rdev->family >= CHIP_BARTS)
5639 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5640 else
5641 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5642 }
5643
5644 if (!disable_l1) {
5645 if (rdev->family >= CHIP_BARTS)
5646 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5647 else
5648 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5649
5650 if (!disable_plloff_in_l1) {
5651 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5652 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5653 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5654 if (data != orig)
5655 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5656
5657 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5658 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5659 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5660 if (data != orig)
5661 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5662
5663 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5664 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5665 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5666 if (data != orig)
5667 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5668
5669 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5670 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5671 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5672 if (data != orig)
5673 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5674
5675 if (rdev->family >= CHIP_BARTS) {
5676 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5677 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5678 data |= PLL_RAMP_UP_TIME_0(4);
5679 if (data != orig)
5680 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5681
5682 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5683 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5684 data |= PLL_RAMP_UP_TIME_1(4);
5685 if (data != orig)
5686 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5687
5688 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5689 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5690 data |= PLL_RAMP_UP_TIME_0(4);
5691 if (data != orig)
5692 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5693
5694 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5695 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5696 data |= PLL_RAMP_UP_TIME_1(4);
5697 if (data != orig)
5698 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5699 }
5700
5701 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5702 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5703 data |= LC_DYN_LANES_PWR_STATE(3);
5704 if (data != orig)
5705 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5706
5707 if (rdev->family >= CHIP_BARTS) {
5708 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5709 data &= ~LS2_EXIT_TIME_MASK;
5710 data |= LS2_EXIT_TIME(1);
5711 if (data != orig)
5712 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5713
5714 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5715 data &= ~LS2_EXIT_TIME_MASK;
5716 data |= LS2_EXIT_TIME(1);
5717 if (data != orig)
5718 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5719 }
5720 }
5721 }
5722
5723 /* evergreen parts only */
5724 if (rdev->family < CHIP_BARTS)
5725 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5726
5727 if (pcie_lc_cntl != pcie_lc_cntl_old)
5728 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5729}
Linux kernel - Deliverables
This page took 0.859011 seconds and 5 git commands to generate.