projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Add support for programming the FMT blocks
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
bcc1c2a1 149
d4788db3
AD		 150static const u32 evergreen_golden_registers[] =
151{
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5cc, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194};
195
196static const u32 evergreen_golden_registers2[] =
197{
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212};
213
214static const u32 cypress_mgcg_init[] =
215{
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365};
366
367static const u32 redwood_mgcg_init[] =
368{
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437};
438
439static const u32 cedar_golden_registers[] =
440{
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5cc, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480};
481
482static const u32 cedar_mgcg_init[] =
483{
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534};
535
536static const u32 juniper_mgcg_init[] =
537{
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632};
633
634static const u32 supersumo_golden_registers[] =
635{
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5cc, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707};
708
709static const u32 sumo_golden_registers[] =
710{
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716};
717
718static const u32 wrestler_golden_registers[] =
719{
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5cc, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769};
770
771static const u32 barts_golden_registers[] =
772{
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818};
819
820static const u32 turks_golden_registers[] =
821{
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869};
870
871static const u32 caicos_golden_registers[] =
872{
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920};
921
922static void evergreen_init_golden_registers(struct radeon_device *rdev)
923{
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006}
1007
285484e2
JG		 1008void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011{
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037}
1038
23d33ba3
AD		 1039static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041{
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061}
1062
1063int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064{
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084}
1085
a8b4925c
AD		 1086int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087{
1088 /* start off with something large */
facd112d 1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1090 int r;
1091
4ed10835
CK		 1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
facd112d
CK		 1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
a8b4925c
AD		 1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
facd112d 1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
facd112d 1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
facd112d 1140 if (fb_div < 307200)
a8b4925c
AD		 1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
facd112d 1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173}
1174
d054ac16
AD		 1175void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176{
1177 u16 ctl, v;
32195aec 1178 int err;
d054ac16 1179
32195aec 1180 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1181 if (err)
1182 return;
1183
1184 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1185
1186 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1187 * to avoid hangs or perfomance issues
1188 */
1189 if ((v == 0) || (v == 6) || (v == 7)) {
1190 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1191 ctl |= (2 << 12);
32195aec 1192 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1193 }
1194}
1195
134b480f
AD		 1196void dce4_program_fmt(struct drm_encoder *encoder)
1197{
1198 struct drm_device *dev = encoder->dev;
1199 struct radeon_device *rdev = dev->dev_private;
1200 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1201 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1202 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1203 int bpc = 0;
1204 u32 tmp = 0;
1205 bool dither = false;
1206
1207 if (connector)
1208 bpc = radeon_get_monitor_bpc(connector);
1209
1210 /* LVDS/eDP FMT is set up by atom */
1211 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1212 return;
1213
1214 /* not needed for analog */
1215 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1216 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1217 return;
1218
1219 if (bpc == 0)
1220 return;
1221
1222 switch (bpc) {
1223 case 6:
1224 if (dither)
1225 /* XXX sort out optimal dither settings */
1226 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1227 FMT_SPATIAL_DITHER_EN);
1228 else
1229 tmp |= FMT_TRUNCATE_EN;
1230 break;
1231 case 8:
1232 if (dither)
1233 /* XXX sort out optimal dither settings */
1234 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1235 FMT_RGB_RANDOM_ENABLE |
1236 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1237 else
1238 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1239 break;
1240 case 10:
1241 default:
1242 /* not needed */
1243 break;
1244 }
1245
1246 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1247}
1248
10257a6d
AD		 1249static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1250{
1251 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1252 return true;
1253 else
1254 return false;
1255}
1256
1257static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1258{
1259 u32 pos1, pos2;
1260
1261 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1262 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1263
1264 if (pos1 != pos2)
1265 return true;
1266 else
1267 return false;
1268}
1269
377edc8b
AD		 1270/**
1271 * dce4_wait_for_vblank - vblank wait asic callback.
1272 *
1273 * @rdev: radeon_device pointer
1274 * @crtc: crtc to wait for vblank on
1275 *
1276 * Wait for vblank on the requested crtc (evergreen+).
1277 */
3ae19b75
AD		 1278void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1279{
10257a6d 1280 unsigned i = 0;
3ae19b75 1281
4a15903d
AD		 1282 if (crtc >= rdev->num_crtc)
1283 return;
1284
10257a6d
AD		 1285 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1286 return;
1287
1288 /* depending on when we hit vblank, we may be close to active; if so,
1289 * wait for another frame.
1290 */
1291 while (dce4_is_in_vblank(rdev, crtc)) {
1292 if (i++ % 100 == 0) {
1293 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1294 break;
3ae19b75 1295 }
10257a6d
AD		 1296 }
1297
1298 while (!dce4_is_in_vblank(rdev, crtc)) {
1299 if (i++ % 100 == 0) {
1300 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1301 break;
3ae19b75
AD		 1302 }
1303 }
1304}
1305
377edc8b
AD		 1306/**
1307 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1308 *
1309 * @rdev: radeon_device pointer
1310 * @crtc: crtc to prepare for pageflip on
1311 *
1312 * Pre-pageflip callback (evergreen+).
1313 * Enables the pageflip irq (vblank irq).
1314 */
6f34be50
AD		 1315void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1316{
6f34be50
AD		 1317 /* enable the pflip int */
1318 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1319}
1320
377edc8b
AD		 1321/**
1322 * evergreen_post_page_flip - pos-pageflip callback.
1323 *
1324 * @rdev: radeon_device pointer
1325 * @crtc: crtc to cleanup pageflip on
1326 *
1327 * Post-pageflip callback (evergreen+).
1328 * Disables the pageflip irq (vblank irq).
1329 */
6f34be50
AD		 1330void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1331{
1332 /* disable the pflip int */
1333 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1334}
1335
377edc8b
AD		 1336/**
1337 * evergreen_page_flip - pageflip callback.
1338 *
1339 * @rdev: radeon_device pointer
1340 * @crtc_id: crtc to cleanup pageflip on
1341 * @crtc_base: new address of the crtc (GPU MC address)
1342 *
1343 * Does the actual pageflip (evergreen+).
1344 * During vblank we take the crtc lock and wait for the update_pending
1345 * bit to go high, when it does, we release the lock, and allow the
1346 * double buffered update to take place.
1347 * Returns the current update pending status.
1348 */
6f34be50
AD		 1349u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1350{
1351 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1352 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1353 int i;
6f34be50
AD		 1354
1355 /* Lock the graphics update lock */
1356 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1357 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1358
1359 /* update the scanout addresses */
1360 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1361 upper_32_bits(crtc_base));
1362 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1363 (u32)crtc_base);
1364
1365 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1366 upper_32_bits(crtc_base));
1367 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1368 (u32)crtc_base);
1369
1370 /* Wait for update_pending to go high. */
f6496479
AD		 1371 for (i = 0; i < rdev->usec_timeout; i++) {
1372 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1373 break;
1374 udelay(1);
1375 }
6f34be50
AD		 1376 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1377
1378 /* Unlock the lock, so double-buffering can take place inside vblank */
1379 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1380 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1381
1382 /* Return current update_pending status: */
1383 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1384}
1385
21a8122a 1386/* get temperature in millidegrees */
20d391d7 1387int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1388{
1c88d74f
AD		 1389 u32 temp, toffset;
1390 int actual_temp = 0;
67b3f823
AD		 1391
1392 if (rdev->family == CHIP_JUNIPER) {
1393 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1394 TOFFSET_SHIFT;
1395 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1396 TS0_ADC_DOUT_SHIFT;
1397
1398 if (toffset & 0x100)
1399 actual_temp = temp / 2 - (0x200 - toffset);
1400 else
1401 actual_temp = temp / 2 + toffset;
1402
1403 actual_temp = actual_temp * 1000;
1404
1405 } else {
1406 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1407 ASIC_T_SHIFT;
1408
1409 if (temp & 0x400)
1410 actual_temp = -256;
1411 else if (temp & 0x200)
1412 actual_temp = 255;
1413 else if (temp & 0x100) {
1414 actual_temp = temp & 0x1ff;
1415 actual_temp |= ~0x1ff;
1416 } else
1417 actual_temp = temp & 0xff;
1418
1419 actual_temp = (actual_temp * 1000) / 2;
1420 }
21a8122a 1421
67b3f823 1422 return actual_temp;
21a8122a
AD		 1423}
1424
20d391d7 1425int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1426{
1427 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1428 int actual_temp = temp - 49;
e33df25f
AD		 1429
1430 return actual_temp * 1000;
1431}
1432
377edc8b
AD		 1433/**
1434 * sumo_pm_init_profile - Initialize power profiles callback.
1435 *
1436 * @rdev: radeon_device pointer
1437 *
1438 * Initialize the power states used in profile mode
1439 * (sumo, trinity, SI).
1440 * Used for profile mode only.
1441 */
a4c9e2ee
AD		 1442void sumo_pm_init_profile(struct radeon_device *rdev)
1443{
1444 int idx;
1445
1446 /* default */
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1451
1452 /* low,mid sh/mh */
1453 if (rdev->flags & RADEON_IS_MOBILITY)
1454 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1455 else
1456 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1457
1458 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1462
1463 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1464 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1466 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1467
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1471 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1472
1473 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1476 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1477
1478 /* high sh/mh */
1479 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1480 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1482 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1483 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1484 rdev->pm.power_state[idx].num_clock_modes - 1;
1485
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1490 rdev->pm.power_state[idx].num_clock_modes - 1;
1491}
1492
27810fb2
AD		 1493/**
1494 * btc_pm_init_profile - Initialize power profiles callback.
1495 *
1496 * @rdev: radeon_device pointer
1497 *
1498 * Initialize the power states used in profile mode
1499 * (BTC, cayman).
1500 * Used for profile mode only.
1501 */
1502void btc_pm_init_profile(struct radeon_device *rdev)
1503{
1504 int idx;
1505
1506 /* default */
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1508 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1509 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1511 /* starting with BTC, there is one state that is used for both
1512 * MH and SH. Difference is that we always use the high clock index for
1513 * mclk.
1514 */
1515 if (rdev->flags & RADEON_IS_MOBILITY)
1516 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1517 else
1518 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1519 /* low sh */
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1524 /* mid sh */
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1529 /* high sh */
1530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1533 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1534 /* low mh */
1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1538 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1539 /* mid mh */
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1544 /* high mh */
1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1549}
1550
377edc8b
AD		 1551/**
1552 * evergreen_pm_misc - set additional pm hw parameters callback.
1553 *
1554 * @rdev: radeon_device pointer
1555 *
1556 * Set non-clock parameters associated with a power state
1557 * (voltage, etc.) (evergreen+).
1558 */
49e02b73
AD		 1559void evergreen_pm_misc(struct radeon_device *rdev)
1560{
a081a9d6
RM		 1561 int req_ps_idx = rdev->pm.requested_power_state_index;
1562 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1563 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1564 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1565
2feea49a 1566 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1567 /* 0xff0x are flags rather then an actual voltage */
1568 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1569 return;
2feea49a 1570 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1571 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1572 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1573 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1574 }
7ae764b1
AD		 1575
1576 /* starting with BTC, there is one state that is used for both
1577 * MH and SH. Difference is that we always use the high clock index for
1578 * mclk and vddci.
1579 */
1580 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1581 (rdev->family >= CHIP_BARTS) &&
1582 rdev->pm.active_crtc_count &&
1583 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1584 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1585 voltage = &rdev->pm.power_state[req_ps_idx].
1586 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1587
c6cf7777
AD		 1588 /* 0xff0x are flags rather then an actual voltage */
1589 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1590 return;
2feea49a
AD		 1591 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1592 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1593 rdev->pm.current_vddci = voltage->vddci;
1594 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1595 }
1596 }
49e02b73
AD		 1597}
1598
377edc8b
AD		 1599/**
1600 * evergreen_pm_prepare - pre-power state change callback.
1601 *
1602 * @rdev: radeon_device pointer
1603 *
1604 * Prepare for a power state change (evergreen+).
1605 */
49e02b73
AD		 1606void evergreen_pm_prepare(struct radeon_device *rdev)
1607{
1608 struct drm_device *ddev = rdev->ddev;
1609 struct drm_crtc *crtc;
1610 struct radeon_crtc *radeon_crtc;
1611 u32 tmp;
1612
1613 /* disable any active CRTCs */
1614 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1615 radeon_crtc = to_radeon_crtc(crtc);
1616 if (radeon_crtc->enabled) {
1617 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1618 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1619 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1620 }
1621 }
1622}
1623
377edc8b
AD		 1624/**
1625 * evergreen_pm_finish - post-power state change callback.
1626 *
1627 * @rdev: radeon_device pointer
1628 *
1629 * Clean up after a power state change (evergreen+).
1630 */
49e02b73
AD		 1631void evergreen_pm_finish(struct radeon_device *rdev)
1632{
1633 struct drm_device *ddev = rdev->ddev;
1634 struct drm_crtc *crtc;
1635 struct radeon_crtc *radeon_crtc;
1636 u32 tmp;
1637
1638 /* enable any active CRTCs */
1639 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1640 radeon_crtc = to_radeon_crtc(crtc);
1641 if (radeon_crtc->enabled) {
1642 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1643 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1644 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1645 }
1646 }
1647}
1648
377edc8b
AD		 1649/**
1650 * evergreen_hpd_sense - hpd sense callback.
1651 *
1652 * @rdev: radeon_device pointer
1653 * @hpd: hpd (hotplug detect) pin
1654 *
1655 * Checks if a digital monitor is connected (evergreen+).
1656 * Returns true if connected, false if not connected.
1657 */
bcc1c2a1
AD		 1658bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1659{
1660 bool connected = false;
0ca2ab52
AD		 1661
1662 switch (hpd) {
1663 case RADEON_HPD_1:
1664 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
1666 break;
1667 case RADEON_HPD_2:
1668 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1669 connected = true;
1670 break;
1671 case RADEON_HPD_3:
1672 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1673 connected = true;
1674 break;
1675 case RADEON_HPD_4:
1676 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1677 connected = true;
1678 break;
1679 case RADEON_HPD_5:
1680 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1681 connected = true;
1682 break;
1683 case RADEON_HPD_6:
1684 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1685 connected = true;
1686 break;
1687 default:
1688 break;
1689 }
1690
bcc1c2a1
AD		 1691 return connected;
1692}
1693
377edc8b
AD		 1694/**
1695 * evergreen_hpd_set_polarity - hpd set polarity callback.
1696 *
1697 * @rdev: radeon_device pointer
1698 * @hpd: hpd (hotplug detect) pin
1699 *
1700 * Set the polarity of the hpd pin (evergreen+).
1701 */
bcc1c2a1
AD		 1702void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1703 enum radeon_hpd_id hpd)
1704{
0ca2ab52
AD		 1705 u32 tmp;
1706 bool connected = evergreen_hpd_sense(rdev, hpd);
1707
1708 switch (hpd) {
1709 case RADEON_HPD_1:
1710 tmp = RREG32(DC_HPD1_INT_CONTROL);
1711 if (connected)
1712 tmp &= ~DC_HPDx_INT_POLARITY;
1713 else
1714 tmp |= DC_HPDx_INT_POLARITY;
1715 WREG32(DC_HPD1_INT_CONTROL, tmp);
1716 break;
1717 case RADEON_HPD_2:
1718 tmp = RREG32(DC_HPD2_INT_CONTROL);
1719 if (connected)
1720 tmp &= ~DC_HPDx_INT_POLARITY;
1721 else
1722 tmp |= DC_HPDx_INT_POLARITY;
1723 WREG32(DC_HPD2_INT_CONTROL, tmp);
1724 break;
1725 case RADEON_HPD_3:
1726 tmp = RREG32(DC_HPD3_INT_CONTROL);
1727 if (connected)
1728 tmp &= ~DC_HPDx_INT_POLARITY;
1729 else
1730 tmp |= DC_HPDx_INT_POLARITY;
1731 WREG32(DC_HPD3_INT_CONTROL, tmp);
1732 break;
1733 case RADEON_HPD_4:
1734 tmp = RREG32(DC_HPD4_INT_CONTROL);
1735 if (connected)
1736 tmp &= ~DC_HPDx_INT_POLARITY;
1737 else
1738 tmp |= DC_HPDx_INT_POLARITY;
1739 WREG32(DC_HPD4_INT_CONTROL, tmp);
1740 break;
1741 case RADEON_HPD_5:
1742 tmp = RREG32(DC_HPD5_INT_CONTROL);
1743 if (connected)
1744 tmp &= ~DC_HPDx_INT_POLARITY;
1745 else
1746 tmp |= DC_HPDx_INT_POLARITY;
1747 WREG32(DC_HPD5_INT_CONTROL, tmp);
1748 break;
1749 case RADEON_HPD_6:
1750 tmp = RREG32(DC_HPD6_INT_CONTROL);
1751 if (connected)
1752 tmp &= ~DC_HPDx_INT_POLARITY;
1753 else
1754 tmp |= DC_HPDx_INT_POLARITY;
1755 WREG32(DC_HPD6_INT_CONTROL, tmp);
1756 break;
1757 default:
1758 break;
1759 }
bcc1c2a1
AD		 1760}
1761
377edc8b
AD		 1762/**
1763 * evergreen_hpd_init - hpd setup callback.
1764 *
1765 * @rdev: radeon_device pointer
1766 *
1767 * Setup the hpd pins used by the card (evergreen+).
1768 * Enable the pin, set the polarity, and enable the hpd interrupts.
1769 */
bcc1c2a1
AD		 1770void evergreen_hpd_init(struct radeon_device *rdev)
1771{
0ca2ab52
AD		 1772 struct drm_device *dev = rdev->ddev;
1773 struct drm_connector *connector;
fb98257a 1774 unsigned enabled = 0;
0ca2ab52
AD		 1775 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1776 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1777
0ca2ab52
AD		 1778 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1779 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1780
1781 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1782 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1783 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1784 * aux dp channel on imac and help (but not completely fix)
1785 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1786 * also avoid interrupt storms during dpms.
1787 */
1788 continue;
1789 }
0ca2ab52
AD		 1790 switch (radeon_connector->hpd.hpd) {
1791 case RADEON_HPD_1:
1792 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1793 break;
1794 case RADEON_HPD_2:
1795 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1796 break;
1797 case RADEON_HPD_3:
1798 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1799 break;
1800 case RADEON_HPD_4:
1801 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1802 break;
1803 case RADEON_HPD_5:
1804 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1805 break;
1806 case RADEON_HPD_6:
1807 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1808 break;
1809 default:
1810 break;
1811 }
64912e99 1812 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1813 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1814 }
fb98257a 1815 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1816}
1817
377edc8b
AD		 1818/**
1819 * evergreen_hpd_fini - hpd tear down callback.
1820 *
1821 * @rdev: radeon_device pointer
1822 *
1823 * Tear down the hpd pins used by the card (evergreen+).
1824 * Disable the hpd interrupts.
1825 */
0ca2ab52 1826void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1827{
0ca2ab52
AD		 1828 struct drm_device *dev = rdev->ddev;
1829 struct drm_connector *connector;
fb98257a 1830 unsigned disabled = 0;
0ca2ab52
AD		 1831
1832 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1834 switch (radeon_connector->hpd.hpd) {
1835 case RADEON_HPD_1:
1836 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1837 break;
1838 case RADEON_HPD_2:
1839 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1840 break;
1841 case RADEON_HPD_3:
1842 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1843 break;
1844 case RADEON_HPD_4:
1845 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1846 break;
1847 case RADEON_HPD_5:
1848 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1849 break;
1850 case RADEON_HPD_6:
1851 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1852 break;
1853 default:
1854 break;
1855 }
fb98257a 1856 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1857 }
fb98257a 1858 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1859}
1860
f9d9c362
AD		 1861/* watermark setup */
1862
1863static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1864 struct radeon_crtc *radeon_crtc,
1865 struct drm_display_mode *mode,
1866 struct drm_display_mode *other_mode)
1867{
0b31e023
AD		 1868 u32 tmp, buffer_alloc, i;
1869 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1870 /*
1871 * Line Buffer Setup
1872 * There are 3 line buffers, each one shared by 2 display controllers.
1873 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1874 * the display controllers. The paritioning is done via one of four
1875 * preset allocations specified in bits 2:0:
1876 * first display controller
1877 * 0 - first half of lb (3840 * 2)
1878 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1879 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1880 * 3 - first 1/4 of lb (1920 * 2)
1881 * second display controller
1882 * 4 - second half of lb (3840 * 2)
1883 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1884 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1885 * 7 - last 1/4 of lb (1920 * 2)
1886 */
12dfc843
AD		 1887 /* this can get tricky if we have two large displays on a paired group
1888 * of crtcs. Ideally for multiple large displays we'd assign them to
1889 * non-linked crtcs for maximum line buffer allocation.
1890 */
1891 if (radeon_crtc->base.enabled && mode) {
0b31e023 1892 if (other_mode) {
f9d9c362 1893 tmp = 0; /* 1/2 */
0b31e023
AD		 1894 buffer_alloc = 1;
1895 } else {
12dfc843 1896 tmp = 2; /* whole */
0b31e023
AD		 1897 buffer_alloc = 2;
1898 }
1899 } else {
12dfc843 1900 tmp = 0;
0b31e023
AD		 1901 buffer_alloc = 0;
1902 }
f9d9c362
AD		 1903
1904 /* second controller of the pair uses second half of the lb */
1905 if (radeon_crtc->crtc_id % 2)
1906 tmp += 4;
1907 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1908
0b31e023
AD		 1909 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1910 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1911 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1912 for (i = 0; i < rdev->usec_timeout; i++) {
1913 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1914 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1915 break;
1916 udelay(1);
1917 }
1918 }
1919
12dfc843
AD		 1920 if (radeon_crtc->base.enabled && mode) {
1921 switch (tmp) {
1922 case 0:
1923 case 4:
1924 default:
1925 if (ASIC_IS_DCE5(rdev))
1926 return 4096 * 2;
1927 else
1928 return 3840 * 2;
1929 case 1:
1930 case 5:
1931 if (ASIC_IS_DCE5(rdev))
1932 return 6144 * 2;
1933 else
1934 return 5760 * 2;
1935 case 2:
1936 case 6:
1937 if (ASIC_IS_DCE5(rdev))
1938 return 8192 * 2;
1939 else
1940 return 7680 * 2;
1941 case 3:
1942 case 7:
1943 if (ASIC_IS_DCE5(rdev))
1944 return 2048 * 2;
1945 else
1946 return 1920 * 2;
1947 }
f9d9c362 1948 }
12dfc843
AD		 1949
1950 /* controller not enabled, so no lb used */
1951 return 0;
f9d9c362
AD		 1952}
1953
ca7db22b 1954u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1955{
1956 u32 tmp = RREG32(MC_SHARED_CHMAP);
1957
1958 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1959 case 0:
1960 default:
1961 return 1;
1962 case 1:
1963 return 2;
1964 case 2:
1965 return 4;
1966 case 3:
1967 return 8;
1968 }
1969}
1970
1971struct evergreen_wm_params {
1972 u32 dram_channels; /* number of dram channels */
1973 u32 yclk; /* bandwidth per dram data pin in kHz */
1974 u32 sclk; /* engine clock in kHz */
1975 u32 disp_clk; /* display clock in kHz */
1976 u32 src_width; /* viewport width */
1977 u32 active_time; /* active display time in ns */
1978 u32 blank_time; /* blank time in ns */
1979 bool interlaced; /* mode is interlaced */
1980 fixed20_12 vsc; /* vertical scale ratio */
1981 u32 num_heads; /* number of active crtcs */
1982 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1983 u32 lb_size; /* line buffer allocated to pipe */
1984 u32 vtaps; /* vertical scaler taps */
1985};
1986
1987static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1988{
1989 /* Calculate DRAM Bandwidth and the part allocated to display. */
1990 fixed20_12 dram_efficiency; /* 0.7 */
1991 fixed20_12 yclk, dram_channels, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 yclk.full = dfixed_const(wm->yclk);
1996 yclk.full = dfixed_div(yclk, a);
1997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998 a.full = dfixed_const(10);
1999 dram_efficiency.full = dfixed_const(7);
2000 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2001 bandwidth.full = dfixed_mul(dram_channels, yclk);
2002 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2003
2004 return dfixed_trunc(bandwidth);
2005}
2006
2007static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2008{
2009 /* Calculate DRAM Bandwidth and the part allocated to display. */
2010 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2011 fixed20_12 yclk, dram_channels, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 yclk.full = dfixed_const(wm->yclk);
2016 yclk.full = dfixed_div(yclk, a);
2017 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2018 a.full = dfixed_const(10);
2019 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2020 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2021 bandwidth.full = dfixed_mul(dram_channels, yclk);
2022 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2023
2024 return dfixed_trunc(bandwidth);
2025}
2026
2027static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2028{
2029 /* Calculate the display Data return Bandwidth */
2030 fixed20_12 return_efficiency; /* 0.8 */
2031 fixed20_12 sclk, bandwidth;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1000);
2035 sclk.full = dfixed_const(wm->sclk);
2036 sclk.full = dfixed_div(sclk, a);
2037 a.full = dfixed_const(10);
2038 return_efficiency.full = dfixed_const(8);
2039 return_efficiency.full = dfixed_div(return_efficiency, a);
2040 a.full = dfixed_const(32);
2041 bandwidth.full = dfixed_mul(a, sclk);
2042 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2043
2044 return dfixed_trunc(bandwidth);
2045}
2046
2047static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2048{
2049 /* Calculate the DMIF Request Bandwidth */
2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2051 fixed20_12 disp_clk, bandwidth;
2052 fixed20_12 a;
2053
2054 a.full = dfixed_const(1000);
2055 disp_clk.full = dfixed_const(wm->disp_clk);
2056 disp_clk.full = dfixed_div(disp_clk, a);
2057 a.full = dfixed_const(10);
2058 disp_clk_request_efficiency.full = dfixed_const(8);
2059 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2060 a.full = dfixed_const(32);
2061 bandwidth.full = dfixed_mul(a, disp_clk);
2062 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2063
2064 return dfixed_trunc(bandwidth);
2065}
2066
2067static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2068{
2069 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2070 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2071 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2072 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2073
2074 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2075}
2076
2077static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2078{
2079 /* Calculate the display mode Average Bandwidth
2080 * DisplayMode should contain the source and destination dimensions,
2081 * timing, etc.
2082 */
2083 fixed20_12 bpp;
2084 fixed20_12 line_time;
2085 fixed20_12 src_width;
2086 fixed20_12 bandwidth;
2087 fixed20_12 a;
2088
2089 a.full = dfixed_const(1000);
2090 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2091 line_time.full = dfixed_div(line_time, a);
2092 bpp.full = dfixed_const(wm->bytes_per_pixel);
2093 src_width.full = dfixed_const(wm->src_width);
2094 bandwidth.full = dfixed_mul(src_width, bpp);
2095 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2096 bandwidth.full = dfixed_div(bandwidth, line_time);
2097
2098 return dfixed_trunc(bandwidth);
2099}
2100
2101static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2102{
2103 /* First calcualte the latency in ns */
2104 u32 mc_latency = 2000; /* 2000 ns. */
2105 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2106 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2107 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2108 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2109 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2110 (wm->num_heads * cursor_line_pair_return_time);
2111 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2112 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2113 fixed20_12 a, b, c;
2114
2115 if (wm->num_heads == 0)
2116 return 0;
2117
2118 a.full = dfixed_const(2);
2119 b.full = dfixed_const(1);
2120 if ((wm->vsc.full > a.full) ||
2121 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2122 (wm->vtaps >= 5) ||
2123 ((wm->vsc.full >= a.full) && wm->interlaced))
2124 max_src_lines_per_dst_line = 4;
2125 else
2126 max_src_lines_per_dst_line = 2;
2127
2128 a.full = dfixed_const(available_bandwidth);
2129 b.full = dfixed_const(wm->num_heads);
2130 a.full = dfixed_div(a, b);
2131
2132 b.full = dfixed_const(1000);
2133 c.full = dfixed_const(wm->disp_clk);
2134 b.full = dfixed_div(c, b);
2135 c.full = dfixed_const(wm->bytes_per_pixel);
2136 b.full = dfixed_mul(b, c);
2137
2138 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2139
2140 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2141 b.full = dfixed_const(1000);
2142 c.full = dfixed_const(lb_fill_bw);
2143 b.full = dfixed_div(c, b);
2144 a.full = dfixed_div(a, b);
2145 line_fill_time = dfixed_trunc(a);
2146
2147 if (line_fill_time < wm->active_time)
2148 return latency;
2149 else
2150 return latency + (line_fill_time - wm->active_time);
2151
2152}
2153
2154static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2155{
2156 if (evergreen_average_bandwidth(wm) <=
2157 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2158 return true;
2159 else
2160 return false;
2161};
2162
2163static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2164{
2165 if (evergreen_average_bandwidth(wm) <=
2166 (evergreen_available_bandwidth(wm) / wm->num_heads))
2167 return true;
2168 else
2169 return false;
2170};
2171
2172static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2173{
2174 u32 lb_partitions = wm->lb_size / wm->src_width;
2175 u32 line_time = wm->active_time + wm->blank_time;
2176 u32 latency_tolerant_lines;
2177 u32 latency_hiding;
2178 fixed20_12 a;
2179
2180 a.full = dfixed_const(1);
2181 if (wm->vsc.full > a.full)
2182 latency_tolerant_lines = 1;
2183 else {
2184 if (lb_partitions <= (wm->vtaps + 1))
2185 latency_tolerant_lines = 1;
2186 else
2187 latency_tolerant_lines = 2;
2188 }
2189
2190 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2191
2192 if (evergreen_latency_watermark(wm) <= latency_hiding)
2193 return true;
2194 else
2195 return false;
2196}
2197
2198static void evergreen_program_watermarks(struct radeon_device *rdev,
2199 struct radeon_crtc *radeon_crtc,
2200 u32 lb_size, u32 num_heads)
2201{
2202 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2203 struct evergreen_wm_params wm_low, wm_high;
2204 u32 dram_channels;
f9d9c362
AD		 2205 u32 pixel_period;
2206 u32 line_time = 0;
2207 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2208 u32 priority_a_mark = 0, priority_b_mark = 0;
2209 u32 priority_a_cnt = PRIORITY_OFF;
2210 u32 priority_b_cnt = PRIORITY_OFF;
2211 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2212 u32 tmp, arb_control3;
2213 fixed20_12 a, b, c;
2214
2215 if (radeon_crtc->base.enabled && num_heads && mode) {
2216 pixel_period = 1000000 / (u32)mode->clock;
2217 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2218 priority_a_cnt = 0;
2219 priority_b_cnt = 0;
cf0cfdd7
AD		 2220 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2221
2222 /* watermark for high clocks */
2223 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2224 wm_high.yclk =
2225 radeon_dpm_get_mclk(rdev, false) * 10;
2226 wm_high.sclk =
2227 radeon_dpm_get_sclk(rdev, false) * 10;
2228 } else {
2229 wm_high.yclk = rdev->pm.current_mclk * 10;
2230 wm_high.sclk = rdev->pm.current_sclk * 10;
2231 }
f9d9c362 2232
cf0cfdd7
AD		 2233 wm_high.disp_clk = mode->clock;
2234 wm_high.src_width = mode->crtc_hdisplay;
2235 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2236 wm_high.blank_time = line_time - wm_high.active_time;
2237 wm_high.interlaced = false;
f9d9c362 2238 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2239 wm_high.interlaced = true;
2240 wm_high.vsc = radeon_crtc->vsc;
2241 wm_high.vtaps = 1;
f9d9c362 2242 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2243 wm_high.vtaps = 2;
2244 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2245 wm_high.lb_size = lb_size;
2246 wm_high.dram_channels = dram_channels;
2247 wm_high.num_heads = num_heads;
2248
2249 /* watermark for low clocks */
2250 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2251 wm_low.yclk =
2252 radeon_dpm_get_mclk(rdev, true) * 10;
2253 wm_low.sclk =
2254 radeon_dpm_get_sclk(rdev, true) * 10;
2255 } else {
2256 wm_low.yclk = rdev->pm.current_mclk * 10;
2257 wm_low.sclk = rdev->pm.current_sclk * 10;
2258 }
2259
2260 wm_low.disp_clk = mode->clock;
2261 wm_low.src_width = mode->crtc_hdisplay;
2262 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2263 wm_low.blank_time = line_time - wm_low.active_time;
2264 wm_low.interlaced = false;
2265 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2266 wm_low.interlaced = true;
2267 wm_low.vsc = radeon_crtc->vsc;
2268 wm_low.vtaps = 1;
2269 if (radeon_crtc->rmx_type != RMX_OFF)
2270 wm_low.vtaps = 2;
2271 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2272 wm_low.lb_size = lb_size;
2273 wm_low.dram_channels = dram_channels;
2274 wm_low.num_heads = num_heads;
f9d9c362
AD		 2275
2276 /* set for high clocks */
cf0cfdd7 2277 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2278 /* set for low clocks */
cf0cfdd7 2279 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2280
2281 /* possibly force display priority to high */
2282 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2283 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2284 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2285 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2286 (rdev->disp_priority == 2)) {
cf0cfdd7 2287 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2288 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2289 }
2290 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2291 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2292 !evergreen_check_latency_hiding(&wm_low) ||
2293 (rdev->disp_priority == 2)) {
2294 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2295 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2296 }
2297
2298 a.full = dfixed_const(1000);
2299 b.full = dfixed_const(mode->clock);
2300 b.full = dfixed_div(b, a);
2301 c.full = dfixed_const(latency_watermark_a);
2302 c.full = dfixed_mul(c, b);
2303 c.full = dfixed_mul(c, radeon_crtc->hsc);
2304 c.full = dfixed_div(c, a);
2305 a.full = dfixed_const(16);
2306 c.full = dfixed_div(c, a);
2307 priority_a_mark = dfixed_trunc(c);
2308 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2309
2310 a.full = dfixed_const(1000);
2311 b.full = dfixed_const(mode->clock);
2312 b.full = dfixed_div(b, a);
2313 c.full = dfixed_const(latency_watermark_b);
2314 c.full = dfixed_mul(c, b);
2315 c.full = dfixed_mul(c, radeon_crtc->hsc);
2316 c.full = dfixed_div(c, a);
2317 a.full = dfixed_const(16);
2318 c.full = dfixed_div(c, a);
2319 priority_b_mark = dfixed_trunc(c);
2320 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2321 }
2322
2323 /* select wm A */
2324 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2325 tmp = arb_control3;
2326 tmp &= ~LATENCY_WATERMARK_MASK(3);
2327 tmp |= LATENCY_WATERMARK_MASK(1);
2328 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2329 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2330 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2331 LATENCY_HIGH_WATERMARK(line_time)));
2332 /* select wm B */
2333 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2334 tmp &= ~LATENCY_WATERMARK_MASK(3);
2335 tmp |= LATENCY_WATERMARK_MASK(2);
2336 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2337 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2338 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2339 LATENCY_HIGH_WATERMARK(line_time)));
2340 /* restore original selection */
2341 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2342
2343 /* write the priority marks */
2344 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2345 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2346
7178d2a6
AD		 2347 /* save values for DPM */
2348 radeon_crtc->line_time = line_time;
2349 radeon_crtc->wm_high = latency_watermark_a;
2350 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2351}
2352
377edc8b
AD		 2353/**
2354 * evergreen_bandwidth_update - update display watermarks callback.
2355 *
2356 * @rdev: radeon_device pointer
2357 *
2358 * Update the display watermarks based on the requested mode(s)
2359 * (evergreen+).
2360 */
0ca2ab52 2361void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2362{
f9d9c362
AD		 2363 struct drm_display_mode *mode0 = NULL;
2364 struct drm_display_mode *mode1 = NULL;
2365 u32 num_heads = 0, lb_size;
2366 int i;
2367
2368 radeon_update_display_priority(rdev);
2369
2370 for (i = 0; i < rdev->num_crtc; i++) {
2371 if (rdev->mode_info.crtcs[i]->base.enabled)
2372 num_heads++;
2373 }
2374 for (i = 0; i < rdev->num_crtc; i += 2) {
2375 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2376 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2377 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2378 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2379 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2380 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2381 }
bcc1c2a1
AD		 2382}
2383
377edc8b
AD		 2384/**
2385 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2386 *
2387 * @rdev: radeon_device pointer
2388 *
2389 * Wait for the MC (memory controller) to be idle.
2390 * (evergreen+).
2391 * Returns 0 if the MC is idle, -1 if not.
2392 */
b9952a8a 2393int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2394{
2395 unsigned i;
2396 u32 tmp;
2397
2398 for (i = 0; i < rdev->usec_timeout; i++) {
2399 /* read MC_STATUS */
2400 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2401 if (!tmp)
2402 return 0;
2403 udelay(1);
2404 }
2405 return -1;
2406}
2407
2408/*
2409 * GART
2410 */
0fcdb61e
AD		 2411void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2412{
2413 unsigned i;
2414 u32 tmp;
2415
6f2f48a9
AD		 2416 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2417
0fcdb61e
AD		 2418 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2419 for (i = 0; i < rdev->usec_timeout; i++) {
2420 /* read MC_STATUS */
2421 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2422 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2423 if (tmp == 2) {
2424 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2425 return;
2426 }
2427 if (tmp) {
2428 return;
2429 }
2430 udelay(1);
2431 }
2432}
2433
1109ca09 2434static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2435{
2436 u32 tmp;
0fcdb61e 2437 int r;
bcc1c2a1 2438
c9a1be96 2439 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2440 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2441 return -EINVAL;
2442 }
2443 r = radeon_gart_table_vram_pin(rdev);
2444 if (r)
2445 return r;
82568565 2446 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2447 /* Setup L2 cache */
2448 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2449 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2450 EFFECTIVE_L2_QUEUE_SIZE(7));
2451 WREG32(VM_L2_CNTL2, 0);
2452 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2453 /* Setup TLB control */
2454 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2455 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2456 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2457 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2458 if (rdev->flags & RADEON_IS_IGP) {
2459 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2462 } else {
2463 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2464 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2465 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2466 if ((rdev->family == CHIP_JUNIPER) ||
2467 (rdev->family == CHIP_CYPRESS) ||
2468 (rdev->family == CHIP_HEMLOCK) ||
2469 (rdev->family == CHIP_BARTS))
2470 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2471 }
bcc1c2a1
AD		 2472 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2473 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2474 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2475 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2476 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2477 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2478 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2479 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2480 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2481 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2482 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2483 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2484
0fcdb61e 2485 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2486 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2487 (unsigned)(rdev->mc.gtt_size >> 20),
2488 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2489 rdev->gart.ready = true;
2490 return 0;
2491}
2492
1109ca09 2493static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2494{
2495 u32 tmp;
bcc1c2a1
AD		 2496
2497 /* Disable all tables */
0fcdb61e
AD		 2498 WREG32(VM_CONTEXT0_CNTL, 0);
2499 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2500
2501 /* Setup L2 cache */
2502 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2503 EFFECTIVE_L2_QUEUE_SIZE(7));
2504 WREG32(VM_L2_CNTL2, 0);
2505 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2506 /* Setup TLB control */
2507 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2508 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2509 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2510 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2513 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2515 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2516}
2517
1109ca09 2518static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2519{
2520 evergreen_pcie_gart_disable(rdev);
2521 radeon_gart_table_vram_free(rdev);
2522 radeon_gart_fini(rdev);
2523}
2524
2525
1109ca09 2526static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2527{
2528 u32 tmp;
bcc1c2a1
AD		 2529
2530 /* Setup L2 cache */
2531 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2532 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2533 EFFECTIVE_L2_QUEUE_SIZE(7));
2534 WREG32(VM_L2_CNTL2, 0);
2535 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2536 /* Setup TLB control */
2537 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2538 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2539 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2540 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2541 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2542 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2543 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2544 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2545 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2546 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2547 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2548 WREG32(VM_CONTEXT0_CNTL, 0);
2549 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2550}
2551
b9952a8a 2552void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2553{
62444b74
AD		 2554 u32 crtc_enabled, tmp, frame_count, blackout;
2555 int i, j;
2556
5153550a
AD		 2557 if (!ASIC_IS_NODCE(rdev)) {
2558 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2559 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2560
5153550a
AD		 2561 /* disable VGA render */
2562 WREG32(VGA_RENDER_CONTROL, 0);
2563 }
62444b74
AD		 2564 /* blank the display controllers */
2565 for (i = 0; i < rdev->num_crtc; i++) {
2566 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2567 if (crtc_enabled) {
2568 save->crtc_enabled[i] = true;
2569 if (ASIC_IS_DCE6(rdev)) {
2570 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2571 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2572 radeon_wait_for_vblank(rdev, i);
abf1457b 2573 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2574 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2575 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2576 }
2577 } else {
2578 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2579 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2580 radeon_wait_for_vblank(rdev, i);
abf1457b 2581 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2582 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2583 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2585 }
2586 }
2587 /* wait for the next frame */
2588 frame_count = radeon_get_vblank_counter(rdev, i);
2589 for (j = 0; j < rdev->usec_timeout; j++) {
2590 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2591 break;
2592 udelay(1);
2593 }
abf1457b
AD		 2594
2595 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2596 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2597 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2598 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2599 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2600 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2601 save->crtc_enabled[i] = false;
2602 /* ***** */
804cc4a0
AD		 2603 } else {
2604 save->crtc_enabled[i] = false;
62444b74 2605 }
18007401 2606 }
bcc1c2a1 2607
62444b74
AD		 2608 radeon_mc_wait_for_idle(rdev);
2609
2610 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2611 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2612 /* Block CPU access */
2613 WREG32(BIF_FB_EN, 0);
2614 /* blackout the MC */
2615 blackout &= ~BLACKOUT_MODE_MASK;
2616 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2617 }
ed39fadd
AD		 2618 /* wait for the MC to settle */
2619 udelay(100);
968c0166
AD		 2620
2621 /* lock double buffered regs */
2622 for (i = 0; i < rdev->num_crtc; i++) {
2623 if (save->crtc_enabled[i]) {
2624 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2625 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2626 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2627 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2628 }
2629 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2630 if (!(tmp & 1)) {
2631 tmp |= 1;
2632 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2633 }
2634 }
2635 }
bcc1c2a1
AD		 2636}
2637
b9952a8a 2638void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2639{
62444b74
AD		 2640 u32 tmp, frame_count;
2641 int i, j;
18007401 2642
62444b74
AD		 2643 /* update crtc base addresses */
2644 for (i = 0; i < rdev->num_crtc; i++) {
2645 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2646 upper_32_bits(rdev->mc.vram_start));
62444b74 2647 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2648 upper_32_bits(rdev->mc.vram_start));
62444b74 2649 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2650 (u32)rdev->mc.vram_start);
62444b74 2651 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2652 (u32)rdev->mc.vram_start);
2653 }
5153550a
AD		 2654
2655 if (!ASIC_IS_NODCE(rdev)) {
2656 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2657 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2658 }
62444b74 2659
968c0166
AD		 2660 /* unlock regs and wait for update */
2661 for (i = 0; i < rdev->num_crtc; i++) {
2662 if (save->crtc_enabled[i]) {
2663 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2664 if ((tmp & 0x3) != 0) {
2665 tmp &= ~0x3;
2666 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2667 }
2668 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2669 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2670 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2671 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2672 }
2673 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2674 if (tmp & 1) {
2675 tmp &= ~1;
2676 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2677 }
2678 for (j = 0; j < rdev->usec_timeout; j++) {
2679 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2680 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2681 break;
2682 udelay(1);
2683 }
2684 }
2685 }
2686
62444b74
AD		 2687 /* unblackout the MC */
2688 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2689 tmp &= ~BLACKOUT_MODE_MASK;
2690 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2691 /* allow CPU access */
2692 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2693
2694 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2695 if (save->crtc_enabled[i]) {
62444b74
AD		 2696 if (ASIC_IS_DCE6(rdev)) {
2697 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2698 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2699 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2700 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2701 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2702 } else {
2703 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2704 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2705 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2706 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2707 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2708 }
2709 /* wait for the next frame */
2710 frame_count = radeon_get_vblank_counter(rdev, i);
2711 for (j = 0; j < rdev->usec_timeout; j++) {
2712 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2713 break;
2714 udelay(1);
2715 }
2716 }
2717 }
5153550a
AD		 2718 if (!ASIC_IS_NODCE(rdev)) {
2719 /* Unlock vga access */
2720 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2721 mdelay(1);
2722 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2723 }
bcc1c2a1
AD		 2724}
2725
755d819e 2726void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2727{
2728 struct evergreen_mc_save save;
2729 u32 tmp;
2730 int i, j;
2731
2732 /* Initialize HDP */
2733 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2734 WREG32((0x2c14 + j), 0x00000000);
2735 WREG32((0x2c18 + j), 0x00000000);
2736 WREG32((0x2c1c + j), 0x00000000);
2737 WREG32((0x2c20 + j), 0x00000000);
2738 WREG32((0x2c24 + j), 0x00000000);
2739 }
2740 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2741
2742 evergreen_mc_stop(rdev, &save);
2743 if (evergreen_mc_wait_for_idle(rdev)) {
2744 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2745 }
2746 /* Lockout access through VGA aperture*/
2747 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2748 /* Update configuration */
2749 if (rdev->flags & RADEON_IS_AGP) {
2750 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2751 /* VRAM before AGP */
2752 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2753 rdev->mc.vram_start >> 12);
2754 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2755 rdev->mc.gtt_end >> 12);
2756 } else {
2757 /* VRAM after AGP */
2758 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2759 rdev->mc.gtt_start >> 12);
2760 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2761 rdev->mc.vram_end >> 12);
2762 }
2763 } else {
2764 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2765 rdev->mc.vram_start >> 12);
2766 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2767 rdev->mc.vram_end >> 12);
2768 }
3b9832f6 2769 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2770 /* llano/ontario only */
2771 if ((rdev->family == CHIP_PALM) ||
2772 (rdev->family == CHIP_SUMO) ||
2773 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2774 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2775 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2776 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2777 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2778 }
bcc1c2a1
AD		 2779 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2780 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2781 WREG32(MC_VM_FB_LOCATION, tmp);
2782 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2783 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2784 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2785 if (rdev->flags & RADEON_IS_AGP) {
2786 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2787 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2788 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2789 } else {
2790 WREG32(MC_VM_AGP_BASE, 0);
2791 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2792 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2793 }
2794 if (evergreen_mc_wait_for_idle(rdev)) {
2795 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2796 }
2797 evergreen_mc_resume(rdev, &save);
2798 /* we need to own VRAM, so turn off the VGA renderer here
2799 * to stop it overwriting our objects */
2800 rv515_vga_render_disable(rdev);
2801}
2802
bcc1c2a1
AD		 2803/*
2804 * CP.
2805 */
12920591
AD		 2806void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2807{
876dc9f3 2808 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2809 u32 next_rptr;
7b1f2485 2810
12920591 2811 /* set to DX10/11 mode */
e32eb50d
CK		 2812 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2813 radeon_ring_write(ring, 1);
45df6803
CK		 2814
2815 if (ring->rptr_save_reg) {
89d35807 2816 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2817 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2818 radeon_ring_write(ring, ((ring->rptr_save_reg -
2819 PACKET3_SET_CONFIG_REG_START) >> 2));
2820 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2821 } else if (rdev->wb.enabled) {
2822 next_rptr = ring->wptr + 5 + 4;
2823 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2824 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2825 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2826 radeon_ring_write(ring, next_rptr);
2827 radeon_ring_write(ring, 0);
45df6803
CK		 2828 }
2829
e32eb50d
CK		 2830 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2831 radeon_ring_write(ring,
0f234f5f
AD		 2832#ifdef __BIG_ENDIAN
2833 (2 << 0) |
2834#endif
2835 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2836 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2837 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2838}
2839
bcc1c2a1
AD		 2840
2841static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2842{
fe251e2f
AD		 2843 const __be32 *fw_data;
2844 int i;
2845
2846 if (!rdev->me_fw || !rdev->pfp_fw)
2847 return -EINVAL;
bcc1c2a1 2848
fe251e2f 2849 r700_cp_stop(rdev);
0f234f5f
AD		 2850 WREG32(CP_RB_CNTL,
2851#ifdef __BIG_ENDIAN
2852 BUF_SWAP_32BIT |
2853#endif
2854 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2855
2856 fw_data = (const __be32 *)rdev->pfp_fw->data;
2857 WREG32(CP_PFP_UCODE_ADDR, 0);
2858 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2859 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2860 WREG32(CP_PFP_UCODE_ADDR, 0);
2861
2862 fw_data = (const __be32 *)rdev->me_fw->data;
2863 WREG32(CP_ME_RAM_WADDR, 0);
2864 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2865 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2866
2867 WREG32(CP_PFP_UCODE_ADDR, 0);
2868 WREG32(CP_ME_RAM_WADDR, 0);
2869 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2870 return 0;
2871}
2872
7e7b41d2
AD		 2873static int evergreen_cp_start(struct radeon_device *rdev)
2874{
e32eb50d 2875 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2876 int r, i;
7e7b41d2
AD		 2877 uint32_t cp_me;
2878
e32eb50d 2879 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2880 if (r) {
2881 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2882 return r;
2883 }
e32eb50d
CK		 2884 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2885 radeon_ring_write(ring, 0x1);
2886 radeon_ring_write(ring, 0x0);
2887 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2888 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2889 radeon_ring_write(ring, 0);
2890 radeon_ring_write(ring, 0);
2891 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2892
2893 cp_me = 0xff;
2894 WREG32(CP_ME_CNTL, cp_me);
2895
e32eb50d 2896 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2897 if (r) {
2898 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2899 return r;
2900 }
2281a378
AD		 2901
2902 /* setup clear context state */
e32eb50d
CK		 2903 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2904 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2905
2906 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2907 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2908
e32eb50d
CK		 2909 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2910 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2911
2912 /* set clear context state */
e32eb50d
CK		 2913 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2914 radeon_ring_write(ring, 0);
2281a378
AD		 2915
2916 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2917 radeon_ring_write(ring, 0xc0026f00);
2918 radeon_ring_write(ring, 0x00000000);
2919 radeon_ring_write(ring, 0x00000000);
2920 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2921
2922 /* Clear consts */
e32eb50d
CK		 2923 radeon_ring_write(ring, 0xc0036f00);
2924 radeon_ring_write(ring, 0x00000bc4);
2925 radeon_ring_write(ring, 0xffffffff);
2926 radeon_ring_write(ring, 0xffffffff);
2927 radeon_ring_write(ring, 0xffffffff);
2281a378 2928
e32eb50d
CK		 2929 radeon_ring_write(ring, 0xc0026900);
2930 radeon_ring_write(ring, 0x00000316);
2931 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2932 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2933
e32eb50d 2934 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2935
2936 return 0;
2937}
2938
1109ca09 2939static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2940{
e32eb50d 2941 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2942 u32 tmp;
2943 u32 rb_bufsz;
2944 int r;
2945
2946 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2947 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2948 SOFT_RESET_PA |
2949 SOFT_RESET_SH |
2950 SOFT_RESET_VGT |
a49a50da 2951 SOFT_RESET_SPI |
fe251e2f
AD		 2952 SOFT_RESET_SX));
2953 RREG32(GRBM_SOFT_RESET);
2954 mdelay(15);
2955 WREG32(GRBM_SOFT_RESET, 0);
2956 RREG32(GRBM_SOFT_RESET);
2957
2958 /* Set ring buffer size */
b72a8925
DV		 2959 rb_bufsz = order_base_2(ring->ring_size / 8);
2960 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2961#ifdef __BIG_ENDIAN
2962 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2963#endif
fe251e2f 2964 WREG32(CP_RB_CNTL, tmp);
15d3332f 2965 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2966 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2967
2968 /* Set the write pointer delay */
2969 WREG32(CP_RB_WPTR_DELAY, 0);
2970
2971 /* Initialize the ring buffer's read and write pointers */
2972 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2973 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2974 ring->wptr = 0;
2975 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2976
48fc7f7e 2977 /* set the wb address whether it's enabled or not */
0f234f5f 2978 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2979 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2980 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2981 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2982
2983 if (rdev->wb.enabled)
2984 WREG32(SCRATCH_UMSK, 0xff);
2985 else {
2986 tmp |= RB_NO_UPDATE;
2987 WREG32(SCRATCH_UMSK, 0);
2988 }
2989
fe251e2f
AD		 2990 mdelay(1);
2991 WREG32(CP_RB_CNTL, tmp);
2992
e32eb50d 2993 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2994 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2995
e32eb50d 2996 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2997
7e7b41d2 2998 evergreen_cp_start(rdev);
e32eb50d 2999 ring->ready = true;
f712812e 3000 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3001 if (r) {
e32eb50d 3002 ring->ready = false;
fe251e2f
AD		 3003 return r;
3004 }
3005 return 0;
3006}
bcc1c2a1
AD		 3007
3008/*
3009 * Core functions
3010 */
bcc1c2a1
AD		 3011static void evergreen_gpu_init(struct radeon_device *rdev)
3012{
416a2bd2 3013 u32 gb_addr_config;
32fcdbf4 3014 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3015 u32 sx_debug_1;
3016 u32 smx_dc_ctl0;
3017 u32 sq_config;
3018 u32 sq_lds_resource_mgmt;
3019 u32 sq_gpr_resource_mgmt_1;
3020 u32 sq_gpr_resource_mgmt_2;
3021 u32 sq_gpr_resource_mgmt_3;
3022 u32 sq_thread_resource_mgmt;
3023 u32 sq_thread_resource_mgmt_2;
3024 u32 sq_stack_resource_mgmt_1;
3025 u32 sq_stack_resource_mgmt_2;
3026 u32 sq_stack_resource_mgmt_3;
3027 u32 vgt_cache_invalidation;
f25a5c63 3028 u32 hdp_host_path_cntl, tmp;
416a2bd2 3029 u32 disabled_rb_mask;
32fcdbf4
AD		 3030 int i, j, num_shader_engines, ps_thread_count;
3031
3032 switch (rdev->family) {
3033 case CHIP_CYPRESS:
3034 case CHIP_HEMLOCK:
3035 rdev->config.evergreen.num_ses = 2;
3036 rdev->config.evergreen.max_pipes = 4;
3037 rdev->config.evergreen.max_tile_pipes = 8;
3038 rdev->config.evergreen.max_simds = 10;
3039 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3040 rdev->config.evergreen.max_gprs = 256;
3041 rdev->config.evergreen.max_threads = 248;
3042 rdev->config.evergreen.max_gs_threads = 32;
3043 rdev->config.evergreen.max_stack_entries = 512;
3044 rdev->config.evergreen.sx_num_of_sets = 4;
3045 rdev->config.evergreen.sx_max_export_size = 256;
3046 rdev->config.evergreen.sx_max_export_pos_size = 64;
3047 rdev->config.evergreen.sx_max_export_smx_size = 192;
3048 rdev->config.evergreen.max_hw_contexts = 8;
3049 rdev->config.evergreen.sq_num_cf_insts = 2;
3050
3051 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3052 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3053 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3054 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3055 break;
3056 case CHIP_JUNIPER:
3057 rdev->config.evergreen.num_ses = 1;
3058 rdev->config.evergreen.max_pipes = 4;
3059 rdev->config.evergreen.max_tile_pipes = 4;
3060 rdev->config.evergreen.max_simds = 10;
3061 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3062 rdev->config.evergreen.max_gprs = 256;
3063 rdev->config.evergreen.max_threads = 248;
3064 rdev->config.evergreen.max_gs_threads = 32;
3065 rdev->config.evergreen.max_stack_entries = 512;
3066 rdev->config.evergreen.sx_num_of_sets = 4;
3067 rdev->config.evergreen.sx_max_export_size = 256;
3068 rdev->config.evergreen.sx_max_export_pos_size = 64;
3069 rdev->config.evergreen.sx_max_export_smx_size = 192;
3070 rdev->config.evergreen.max_hw_contexts = 8;
3071 rdev->config.evergreen.sq_num_cf_insts = 2;
3072
3073 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3074 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3075 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3076 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3077 break;
3078 case CHIP_REDWOOD:
3079 rdev->config.evergreen.num_ses = 1;
3080 rdev->config.evergreen.max_pipes = 4;
3081 rdev->config.evergreen.max_tile_pipes = 4;
3082 rdev->config.evergreen.max_simds = 5;
3083 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3084 rdev->config.evergreen.max_gprs = 256;
3085 rdev->config.evergreen.max_threads = 248;
3086 rdev->config.evergreen.max_gs_threads = 32;
3087 rdev->config.evergreen.max_stack_entries = 256;
3088 rdev->config.evergreen.sx_num_of_sets = 4;
3089 rdev->config.evergreen.sx_max_export_size = 256;
3090 rdev->config.evergreen.sx_max_export_pos_size = 64;
3091 rdev->config.evergreen.sx_max_export_smx_size = 192;
3092 rdev->config.evergreen.max_hw_contexts = 8;
3093 rdev->config.evergreen.sq_num_cf_insts = 2;
3094
3095 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3096 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3097 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3098 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3099 break;
3100 case CHIP_CEDAR:
3101 default:
3102 rdev->config.evergreen.num_ses = 1;
3103 rdev->config.evergreen.max_pipes = 2;
3104 rdev->config.evergreen.max_tile_pipes = 2;
3105 rdev->config.evergreen.max_simds = 2;
3106 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3107 rdev->config.evergreen.max_gprs = 256;
3108 rdev->config.evergreen.max_threads = 192;
3109 rdev->config.evergreen.max_gs_threads = 16;
3110 rdev->config.evergreen.max_stack_entries = 256;
3111 rdev->config.evergreen.sx_num_of_sets = 4;
3112 rdev->config.evergreen.sx_max_export_size = 128;
3113 rdev->config.evergreen.sx_max_export_pos_size = 32;
3114 rdev->config.evergreen.sx_max_export_smx_size = 96;
3115 rdev->config.evergreen.max_hw_contexts = 4;
3116 rdev->config.evergreen.sq_num_cf_insts = 1;
3117
d5e455e4
AD		 3118 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3119 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3120 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3121 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3122 break;
3123 case CHIP_PALM:
3124 rdev->config.evergreen.num_ses = 1;
3125 rdev->config.evergreen.max_pipes = 2;
3126 rdev->config.evergreen.max_tile_pipes = 2;
3127 rdev->config.evergreen.max_simds = 2;
3128 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3129 rdev->config.evergreen.max_gprs = 256;
3130 rdev->config.evergreen.max_threads = 192;
3131 rdev->config.evergreen.max_gs_threads = 16;
3132 rdev->config.evergreen.max_stack_entries = 256;
3133 rdev->config.evergreen.sx_num_of_sets = 4;
3134 rdev->config.evergreen.sx_max_export_size = 128;
3135 rdev->config.evergreen.sx_max_export_pos_size = 32;
3136 rdev->config.evergreen.sx_max_export_smx_size = 96;
3137 rdev->config.evergreen.max_hw_contexts = 4;
3138 rdev->config.evergreen.sq_num_cf_insts = 1;
3139
d5c5a72f
AD		 3140 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3141 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3142 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3143 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3144 break;
3145 case CHIP_SUMO:
3146 rdev->config.evergreen.num_ses = 1;
3147 rdev->config.evergreen.max_pipes = 4;
bd25f078 3148 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3149 if (rdev->pdev->device == 0x9648)
3150 rdev->config.evergreen.max_simds = 3;
3151 else if ((rdev->pdev->device == 0x9647) ||
3152 (rdev->pdev->device == 0x964a))
3153 rdev->config.evergreen.max_simds = 4;
3154 else
3155 rdev->config.evergreen.max_simds = 5;
3156 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3157 rdev->config.evergreen.max_gprs = 256;
3158 rdev->config.evergreen.max_threads = 248;
3159 rdev->config.evergreen.max_gs_threads = 32;
3160 rdev->config.evergreen.max_stack_entries = 256;
3161 rdev->config.evergreen.sx_num_of_sets = 4;
3162 rdev->config.evergreen.sx_max_export_size = 256;
3163 rdev->config.evergreen.sx_max_export_pos_size = 64;
3164 rdev->config.evergreen.sx_max_export_smx_size = 192;
3165 rdev->config.evergreen.max_hw_contexts = 8;
3166 rdev->config.evergreen.sq_num_cf_insts = 2;
3167
3168 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3169 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3170 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3171 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3172 break;
3173 case CHIP_SUMO2:
3174 rdev->config.evergreen.num_ses = 1;
3175 rdev->config.evergreen.max_pipes = 4;
3176 rdev->config.evergreen.max_tile_pipes = 4;
3177 rdev->config.evergreen.max_simds = 2;
3178 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3179 rdev->config.evergreen.max_gprs = 256;
3180 rdev->config.evergreen.max_threads = 248;
3181 rdev->config.evergreen.max_gs_threads = 32;
3182 rdev->config.evergreen.max_stack_entries = 512;
3183 rdev->config.evergreen.sx_num_of_sets = 4;
3184 rdev->config.evergreen.sx_max_export_size = 256;
3185 rdev->config.evergreen.sx_max_export_pos_size = 64;
3186 rdev->config.evergreen.sx_max_export_smx_size = 192;
3187 rdev->config.evergreen.max_hw_contexts = 8;
3188 rdev->config.evergreen.sq_num_cf_insts = 2;
3189
adb68fa2
AD		 3190 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3191 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3192 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3193 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3194 break;
3195 case CHIP_BARTS:
3196 rdev->config.evergreen.num_ses = 2;
3197 rdev->config.evergreen.max_pipes = 4;
3198 rdev->config.evergreen.max_tile_pipes = 8;
3199 rdev->config.evergreen.max_simds = 7;
3200 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3201 rdev->config.evergreen.max_gprs = 256;
3202 rdev->config.evergreen.max_threads = 248;
3203 rdev->config.evergreen.max_gs_threads = 32;
3204 rdev->config.evergreen.max_stack_entries = 512;
3205 rdev->config.evergreen.sx_num_of_sets = 4;
3206 rdev->config.evergreen.sx_max_export_size = 256;
3207 rdev->config.evergreen.sx_max_export_pos_size = 64;
3208 rdev->config.evergreen.sx_max_export_smx_size = 192;
3209 rdev->config.evergreen.max_hw_contexts = 8;
3210 rdev->config.evergreen.sq_num_cf_insts = 2;
3211
3212 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3213 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3214 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3215 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3216 break;
3217 case CHIP_TURKS:
3218 rdev->config.evergreen.num_ses = 1;
3219 rdev->config.evergreen.max_pipes = 4;
3220 rdev->config.evergreen.max_tile_pipes = 4;
3221 rdev->config.evergreen.max_simds = 6;
3222 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3223 rdev->config.evergreen.max_gprs = 256;
3224 rdev->config.evergreen.max_threads = 248;
3225 rdev->config.evergreen.max_gs_threads = 32;
3226 rdev->config.evergreen.max_stack_entries = 256;
3227 rdev->config.evergreen.sx_num_of_sets = 4;
3228 rdev->config.evergreen.sx_max_export_size = 256;
3229 rdev->config.evergreen.sx_max_export_pos_size = 64;
3230 rdev->config.evergreen.sx_max_export_smx_size = 192;
3231 rdev->config.evergreen.max_hw_contexts = 8;
3232 rdev->config.evergreen.sq_num_cf_insts = 2;
3233
3234 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3235 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3236 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3237 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3238 break;
3239 case CHIP_CAICOS:
3240 rdev->config.evergreen.num_ses = 1;
bd25f078 3241 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3242 rdev->config.evergreen.max_tile_pipes = 2;
3243 rdev->config.evergreen.max_simds = 2;
3244 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3245 rdev->config.evergreen.max_gprs = 256;
3246 rdev->config.evergreen.max_threads = 192;
3247 rdev->config.evergreen.max_gs_threads = 16;
3248 rdev->config.evergreen.max_stack_entries = 256;
3249 rdev->config.evergreen.sx_num_of_sets = 4;
3250 rdev->config.evergreen.sx_max_export_size = 128;
3251 rdev->config.evergreen.sx_max_export_pos_size = 32;
3252 rdev->config.evergreen.sx_max_export_smx_size = 96;
3253 rdev->config.evergreen.max_hw_contexts = 4;
3254 rdev->config.evergreen.sq_num_cf_insts = 1;
3255
32fcdbf4
AD		 3256 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3257 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3258 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3259 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3260 break;
3261 }
3262
3263 /* Initialize HDP */
3264 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3265 WREG32((0x2c14 + j), 0x00000000);
3266 WREG32((0x2c18 + j), 0x00000000);
3267 WREG32((0x2c1c + j), 0x00000000);
3268 WREG32((0x2c20 + j), 0x00000000);
3269 WREG32((0x2c24 + j), 0x00000000);
3270 }
3271
3272 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3273
d054ac16
AD		 3274 evergreen_fix_pci_max_read_req_size(rdev);
3275
32fcdbf4 3276 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3277 if ((rdev->family == CHIP_PALM) ||
3278 (rdev->family == CHIP_SUMO) ||
3279 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3280 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3281 else
3282 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3283
1aa52bd3
AD		 3284 /* setup tiling info dword. gb_addr_config is not adequate since it does
3285 * not have bank info, so create a custom tiling dword.
3286 * bits 3:0 num_pipes
3287 * bits 7:4 num_banks
3288 * bits 11:8 group_size
3289 * bits 15:12 row_size
3290 */
3291 rdev->config.evergreen.tile_config = 0;
3292 switch (rdev->config.evergreen.max_tile_pipes) {
3293 case 1:
3294 default:
3295 rdev->config.evergreen.tile_config |= (0 << 0);
3296 break;
3297 case 2:
3298 rdev->config.evergreen.tile_config |= (1 << 0);
3299 break;
3300 case 4:
3301 rdev->config.evergreen.tile_config |= (2 << 0);
3302 break;
3303 case 8:
3304 rdev->config.evergreen.tile_config |= (3 << 0);
3305 break;
3306 }
d698a34d 3307 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3308 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3309 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3310 else {
c8d15edc
AD		 3311 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3312 case 0: /* four banks */
29d65406 3313 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3314 break;
3315 case 1: /* eight banks */
3316 rdev->config.evergreen.tile_config |= 1 << 4;
3317 break;
3318 case 2: /* sixteen banks */
3319 default:
3320 rdev->config.evergreen.tile_config |= 2 << 4;
3321 break;
3322 }
29d65406 3323 }
416a2bd2 3324 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3325 rdev->config.evergreen.tile_config |=
3326 ((gb_addr_config & 0x30000000) >> 28) << 12;
3327
416a2bd2 3328 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3329
416a2bd2
AD		 3330 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3331 u32 efuse_straps_4;
3332 u32 efuse_straps_3;
32fcdbf4 3333
ff82bbc4
AD		 3334 efuse_straps_4 = RREG32_RCU(0x204);
3335 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3336 tmp = (((efuse_straps_4 & 0xf) << 4) |
3337 ((efuse_straps_3 & 0xf0000000) >> 28));
3338 } else {
3339 tmp = 0;
3340 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3341 u32 rb_disable_bitmap;
3342
3343 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3344 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3345 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3346 tmp <<= 4;
3347 tmp |= rb_disable_bitmap;
32fcdbf4 3348 }
416a2bd2
AD		 3349 }
3350 /* enabled rb are just the one not disabled :) */
3351 disabled_rb_mask = tmp;
cedb655a
AD		 3352 tmp = 0;
3353 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3354 tmp |= (1 << i);
3355 /* if all the backends are disabled, fix it up here */
3356 if ((disabled_rb_mask & tmp) == tmp) {
3357 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3358 disabled_rb_mask &= ~(1 << i);
3359 }
32fcdbf4 3360
416a2bd2
AD		 3361 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3362 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3363
416a2bd2
AD		 3364 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3365 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3366 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3367 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3368 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3369 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3370 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3371
f7eb9730
AD		 3372 if ((rdev->config.evergreen.max_backends == 1) &&
3373 (rdev->flags & RADEON_IS_IGP)) {
3374 if ((disabled_rb_mask & 3) == 1) {
3375 /* RB0 disabled, RB1 enabled */
3376 tmp = 0x11111111;
3377 } else {
3378 /* RB1 disabled, RB0 enabled */
3379 tmp = 0x00000000;
3380 }
3381 } else {
3382 tmp = gb_addr_config & NUM_PIPES_MASK;
3383 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3384 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3385 }
416a2bd2 3386 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3387
3388 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3389 WREG32(CGTS_TCC_DISABLE, 0);
3390 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3391 WREG32(CGTS_USER_TCC_DISABLE, 0);
3392
3393 /* set HW defaults for 3D engine */
3394 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3395 ROQ_IB2_START(0x2b)));
3396
3397 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3398
3399 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3400 SYNC_GRADIENT |
3401 SYNC_WALKER |
3402 SYNC_ALIGNER));
3403
3404 sx_debug_1 = RREG32(SX_DEBUG_1);
3405 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3406 WREG32(SX_DEBUG_1, sx_debug_1);
3407
3408
3409 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3410 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3411 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3412 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3413
b866d133
AD		 3414 if (rdev->family <= CHIP_SUMO2)
3415 WREG32(SMX_SAR_CTL0, 0x00010000);
3416
32fcdbf4
AD		 3417 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3418 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3419 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3420
3421 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3422 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3423 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3424
3425 WREG32(VGT_NUM_INSTANCES, 1);
3426 WREG32(SPI_CONFIG_CNTL, 0);
3427 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3428 WREG32(CP_PERFMON_CNTL, 0);
3429
3430 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3431 FETCH_FIFO_HIWATER(0x4) |
3432 DONE_FIFO_HIWATER(0xe0) |
3433 ALU_UPDATE_FIFO_HIWATER(0x8)));
3434
3435 sq_config = RREG32(SQ_CONFIG);
3436 sq_config &= ~(PS_PRIO(3) |
3437 VS_PRIO(3) |
3438 GS_PRIO(3) |
3439 ES_PRIO(3));
3440 sq_config |= (VC_ENABLE |
3441 EXPORT_SRC_C |
3442 PS_PRIO(0) |
3443 VS_PRIO(1) |
3444 GS_PRIO(2) |
3445 ES_PRIO(3));
3446
d5e455e4
AD		 3447 switch (rdev->family) {
3448 case CHIP_CEDAR:
3449 case CHIP_PALM:
d5c5a72f
AD		 3450 case CHIP_SUMO:
3451 case CHIP_SUMO2:
adb68fa2 3452 case CHIP_CAICOS:
32fcdbf4
AD		 3453 /* no vertex cache */
3454 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3455 break;
3456 default:
3457 break;
3458 }
32fcdbf4
AD		 3459
3460 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3461
3462 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3463 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3464 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3465 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3466 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3467 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3468 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3469
d5e455e4
AD		 3470 switch (rdev->family) {
3471 case CHIP_CEDAR:
3472 case CHIP_PALM:
d5c5a72f
AD		 3473 case CHIP_SUMO:
3474 case CHIP_SUMO2:
32fcdbf4 3475 ps_thread_count = 96;
d5e455e4
AD		 3476 break;
3477 default:
32fcdbf4 3478 ps_thread_count = 128;
d5e455e4
AD		 3479 break;
3480 }
32fcdbf4
AD		 3481
3482 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3483 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3484 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3485 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3486 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3487 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3488
3489 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3490 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3491 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3492 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3493 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3494 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3495
3496 WREG32(SQ_CONFIG, sq_config);
3497 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3498 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3499 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3500 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3501 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3502 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3503 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3504 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3505 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3506 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3507
3508 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3509 FORCE_EOV_MAX_REZ_CNT(255)));
3510
d5e455e4
AD		 3511 switch (rdev->family) {
3512 case CHIP_CEDAR:
3513 case CHIP_PALM:
d5c5a72f
AD		 3514 case CHIP_SUMO:
3515 case CHIP_SUMO2:
adb68fa2 3516 case CHIP_CAICOS:
32fcdbf4 3517 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3518 break;
3519 default:
32fcdbf4 3520 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3521 break;
3522 }
32fcdbf4
AD		 3523 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3524 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3525
3526 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3527 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3528 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3529
60a4a3e0
AD		 3530 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3531 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3532
32fcdbf4
AD		 3533 WREG32(CB_PERF_CTR0_SEL_0, 0);
3534 WREG32(CB_PERF_CTR0_SEL_1, 0);
3535 WREG32(CB_PERF_CTR1_SEL_0, 0);
3536 WREG32(CB_PERF_CTR1_SEL_1, 0);
3537 WREG32(CB_PERF_CTR2_SEL_0, 0);
3538 WREG32(CB_PERF_CTR2_SEL_1, 0);
3539 WREG32(CB_PERF_CTR3_SEL_0, 0);
3540 WREG32(CB_PERF_CTR3_SEL_1, 0);
3541
60a4a3e0
AD		 3542 /* clear render buffer base addresses */
3543 WREG32(CB_COLOR0_BASE, 0);
3544 WREG32(CB_COLOR1_BASE, 0);
3545 WREG32(CB_COLOR2_BASE, 0);
3546 WREG32(CB_COLOR3_BASE, 0);
3547 WREG32(CB_COLOR4_BASE, 0);
3548 WREG32(CB_COLOR5_BASE, 0);
3549 WREG32(CB_COLOR6_BASE, 0);
3550 WREG32(CB_COLOR7_BASE, 0);
3551 WREG32(CB_COLOR8_BASE, 0);
3552 WREG32(CB_COLOR9_BASE, 0);
3553 WREG32(CB_COLOR10_BASE, 0);
3554 WREG32(CB_COLOR11_BASE, 0);
3555
3556 /* set the shader const cache sizes to 0 */
3557 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3558 WREG32(i, 0);
3559 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3560 WREG32(i, 0);
3561
f25a5c63
AD		 3562 tmp = RREG32(HDP_MISC_CNTL);
3563 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3564 WREG32(HDP_MISC_CNTL, tmp);
3565
32fcdbf4
AD		 3566 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3567 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3568
3569 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3570
3571 udelay(50);
3572
bcc1c2a1
AD		 3573}
3574
3575int evergreen_mc_init(struct radeon_device *rdev)
3576{
bcc1c2a1
AD		 3577 u32 tmp;
3578 int chansize, numchan;
bcc1c2a1
AD		 3579
3580 /* Get VRAM informations */
3581 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3582 if ((rdev->family == CHIP_PALM) ||
3583 (rdev->family == CHIP_SUMO) ||
3584 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3585 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3586 else
3587 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3588 if (tmp & CHANSIZE_OVERRIDE) {
3589 chansize = 16;
3590 } else if (tmp & CHANSIZE_MASK) {
3591 chansize = 64;
3592 } else {
3593 chansize = 32;
3594 }
3595 tmp = RREG32(MC_SHARED_CHMAP);
3596 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3597 case 0:
3598 default:
3599 numchan = 1;
3600 break;
3601 case 1:
3602 numchan = 2;
3603 break;
3604 case 2:
3605 numchan = 4;
3606 break;
3607 case 3:
3608 numchan = 8;
3609 break;
3610 }
3611 rdev->mc.vram_width = numchan * chansize;
3612 /* Could aper size report 0 ? */
01d73a69
JC		 3613 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3614 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3615 /* Setup GPU memory space */
05b3ef69
AD		 3616 if ((rdev->family == CHIP_PALM) ||
3617 (rdev->family == CHIP_SUMO) ||
3618 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3619 /* size in bytes on fusion */
3620 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3621 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3622 } else {
05b3ef69 3623 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3624 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3625 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3626 }
51e5fcd3 3627 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3628 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3629 radeon_update_bandwidth_info(rdev);
3630
bcc1c2a1
AD		 3631 return 0;
3632}
d594e46a 3633
187e3593 3634void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3635{
64c56e8c 3636 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3637 RREG32(GRBM_STATUS));
64c56e8c 3638 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3639 RREG32(GRBM_STATUS_SE0));
64c56e8c 3640 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3641 RREG32(GRBM_STATUS_SE1));
64c56e8c 3642 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3643 RREG32(SRBM_STATUS));
a65a4369
AD		 3644 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3645 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3646 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3647 RREG32(CP_STALLED_STAT1));
3648 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3649 RREG32(CP_STALLED_STAT2));
3650 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3651 RREG32(CP_BUSY_STAT));
3652 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3653 RREG32(CP_STAT));
eaaa6983
JG		 3654 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3655 RREG32(DMA_STATUS_REG));
168757ea
AD		 3656 if (rdev->family >= CHIP_CAYMAN) {
3657 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3658 RREG32(DMA_STATUS_REG + 0x800));
3659 }
0ecebb9e
AD		 3660}
3661
168757ea 3662bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3663{
a65a4369
AD		 3664 u32 crtc_hung = 0;
3665 u32 crtc_status[6];
3666 u32 i, j, tmp;
3667
3668 for (i = 0; i < rdev->num_crtc; i++) {
3669 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3670 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3671 crtc_hung |= (1 << i);
3672 }
3673 }
3674
3675 for (j = 0; j < 10; j++) {
3676 for (i = 0; i < rdev->num_crtc; i++) {
3677 if (crtc_hung & (1 << i)) {
3678 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3679 if (tmp != crtc_status[i])
3680 crtc_hung &= ~(1 << i);
3681 }
3682 }
3683 if (crtc_hung == 0)
3684 return false;
3685 udelay(100);
3686 }
3687
3688 return true;
3689}
3690
2483b4ea 3691u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3692{
3693 u32 reset_mask = 0;
b7630473 3694 u32 tmp;
0ecebb9e 3695
a65a4369
AD		 3696 /* GRBM_STATUS */
3697 tmp = RREG32(GRBM_STATUS);
3698 if (tmp & (PA_BUSY | SC_BUSY |
3699 SH_BUSY | SX_BUSY |
3700 TA_BUSY | VGT_BUSY |
3701 DB_BUSY | CB_BUSY |
3702 SPI_BUSY | VGT_BUSY_NO_DMA))
3703 reset_mask |= RADEON_RESET_GFX;
3704
3705 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3706 CP_BUSY | CP_COHERENCY_BUSY))
3707 reset_mask |= RADEON_RESET_CP;
3708
3709 if (tmp & GRBM_EE_BUSY)
3710 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3711
a65a4369
AD		 3712 /* DMA_STATUS_REG */
3713 tmp = RREG32(DMA_STATUS_REG);
3714 if (!(tmp & DMA_IDLE))
3715 reset_mask |= RADEON_RESET_DMA;
3716
3717 /* SRBM_STATUS2 */
3718 tmp = RREG32(SRBM_STATUS2);
3719 if (tmp & DMA_BUSY)
3720 reset_mask |= RADEON_RESET_DMA;
3721
3722 /* SRBM_STATUS */
3723 tmp = RREG32(SRBM_STATUS);
3724 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3725 reset_mask |= RADEON_RESET_RLC;
3726
3727 if (tmp & IH_BUSY)
3728 reset_mask |= RADEON_RESET_IH;
3729
3730 if (tmp & SEM_BUSY)
3731 reset_mask |= RADEON_RESET_SEM;
3732
3733 if (tmp & GRBM_RQ_PENDING)
3734 reset_mask |= RADEON_RESET_GRBM;
3735
3736 if (tmp & VMC_BUSY)
3737 reset_mask |= RADEON_RESET_VMC;
3738
3739 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3740 MCC_BUSY | MCD_BUSY))
3741 reset_mask |= RADEON_RESET_MC;
3742
3743 if (evergreen_is_display_hung(rdev))
3744 reset_mask |= RADEON_RESET_DISPLAY;
3745
3746 /* VM_L2_STATUS */
3747 tmp = RREG32(VM_L2_STATUS);
3748 if (tmp & L2_BUSY)
3749 reset_mask |= RADEON_RESET_VMC;
3750
d808fc88
AD		 3751 /* Skip MC reset as it's mostly likely not hung, just busy */
3752 if (reset_mask & RADEON_RESET_MC) {
3753 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3754 reset_mask &= ~RADEON_RESET_MC;
3755 }
3756
a65a4369
AD		 3757 return reset_mask;
3758}
3759
3760static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3761{
3762 struct evergreen_mc_save save;
3763 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3764 u32 tmp;
19fc42ed 3765
0ecebb9e 3766 if (reset_mask == 0)
a65a4369 3767 return;
0ecebb9e
AD		 3768
3769 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3770
b7630473
AD		 3771 evergreen_print_gpu_status_regs(rdev);
3772
b7630473
AD		 3773 /* Disable CP parsing/prefetching */
3774 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3775
3776 if (reset_mask & RADEON_RESET_DMA) {
3777 /* Disable DMA */
3778 tmp = RREG32(DMA_RB_CNTL);
3779 tmp &= ~DMA_RB_ENABLE;
3780 WREG32(DMA_RB_CNTL, tmp);
3781 }
3782
b21b6e7a
AD		 3783 udelay(50);
3784
3785 evergreen_mc_stop(rdev, &save);
3786 if (evergreen_mc_wait_for_idle(rdev)) {
3787 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3788 }
3789
b7630473
AD		 3790 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3791 grbm_soft_reset |= SOFT_RESET_DB |
3792 SOFT_RESET_CB |
3793 SOFT_RESET_PA |
3794 SOFT_RESET_SC |
3795 SOFT_RESET_SPI |
3796 SOFT_RESET_SX |
3797 SOFT_RESET_SH |
3798 SOFT_RESET_TC |
3799 SOFT_RESET_TA |
3800 SOFT_RESET_VC |
3801 SOFT_RESET_VGT;
3802 }
3803
3804 if (reset_mask & RADEON_RESET_CP) {
3805 grbm_soft_reset |= SOFT_RESET_CP |
3806 SOFT_RESET_VGT;
3807
3808 srbm_soft_reset |= SOFT_RESET_GRBM;
3809 }
0ecebb9e
AD		 3810
3811 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3812 srbm_soft_reset |= SOFT_RESET_DMA;
3813
a65a4369
AD		 3814 if (reset_mask & RADEON_RESET_DISPLAY)
3815 srbm_soft_reset |= SOFT_RESET_DC;
3816
3817 if (reset_mask & RADEON_RESET_RLC)
3818 srbm_soft_reset |= SOFT_RESET_RLC;
3819
3820 if (reset_mask & RADEON_RESET_SEM)
3821 srbm_soft_reset |= SOFT_RESET_SEM;
3822
3823 if (reset_mask & RADEON_RESET_IH)
3824 srbm_soft_reset |= SOFT_RESET_IH;
3825
3826 if (reset_mask & RADEON_RESET_GRBM)
3827 srbm_soft_reset |= SOFT_RESET_GRBM;
3828
3829 if (reset_mask & RADEON_RESET_VMC)
3830 srbm_soft_reset |= SOFT_RESET_VMC;
3831
24178ec4
AD		 3832 if (!(rdev->flags & RADEON_IS_IGP)) {
3833 if (reset_mask & RADEON_RESET_MC)
3834 srbm_soft_reset |= SOFT_RESET_MC;
3835 }
a65a4369 3836
b7630473
AD		 3837 if (grbm_soft_reset) {
3838 tmp = RREG32(GRBM_SOFT_RESET);
3839 tmp |= grbm_soft_reset;
3840 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3841 WREG32(GRBM_SOFT_RESET, tmp);
3842 tmp = RREG32(GRBM_SOFT_RESET);
3843
3844 udelay(50);
3845
3846 tmp &= ~grbm_soft_reset;
3847 WREG32(GRBM_SOFT_RESET, tmp);
3848 tmp = RREG32(GRBM_SOFT_RESET);
3849 }
3850
3851 if (srbm_soft_reset) {
3852 tmp = RREG32(SRBM_SOFT_RESET);
3853 tmp |= srbm_soft_reset;
3854 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3855 WREG32(SRBM_SOFT_RESET, tmp);
3856 tmp = RREG32(SRBM_SOFT_RESET);
3857
3858 udelay(50);
3859
3860 tmp &= ~srbm_soft_reset;
3861 WREG32(SRBM_SOFT_RESET, tmp);
3862 tmp = RREG32(SRBM_SOFT_RESET);
3863 }
0ecebb9e
AD		 3864
3865 /* Wait a little for things to settle down */
3866 udelay(50);
3867
747943ea 3868 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3869 udelay(50);
3870
b7630473 3871 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3872}
3873
a2d07b74 3874int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3875{
a65a4369
AD		 3876 u32 reset_mask;
3877
3878 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3879
3880 if (reset_mask)
3881 r600_set_bios_scratch_engine_hung(rdev, true);
3882
3883 evergreen_gpu_soft_reset(rdev, reset_mask);
3884
3885 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3886
3887 if (!reset_mask)
3888 r600_set_bios_scratch_engine_hung(rdev, false);
3889
3890 return 0;
747943ea
AD		 3891}
3892
123bc183
AD		 3893/**
3894 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3895 *
3896 * @rdev: radeon_device pointer
3897 * @ring: radeon_ring structure holding ring information
3898 *
3899 * Check if the GFX engine is locked up.
3900 * Returns true if the engine appears to be locked up, false if not.
3901 */
3902bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3903{
3904 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3905
3906 if (!(reset_mask & (RADEON_RESET_GFX |
3907 RADEON_RESET_COMPUTE |
3908 RADEON_RESET_CP))) {
3909 radeon_ring_lockup_update(ring);
3910 return false;
3911 }
3912 /* force CP activities */
3913 radeon_ring_force_activity(rdev, ring);
3914 return radeon_ring_test_lockup(rdev, ring);
3915}
3916
2948f5e6
AD		 3917/*
3918 * RLC
3919 */
3920#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3921#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3922
3923void sumo_rlc_fini(struct radeon_device *rdev)
3924{
3925 int r;
3926
3927 /* save restore block */
3928 if (rdev->rlc.save_restore_obj) {
3929 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3930 if (unlikely(r != 0))
3931 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3932 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3933 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3934
3935 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3936 rdev->rlc.save_restore_obj = NULL;
3937 }
3938
3939 /* clear state block */
3940 if (rdev->rlc.clear_state_obj) {
3941 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3942 if (unlikely(r != 0))
3943 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3944 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3945 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3946
3947 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3948 rdev->rlc.clear_state_obj = NULL;
3949 }
22c775ce
AD		 3950
3951 /* clear state block */
3952 if (rdev->rlc.cp_table_obj) {
3953 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3954 if (unlikely(r != 0))
3955 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3956 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3957 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3958
3959 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3960 rdev->rlc.cp_table_obj = NULL;
3961 }
2948f5e6
AD		 3962}
3963
22c775ce
AD		 3964#define CP_ME_TABLE_SIZE 96
3965
2948f5e6
AD		 3966int sumo_rlc_init(struct radeon_device *rdev)
3967{
1fd11777 3968 const u32 *src_ptr;
2948f5e6
AD		 3969 volatile u32 *dst_ptr;
3970 u32 dws, data, i, j, k, reg_num;
59a82d0e 3971 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 3972 u64 reg_list_mc_addr;
1fd11777 3973 const struct cs_section_def *cs_data;
2948f5e6
AD		 3974 int r;
3975
3976 src_ptr = rdev->rlc.reg_list;
3977 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 3978 if (rdev->family >= CHIP_BONAIRE) {
3979 dws += (5 * 16) + 48 + 48 + 64;
3980 }
2948f5e6
AD		 3981 cs_data = rdev->rlc.cs_data;
3982
10b7ca7e
AD		 3983 if (src_ptr) {
3984 /* save restore block */
3985 if (rdev->rlc.save_restore_obj == NULL) {
3986 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3987 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3988 if (r) {
3989 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3990 return r;
3991 }
3992 }
3993
3994 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3995 if (unlikely(r != 0)) {
3996 sumo_rlc_fini(rdev);
3997 return r;
3998 }
3999 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4000 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4001 if (r) {
10b7ca7e
AD		 4002 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4003 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4004 sumo_rlc_fini(rdev);
2948f5e6
AD		 4005 return r;
4006 }
2948f5e6 4007
10b7ca7e
AD		 4008 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4009 if (r) {
4010 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4011 sumo_rlc_fini(rdev);
4012 return r;
4013 }
4014 /* write the sr buffer */
4015 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4016 if (rdev->family >= CHIP_TAHITI) {
4017 /* SI */
59a82d0e 4018 for (i = 0; i < rdev->rlc.reg_list_size; i++)
1fd11777
AD		 4019 dst_ptr[i] = src_ptr[i];
4020 } else {
4021 /* ON/LN/TN */
4022 /* format:
4023 * dw0: (reg2 << 16) | reg1
4024 * dw1: reg1 save space
4025 * dw2: reg2 save space
4026 */
4027 for (i = 0; i < dws; i++) {
4028 data = src_ptr[i] >> 2;
4029 i++;
4030 if (i < dws)
4031 data |= (src_ptr[i] >> 2) << 16;
4032 j = (((i - 1) * 3) / 2);
4033 dst_ptr[j] = data;
4034 }
4035 j = ((i * 3) / 2);
4036 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
10b7ca7e 4037 }
10b7ca7e 4038 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4039 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4040 }
2948f5e6 4041
10b7ca7e
AD		 4042 if (cs_data) {
4043 /* clear state block */
a0f38609
AD		 4044 if (rdev->family >= CHIP_BONAIRE) {
4045 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4046 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4047 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4048 dws = rdev->rlc.clear_state_size + (256 / 4);
4049 } else {
4050 reg_list_num = 0;
4051 dws = 0;
4052 for (i = 0; cs_data[i].section != NULL; i++) {
4053 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4054 reg_list_num++;
4055 dws += cs_data[i].section[j].reg_count;
4056 }
10b7ca7e 4057 }
59a82d0e
AD		 4058 reg_list_blk_index = (3 * reg_list_num + 2);
4059 dws += reg_list_blk_index;
4060 rdev->rlc.clear_state_size = dws;
2948f5e6 4061 }
2948f5e6 4062
10b7ca7e 4063 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4064 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
10b7ca7e
AD		 4065 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4066 if (r) {
4067 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4068 sumo_rlc_fini(rdev);
4069 return r;
4070 }
4071 }
4072 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4073 if (unlikely(r != 0)) {
4074 sumo_rlc_fini(rdev);
4075 return r;
4076 }
4077 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4078 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4079 if (r) {
10b7ca7e
AD		 4080 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4081 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4082 sumo_rlc_fini(rdev);
4083 return r;
4084 }
2948f5e6 4085
10b7ca7e
AD		 4086 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4087 if (r) {
4088 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4089 sumo_rlc_fini(rdev);
4090 return r;
4091 }
4092 /* set up the cs buffer */
4093 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4094 if (rdev->family >= CHIP_BONAIRE) {
4095 cik_get_csb_buffer(rdev, dst_ptr);
4096 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4097 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4098 dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
4099 dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
4100 dst_ptr[2] = rdev->rlc.clear_state_size;
4101 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4102 } else {
4103 reg_list_hdr_blk_index = 0;
4104 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4105 data = upper_32_bits(reg_list_mc_addr);
2948f5e6
AD		 4106 dst_ptr[reg_list_hdr_blk_index] = data;
4107 reg_list_hdr_blk_index++;
59a82d0e
AD		 4108 for (i = 0; cs_data[i].section != NULL; i++) {
4109 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4110 reg_num = cs_data[i].section[j].reg_count;
4111 data = reg_list_mc_addr & 0xffffffff;
4112 dst_ptr[reg_list_hdr_blk_index] = data;
4113 reg_list_hdr_blk_index++;
4114
4115 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4116 dst_ptr[reg_list_hdr_blk_index] = data;
4117 reg_list_hdr_blk_index++;
4118
4119 data = 0x08000000 | (reg_num * 4);
4120 dst_ptr[reg_list_hdr_blk_index] = data;
4121 reg_list_hdr_blk_index++;
4122
4123 for (k = 0; k < reg_num; k++) {
4124 data = cs_data[i].section[j].extent[k];
4125 dst_ptr[reg_list_blk_index + k] = data;
4126 }
4127 reg_list_mc_addr += reg_num * 4;
4128 reg_list_blk_index += reg_num;
10b7ca7e 4129 }
2948f5e6 4130 }
59a82d0e 4131 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
2948f5e6 4132 }
10b7ca7e
AD		 4133 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4134 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4135 }
2948f5e6 4136
22c775ce
AD		 4137 if (rdev->rlc.cp_table_size) {
4138 if (rdev->rlc.cp_table_obj == NULL) {
4139 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4140 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4141 if (r) {
4142 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4143 sumo_rlc_fini(rdev);
4144 return r;
4145 }
4146 }
4147
4148 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4149 if (unlikely(r != 0)) {
4150 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4151 sumo_rlc_fini(rdev);
4152 return r;
4153 }
4154 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4155 &rdev->rlc.cp_table_gpu_addr);
4156 if (r) {
4157 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4158 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4159 sumo_rlc_fini(rdev);
4160 return r;
4161 }
4162 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4163 if (r) {
4164 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4165 sumo_rlc_fini(rdev);
4166 return r;
4167 }
4168
4169 cik_init_cp_pg_table(rdev);
4170
4171 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4172 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4173
4174 }
2948f5e6
AD		 4175
4176 return 0;
4177}
4178
4179static void evergreen_rlc_start(struct radeon_device *rdev)
4180{
8ba10463
AD		 4181 u32 mask = RLC_ENABLE;
4182
4183 if (rdev->flags & RADEON_IS_IGP) {
4184 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4185 }
4186
4187 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4188}
4189
4190int evergreen_rlc_resume(struct radeon_device *rdev)
4191{
4192 u32 i;
4193 const __be32 *fw_data;
4194
4195 if (!rdev->rlc_fw)
4196 return -EINVAL;
4197
4198 r600_rlc_stop(rdev);
4199
4200 WREG32(RLC_HB_CNTL, 0);
4201
4202 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4203 if (rdev->family == CHIP_ARUBA) {
4204 u32 always_on_bitmap =
4205 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4206 /* find out the number of active simds */
4207 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4208 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4209 tmp = hweight32(~tmp);
4210 if (tmp == rdev->config.cayman.max_simds_per_se) {
4211 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4212 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4213 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4214 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4215 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4216 }
4217 } else {
4218 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4219 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4220 }
2948f5e6
AD		 4221 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4222 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4223 } else {
4224 WREG32(RLC_HB_BASE, 0);
4225 WREG32(RLC_HB_RPTR, 0);
4226 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4227 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4228 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4229 }
2948f5e6
AD		 4230 WREG32(RLC_MC_CNTL, 0);
4231 WREG32(RLC_UCODE_CNTL, 0);
4232
4233 fw_data = (const __be32 *)rdev->rlc_fw->data;
4234 if (rdev->family >= CHIP_ARUBA) {
4235 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4236 WREG32(RLC_UCODE_ADDR, i);
4237 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4238 }
4239 } else if (rdev->family >= CHIP_CAYMAN) {
4240 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4241 WREG32(RLC_UCODE_ADDR, i);
4242 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4243 }
4244 } else {
4245 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4246 WREG32(RLC_UCODE_ADDR, i);
4247 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4248 }
4249 }
4250 WREG32(RLC_UCODE_ADDR, 0);
4251
4252 evergreen_rlc_start(rdev);
4253
4254 return 0;
4255}
4256
45f9a39b
AD		 4257/* Interrupts */
4258
4259u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4260{
46437057 4261 if (crtc >= rdev->num_crtc)
45f9a39b 4262 return 0;
46437057
AD		 4263 else
4264 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4265}
4266
4267void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4268{
4269 u32 tmp;
4270
1b37078b
AD		 4271 if (rdev->family >= CHIP_CAYMAN) {
4272 cayman_cp_int_cntl_setup(rdev, 0,
4273 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4274 cayman_cp_int_cntl_setup(rdev, 1, 0);
4275 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4276 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4277 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4278 } else
4279 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4280 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4281 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4282 WREG32(GRBM_INT_CNTL, 0);
4283 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4284 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4285 if (rdev->num_crtc >= 4) {
18007401
AD		 4286 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4287 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4288 }
4289 if (rdev->num_crtc >= 6) {
18007401
AD		 4290 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4291 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4292 }
45f9a39b
AD		 4293
4294 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4296 if (rdev->num_crtc >= 4) {
18007401
AD		 4297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4298 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4299 }
4300 if (rdev->num_crtc >= 6) {
18007401
AD		 4301 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4302 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4303 }
45f9a39b 4304
05b3ef69
AD		 4305 /* only one DAC on DCE6 */
4306 if (!ASIC_IS_DCE6(rdev))
4307 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4308 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4309
4310 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4311 WREG32(DC_HPD1_INT_CONTROL, tmp);
4312 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4313 WREG32(DC_HPD2_INT_CONTROL, tmp);
4314 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4315 WREG32(DC_HPD3_INT_CONTROL, tmp);
4316 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4317 WREG32(DC_HPD4_INT_CONTROL, tmp);
4318 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4319 WREG32(DC_HPD5_INT_CONTROL, tmp);
4320 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4321 WREG32(DC_HPD6_INT_CONTROL, tmp);
4322
4323}
4324
4325int evergreen_irq_set(struct radeon_device *rdev)
4326{
4327 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4328 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4329 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4330 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4331 u32 grbm_int_cntl = 0;
6f34be50 4332 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4333 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4334 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4335 u32 thermal_int = 0;
45f9a39b
AD		 4336
4337 if (!rdev->irq.installed) {
fce7d61b 4338 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4339 return -EINVAL;
4340 }
4341 /* don't enable anything if the ih is disabled */
4342 if (!rdev->ih.enabled) {
4343 r600_disable_interrupts(rdev);
4344 /* force the active interrupt state to all disabled */
4345 evergreen_disable_interrupt_state(rdev);
4346 return 0;
4347 }
4348
4349 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4350 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4351 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4352 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4353 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4354 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4355 if (rdev->family == CHIP_ARUBA)
4356 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4357 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4358 else
4359 thermal_int = RREG32(CG_THERMAL_INT) &
4360 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4361
f122c610
AD		 4362 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4363 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4364 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4365 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4366 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4367 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4368
233d1ad5
AD		 4369 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4370
1b37078b
AD		 4371 if (rdev->family >= CHIP_CAYMAN) {
4372 /* enable CP interrupts on all rings */
736fc37f 4373 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4374 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4375 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4376 }
736fc37f 4377 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4378 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4379 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4380 }
736fc37f 4381 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4382 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4383 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4384 }
4385 } else {
736fc37f 4386 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4387 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4388 cp_int_cntl |= RB_INT_ENABLE;
4389 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4390 }
45f9a39b 4391 }
1b37078b 4392
233d1ad5
AD		 4393 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4394 DRM_DEBUG("r600_irq_set: sw int dma\n");
4395 dma_cntl |= TRAP_ENABLE;
4396 }
4397
f60cbd11
AD		 4398 if (rdev->family >= CHIP_CAYMAN) {
4399 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4400 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4401 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4402 dma_cntl1 |= TRAP_ENABLE;
4403 }
4404 }
4405
dc50ba7f
AD		 4406 if (rdev->irq.dpm_thermal) {
4407 DRM_DEBUG("dpm thermal\n");
4408 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4409 }
4410
6f34be50 4411 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4412 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4413 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4414 crtc1 |= VBLANK_INT_MASK;
4415 }
6f34be50 4416 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4417 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4418 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4419 crtc2 |= VBLANK_INT_MASK;
4420 }
6f34be50 4421 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4422 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4423 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4424 crtc3 |= VBLANK_INT_MASK;
4425 }
6f34be50 4426 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4427 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4428 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4429 crtc4 |= VBLANK_INT_MASK;
4430 }
6f34be50 4431 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4432 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4433 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4434 crtc5 |= VBLANK_INT_MASK;
4435 }
6f34be50 4436 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4437 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4438 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4439 crtc6 |= VBLANK_INT_MASK;
4440 }
4441 if (rdev->irq.hpd[0]) {
4442 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4443 hpd1 |= DC_HPDx_INT_EN;
4444 }
4445 if (rdev->irq.hpd[1]) {
4446 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4447 hpd2 |= DC_HPDx_INT_EN;
4448 }
4449 if (rdev->irq.hpd[2]) {
4450 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4451 hpd3 |= DC_HPDx_INT_EN;
4452 }
4453 if (rdev->irq.hpd[3]) {
4454 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4455 hpd4 |= DC_HPDx_INT_EN;
4456 }
4457 if (rdev->irq.hpd[4]) {
4458 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4459 hpd5 |= DC_HPDx_INT_EN;
4460 }
4461 if (rdev->irq.hpd[5]) {
4462 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4463 hpd6 |= DC_HPDx_INT_EN;
4464 }
f122c610
AD		 4465 if (rdev->irq.afmt[0]) {
4466 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4467 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4468 }
4469 if (rdev->irq.afmt[1]) {
4470 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4471 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4472 }
4473 if (rdev->irq.afmt[2]) {
4474 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4475 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4476 }
4477 if (rdev->irq.afmt[3]) {
4478 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4479 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4480 }
4481 if (rdev->irq.afmt[4]) {
4482 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4483 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4484 }
4485 if (rdev->irq.afmt[5]) {
4486 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4487 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4488 }
45f9a39b 4489
1b37078b
AD		 4490 if (rdev->family >= CHIP_CAYMAN) {
4491 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4492 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4493 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4494 } else
4495 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4496
4497 WREG32(DMA_CNTL, dma_cntl);
4498
f60cbd11
AD		 4499 if (rdev->family >= CHIP_CAYMAN)
4500 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4501
2031f77c 4502 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4503
4504 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4505 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4506 if (rdev->num_crtc >= 4) {
18007401
AD		 4507 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4508 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4509 }
4510 if (rdev->num_crtc >= 6) {
18007401
AD		 4511 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4512 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4513 }
45f9a39b 4514
6f34be50
AD		 4515 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4516 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4517 if (rdev->num_crtc >= 4) {
4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4520 }
4521 if (rdev->num_crtc >= 6) {
4522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4523 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4524 }
6f34be50 4525
45f9a39b
AD		 4526 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4527 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4528 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4529 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4530 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4531 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4532 if (rdev->family == CHIP_ARUBA)
4533 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4534 else
4535 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4536
f122c610
AD		 4537 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4538 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4539 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4540 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4541 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4542 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4543
bcc1c2a1
AD		 4544 return 0;
4545}
4546
cbdd4501 4547static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4548{
4549 u32 tmp;
4550
6f34be50
AD		 4551 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4552 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4553 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4554 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4555 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4556 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4557 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4558 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4559 if (rdev->num_crtc >= 4) {
4560 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4561 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4562 }
4563 if (rdev->num_crtc >= 6) {
4564 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4565 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4566 }
6f34be50 4567
f122c610
AD		 4568 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4569 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4570 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4571 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4572 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4573 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4574
6f34be50
AD		 4575 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4576 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4577 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4578 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4579 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4580 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4581 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4582 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4583 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4584 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4585 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4586 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4587
b7eff394
AD		 4588 if (rdev->num_crtc >= 4) {
4589 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4590 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4591 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4592 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4593 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4594 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4595 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4596 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4597 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4598 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4599 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4600 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4601 }
4602
4603 if (rdev->num_crtc >= 6) {
4604 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4605 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4606 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4607 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4608 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4609 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4610 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4611 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4612 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4613 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4614 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4615 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4616 }
45f9a39b 4617
6f34be50 4618 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4619 tmp = RREG32(DC_HPD1_INT_CONTROL);
4620 tmp |= DC_HPDx_INT_ACK;
4621 WREG32(DC_HPD1_INT_CONTROL, tmp);
4622 }
6f34be50 4623 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4624 tmp = RREG32(DC_HPD2_INT_CONTROL);
4625 tmp |= DC_HPDx_INT_ACK;
4626 WREG32(DC_HPD2_INT_CONTROL, tmp);
4627 }
6f34be50 4628 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4629 tmp = RREG32(DC_HPD3_INT_CONTROL);
4630 tmp |= DC_HPDx_INT_ACK;
4631 WREG32(DC_HPD3_INT_CONTROL, tmp);
4632 }
6f34be50 4633 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4634 tmp = RREG32(DC_HPD4_INT_CONTROL);
4635 tmp |= DC_HPDx_INT_ACK;
4636 WREG32(DC_HPD4_INT_CONTROL, tmp);
4637 }
6f34be50 4638 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4639 tmp = RREG32(DC_HPD5_INT_CONTROL);
4640 tmp |= DC_HPDx_INT_ACK;
4641 WREG32(DC_HPD5_INT_CONTROL, tmp);
4642 }
6f34be50 4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4644 tmp = RREG32(DC_HPD5_INT_CONTROL);
4645 tmp |= DC_HPDx_INT_ACK;
4646 WREG32(DC_HPD6_INT_CONTROL, tmp);
4647 }
f122c610
AD		 4648 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4649 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4650 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4651 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4652 }
4653 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4654 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4655 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4656 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4657 }
4658 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4659 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4660 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4661 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4662 }
4663 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4664 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4665 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4666 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4667 }
4668 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4669 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4670 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4671 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4672 }
4673 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4674 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4675 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4676 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4677 }
45f9a39b
AD		 4678}
4679
1109ca09 4680static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4681{
45f9a39b
AD		 4682 r600_disable_interrupts(rdev);
4683 /* Wait and acknowledge irq */
4684 mdelay(1);
6f34be50 4685 evergreen_irq_ack(rdev);
45f9a39b
AD		 4686 evergreen_disable_interrupt_state(rdev);
4687}
4688
755d819e 4689void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4690{
4691 evergreen_irq_disable(rdev);
4692 r600_rlc_stop(rdev);
4693}
4694
cbdd4501 4695static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4696{
4697 u32 wptr, tmp;
4698
724c80e1 4699 if (rdev->wb.enabled)
204ae24d 4700 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4701 else
4702 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4703
4704 if (wptr & RB_OVERFLOW) {
4705 /* When a ring buffer overflow happen start parsing interrupt
4706 * from the last not overwritten vector (wptr + 16). Hopefully
4707 * this should allow us to catchup.
4708 */
4709 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4710 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4711 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4712 tmp = RREG32(IH_RB_CNTL);
4713 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4714 WREG32(IH_RB_CNTL, tmp);
4715 }
4716 return (wptr & rdev->ih.ptr_mask);
4717}
4718
4719int evergreen_irq_process(struct radeon_device *rdev)
4720{
682f1a54
DA		 4721 u32 wptr;
4722 u32 rptr;
45f9a39b
AD		 4723 u32 src_id, src_data;
4724 u32 ring_index;
45f9a39b 4725 bool queue_hotplug = false;
f122c610 4726 bool queue_hdmi = false;
dc50ba7f 4727 bool queue_thermal = false;
54e2e49c 4728 u32 status, addr;
45f9a39b 4729
682f1a54 4730 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4731 return IRQ_NONE;
4732
682f1a54 4733 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4734
4735restart_ih:
4736 /* is somebody else already processing irqs? */
4737 if (atomic_xchg(&rdev->ih.lock, 1))
4738 return IRQ_NONE;
4739
682f1a54
DA		 4740 rptr = rdev->ih.rptr;
4741 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4742
964f6645
BH		 4743 /* Order reading of wptr vs. reading of IH ring data */
4744 rmb();
4745
45f9a39b 4746 /* display interrupts */
6f34be50 4747 evergreen_irq_ack(rdev);
45f9a39b 4748
45f9a39b
AD		 4749 while (rptr != wptr) {
4750 /* wptr/rptr are in bytes! */
4751 ring_index = rptr / 4;
0f234f5f
AD		 4752 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4753 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4754
4755 switch (src_id) {
4756 case 1: /* D1 vblank/vline */
4757 switch (src_data) {
4758 case 0: /* D1 vblank */
6f34be50 4759 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4760 if (rdev->irq.crtc_vblank_int[0]) {
4761 drm_handle_vblank(rdev->ddev, 0);
4762 rdev->pm.vblank_sync = true;
4763 wake_up(&rdev->irq.vblank_queue);
4764 }
736fc37f 4765 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4766 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4767 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4768 DRM_DEBUG("IH: D1 vblank\n");
4769 }
4770 break;
4771 case 1: /* D1 vline */
6f34be50
AD		 4772 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4773 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4774 DRM_DEBUG("IH: D1 vline\n");
4775 }
4776 break;
4777 default:
4778 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4779 break;
4780 }
4781 break;
4782 case 2: /* D2 vblank/vline */
4783 switch (src_data) {
4784 case 0: /* D2 vblank */
6f34be50 4785 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4786 if (rdev->irq.crtc_vblank_int[1]) {
4787 drm_handle_vblank(rdev->ddev, 1);
4788 rdev->pm.vblank_sync = true;
4789 wake_up(&rdev->irq.vblank_queue);
4790 }
736fc37f 4791 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4792 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4793 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4794 DRM_DEBUG("IH: D2 vblank\n");
4795 }
4796 break;
4797 case 1: /* D2 vline */
6f34be50
AD		 4798 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4799 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4800 DRM_DEBUG("IH: D2 vline\n");
4801 }
4802 break;
4803 default:
4804 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4805 break;
4806 }
4807 break;
4808 case 3: /* D3 vblank/vline */
4809 switch (src_data) {
4810 case 0: /* D3 vblank */
6f34be50
AD		 4811 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4812 if (rdev->irq.crtc_vblank_int[2]) {
4813 drm_handle_vblank(rdev->ddev, 2);
4814 rdev->pm.vblank_sync = true;
4815 wake_up(&rdev->irq.vblank_queue);
4816 }
736fc37f 4817 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4818 radeon_crtc_handle_flip(rdev, 2);
4819 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4820 DRM_DEBUG("IH: D3 vblank\n");
4821 }
4822 break;
4823 case 1: /* D3 vline */
6f34be50
AD		 4824 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4825 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4826 DRM_DEBUG("IH: D3 vline\n");
4827 }
4828 break;
4829 default:
4830 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4831 break;
4832 }
4833 break;
4834 case 4: /* D4 vblank/vline */
4835 switch (src_data) {
4836 case 0: /* D4 vblank */
6f34be50
AD		 4837 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4838 if (rdev->irq.crtc_vblank_int[3]) {
4839 drm_handle_vblank(rdev->ddev, 3);
4840 rdev->pm.vblank_sync = true;
4841 wake_up(&rdev->irq.vblank_queue);
4842 }
736fc37f 4843 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4844 radeon_crtc_handle_flip(rdev, 3);
4845 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4846 DRM_DEBUG("IH: D4 vblank\n");
4847 }
4848 break;
4849 case 1: /* D4 vline */
6f34be50
AD		 4850 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4851 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4852 DRM_DEBUG("IH: D4 vline\n");
4853 }
4854 break;
4855 default:
4856 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4857 break;
4858 }
4859 break;
4860 case 5: /* D5 vblank/vline */
4861 switch (src_data) {
4862 case 0: /* D5 vblank */
6f34be50
AD		 4863 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4864 if (rdev->irq.crtc_vblank_int[4]) {
4865 drm_handle_vblank(rdev->ddev, 4);
4866 rdev->pm.vblank_sync = true;
4867 wake_up(&rdev->irq.vblank_queue);
4868 }
736fc37f 4869 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4870 radeon_crtc_handle_flip(rdev, 4);
4871 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4872 DRM_DEBUG("IH: D5 vblank\n");
4873 }
4874 break;
4875 case 1: /* D5 vline */
6f34be50
AD		 4876 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4877 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4878 DRM_DEBUG("IH: D5 vline\n");
4879 }
4880 break;
4881 default:
4882 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4883 break;
4884 }
4885 break;
4886 case 6: /* D6 vblank/vline */
4887 switch (src_data) {
4888 case 0: /* D6 vblank */
6f34be50
AD		 4889 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4890 if (rdev->irq.crtc_vblank_int[5]) {
4891 drm_handle_vblank(rdev->ddev, 5);
4892 rdev->pm.vblank_sync = true;
4893 wake_up(&rdev->irq.vblank_queue);
4894 }
736fc37f 4895 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4896 radeon_crtc_handle_flip(rdev, 5);
4897 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4898 DRM_DEBUG("IH: D6 vblank\n");
4899 }
4900 break;
4901 case 1: /* D6 vline */
6f34be50
AD		 4902 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4903 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4904 DRM_DEBUG("IH: D6 vline\n");
4905 }
4906 break;
4907 default:
4908 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4909 break;
4910 }
4911 break;
4912 case 42: /* HPD hotplug */
4913 switch (src_data) {
4914 case 0:
6f34be50
AD		 4915 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4916 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4917 queue_hotplug = true;
4918 DRM_DEBUG("IH: HPD1\n");
4919 }
4920 break;
4921 case 1:
6f34be50
AD		 4922 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4923 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4924 queue_hotplug = true;
4925 DRM_DEBUG("IH: HPD2\n");
4926 }
4927 break;
4928 case 2:
6f34be50
AD		 4929 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4930 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4931 queue_hotplug = true;
4932 DRM_DEBUG("IH: HPD3\n");
4933 }
4934 break;
4935 case 3:
6f34be50
AD		 4936 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4937 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4938 queue_hotplug = true;
4939 DRM_DEBUG("IH: HPD4\n");
4940 }
4941 break;
4942 case 4:
6f34be50
AD		 4943 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4944 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4945 queue_hotplug = true;
4946 DRM_DEBUG("IH: HPD5\n");
4947 }
4948 break;
4949 case 5:
6f34be50
AD		 4950 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4951 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4952 queue_hotplug = true;
4953 DRM_DEBUG("IH: HPD6\n");
4954 }
4955 break;
4956 default:
4957 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4958 break;
4959 }
4960 break;
f122c610
AD		 4961 case 44: /* hdmi */
4962 switch (src_data) {
4963 case 0:
4964 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4965 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4966 queue_hdmi = true;
4967 DRM_DEBUG("IH: HDMI0\n");
4968 }
4969 break;
4970 case 1:
4971 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4972 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4973 queue_hdmi = true;
4974 DRM_DEBUG("IH: HDMI1\n");
4975 }
4976 break;
4977 case 2:
4978 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4979 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4980 queue_hdmi = true;
4981 DRM_DEBUG("IH: HDMI2\n");
4982 }
4983 break;
4984 case 3:
4985 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4986 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4987 queue_hdmi = true;
4988 DRM_DEBUG("IH: HDMI3\n");
4989 }
4990 break;
4991 case 4:
4992 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4993 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4994 queue_hdmi = true;
4995 DRM_DEBUG("IH: HDMI4\n");
4996 }
4997 break;
4998 case 5:
4999 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5000 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5001 queue_hdmi = true;
5002 DRM_DEBUG("IH: HDMI5\n");
5003 }
5004 break;
5005 default:
5006 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5007 break;
5008 }
f2ba57b5
CK		 5009 case 124: /* UVD */
5010 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5011 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5012 break;
ae133a11
CK		 5013 case 146:
5014 case 147:
54e2e49c
AD		 5015 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5016 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 5017 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5018 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5019 addr);
ae133a11 5020 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5021 status);
5022 cayman_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 5023 /* reset addr and status */
5024 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5025 break;
45f9a39b
AD		 5026 case 176: /* CP_INT in ring buffer */
5027 case 177: /* CP_INT in IB1 */
5028 case 178: /* CP_INT in IB2 */
5029 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5030 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5031 break;
5032 case 181: /* CP EOP event */
5033 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5034 if (rdev->family >= CHIP_CAYMAN) {
5035 switch (src_data) {
5036 case 0:
5037 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5038 break;
5039 case 1:
5040 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5041 break;
5042 case 2:
5043 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5044 break;
5045 }
5046 } else
5047 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5048 break;
233d1ad5
AD		 5049 case 224: /* DMA trap event */
5050 DRM_DEBUG("IH: DMA trap\n");
5051 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5052 break;
dc50ba7f
AD		 5053 case 230: /* thermal low to high */
5054 DRM_DEBUG("IH: thermal low to high\n");
5055 rdev->pm.dpm.thermal.high_to_low = false;
5056 queue_thermal = true;
5057 break;
5058 case 231: /* thermal high to low */
5059 DRM_DEBUG("IH: thermal high to low\n");
5060 rdev->pm.dpm.thermal.high_to_low = true;
5061 queue_thermal = true;
5062 break;
2031f77c 5063 case 233: /* GUI IDLE */
303c805c 5064 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5065 break;
f60cbd11
AD		 5066 case 244: /* DMA trap event */
5067 if (rdev->family >= CHIP_CAYMAN) {
5068 DRM_DEBUG("IH: DMA1 trap\n");
5069 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5070 }
5071 break;
45f9a39b
AD		 5072 default:
5073 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5074 break;
5075 }
5076
5077 /* wptr/rptr are in bytes! */
5078 rptr += 16;
5079 rptr &= rdev->ih.ptr_mask;
5080 }
45f9a39b 5081 if (queue_hotplug)
32c87fca 5082 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5083 if (queue_hdmi)
5084 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5085 if (queue_thermal && rdev->pm.dpm_enabled)
5086 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 5087 rdev->ih.rptr = rptr;
5088 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5089 atomic_set(&rdev->ih.lock, 0);
5090
5091 /* make sure wptr hasn't changed while processing */
5092 wptr = evergreen_get_ih_wptr(rdev);
5093 if (wptr != rptr)
5094 goto restart_ih;
5095
45f9a39b
AD		 5096 return IRQ_HANDLED;
5097}
5098
bcc1c2a1
AD		 5099static int evergreen_startup(struct radeon_device *rdev)
5100{
f2ba57b5 5101 struct radeon_ring *ring;
bcc1c2a1
AD		 5102 int r;
5103
9e46a48d 5104 /* enable pcie gen2 link */
cd54033a 5105 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5106 /* enable aspm */
5107 evergreen_program_aspm(rdev);
9e46a48d 5108
e5903d39
AD		 5109 /* scratch needs to be initialized before MC */
5110 r = r600_vram_scratch_init(rdev);
5111 if (r)
5112 return r;
5113
6fab3feb
AD		 5114 evergreen_mc_program(rdev);
5115
0af62b01
AD		 5116 if (ASIC_IS_DCE5(rdev)) {
5117 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5118 r = ni_init_microcode(rdev);
5119 if (r) {
5120 DRM_ERROR("Failed to load firmware!\n");
5121 return r;
5122 }
5123 }
755d819e 5124 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5125 if (r) {
0af62b01 5126 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5127 return r;
5128 }
0af62b01
AD		 5129 } else {
5130 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5131 r = r600_init_microcode(rdev);
5132 if (r) {
5133 DRM_ERROR("Failed to load firmware!\n");
5134 return r;
5135 }
5136 }
bcc1c2a1 5137 }
fe251e2f 5138
bcc1c2a1 5139 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5140 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5141 } else {
5142 r = evergreen_pcie_gart_enable(rdev);
5143 if (r)
5144 return r;
5145 }
bcc1c2a1 5146 evergreen_gpu_init(rdev);
bcc1c2a1 5147
2948f5e6
AD		 5148 /* allocate rlc buffers */
5149 if (rdev->flags & RADEON_IS_IGP) {
5150 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5151 rdev->rlc.reg_list_size =
5152 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5153 rdev->rlc.cs_data = evergreen_cs_data;
5154 r = sumo_rlc_init(rdev);
5155 if (r) {
5156 DRM_ERROR("Failed to init rlc BOs!\n");
5157 return r;
5158 }
5159 }
5160
724c80e1
AD		 5161 /* allocate wb buffer */
5162 r = radeon_wb_init(rdev);
5163 if (r)
5164 return r;
5165
30eb77f4
JG		 5166 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5167 if (r) {
5168 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5169 return r;
5170 }
5171
233d1ad5
AD		 5172 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5173 if (r) {
5174 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5175 return r;
5176 }
5177
e409b128 5178 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5179 if (!r) {
5180 r = radeon_fence_driver_start_ring(rdev,
5181 R600_RING_TYPE_UVD_INDEX);
5182 if (r)
5183 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5184 }
5185
5186 if (r)
5187 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5188
bcc1c2a1 5189 /* Enable IRQ */
e49f3959
AH		 5190 if (!rdev->irq.installed) {
5191 r = radeon_irq_kms_init(rdev);
5192 if (r)
5193 return r;
5194 }
5195
bcc1c2a1
AD		 5196 r = r600_irq_init(rdev);
5197 if (r) {
5198 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5199 radeon_irq_kms_fini(rdev);
5200 return r;
5201 }
45f9a39b 5202 evergreen_irq_set(rdev);
bcc1c2a1 5203
f2ba57b5 5204 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5205 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a 5206 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2e1e6dad 5207 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5208 if (r)
5209 return r;
233d1ad5
AD		 5210
5211 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5212 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5213 DMA_RB_RPTR, DMA_RB_WPTR,
2e1e6dad 5214 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5215 if (r)
5216 return r;
5217
bcc1c2a1
AD		 5218 r = evergreen_cp_load_microcode(rdev);
5219 if (r)
5220 return r;
fe251e2f 5221 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5222 if (r)
5223 return r;
5224 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5225 if (r)
5226 return r;
fe251e2f 5227
f2ba57b5
CK		 5228 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5229 if (ring->ring_size) {
02c9f7fa 5230 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
f2ba57b5 5231 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 5232 RADEON_CP_PACKET2);
f2ba57b5 5233 if (!r)
e409b128 5234 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5235
5236 if (r)
5237 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5238 }
5239
2898c348
CK		 5240 r = radeon_ib_pool_init(rdev);
5241 if (r) {
5242 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5243 return r;
2898c348 5244 }
b15ba512 5245
69d2ae57
RM		 5246 r = r600_audio_init(rdev);
5247 if (r) {
5248 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5249 return r;
5250 }
5251
bcc1c2a1
AD		 5252 return 0;
5253}
5254
5255int evergreen_resume(struct radeon_device *rdev)
5256{
5257 int r;
5258
86f5c9ed
AD		 5259 /* reset the asic, the gfx blocks are often in a bad state
5260 * after the driver is unloaded or after a resume
5261 */
5262 if (radeon_asic_reset(rdev))
5263 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5264 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5265 * posting will perform necessary task to bring back GPU into good
5266 * shape.
5267 */
5268 /* post card */
5269 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5270
d4788db3
AD		 5271 /* init golden registers */
5272 evergreen_init_golden_registers(rdev);
5273
b15ba512 5274 rdev->accel_working = true;
bcc1c2a1
AD		 5275 r = evergreen_startup(rdev);
5276 if (r) {
755d819e 5277 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5278 rdev->accel_working = false;
bcc1c2a1
AD		 5279 return r;
5280 }
fe251e2f 5281
bcc1c2a1
AD		 5282 return r;
5283
5284}
5285
5286int evergreen_suspend(struct radeon_device *rdev)
5287{
69d2ae57 5288 r600_audio_fini(rdev);
e409b128 5289 uvd_v1_0_fini(rdev);
f2ba57b5 5290 radeon_uvd_suspend(rdev);
bcc1c2a1 5291 r700_cp_stop(rdev);
233d1ad5 5292 r600_dma_stop(rdev);
45f9a39b 5293 evergreen_irq_suspend(rdev);
724c80e1 5294 radeon_wb_disable(rdev);
bcc1c2a1 5295 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5296
5297 return 0;
5298}
5299
bcc1c2a1
AD		 5300/* Plan is to move initialization in that function and use
5301 * helper function so that radeon_device_init pretty much
5302 * do nothing more than calling asic specific function. This
5303 * should also allow to remove a bunch of callback function
5304 * like vram_info.
5305 */
5306int evergreen_init(struct radeon_device *rdev)
5307{
5308 int r;
5309
bcc1c2a1
AD		 5310 /* Read BIOS */
5311 if (!radeon_get_bios(rdev)) {
5312 if (ASIC_IS_AVIVO(rdev))
5313 return -EINVAL;
5314 }
5315 /* Must be an ATOMBIOS */
5316 if (!rdev->is_atom_bios) {
755d819e 5317 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5318 return -EINVAL;
5319 }
5320 r = radeon_atombios_init(rdev);
5321 if (r)
5322 return r;
86f5c9ed
AD		 5323 /* reset the asic, the gfx blocks are often in a bad state
5324 * after the driver is unloaded or after a resume
5325 */
5326 if (radeon_asic_reset(rdev))
5327 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5328 /* Post card if necessary */
fd909c37 5329 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5330 if (!rdev->bios) {
5331 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5332 return -EINVAL;
5333 }
5334 DRM_INFO("GPU not posted. posting now...\n");
5335 atom_asic_init(rdev->mode_info.atom_context);
5336 }
d4788db3
AD		 5337 /* init golden registers */
5338 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5339 /* Initialize scratch registers */
5340 r600_scratch_init(rdev);
5341 /* Initialize surface registers */
5342 radeon_surface_init(rdev);
5343 /* Initialize clocks */
5344 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5345 /* Fence driver */
5346 r = radeon_fence_driver_init(rdev);
5347 if (r)
5348 return r;
d594e46a
JG		 5349 /* initialize AGP */
5350 if (rdev->flags & RADEON_IS_AGP) {
5351 r = radeon_agp_init(rdev);
5352 if (r)
5353 radeon_agp_disable(rdev);
5354 }
5355 /* initialize memory controller */
bcc1c2a1
AD		 5356 r = evergreen_mc_init(rdev);
5357 if (r)
5358 return r;
5359 /* Memory manager */
5360 r = radeon_bo_init(rdev);
5361 if (r)
5362 return r;
45f9a39b 5363
e32eb50d
CK		 5364 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5365 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5366
233d1ad5
AD		 5367 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5368 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5369
f2ba57b5
CK		 5370 r = radeon_uvd_init(rdev);
5371 if (!r) {
5372 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5373 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5374 4096);
5375 }
5376
bcc1c2a1
AD		 5377 rdev->ih.ring_obj = NULL;
5378 r600_ih_ring_init(rdev, 64 * 1024);
5379
5380 r = r600_pcie_gart_init(rdev);
5381 if (r)
5382 return r;
0fcdb61e 5383
148a03bc 5384 rdev->accel_working = true;
bcc1c2a1
AD		 5385 r = evergreen_startup(rdev);
5386 if (r) {
fe251e2f
AD		 5387 dev_err(rdev->dev, "disabling GPU acceleration\n");
5388 r700_cp_fini(rdev);
233d1ad5 5389 r600_dma_fini(rdev);
fe251e2f 5390 r600_irq_fini(rdev);
2948f5e6
AD		 5391 if (rdev->flags & RADEON_IS_IGP)
5392 sumo_rlc_fini(rdev);
724c80e1 5393 radeon_wb_fini(rdev);
2898c348 5394 radeon_ib_pool_fini(rdev);
fe251e2f 5395 radeon_irq_kms_fini(rdev);
0fcdb61e 5396 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5397 rdev->accel_working = false;
5398 }
77e00f2e
AD		 5399
5400 /* Don't start up if the MC ucode is missing on BTC parts.
5401 * The default clocks and voltages before the MC ucode
5402 * is loaded are not suffient for advanced operations.
5403 */
5404 if (ASIC_IS_DCE5(rdev)) {
5405 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5406 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5407 return -EINVAL;
5408 }
5409 }
5410
bcc1c2a1
AD		 5411 return 0;
5412}
5413
5414void evergreen_fini(struct radeon_device *rdev)
5415{
69d2ae57 5416 r600_audio_fini(rdev);
45f9a39b 5417 r700_cp_fini(rdev);
233d1ad5 5418 r600_dma_fini(rdev);
bcc1c2a1 5419 r600_irq_fini(rdev);
2948f5e6
AD		 5420 if (rdev->flags & RADEON_IS_IGP)
5421 sumo_rlc_fini(rdev);
724c80e1 5422 radeon_wb_fini(rdev);
2898c348 5423 radeon_ib_pool_fini(rdev);
bcc1c2a1 5424 radeon_irq_kms_fini(rdev);
bcc1c2a1 5425 evergreen_pcie_gart_fini(rdev);
e409b128 5426 uvd_v1_0_fini(rdev);
f2ba57b5 5427 radeon_uvd_fini(rdev);
16cdf04d 5428 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5429 radeon_gem_fini(rdev);
5430 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5431 radeon_agp_fini(rdev);
5432 radeon_bo_fini(rdev);
5433 radeon_atombios_fini(rdev);
5434 kfree(rdev->bios);
5435 rdev->bios = NULL;
bcc1c2a1 5436}
9e46a48d 5437
b07759bf 5438void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5439{
7e0e4196 5440 u32 link_width_cntl, speed_cntl;
9e46a48d 5441
d42dd579
AD		 5442 if (radeon_pcie_gen2 == 0)
5443 return;
5444
9e46a48d
AD		 5445 if (rdev->flags & RADEON_IS_IGP)
5446 return;
5447
5448 if (!(rdev->flags & RADEON_IS_PCIE))
5449 return;
5450
5451 /* x2 cards have a special sequence */
5452 if (ASIC_IS_X2(rdev))
5453 return;
5454
7e0e4196
KSS		 5455 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5456 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5457 return;
5458
492d2b61 5459 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5460 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5461 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5462 return;
5463 }
5464
197bbb3d
DA		 5465 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5466
9e46a48d
AD		 5467 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5468 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5469
492d2b61 5470 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5471 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5472 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5473
492d2b61 5474 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5475 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5476 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5477
492d2b61 5478 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5479 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5480 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5481
492d2b61 5482 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5483 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5484 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5485
492d2b61 5486 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5487 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5488 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5489
5490 } else {
492d2b61 5491 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5492 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5493 if (1)
5494 link_width_cntl |= LC_UPCONFIGURE_DIS;
5495 else
5496 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5497 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5498 }
5499}
f52382d7
AD		 5500
5501void evergreen_program_aspm(struct radeon_device *rdev)
5502{
5503 u32 data, orig;
5504 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5505 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5506 /* fusion_platform = true
5507 * if the system is a fusion system
5508 * (APU or DGPU in a fusion system).
5509 * todo: check if the system is a fusion platform.
5510 */
5511 bool fusion_platform = false;
5512
1294d4a3
AD		 5513 if (radeon_aspm == 0)
5514 return;
5515
f52382d7
AD		 5516 if (!(rdev->flags & RADEON_IS_PCIE))
5517 return;
5518
5519 switch (rdev->family) {
5520 case CHIP_CYPRESS:
5521 case CHIP_HEMLOCK:
5522 case CHIP_JUNIPER:
5523 case CHIP_REDWOOD:
5524 case CHIP_CEDAR:
5525 case CHIP_SUMO:
5526 case CHIP_SUMO2:
5527 case CHIP_PALM:
5528 case CHIP_ARUBA:
5529 disable_l0s = true;
5530 break;
5531 default:
5532 disable_l0s = false;
5533 break;
5534 }
5535
5536 if (rdev->flags & RADEON_IS_IGP)
5537 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5538
5539 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5540 if (fusion_platform)
5541 data &= ~MULTI_PIF;
5542 else
5543 data |= MULTI_PIF;
5544 if (data != orig)
5545 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5546
5547 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5548 if (fusion_platform)
5549 data &= ~MULTI_PIF;
5550 else
5551 data |= MULTI_PIF;
5552 if (data != orig)
5553 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5554
5555 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5556 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5557 if (!disable_l0s) {
5558 if (rdev->family >= CHIP_BARTS)
5559 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5560 else
5561 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5562 }
5563
5564 if (!disable_l1) {
5565 if (rdev->family >= CHIP_BARTS)
5566 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5567 else
5568 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5569
5570 if (!disable_plloff_in_l1) {
5571 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5572 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5573 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5574 if (data != orig)
5575 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5576
5577 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5578 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5579 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5580 if (data != orig)
5581 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5582
5583 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5584 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5585 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5586 if (data != orig)
5587 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5588
5589 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5590 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5591 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5592 if (data != orig)
5593 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5594
5595 if (rdev->family >= CHIP_BARTS) {
5596 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5597 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5598 data |= PLL_RAMP_UP_TIME_0(4);
5599 if (data != orig)
5600 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5601
5602 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5603 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5604 data |= PLL_RAMP_UP_TIME_1(4);
5605 if (data != orig)
5606 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5607
5608 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5609 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5610 data |= PLL_RAMP_UP_TIME_0(4);
5611 if (data != orig)
5612 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5613
5614 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5615 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5616 data |= PLL_RAMP_UP_TIME_1(4);
5617 if (data != orig)
5618 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5619 }
5620
5621 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5622 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5623 data |= LC_DYN_LANES_PWR_STATE(3);
5624 if (data != orig)
5625 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5626
5627 if (rdev->family >= CHIP_BARTS) {
5628 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5629 data &= ~LS2_EXIT_TIME_MASK;
5630 data |= LS2_EXIT_TIME(1);
5631 if (data != orig)
5632 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5633
5634 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5635 data &= ~LS2_EXIT_TIME_MASK;
5636 data |= LS2_EXIT_TIME(1);
5637 if (data != orig)
5638 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5639 }
5640 }
5641 }
5642
5643 /* evergreen parts only */
5644 if (rdev->family < CHIP_BARTS)
5645 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5646
5647 if (pcie_lc_cntl != pcie_lc_cntl_old)
5648 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5649}
Linux kernel - Deliverables
This page took 0.616925 seconds and 5 git commands to generate.