projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/kms: add dpm support for trinity asics
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
50static u32 sumo_rlc_save_restore_register_list[] =
51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
134static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
bcc1c2a1
AD		 136static void evergreen_gpu_init(struct radeon_device *rdev);
137void evergreen_fini(struct radeon_device *rdev);
b07759bf 138void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
bcc1c2a1 141
d4788db3
AD		 142static const u32 evergreen_golden_registers[] =
143{
144 0x3f90, 0xffff0000, 0xff000000,
145 0x9148, 0xffff0000, 0xff000000,
146 0x3f94, 0xffff0000, 0xff000000,
147 0x914c, 0xffff0000, 0xff000000,
148 0x9b7c, 0xffffffff, 0x00000000,
149 0x8a14, 0xffffffff, 0x00000007,
150 0x8b10, 0xffffffff, 0x00000000,
151 0x960c, 0xffffffff, 0x54763210,
152 0x88c4, 0xffffffff, 0x000000c2,
153 0x88d4, 0xffffffff, 0x00000010,
154 0x8974, 0xffffffff, 0x00000000,
155 0xc78, 0x00000080, 0x00000080,
156 0x5eb4, 0xffffffff, 0x00000002,
157 0x5e78, 0xffffffff, 0x001000f0,
158 0x6104, 0x01000300, 0x00000000,
159 0x5bc0, 0x00300000, 0x00000000,
160 0x7030, 0xffffffff, 0x00000011,
161 0x7c30, 0xffffffff, 0x00000011,
162 0x10830, 0xffffffff, 0x00000011,
163 0x11430, 0xffffffff, 0x00000011,
164 0x12030, 0xffffffff, 0x00000011,
165 0x12c30, 0xffffffff, 0x00000011,
166 0xd02c, 0xffffffff, 0x08421000,
167 0x240c, 0xffffffff, 0x00000380,
168 0x8b24, 0xffffffff, 0x00ff0fff,
169 0x28a4c, 0x06000000, 0x06000000,
170 0x10c, 0x00000001, 0x00000001,
171 0x8d00, 0xffffffff, 0x100e4848,
172 0x8d04, 0xffffffff, 0x00164745,
173 0x8c00, 0xffffffff, 0xe4000003,
174 0x8c04, 0xffffffff, 0x40600060,
175 0x8c08, 0xffffffff, 0x001c001c,
176 0x8cf0, 0xffffffff, 0x08e00620,
177 0x8c20, 0xffffffff, 0x00800080,
178 0x8c24, 0xffffffff, 0x00800080,
179 0x8c18, 0xffffffff, 0x20202078,
180 0x8c1c, 0xffffffff, 0x00001010,
181 0x28350, 0xffffffff, 0x00000000,
182 0xa008, 0xffffffff, 0x00010000,
183 0x5cc, 0xffffffff, 0x00000001,
184 0x9508, 0xffffffff, 0x00000002,
185 0x913c, 0x0000000f, 0x0000000a
186};
187
188static const u32 evergreen_golden_registers2[] =
189{
190 0x2f4c, 0xffffffff, 0x00000000,
191 0x54f4, 0xffffffff, 0x00000000,
192 0x54f0, 0xffffffff, 0x00000000,
193 0x5498, 0xffffffff, 0x00000000,
194 0x549c, 0xffffffff, 0x00000000,
195 0x5494, 0xffffffff, 0x00000000,
196 0x53cc, 0xffffffff, 0x00000000,
197 0x53c8, 0xffffffff, 0x00000000,
198 0x53c4, 0xffffffff, 0x00000000,
199 0x53c0, 0xffffffff, 0x00000000,
200 0x53bc, 0xffffffff, 0x00000000,
201 0x53b8, 0xffffffff, 0x00000000,
202 0x53b4, 0xffffffff, 0x00000000,
203 0x53b0, 0xffffffff, 0x00000000
204};
205
206static const u32 cypress_mgcg_init[] =
207{
208 0x802c, 0xffffffff, 0xc0000000,
209 0x5448, 0xffffffff, 0x00000100,
210 0x55e4, 0xffffffff, 0x00000100,
211 0x160c, 0xffffffff, 0x00000100,
212 0x5644, 0xffffffff, 0x00000100,
213 0xc164, 0xffffffff, 0x00000100,
214 0x8a18, 0xffffffff, 0x00000100,
215 0x897c, 0xffffffff, 0x06000100,
216 0x8b28, 0xffffffff, 0x00000100,
217 0x9144, 0xffffffff, 0x00000100,
218 0x9a60, 0xffffffff, 0x00000100,
219 0x9868, 0xffffffff, 0x00000100,
220 0x8d58, 0xffffffff, 0x00000100,
221 0x9510, 0xffffffff, 0x00000100,
222 0x949c, 0xffffffff, 0x00000100,
223 0x9654, 0xffffffff, 0x00000100,
224 0x9030, 0xffffffff, 0x00000100,
225 0x9034, 0xffffffff, 0x00000100,
226 0x9038, 0xffffffff, 0x00000100,
227 0x903c, 0xffffffff, 0x00000100,
228 0x9040, 0xffffffff, 0x00000100,
229 0xa200, 0xffffffff, 0x00000100,
230 0xa204, 0xffffffff, 0x00000100,
231 0xa208, 0xffffffff, 0x00000100,
232 0xa20c, 0xffffffff, 0x00000100,
233 0x971c, 0xffffffff, 0x00000100,
234 0x977c, 0xffffffff, 0x00000100,
235 0x3f80, 0xffffffff, 0x00000100,
236 0xa210, 0xffffffff, 0x00000100,
237 0xa214, 0xffffffff, 0x00000100,
238 0x4d8, 0xffffffff, 0x00000100,
239 0x9784, 0xffffffff, 0x00000100,
240 0x9698, 0xffffffff, 0x00000100,
241 0x4d4, 0xffffffff, 0x00000200,
242 0x30cc, 0xffffffff, 0x00000100,
243 0xd0c0, 0xffffffff, 0xff000100,
244 0x802c, 0xffffffff, 0x40000000,
245 0x915c, 0xffffffff, 0x00010000,
246 0x9160, 0xffffffff, 0x00030002,
247 0x9178, 0xffffffff, 0x00070000,
248 0x917c, 0xffffffff, 0x00030002,
249 0x9180, 0xffffffff, 0x00050004,
250 0x918c, 0xffffffff, 0x00010006,
251 0x9190, 0xffffffff, 0x00090008,
252 0x9194, 0xffffffff, 0x00070000,
253 0x9198, 0xffffffff, 0x00030002,
254 0x919c, 0xffffffff, 0x00050004,
255 0x91a8, 0xffffffff, 0x00010006,
256 0x91ac, 0xffffffff, 0x00090008,
257 0x91b0, 0xffffffff, 0x00070000,
258 0x91b4, 0xffffffff, 0x00030002,
259 0x91b8, 0xffffffff, 0x00050004,
260 0x91c4, 0xffffffff, 0x00010006,
261 0x91c8, 0xffffffff, 0x00090008,
262 0x91cc, 0xffffffff, 0x00070000,
263 0x91d0, 0xffffffff, 0x00030002,
264 0x91d4, 0xffffffff, 0x00050004,
265 0x91e0, 0xffffffff, 0x00010006,
266 0x91e4, 0xffffffff, 0x00090008,
267 0x91e8, 0xffffffff, 0x00000000,
268 0x91ec, 0xffffffff, 0x00070000,
269 0x91f0, 0xffffffff, 0x00030002,
270 0x91f4, 0xffffffff, 0x00050004,
271 0x9200, 0xffffffff, 0x00010006,
272 0x9204, 0xffffffff, 0x00090008,
273 0x9208, 0xffffffff, 0x00070000,
274 0x920c, 0xffffffff, 0x00030002,
275 0x9210, 0xffffffff, 0x00050004,
276 0x921c, 0xffffffff, 0x00010006,
277 0x9220, 0xffffffff, 0x00090008,
278 0x9224, 0xffffffff, 0x00070000,
279 0x9228, 0xffffffff, 0x00030002,
280 0x922c, 0xffffffff, 0x00050004,
281 0x9238, 0xffffffff, 0x00010006,
282 0x923c, 0xffffffff, 0x00090008,
283 0x9240, 0xffffffff, 0x00070000,
284 0x9244, 0xffffffff, 0x00030002,
285 0x9248, 0xffffffff, 0x00050004,
286 0x9254, 0xffffffff, 0x00010006,
287 0x9258, 0xffffffff, 0x00090008,
288 0x925c, 0xffffffff, 0x00070000,
289 0x9260, 0xffffffff, 0x00030002,
290 0x9264, 0xffffffff, 0x00050004,
291 0x9270, 0xffffffff, 0x00010006,
292 0x9274, 0xffffffff, 0x00090008,
293 0x9278, 0xffffffff, 0x00070000,
294 0x927c, 0xffffffff, 0x00030002,
295 0x9280, 0xffffffff, 0x00050004,
296 0x928c, 0xffffffff, 0x00010006,
297 0x9290, 0xffffffff, 0x00090008,
298 0x9294, 0xffffffff, 0x00000000,
299 0x929c, 0xffffffff, 0x00000001,
300 0x802c, 0xffffffff, 0x40010000,
301 0x915c, 0xffffffff, 0x00010000,
302 0x9160, 0xffffffff, 0x00030002,
303 0x9178, 0xffffffff, 0x00070000,
304 0x917c, 0xffffffff, 0x00030002,
305 0x9180, 0xffffffff, 0x00050004,
306 0x918c, 0xffffffff, 0x00010006,
307 0x9190, 0xffffffff, 0x00090008,
308 0x9194, 0xffffffff, 0x00070000,
309 0x9198, 0xffffffff, 0x00030002,
310 0x919c, 0xffffffff, 0x00050004,
311 0x91a8, 0xffffffff, 0x00010006,
312 0x91ac, 0xffffffff, 0x00090008,
313 0x91b0, 0xffffffff, 0x00070000,
314 0x91b4, 0xffffffff, 0x00030002,
315 0x91b8, 0xffffffff, 0x00050004,
316 0x91c4, 0xffffffff, 0x00010006,
317 0x91c8, 0xffffffff, 0x00090008,
318 0x91cc, 0xffffffff, 0x00070000,
319 0x91d0, 0xffffffff, 0x00030002,
320 0x91d4, 0xffffffff, 0x00050004,
321 0x91e0, 0xffffffff, 0x00010006,
322 0x91e4, 0xffffffff, 0x00090008,
323 0x91e8, 0xffffffff, 0x00000000,
324 0x91ec, 0xffffffff, 0x00070000,
325 0x91f0, 0xffffffff, 0x00030002,
326 0x91f4, 0xffffffff, 0x00050004,
327 0x9200, 0xffffffff, 0x00010006,
328 0x9204, 0xffffffff, 0x00090008,
329 0x9208, 0xffffffff, 0x00070000,
330 0x920c, 0xffffffff, 0x00030002,
331 0x9210, 0xffffffff, 0x00050004,
332 0x921c, 0xffffffff, 0x00010006,
333 0x9220, 0xffffffff, 0x00090008,
334 0x9224, 0xffffffff, 0x00070000,
335 0x9228, 0xffffffff, 0x00030002,
336 0x922c, 0xffffffff, 0x00050004,
337 0x9238, 0xffffffff, 0x00010006,
338 0x923c, 0xffffffff, 0x00090008,
339 0x9240, 0xffffffff, 0x00070000,
340 0x9244, 0xffffffff, 0x00030002,
341 0x9248, 0xffffffff, 0x00050004,
342 0x9254, 0xffffffff, 0x00010006,
343 0x9258, 0xffffffff, 0x00090008,
344 0x925c, 0xffffffff, 0x00070000,
345 0x9260, 0xffffffff, 0x00030002,
346 0x9264, 0xffffffff, 0x00050004,
347 0x9270, 0xffffffff, 0x00010006,
348 0x9274, 0xffffffff, 0x00090008,
349 0x9278, 0xffffffff, 0x00070000,
350 0x927c, 0xffffffff, 0x00030002,
351 0x9280, 0xffffffff, 0x00050004,
352 0x928c, 0xffffffff, 0x00010006,
353 0x9290, 0xffffffff, 0x00090008,
354 0x9294, 0xffffffff, 0x00000000,
355 0x929c, 0xffffffff, 0x00000001,
356 0x802c, 0xffffffff, 0xc0000000
357};
358
359static const u32 redwood_mgcg_init[] =
360{
361 0x802c, 0xffffffff, 0xc0000000,
362 0x5448, 0xffffffff, 0x00000100,
363 0x55e4, 0xffffffff, 0x00000100,
364 0x160c, 0xffffffff, 0x00000100,
365 0x5644, 0xffffffff, 0x00000100,
366 0xc164, 0xffffffff, 0x00000100,
367 0x8a18, 0xffffffff, 0x00000100,
368 0x897c, 0xffffffff, 0x06000100,
369 0x8b28, 0xffffffff, 0x00000100,
370 0x9144, 0xffffffff, 0x00000100,
371 0x9a60, 0xffffffff, 0x00000100,
372 0x9868, 0xffffffff, 0x00000100,
373 0x8d58, 0xffffffff, 0x00000100,
374 0x9510, 0xffffffff, 0x00000100,
375 0x949c, 0xffffffff, 0x00000100,
376 0x9654, 0xffffffff, 0x00000100,
377 0x9030, 0xffffffff, 0x00000100,
378 0x9034, 0xffffffff, 0x00000100,
379 0x9038, 0xffffffff, 0x00000100,
380 0x903c, 0xffffffff, 0x00000100,
381 0x9040, 0xffffffff, 0x00000100,
382 0xa200, 0xffffffff, 0x00000100,
383 0xa204, 0xffffffff, 0x00000100,
384 0xa208, 0xffffffff, 0x00000100,
385 0xa20c, 0xffffffff, 0x00000100,
386 0x971c, 0xffffffff, 0x00000100,
387 0x977c, 0xffffffff, 0x00000100,
388 0x3f80, 0xffffffff, 0x00000100,
389 0xa210, 0xffffffff, 0x00000100,
390 0xa214, 0xffffffff, 0x00000100,
391 0x4d8, 0xffffffff, 0x00000100,
392 0x9784, 0xffffffff, 0x00000100,
393 0x9698, 0xffffffff, 0x00000100,
394 0x4d4, 0xffffffff, 0x00000200,
395 0x30cc, 0xffffffff, 0x00000100,
396 0xd0c0, 0xffffffff, 0xff000100,
397 0x802c, 0xffffffff, 0x40000000,
398 0x915c, 0xffffffff, 0x00010000,
399 0x9160, 0xffffffff, 0x00030002,
400 0x9178, 0xffffffff, 0x00070000,
401 0x917c, 0xffffffff, 0x00030002,
402 0x9180, 0xffffffff, 0x00050004,
403 0x918c, 0xffffffff, 0x00010006,
404 0x9190, 0xffffffff, 0x00090008,
405 0x9194, 0xffffffff, 0x00070000,
406 0x9198, 0xffffffff, 0x00030002,
407 0x919c, 0xffffffff, 0x00050004,
408 0x91a8, 0xffffffff, 0x00010006,
409 0x91ac, 0xffffffff, 0x00090008,
410 0x91b0, 0xffffffff, 0x00070000,
411 0x91b4, 0xffffffff, 0x00030002,
412 0x91b8, 0xffffffff, 0x00050004,
413 0x91c4, 0xffffffff, 0x00010006,
414 0x91c8, 0xffffffff, 0x00090008,
415 0x91cc, 0xffffffff, 0x00070000,
416 0x91d0, 0xffffffff, 0x00030002,
417 0x91d4, 0xffffffff, 0x00050004,
418 0x91e0, 0xffffffff, 0x00010006,
419 0x91e4, 0xffffffff, 0x00090008,
420 0x91e8, 0xffffffff, 0x00000000,
421 0x91ec, 0xffffffff, 0x00070000,
422 0x91f0, 0xffffffff, 0x00030002,
423 0x91f4, 0xffffffff, 0x00050004,
424 0x9200, 0xffffffff, 0x00010006,
425 0x9204, 0xffffffff, 0x00090008,
426 0x9294, 0xffffffff, 0x00000000,
427 0x929c, 0xffffffff, 0x00000001,
428 0x802c, 0xffffffff, 0xc0000000
429};
430
431static const u32 cedar_golden_registers[] =
432{
433 0x3f90, 0xffff0000, 0xff000000,
434 0x9148, 0xffff0000, 0xff000000,
435 0x3f94, 0xffff0000, 0xff000000,
436 0x914c, 0xffff0000, 0xff000000,
437 0x9b7c, 0xffffffff, 0x00000000,
438 0x8a14, 0xffffffff, 0x00000007,
439 0x8b10, 0xffffffff, 0x00000000,
440 0x960c, 0xffffffff, 0x54763210,
441 0x88c4, 0xffffffff, 0x000000c2,
442 0x88d4, 0xffffffff, 0x00000000,
443 0x8974, 0xffffffff, 0x00000000,
444 0xc78, 0x00000080, 0x00000080,
445 0x5eb4, 0xffffffff, 0x00000002,
446 0x5e78, 0xffffffff, 0x001000f0,
447 0x6104, 0x01000300, 0x00000000,
448 0x5bc0, 0x00300000, 0x00000000,
449 0x7030, 0xffffffff, 0x00000011,
450 0x7c30, 0xffffffff, 0x00000011,
451 0x10830, 0xffffffff, 0x00000011,
452 0x11430, 0xffffffff, 0x00000011,
453 0xd02c, 0xffffffff, 0x08421000,
454 0x240c, 0xffffffff, 0x00000380,
455 0x8b24, 0xffffffff, 0x00ff0fff,
456 0x28a4c, 0x06000000, 0x06000000,
457 0x10c, 0x00000001, 0x00000001,
458 0x8d00, 0xffffffff, 0x100e4848,
459 0x8d04, 0xffffffff, 0x00164745,
460 0x8c00, 0xffffffff, 0xe4000003,
461 0x8c04, 0xffffffff, 0x40600060,
462 0x8c08, 0xffffffff, 0x001c001c,
463 0x8cf0, 0xffffffff, 0x08e00410,
464 0x8c20, 0xffffffff, 0x00800080,
465 0x8c24, 0xffffffff, 0x00800080,
466 0x8c18, 0xffffffff, 0x20202078,
467 0x8c1c, 0xffffffff, 0x00001010,
468 0x28350, 0xffffffff, 0x00000000,
469 0xa008, 0xffffffff, 0x00010000,
470 0x5cc, 0xffffffff, 0x00000001,
471 0x9508, 0xffffffff, 0x00000002
472};
473
474static const u32 cedar_mgcg_init[] =
475{
476 0x802c, 0xffffffff, 0xc0000000,
477 0x5448, 0xffffffff, 0x00000100,
478 0x55e4, 0xffffffff, 0x00000100,
479 0x160c, 0xffffffff, 0x00000100,
480 0x5644, 0xffffffff, 0x00000100,
481 0xc164, 0xffffffff, 0x00000100,
482 0x8a18, 0xffffffff, 0x00000100,
483 0x897c, 0xffffffff, 0x06000100,
484 0x8b28, 0xffffffff, 0x00000100,
485 0x9144, 0xffffffff, 0x00000100,
486 0x9a60, 0xffffffff, 0x00000100,
487 0x9868, 0xffffffff, 0x00000100,
488 0x8d58, 0xffffffff, 0x00000100,
489 0x9510, 0xffffffff, 0x00000100,
490 0x949c, 0xffffffff, 0x00000100,
491 0x9654, 0xffffffff, 0x00000100,
492 0x9030, 0xffffffff, 0x00000100,
493 0x9034, 0xffffffff, 0x00000100,
494 0x9038, 0xffffffff, 0x00000100,
495 0x903c, 0xffffffff, 0x00000100,
496 0x9040, 0xffffffff, 0x00000100,
497 0xa200, 0xffffffff, 0x00000100,
498 0xa204, 0xffffffff, 0x00000100,
499 0xa208, 0xffffffff, 0x00000100,
500 0xa20c, 0xffffffff, 0x00000100,
501 0x971c, 0xffffffff, 0x00000100,
502 0x977c, 0xffffffff, 0x00000100,
503 0x3f80, 0xffffffff, 0x00000100,
504 0xa210, 0xffffffff, 0x00000100,
505 0xa214, 0xffffffff, 0x00000100,
506 0x4d8, 0xffffffff, 0x00000100,
507 0x9784, 0xffffffff, 0x00000100,
508 0x9698, 0xffffffff, 0x00000100,
509 0x4d4, 0xffffffff, 0x00000200,
510 0x30cc, 0xffffffff, 0x00000100,
511 0xd0c0, 0xffffffff, 0xff000100,
512 0x802c, 0xffffffff, 0x40000000,
513 0x915c, 0xffffffff, 0x00010000,
514 0x9178, 0xffffffff, 0x00050000,
515 0x917c, 0xffffffff, 0x00030002,
516 0x918c, 0xffffffff, 0x00010004,
517 0x9190, 0xffffffff, 0x00070006,
518 0x9194, 0xffffffff, 0x00050000,
519 0x9198, 0xffffffff, 0x00030002,
520 0x91a8, 0xffffffff, 0x00010004,
521 0x91ac, 0xffffffff, 0x00070006,
522 0x91e8, 0xffffffff, 0x00000000,
523 0x9294, 0xffffffff, 0x00000000,
524 0x929c, 0xffffffff, 0x00000001,
525 0x802c, 0xffffffff, 0xc0000000
526};
527
528static const u32 juniper_mgcg_init[] =
529{
530 0x802c, 0xffffffff, 0xc0000000,
531 0x5448, 0xffffffff, 0x00000100,
532 0x55e4, 0xffffffff, 0x00000100,
533 0x160c, 0xffffffff, 0x00000100,
534 0x5644, 0xffffffff, 0x00000100,
535 0xc164, 0xffffffff, 0x00000100,
536 0x8a18, 0xffffffff, 0x00000100,
537 0x897c, 0xffffffff, 0x06000100,
538 0x8b28, 0xffffffff, 0x00000100,
539 0x9144, 0xffffffff, 0x00000100,
540 0x9a60, 0xffffffff, 0x00000100,
541 0x9868, 0xffffffff, 0x00000100,
542 0x8d58, 0xffffffff, 0x00000100,
543 0x9510, 0xffffffff, 0x00000100,
544 0x949c, 0xffffffff, 0x00000100,
545 0x9654, 0xffffffff, 0x00000100,
546 0x9030, 0xffffffff, 0x00000100,
547 0x9034, 0xffffffff, 0x00000100,
548 0x9038, 0xffffffff, 0x00000100,
549 0x903c, 0xffffffff, 0x00000100,
550 0x9040, 0xffffffff, 0x00000100,
551 0xa200, 0xffffffff, 0x00000100,
552 0xa204, 0xffffffff, 0x00000100,
553 0xa208, 0xffffffff, 0x00000100,
554 0xa20c, 0xffffffff, 0x00000100,
555 0x971c, 0xffffffff, 0x00000100,
556 0xd0c0, 0xffffffff, 0xff000100,
557 0x802c, 0xffffffff, 0x40000000,
558 0x915c, 0xffffffff, 0x00010000,
559 0x9160, 0xffffffff, 0x00030002,
560 0x9178, 0xffffffff, 0x00070000,
561 0x917c, 0xffffffff, 0x00030002,
562 0x9180, 0xffffffff, 0x00050004,
563 0x918c, 0xffffffff, 0x00010006,
564 0x9190, 0xffffffff, 0x00090008,
565 0x9194, 0xffffffff, 0x00070000,
566 0x9198, 0xffffffff, 0x00030002,
567 0x919c, 0xffffffff, 0x00050004,
568 0x91a8, 0xffffffff, 0x00010006,
569 0x91ac, 0xffffffff, 0x00090008,
570 0x91b0, 0xffffffff, 0x00070000,
571 0x91b4, 0xffffffff, 0x00030002,
572 0x91b8, 0xffffffff, 0x00050004,
573 0x91c4, 0xffffffff, 0x00010006,
574 0x91c8, 0xffffffff, 0x00090008,
575 0x91cc, 0xffffffff, 0x00070000,
576 0x91d0, 0xffffffff, 0x00030002,
577 0x91d4, 0xffffffff, 0x00050004,
578 0x91e0, 0xffffffff, 0x00010006,
579 0x91e4, 0xffffffff, 0x00090008,
580 0x91e8, 0xffffffff, 0x00000000,
581 0x91ec, 0xffffffff, 0x00070000,
582 0x91f0, 0xffffffff, 0x00030002,
583 0x91f4, 0xffffffff, 0x00050004,
584 0x9200, 0xffffffff, 0x00010006,
585 0x9204, 0xffffffff, 0x00090008,
586 0x9208, 0xffffffff, 0x00070000,
587 0x920c, 0xffffffff, 0x00030002,
588 0x9210, 0xffffffff, 0x00050004,
589 0x921c, 0xffffffff, 0x00010006,
590 0x9220, 0xffffffff, 0x00090008,
591 0x9224, 0xffffffff, 0x00070000,
592 0x9228, 0xffffffff, 0x00030002,
593 0x922c, 0xffffffff, 0x00050004,
594 0x9238, 0xffffffff, 0x00010006,
595 0x923c, 0xffffffff, 0x00090008,
596 0x9240, 0xffffffff, 0x00070000,
597 0x9244, 0xffffffff, 0x00030002,
598 0x9248, 0xffffffff, 0x00050004,
599 0x9254, 0xffffffff, 0x00010006,
600 0x9258, 0xffffffff, 0x00090008,
601 0x925c, 0xffffffff, 0x00070000,
602 0x9260, 0xffffffff, 0x00030002,
603 0x9264, 0xffffffff, 0x00050004,
604 0x9270, 0xffffffff, 0x00010006,
605 0x9274, 0xffffffff, 0x00090008,
606 0x9278, 0xffffffff, 0x00070000,
607 0x927c, 0xffffffff, 0x00030002,
608 0x9280, 0xffffffff, 0x00050004,
609 0x928c, 0xffffffff, 0x00010006,
610 0x9290, 0xffffffff, 0x00090008,
611 0x9294, 0xffffffff, 0x00000000,
612 0x929c, 0xffffffff, 0x00000001,
613 0x802c, 0xffffffff, 0xc0000000,
614 0x977c, 0xffffffff, 0x00000100,
615 0x3f80, 0xffffffff, 0x00000100,
616 0xa210, 0xffffffff, 0x00000100,
617 0xa214, 0xffffffff, 0x00000100,
618 0x4d8, 0xffffffff, 0x00000100,
619 0x9784, 0xffffffff, 0x00000100,
620 0x9698, 0xffffffff, 0x00000100,
621 0x4d4, 0xffffffff, 0x00000200,
622 0x30cc, 0xffffffff, 0x00000100,
623 0x802c, 0xffffffff, 0xc0000000
624};
625
626static const u32 supersumo_golden_registers[] =
627{
628 0x5eb4, 0xffffffff, 0x00000002,
629 0x5cc, 0xffffffff, 0x00000001,
630 0x7030, 0xffffffff, 0x00000011,
631 0x7c30, 0xffffffff, 0x00000011,
632 0x6104, 0x01000300, 0x00000000,
633 0x5bc0, 0x00300000, 0x00000000,
634 0x8c04, 0xffffffff, 0x40600060,
635 0x8c08, 0xffffffff, 0x001c001c,
636 0x8c20, 0xffffffff, 0x00800080,
637 0x8c24, 0xffffffff, 0x00800080,
638 0x8c18, 0xffffffff, 0x20202078,
639 0x8c1c, 0xffffffff, 0x00001010,
640 0x918c, 0xffffffff, 0x00010006,
641 0x91a8, 0xffffffff, 0x00010006,
642 0x91c4, 0xffffffff, 0x00010006,
643 0x91e0, 0xffffffff, 0x00010006,
644 0x9200, 0xffffffff, 0x00010006,
645 0x9150, 0xffffffff, 0x6e944040,
646 0x917c, 0xffffffff, 0x00030002,
647 0x9180, 0xffffffff, 0x00050004,
648 0x9198, 0xffffffff, 0x00030002,
649 0x919c, 0xffffffff, 0x00050004,
650 0x91b4, 0xffffffff, 0x00030002,
651 0x91b8, 0xffffffff, 0x00050004,
652 0x91d0, 0xffffffff, 0x00030002,
653 0x91d4, 0xffffffff, 0x00050004,
654 0x91f0, 0xffffffff, 0x00030002,
655 0x91f4, 0xffffffff, 0x00050004,
656 0x915c, 0xffffffff, 0x00010000,
657 0x9160, 0xffffffff, 0x00030002,
658 0x3f90, 0xffff0000, 0xff000000,
659 0x9178, 0xffffffff, 0x00070000,
660 0x9194, 0xffffffff, 0x00070000,
661 0x91b0, 0xffffffff, 0x00070000,
662 0x91cc, 0xffffffff, 0x00070000,
663 0x91ec, 0xffffffff, 0x00070000,
664 0x9148, 0xffff0000, 0xff000000,
665 0x9190, 0xffffffff, 0x00090008,
666 0x91ac, 0xffffffff, 0x00090008,
667 0x91c8, 0xffffffff, 0x00090008,
668 0x91e4, 0xffffffff, 0x00090008,
669 0x9204, 0xffffffff, 0x00090008,
670 0x3f94, 0xffff0000, 0xff000000,
671 0x914c, 0xffff0000, 0xff000000,
672 0x929c, 0xffffffff, 0x00000001,
673 0x8a18, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x5644, 0xffffffff, 0x00000100,
677 0x9b7c, 0xffffffff, 0x00000000,
678 0x8030, 0xffffffff, 0x0000100a,
679 0x8a14, 0xffffffff, 0x00000007,
680 0x8b24, 0xffffffff, 0x00ff0fff,
681 0x8b10, 0xffffffff, 0x00000000,
682 0x28a4c, 0x06000000, 0x06000000,
683 0x4d8, 0xffffffff, 0x00000100,
684 0x913c, 0xffff000f, 0x0100000a,
685 0x960c, 0xffffffff, 0x54763210,
686 0x88c4, 0xffffffff, 0x000000c2,
687 0x88d4, 0xffffffff, 0x00000010,
688 0x8974, 0xffffffff, 0x00000000,
689 0xc78, 0x00000080, 0x00000080,
690 0x5e78, 0xffffffff, 0x001000f0,
691 0xd02c, 0xffffffff, 0x08421000,
692 0xa008, 0xffffffff, 0x00010000,
693 0x8d00, 0xffffffff, 0x100e4848,
694 0x8d04, 0xffffffff, 0x00164745,
695 0x8c00, 0xffffffff, 0xe4000003,
696 0x8cf0, 0x1fffffff, 0x08e00620,
697 0x28350, 0xffffffff, 0x00000000,
698 0x9508, 0xffffffff, 0x00000002
699};
700
701static const u32 sumo_golden_registers[] =
702{
703 0x900c, 0x00ffffff, 0x0017071f,
704 0x8c18, 0xffffffff, 0x10101060,
705 0x8c1c, 0xffffffff, 0x00001010,
706 0x8c30, 0x0000000f, 0x00000005,
707 0x9688, 0x0000000f, 0x00000007
708};
709
710static const u32 wrestler_golden_registers[] =
711{
712 0x5eb4, 0xffffffff, 0x00000002,
713 0x5cc, 0xffffffff, 0x00000001,
714 0x7030, 0xffffffff, 0x00000011,
715 0x7c30, 0xffffffff, 0x00000011,
716 0x6104, 0x01000300, 0x00000000,
717 0x5bc0, 0x00300000, 0x00000000,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x9150, 0xffffffff, 0x6e944040,
721 0x917c, 0xffffffff, 0x00030002,
722 0x9198, 0xffffffff, 0x00030002,
723 0x915c, 0xffffffff, 0x00010000,
724 0x3f90, 0xffff0000, 0xff000000,
725 0x9178, 0xffffffff, 0x00070000,
726 0x9194, 0xffffffff, 0x00070000,
727 0x9148, 0xffff0000, 0xff000000,
728 0x9190, 0xffffffff, 0x00090008,
729 0x91ac, 0xffffffff, 0x00090008,
730 0x3f94, 0xffff0000, 0xff000000,
731 0x914c, 0xffff0000, 0xff000000,
732 0x929c, 0xffffffff, 0x00000001,
733 0x8a18, 0xffffffff, 0x00000100,
734 0x8b28, 0xffffffff, 0x00000100,
735 0x9144, 0xffffffff, 0x00000100,
736 0x9b7c, 0xffffffff, 0x00000000,
737 0x8030, 0xffffffff, 0x0000100a,
738 0x8a14, 0xffffffff, 0x00000001,
739 0x8b24, 0xffffffff, 0x00ff0fff,
740 0x8b10, 0xffffffff, 0x00000000,
741 0x28a4c, 0x06000000, 0x06000000,
742 0x4d8, 0xffffffff, 0x00000100,
743 0x913c, 0xffff000f, 0x0100000a,
744 0x960c, 0xffffffff, 0x54763210,
745 0x88c4, 0xffffffff, 0x000000c2,
746 0x88d4, 0xffffffff, 0x00000010,
747 0x8974, 0xffffffff, 0x00000000,
748 0xc78, 0x00000080, 0x00000080,
749 0x5e78, 0xffffffff, 0x001000f0,
750 0xd02c, 0xffffffff, 0x08421000,
751 0xa008, 0xffffffff, 0x00010000,
752 0x8d00, 0xffffffff, 0x100e4848,
753 0x8d04, 0xffffffff, 0x00164745,
754 0x8c00, 0xffffffff, 0xe4000003,
755 0x8cf0, 0x1fffffff, 0x08e00410,
756 0x28350, 0xffffffff, 0x00000000,
757 0x9508, 0xffffffff, 0x00000002,
758 0x900c, 0xffffffff, 0x0017071f,
759 0x8c18, 0xffffffff, 0x10101060,
760 0x8c1c, 0xffffffff, 0x00001010
761};
762
763static const u32 barts_golden_registers[] =
764{
765 0x5eb4, 0xffffffff, 0x00000002,
766 0x5e78, 0x8f311ff1, 0x001000f0,
767 0x3f90, 0xffff0000, 0xff000000,
768 0x9148, 0xffff0000, 0xff000000,
769 0x3f94, 0xffff0000, 0xff000000,
770 0x914c, 0xffff0000, 0xff000000,
771 0xc78, 0x00000080, 0x00000080,
772 0xbd4, 0x70073777, 0x00010001,
773 0xd02c, 0xbfffff1f, 0x08421000,
774 0xd0b8, 0x03773777, 0x02011003,
775 0x5bc0, 0x00200000, 0x50100000,
776 0x98f8, 0x33773777, 0x02011003,
777 0x98fc, 0xffffffff, 0x76543210,
778 0x7030, 0x31000311, 0x00000011,
779 0x2f48, 0x00000007, 0x02011003,
780 0x6b28, 0x00000010, 0x00000012,
781 0x7728, 0x00000010, 0x00000012,
782 0x10328, 0x00000010, 0x00000012,
783 0x10f28, 0x00000010, 0x00000012,
784 0x11b28, 0x00000010, 0x00000012,
785 0x12728, 0x00000010, 0x00000012,
786 0x240c, 0x000007ff, 0x00000380,
787 0x8a14, 0xf000001f, 0x00000007,
788 0x8b24, 0x3fff3fff, 0x00ff0fff,
789 0x8b10, 0x0000ff0f, 0x00000000,
790 0x28a4c, 0x07ffffff, 0x06000000,
791 0x10c, 0x00000001, 0x00010003,
792 0xa02c, 0xffffffff, 0x0000009b,
793 0x913c, 0x0000000f, 0x0100000a,
794 0x8d00, 0xffff7f7f, 0x100e4848,
795 0x8d04, 0x00ffffff, 0x00164745,
796 0x8c00, 0xfffc0003, 0xe4000003,
797 0x8c04, 0xf8ff00ff, 0x40600060,
798 0x8c08, 0x00ff00ff, 0x001c001c,
799 0x8cf0, 0x1fff1fff, 0x08e00620,
800 0x8c20, 0x0fff0fff, 0x00800080,
801 0x8c24, 0x0fff0fff, 0x00800080,
802 0x8c18, 0xffffffff, 0x20202078,
803 0x8c1c, 0x0000ffff, 0x00001010,
804 0x28350, 0x00000f01, 0x00000000,
805 0x9508, 0x3700001f, 0x00000002,
806 0x960c, 0xffffffff, 0x54763210,
807 0x88c4, 0x001f3ae3, 0x000000c2,
808 0x88d4, 0x0000001f, 0x00000010,
809 0x8974, 0xffffffff, 0x00000000
810};
811
812static const u32 turks_golden_registers[] =
813{
814 0x5eb4, 0xffffffff, 0x00000002,
815 0x5e78, 0x8f311ff1, 0x001000f0,
816 0x8c8, 0x00003000, 0x00001070,
817 0x8cc, 0x000fffff, 0x00040035,
818 0x3f90, 0xffff0000, 0xfff00000,
819 0x9148, 0xffff0000, 0xfff00000,
820 0x3f94, 0xffff0000, 0xfff00000,
821 0x914c, 0xffff0000, 0xfff00000,
822 0xc78, 0x00000080, 0x00000080,
823 0xbd4, 0x00073007, 0x00010002,
824 0xd02c, 0xbfffff1f, 0x08421000,
825 0xd0b8, 0x03773777, 0x02010002,
826 0x5bc0, 0x00200000, 0x50100000,
827 0x98f8, 0x33773777, 0x00010002,
828 0x98fc, 0xffffffff, 0x33221100,
829 0x7030, 0x31000311, 0x00000011,
830 0x2f48, 0x33773777, 0x00010002,
831 0x6b28, 0x00000010, 0x00000012,
832 0x7728, 0x00000010, 0x00000012,
833 0x10328, 0x00000010, 0x00000012,
834 0x10f28, 0x00000010, 0x00000012,
835 0x11b28, 0x00000010, 0x00000012,
836 0x12728, 0x00000010, 0x00000012,
837 0x240c, 0x000007ff, 0x00000380,
838 0x8a14, 0xf000001f, 0x00000007,
839 0x8b24, 0x3fff3fff, 0x00ff0fff,
840 0x8b10, 0x0000ff0f, 0x00000000,
841 0x28a4c, 0x07ffffff, 0x06000000,
842 0x10c, 0x00000001, 0x00010003,
843 0xa02c, 0xffffffff, 0x0000009b,
844 0x913c, 0x0000000f, 0x0100000a,
845 0x8d00, 0xffff7f7f, 0x100e4848,
846 0x8d04, 0x00ffffff, 0x00164745,
847 0x8c00, 0xfffc0003, 0xe4000003,
848 0x8c04, 0xf8ff00ff, 0x40600060,
849 0x8c08, 0x00ff00ff, 0x001c001c,
850 0x8cf0, 0x1fff1fff, 0x08e00410,
851 0x8c20, 0x0fff0fff, 0x00800080,
852 0x8c24, 0x0fff0fff, 0x00800080,
853 0x8c18, 0xffffffff, 0x20202078,
854 0x8c1c, 0x0000ffff, 0x00001010,
855 0x28350, 0x00000f01, 0x00000000,
856 0x9508, 0x3700001f, 0x00000002,
857 0x960c, 0xffffffff, 0x54763210,
858 0x88c4, 0x001f3ae3, 0x000000c2,
859 0x88d4, 0x0000001f, 0x00000010,
860 0x8974, 0xffffffff, 0x00000000
861};
862
863static const u32 caicos_golden_registers[] =
864{
865 0x5eb4, 0xffffffff, 0x00000002,
866 0x5e78, 0x8f311ff1, 0x001000f0,
867 0x8c8, 0x00003420, 0x00001450,
868 0x8cc, 0x000fffff, 0x00040035,
869 0x3f90, 0xffff0000, 0xfffc0000,
870 0x9148, 0xffff0000, 0xfffc0000,
871 0x3f94, 0xffff0000, 0xfffc0000,
872 0x914c, 0xffff0000, 0xfffc0000,
873 0xc78, 0x00000080, 0x00000080,
874 0xbd4, 0x00073007, 0x00010001,
875 0xd02c, 0xbfffff1f, 0x08421000,
876 0xd0b8, 0x03773777, 0x02010001,
877 0x5bc0, 0x00200000, 0x50100000,
878 0x98f8, 0x33773777, 0x02010001,
879 0x98fc, 0xffffffff, 0x33221100,
880 0x7030, 0x31000311, 0x00000011,
881 0x2f48, 0x33773777, 0x02010001,
882 0x6b28, 0x00000010, 0x00000012,
883 0x7728, 0x00000010, 0x00000012,
884 0x10328, 0x00000010, 0x00000012,
885 0x10f28, 0x00000010, 0x00000012,
886 0x11b28, 0x00000010, 0x00000012,
887 0x12728, 0x00000010, 0x00000012,
888 0x240c, 0x000007ff, 0x00000380,
889 0x8a14, 0xf000001f, 0x00000001,
890 0x8b24, 0x3fff3fff, 0x00ff0fff,
891 0x8b10, 0x0000ff0f, 0x00000000,
892 0x28a4c, 0x07ffffff, 0x06000000,
893 0x10c, 0x00000001, 0x00010003,
894 0xa02c, 0xffffffff, 0x0000009b,
895 0x913c, 0x0000000f, 0x0100000a,
896 0x8d00, 0xffff7f7f, 0x100e4848,
897 0x8d04, 0x00ffffff, 0x00164745,
898 0x8c00, 0xfffc0003, 0xe4000003,
899 0x8c04, 0xf8ff00ff, 0x40600060,
900 0x8c08, 0x00ff00ff, 0x001c001c,
901 0x8cf0, 0x1fff1fff, 0x08e00410,
902 0x8c20, 0x0fff0fff, 0x00800080,
903 0x8c24, 0x0fff0fff, 0x00800080,
904 0x8c18, 0xffffffff, 0x20202078,
905 0x8c1c, 0x0000ffff, 0x00001010,
906 0x28350, 0x00000f01, 0x00000000,
907 0x9508, 0x3700001f, 0x00000002,
908 0x960c, 0xffffffff, 0x54763210,
909 0x88c4, 0x001f3ae3, 0x000000c2,
910 0x88d4, 0x0000001f, 0x00000010,
911 0x8974, 0xffffffff, 0x00000000
912};
913
914static void evergreen_init_golden_registers(struct radeon_device *rdev)
915{
916 switch (rdev->family) {
917 case CHIP_CYPRESS:
918 case CHIP_HEMLOCK:
919 radeon_program_register_sequence(rdev,
920 evergreen_golden_registers,
921 (const u32)ARRAY_SIZE(evergreen_golden_registers));
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers2,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
925 radeon_program_register_sequence(rdev,
926 cypress_mgcg_init,
927 (const u32)ARRAY_SIZE(cypress_mgcg_init));
928 break;
929 case CHIP_JUNIPER:
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers));
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers2,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
936 radeon_program_register_sequence(rdev,
937 juniper_mgcg_init,
938 (const u32)ARRAY_SIZE(juniper_mgcg_init));
939 break;
940 case CHIP_REDWOOD:
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers));
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers2,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
947 radeon_program_register_sequence(rdev,
948 redwood_mgcg_init,
949 (const u32)ARRAY_SIZE(redwood_mgcg_init));
950 break;
951 case CHIP_CEDAR:
952 radeon_program_register_sequence(rdev,
953 cedar_golden_registers,
954 (const u32)ARRAY_SIZE(cedar_golden_registers));
955 radeon_program_register_sequence(rdev,
956 evergreen_golden_registers2,
957 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
958 radeon_program_register_sequence(rdev,
959 cedar_mgcg_init,
960 (const u32)ARRAY_SIZE(cedar_mgcg_init));
961 break;
962 case CHIP_PALM:
963 radeon_program_register_sequence(rdev,
964 wrestler_golden_registers,
965 (const u32)ARRAY_SIZE(wrestler_golden_registers));
966 break;
967 case CHIP_SUMO:
968 radeon_program_register_sequence(rdev,
969 supersumo_golden_registers,
970 (const u32)ARRAY_SIZE(supersumo_golden_registers));
971 break;
972 case CHIP_SUMO2:
973 radeon_program_register_sequence(rdev,
974 supersumo_golden_registers,
975 (const u32)ARRAY_SIZE(supersumo_golden_registers));
976 radeon_program_register_sequence(rdev,
977 sumo_golden_registers,
978 (const u32)ARRAY_SIZE(sumo_golden_registers));
979 break;
980 case CHIP_BARTS:
981 radeon_program_register_sequence(rdev,
982 barts_golden_registers,
983 (const u32)ARRAY_SIZE(barts_golden_registers));
984 break;
985 case CHIP_TURKS:
986 radeon_program_register_sequence(rdev,
987 turks_golden_registers,
988 (const u32)ARRAY_SIZE(turks_golden_registers));
989 break;
990 case CHIP_CAICOS:
991 radeon_program_register_sequence(rdev,
992 caicos_golden_registers,
993 (const u32)ARRAY_SIZE(caicos_golden_registers));
994 break;
995 default:
996 break;
997 }
998}
999
285484e2
JG		 1000void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1001 unsigned *bankh, unsigned *mtaspect,
1002 unsigned *tile_split)
1003{
1004 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1005 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1006 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1007 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1008 switch (*bankw) {
1009 default:
1010 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1011 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1012 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1013 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1014 }
1015 switch (*bankh) {
1016 default:
1017 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1018 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1019 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1020 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1021 }
1022 switch (*mtaspect) {
1023 default:
1024 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1025 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1026 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1027 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1028 }
1029}
1030
23d33ba3
AD		 1031static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1032 u32 cntl_reg, u32 status_reg)
1033{
1034 int r, i;
1035 struct atom_clock_dividers dividers;
1036
1037 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1038 clock, false, &dividers);
1039 if (r)
1040 return r;
1041
1042 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1043
1044 for (i = 0; i < 100; i++) {
1045 if (RREG32(status_reg) & DCLK_STATUS)
1046 break;
1047 mdelay(10);
1048 }
1049 if (i == 100)
1050 return -ETIMEDOUT;
1051
1052 return 0;
1053}
1054
1055int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1056{
1057 int r = 0;
1058 u32 cg_scratch = RREG32(CG_SCRATCH1);
1059
1060 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1061 if (r)
1062 goto done;
1063 cg_scratch &= 0xffff0000;
1064 cg_scratch |= vclk / 100; /* Mhz */
1065
1066 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1067 if (r)
1068 goto done;
1069 cg_scratch &= 0x0000ffff;
1070 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1071
1072done:
1073 WREG32(CG_SCRATCH1, cg_scratch);
1074
1075 return r;
1076}
1077
a8b4925c
AD		 1078int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1079{
1080 /* start off with something large */
facd112d 1081 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1082 int r;
1083
4ed10835
CK		 1084 /* bypass vclk and dclk with bclk */
1085 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1086 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1087 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1088
1089 /* put PLL in bypass mode */
1090 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1091
1092 if (!vclk || !dclk) {
1093 /* keep the Bypass mode, put PLL to sleep */
1094 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1095 return 0;
1096 }
1097
facd112d
CK		 1098 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1099 16384, 0x03FFFFFF, 0, 128, 5,
1100 &fb_div, &vclk_div, &dclk_div);
1101 if (r)
1102 return r;
a8b4925c
AD		 1103
1104 /* set VCO_MODE to 1 */
1105 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1106
1107 /* toggle UPLL_SLEEP to 1 then back to 0 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1109 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1110
1111 /* deassert UPLL_RESET */
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1113
1114 mdelay(1);
1115
facd112d 1116 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1117 if (r)
1118 return r;
1119
1120 /* assert UPLL_RESET again */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1122
1123 /* disable spread spectrum. */
1124 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1125
1126 /* set feedback divider */
facd112d 1127 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1128
1129 /* set ref divider to 0 */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1131
facd112d 1132 if (fb_div < 307200)
a8b4925c
AD		 1133 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1134 else
1135 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1136
1137 /* set PDIV_A and PDIV_B */
1138 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1139 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1140 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1141
1142 /* give the PLL some time to settle */
1143 mdelay(15);
1144
1145 /* deassert PLL_RESET */
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1147
1148 mdelay(15);
1149
1150 /* switch from bypass mode to normal mode */
1151 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1152
facd112d 1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1154 if (r)
1155 return r;
1156
1157 /* switch VCLK and DCLK selection */
1158 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1159 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1160 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1161
1162 mdelay(100);
1163
1164 return 0;
1165}
1166
d054ac16
AD		 1167void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1168{
1169 u16 ctl, v;
32195aec 1170 int err;
d054ac16 1171
32195aec 1172 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1173 if (err)
1174 return;
1175
1176 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1177
1178 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1179 * to avoid hangs or perfomance issues
1180 */
1181 if ((v == 0) || (v == 6) || (v == 7)) {
1182 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1183 ctl |= (2 << 12);
32195aec 1184 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1185 }
1186}
1187
10257a6d
AD		 1188static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1189{
1190 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1191 return true;
1192 else
1193 return false;
1194}
1195
1196static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1197{
1198 u32 pos1, pos2;
1199
1200 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1201 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202
1203 if (pos1 != pos2)
1204 return true;
1205 else
1206 return false;
1207}
1208
377edc8b
AD		 1209/**
1210 * dce4_wait_for_vblank - vblank wait asic callback.
1211 *
1212 * @rdev: radeon_device pointer
1213 * @crtc: crtc to wait for vblank on
1214 *
1215 * Wait for vblank on the requested crtc (evergreen+).
1216 */
3ae19b75
AD		 1217void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1218{
10257a6d 1219 unsigned i = 0;
3ae19b75 1220
4a15903d
AD		 1221 if (crtc >= rdev->num_crtc)
1222 return;
1223
10257a6d
AD		 1224 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1225 return;
1226
1227 /* depending on when we hit vblank, we may be close to active; if so,
1228 * wait for another frame.
1229 */
1230 while (dce4_is_in_vblank(rdev, crtc)) {
1231 if (i++ % 100 == 0) {
1232 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1233 break;
3ae19b75 1234 }
10257a6d
AD		 1235 }
1236
1237 while (!dce4_is_in_vblank(rdev, crtc)) {
1238 if (i++ % 100 == 0) {
1239 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1240 break;
3ae19b75
AD		 1241 }
1242 }
1243}
1244
377edc8b
AD		 1245/**
1246 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1247 *
1248 * @rdev: radeon_device pointer
1249 * @crtc: crtc to prepare for pageflip on
1250 *
1251 * Pre-pageflip callback (evergreen+).
1252 * Enables the pageflip irq (vblank irq).
1253 */
6f34be50
AD		 1254void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1255{
6f34be50
AD		 1256 /* enable the pflip int */
1257 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1258}
1259
377edc8b
AD		 1260/**
1261 * evergreen_post_page_flip - pos-pageflip callback.
1262 *
1263 * @rdev: radeon_device pointer
1264 * @crtc: crtc to cleanup pageflip on
1265 *
1266 * Post-pageflip callback (evergreen+).
1267 * Disables the pageflip irq (vblank irq).
1268 */
6f34be50
AD		 1269void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1270{
1271 /* disable the pflip int */
1272 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1273}
1274
377edc8b
AD		 1275/**
1276 * evergreen_page_flip - pageflip callback.
1277 *
1278 * @rdev: radeon_device pointer
1279 * @crtc_id: crtc to cleanup pageflip on
1280 * @crtc_base: new address of the crtc (GPU MC address)
1281 *
1282 * Does the actual pageflip (evergreen+).
1283 * During vblank we take the crtc lock and wait for the update_pending
1284 * bit to go high, when it does, we release the lock, and allow the
1285 * double buffered update to take place.
1286 * Returns the current update pending status.
1287 */
6f34be50
AD		 1288u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1289{
1290 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1291 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1292 int i;
6f34be50
AD		 1293
1294 /* Lock the graphics update lock */
1295 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1296 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1297
1298 /* update the scanout addresses */
1299 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1300 upper_32_bits(crtc_base));
1301 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1302 (u32)crtc_base);
1303
1304 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1305 upper_32_bits(crtc_base));
1306 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1307 (u32)crtc_base);
1308
1309 /* Wait for update_pending to go high. */
f6496479
AD		 1310 for (i = 0; i < rdev->usec_timeout; i++) {
1311 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1312 break;
1313 udelay(1);
1314 }
6f34be50
AD		 1315 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1316
1317 /* Unlock the lock, so double-buffering can take place inside vblank */
1318 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1319 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1320
1321 /* Return current update_pending status: */
1322 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1323}
1324
21a8122a 1325/* get temperature in millidegrees */
20d391d7 1326int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1327{
1c88d74f
AD		 1328 u32 temp, toffset;
1329 int actual_temp = 0;
67b3f823
AD		 1330
1331 if (rdev->family == CHIP_JUNIPER) {
1332 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1333 TOFFSET_SHIFT;
1334 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1335 TS0_ADC_DOUT_SHIFT;
1336
1337 if (toffset & 0x100)
1338 actual_temp = temp / 2 - (0x200 - toffset);
1339 else
1340 actual_temp = temp / 2 + toffset;
1341
1342 actual_temp = actual_temp * 1000;
1343
1344 } else {
1345 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1346 ASIC_T_SHIFT;
1347
1348 if (temp & 0x400)
1349 actual_temp = -256;
1350 else if (temp & 0x200)
1351 actual_temp = 255;
1352 else if (temp & 0x100) {
1353 actual_temp = temp & 0x1ff;
1354 actual_temp |= ~0x1ff;
1355 } else
1356 actual_temp = temp & 0xff;
1357
1358 actual_temp = (actual_temp * 1000) / 2;
1359 }
21a8122a 1360
67b3f823 1361 return actual_temp;
21a8122a
AD		 1362}
1363
20d391d7 1364int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1365{
1366 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1367 int actual_temp = temp - 49;
e33df25f
AD		 1368
1369 return actual_temp * 1000;
1370}
1371
377edc8b
AD		 1372/**
1373 * sumo_pm_init_profile - Initialize power profiles callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 *
1377 * Initialize the power states used in profile mode
1378 * (sumo, trinity, SI).
1379 * Used for profile mode only.
1380 */
a4c9e2ee
AD		 1381void sumo_pm_init_profile(struct radeon_device *rdev)
1382{
1383 int idx;
1384
1385 /* default */
1386 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1390
1391 /* low,mid sh/mh */
1392 if (rdev->flags & RADEON_IS_MOBILITY)
1393 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1394 else
1395 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1396
1397 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1401
1402 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1406
1407 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1411
1412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1416
1417 /* high sh/mh */
1418 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1419 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1423 rdev->pm.power_state[idx].num_clock_modes - 1;
1424
1425 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1429 rdev->pm.power_state[idx].num_clock_modes - 1;
1430}
1431
27810fb2
AD		 1432/**
1433 * btc_pm_init_profile - Initialize power profiles callback.
1434 *
1435 * @rdev: radeon_device pointer
1436 *
1437 * Initialize the power states used in profile mode
1438 * (BTC, cayman).
1439 * Used for profile mode only.
1440 */
1441void btc_pm_init_profile(struct radeon_device *rdev)
1442{
1443 int idx;
1444
1445 /* default */
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1450 /* starting with BTC, there is one state that is used for both
1451 * MH and SH. Difference is that we always use the high clock index for
1452 * mclk.
1453 */
1454 if (rdev->flags & RADEON_IS_MOBILITY)
1455 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1456 else
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1458 /* low sh */
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1463 /* mid sh */
1464 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1468 /* high sh */
1469 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1473 /* low mh */
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1478 /* mid mh */
1479 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1483 /* high mh */
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1488}
1489
377edc8b
AD		 1490/**
1491 * evergreen_pm_misc - set additional pm hw parameters callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Set non-clock parameters associated with a power state
1496 * (voltage, etc.) (evergreen+).
1497 */
49e02b73
AD		 1498void evergreen_pm_misc(struct radeon_device *rdev)
1499{
a081a9d6
RM		 1500 int req_ps_idx = rdev->pm.requested_power_state_index;
1501 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1502 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1503 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1504
2feea49a 1505 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 1506 /* 0xff01 is a flag rather then an actual voltage */
1507 if (voltage->voltage == 0xff01)
1508 return;
2feea49a 1509 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1510 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1511 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1512 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1513 }
7ae764b1
AD		 1514
1515 /* starting with BTC, there is one state that is used for both
1516 * MH and SH. Difference is that we always use the high clock index for
1517 * mclk and vddci.
1518 */
1519 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1520 (rdev->family >= CHIP_BARTS) &&
1521 rdev->pm.active_crtc_count &&
1522 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1523 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1524 voltage = &rdev->pm.power_state[req_ps_idx].
1525 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1526
a377e187
AD		 1527 /* 0xff01 is a flag rather then an actual voltage */
1528 if (voltage->vddci == 0xff01)
1529 return;
2feea49a
AD		 1530 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1531 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1532 rdev->pm.current_vddci = voltage->vddci;
1533 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1534 }
1535 }
49e02b73
AD		 1536}
1537
377edc8b
AD		 1538/**
1539 * evergreen_pm_prepare - pre-power state change callback.
1540 *
1541 * @rdev: radeon_device pointer
1542 *
1543 * Prepare for a power state change (evergreen+).
1544 */
49e02b73
AD		 1545void evergreen_pm_prepare(struct radeon_device *rdev)
1546{
1547 struct drm_device *ddev = rdev->ddev;
1548 struct drm_crtc *crtc;
1549 struct radeon_crtc *radeon_crtc;
1550 u32 tmp;
1551
1552 /* disable any active CRTCs */
1553 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1554 radeon_crtc = to_radeon_crtc(crtc);
1555 if (radeon_crtc->enabled) {
1556 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1557 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1558 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1559 }
1560 }
1561}
1562
377edc8b
AD		 1563/**
1564 * evergreen_pm_finish - post-power state change callback.
1565 *
1566 * @rdev: radeon_device pointer
1567 *
1568 * Clean up after a power state change (evergreen+).
1569 */
49e02b73
AD		 1570void evergreen_pm_finish(struct radeon_device *rdev)
1571{
1572 struct drm_device *ddev = rdev->ddev;
1573 struct drm_crtc *crtc;
1574 struct radeon_crtc *radeon_crtc;
1575 u32 tmp;
1576
1577 /* enable any active CRTCs */
1578 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1579 radeon_crtc = to_radeon_crtc(crtc);
1580 if (radeon_crtc->enabled) {
1581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1582 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1583 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1584 }
1585 }
1586}
1587
377edc8b
AD		 1588/**
1589 * evergreen_hpd_sense - hpd sense callback.
1590 *
1591 * @rdev: radeon_device pointer
1592 * @hpd: hpd (hotplug detect) pin
1593 *
1594 * Checks if a digital monitor is connected (evergreen+).
1595 * Returns true if connected, false if not connected.
1596 */
bcc1c2a1
AD		 1597bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1598{
1599 bool connected = false;
0ca2ab52
AD		 1600
1601 switch (hpd) {
1602 case RADEON_HPD_1:
1603 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1604 connected = true;
1605 break;
1606 case RADEON_HPD_2:
1607 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1608 connected = true;
1609 break;
1610 case RADEON_HPD_3:
1611 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1612 connected = true;
1613 break;
1614 case RADEON_HPD_4:
1615 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1616 connected = true;
1617 break;
1618 case RADEON_HPD_5:
1619 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1620 connected = true;
1621 break;
1622 case RADEON_HPD_6:
1623 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1624 connected = true;
1625 break;
1626 default:
1627 break;
1628 }
1629
bcc1c2a1
AD		 1630 return connected;
1631}
1632
377edc8b
AD		 1633/**
1634 * evergreen_hpd_set_polarity - hpd set polarity callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 * @hpd: hpd (hotplug detect) pin
1638 *
1639 * Set the polarity of the hpd pin (evergreen+).
1640 */
bcc1c2a1
AD		 1641void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1642 enum radeon_hpd_id hpd)
1643{
0ca2ab52
AD		 1644 u32 tmp;
1645 bool connected = evergreen_hpd_sense(rdev, hpd);
1646
1647 switch (hpd) {
1648 case RADEON_HPD_1:
1649 tmp = RREG32(DC_HPD1_INT_CONTROL);
1650 if (connected)
1651 tmp &= ~DC_HPDx_INT_POLARITY;
1652 else
1653 tmp |= DC_HPDx_INT_POLARITY;
1654 WREG32(DC_HPD1_INT_CONTROL, tmp);
1655 break;
1656 case RADEON_HPD_2:
1657 tmp = RREG32(DC_HPD2_INT_CONTROL);
1658 if (connected)
1659 tmp &= ~DC_HPDx_INT_POLARITY;
1660 else
1661 tmp |= DC_HPDx_INT_POLARITY;
1662 WREG32(DC_HPD2_INT_CONTROL, tmp);
1663 break;
1664 case RADEON_HPD_3:
1665 tmp = RREG32(DC_HPD3_INT_CONTROL);
1666 if (connected)
1667 tmp &= ~DC_HPDx_INT_POLARITY;
1668 else
1669 tmp |= DC_HPDx_INT_POLARITY;
1670 WREG32(DC_HPD3_INT_CONTROL, tmp);
1671 break;
1672 case RADEON_HPD_4:
1673 tmp = RREG32(DC_HPD4_INT_CONTROL);
1674 if (connected)
1675 tmp &= ~DC_HPDx_INT_POLARITY;
1676 else
1677 tmp |= DC_HPDx_INT_POLARITY;
1678 WREG32(DC_HPD4_INT_CONTROL, tmp);
1679 break;
1680 case RADEON_HPD_5:
1681 tmp = RREG32(DC_HPD5_INT_CONTROL);
1682 if (connected)
1683 tmp &= ~DC_HPDx_INT_POLARITY;
1684 else
1685 tmp |= DC_HPDx_INT_POLARITY;
1686 WREG32(DC_HPD5_INT_CONTROL, tmp);
1687 break;
1688 case RADEON_HPD_6:
1689 tmp = RREG32(DC_HPD6_INT_CONTROL);
1690 if (connected)
1691 tmp &= ~DC_HPDx_INT_POLARITY;
1692 else
1693 tmp |= DC_HPDx_INT_POLARITY;
1694 WREG32(DC_HPD6_INT_CONTROL, tmp);
1695 break;
1696 default:
1697 break;
1698 }
bcc1c2a1
AD		 1699}
1700
377edc8b
AD		 1701/**
1702 * evergreen_hpd_init - hpd setup callback.
1703 *
1704 * @rdev: radeon_device pointer
1705 *
1706 * Setup the hpd pins used by the card (evergreen+).
1707 * Enable the pin, set the polarity, and enable the hpd interrupts.
1708 */
bcc1c2a1
AD		 1709void evergreen_hpd_init(struct radeon_device *rdev)
1710{
0ca2ab52
AD		 1711 struct drm_device *dev = rdev->ddev;
1712 struct drm_connector *connector;
fb98257a 1713 unsigned enabled = 0;
0ca2ab52
AD		 1714 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1715 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1716
0ca2ab52
AD		 1717 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1718 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1719
1720 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1721 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1722 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1723 * aux dp channel on imac and help (but not completely fix)
1724 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1725 * also avoid interrupt storms during dpms.
1726 */
1727 continue;
1728 }
0ca2ab52
AD		 1729 switch (radeon_connector->hpd.hpd) {
1730 case RADEON_HPD_1:
1731 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1732 break;
1733 case RADEON_HPD_2:
1734 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1735 break;
1736 case RADEON_HPD_3:
1737 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1738 break;
1739 case RADEON_HPD_4:
1740 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1741 break;
1742 case RADEON_HPD_5:
1743 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1744 break;
1745 case RADEON_HPD_6:
1746 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1747 break;
1748 default:
1749 break;
1750 }
64912e99 1751 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1752 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1753 }
fb98257a 1754 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1755}
1756
377edc8b
AD		 1757/**
1758 * evergreen_hpd_fini - hpd tear down callback.
1759 *
1760 * @rdev: radeon_device pointer
1761 *
1762 * Tear down the hpd pins used by the card (evergreen+).
1763 * Disable the hpd interrupts.
1764 */
0ca2ab52 1765void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1766{
0ca2ab52
AD		 1767 struct drm_device *dev = rdev->ddev;
1768 struct drm_connector *connector;
fb98257a 1769 unsigned disabled = 0;
0ca2ab52
AD		 1770
1771 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1772 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1773 switch (radeon_connector->hpd.hpd) {
1774 case RADEON_HPD_1:
1775 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1776 break;
1777 case RADEON_HPD_2:
1778 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1779 break;
1780 case RADEON_HPD_3:
1781 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1782 break;
1783 case RADEON_HPD_4:
1784 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1785 break;
1786 case RADEON_HPD_5:
1787 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1788 break;
1789 case RADEON_HPD_6:
1790 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1791 break;
1792 default:
1793 break;
1794 }
fb98257a 1795 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1796 }
fb98257a 1797 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1798}
1799
f9d9c362
AD		 1800/* watermark setup */
1801
1802static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1803 struct radeon_crtc *radeon_crtc,
1804 struct drm_display_mode *mode,
1805 struct drm_display_mode *other_mode)
1806{
12dfc843 1807 u32 tmp;
f9d9c362
AD		 1808 /*
1809 * Line Buffer Setup
1810 * There are 3 line buffers, each one shared by 2 display controllers.
1811 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1812 * the display controllers. The paritioning is done via one of four
1813 * preset allocations specified in bits 2:0:
1814 * first display controller
1815 * 0 - first half of lb (3840 * 2)
1816 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1817 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1818 * 3 - first 1/4 of lb (1920 * 2)
1819 * second display controller
1820 * 4 - second half of lb (3840 * 2)
1821 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1822 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1823 * 7 - last 1/4 of lb (1920 * 2)
1824 */
12dfc843
AD		 1825 /* this can get tricky if we have two large displays on a paired group
1826 * of crtcs. Ideally for multiple large displays we'd assign them to
1827 * non-linked crtcs for maximum line buffer allocation.
1828 */
1829 if (radeon_crtc->base.enabled && mode) {
1830 if (other_mode)
f9d9c362 1831 tmp = 0; /* 1/2 */
12dfc843
AD		 1832 else
1833 tmp = 2; /* whole */
1834 } else
1835 tmp = 0;
f9d9c362
AD		 1836
1837 /* second controller of the pair uses second half of the lb */
1838 if (radeon_crtc->crtc_id % 2)
1839 tmp += 4;
1840 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1841
12dfc843
AD		 1842 if (radeon_crtc->base.enabled && mode) {
1843 switch (tmp) {
1844 case 0:
1845 case 4:
1846 default:
1847 if (ASIC_IS_DCE5(rdev))
1848 return 4096 * 2;
1849 else
1850 return 3840 * 2;
1851 case 1:
1852 case 5:
1853 if (ASIC_IS_DCE5(rdev))
1854 return 6144 * 2;
1855 else
1856 return 5760 * 2;
1857 case 2:
1858 case 6:
1859 if (ASIC_IS_DCE5(rdev))
1860 return 8192 * 2;
1861 else
1862 return 7680 * 2;
1863 case 3:
1864 case 7:
1865 if (ASIC_IS_DCE5(rdev))
1866 return 2048 * 2;
1867 else
1868 return 1920 * 2;
1869 }
f9d9c362 1870 }
12dfc843
AD		 1871
1872 /* controller not enabled, so no lb used */
1873 return 0;
f9d9c362
AD		 1874}
1875
ca7db22b 1876u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1877{
1878 u32 tmp = RREG32(MC_SHARED_CHMAP);
1879
1880 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1881 case 0:
1882 default:
1883 return 1;
1884 case 1:
1885 return 2;
1886 case 2:
1887 return 4;
1888 case 3:
1889 return 8;
1890 }
1891}
1892
1893struct evergreen_wm_params {
1894 u32 dram_channels; /* number of dram channels */
1895 u32 yclk; /* bandwidth per dram data pin in kHz */
1896 u32 sclk; /* engine clock in kHz */
1897 u32 disp_clk; /* display clock in kHz */
1898 u32 src_width; /* viewport width */
1899 u32 active_time; /* active display time in ns */
1900 u32 blank_time; /* blank time in ns */
1901 bool interlaced; /* mode is interlaced */
1902 fixed20_12 vsc; /* vertical scale ratio */
1903 u32 num_heads; /* number of active crtcs */
1904 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1905 u32 lb_size; /* line buffer allocated to pipe */
1906 u32 vtaps; /* vertical scaler taps */
1907};
1908
1909static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1910{
1911 /* Calculate DRAM Bandwidth and the part allocated to display. */
1912 fixed20_12 dram_efficiency; /* 0.7 */
1913 fixed20_12 yclk, dram_channels, bandwidth;
1914 fixed20_12 a;
1915
1916 a.full = dfixed_const(1000);
1917 yclk.full = dfixed_const(wm->yclk);
1918 yclk.full = dfixed_div(yclk, a);
1919 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1920 a.full = dfixed_const(10);
1921 dram_efficiency.full = dfixed_const(7);
1922 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1923 bandwidth.full = dfixed_mul(dram_channels, yclk);
1924 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1925
1926 return dfixed_trunc(bandwidth);
1927}
1928
1929static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1930{
1931 /* Calculate DRAM Bandwidth and the part allocated to display. */
1932 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1933 fixed20_12 yclk, dram_channels, bandwidth;
1934 fixed20_12 a;
1935
1936 a.full = dfixed_const(1000);
1937 yclk.full = dfixed_const(wm->yclk);
1938 yclk.full = dfixed_div(yclk, a);
1939 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940 a.full = dfixed_const(10);
1941 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1942 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1943 bandwidth.full = dfixed_mul(dram_channels, yclk);
1944 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1945
1946 return dfixed_trunc(bandwidth);
1947}
1948
1949static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1950{
1951 /* Calculate the display Data return Bandwidth */
1952 fixed20_12 return_efficiency; /* 0.8 */
1953 fixed20_12 sclk, bandwidth;
1954 fixed20_12 a;
1955
1956 a.full = dfixed_const(1000);
1957 sclk.full = dfixed_const(wm->sclk);
1958 sclk.full = dfixed_div(sclk, a);
1959 a.full = dfixed_const(10);
1960 return_efficiency.full = dfixed_const(8);
1961 return_efficiency.full = dfixed_div(return_efficiency, a);
1962 a.full = dfixed_const(32);
1963 bandwidth.full = dfixed_mul(a, sclk);
1964 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1965
1966 return dfixed_trunc(bandwidth);
1967}
1968
1969static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1970{
1971 /* Calculate the DMIF Request Bandwidth */
1972 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1973 fixed20_12 disp_clk, bandwidth;
1974 fixed20_12 a;
1975
1976 a.full = dfixed_const(1000);
1977 disp_clk.full = dfixed_const(wm->disp_clk);
1978 disp_clk.full = dfixed_div(disp_clk, a);
1979 a.full = dfixed_const(10);
1980 disp_clk_request_efficiency.full = dfixed_const(8);
1981 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1982 a.full = dfixed_const(32);
1983 bandwidth.full = dfixed_mul(a, disp_clk);
1984 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1985
1986 return dfixed_trunc(bandwidth);
1987}
1988
1989static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1990{
1991 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1992 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1993 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1994 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1995
1996 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1997}
1998
1999static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2000{
2001 /* Calculate the display mode Average Bandwidth
2002 * DisplayMode should contain the source and destination dimensions,
2003 * timing, etc.
2004 */
2005 fixed20_12 bpp;
2006 fixed20_12 line_time;
2007 fixed20_12 src_width;
2008 fixed20_12 bandwidth;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1000);
2012 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2013 line_time.full = dfixed_div(line_time, a);
2014 bpp.full = dfixed_const(wm->bytes_per_pixel);
2015 src_width.full = dfixed_const(wm->src_width);
2016 bandwidth.full = dfixed_mul(src_width, bpp);
2017 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2018 bandwidth.full = dfixed_div(bandwidth, line_time);
2019
2020 return dfixed_trunc(bandwidth);
2021}
2022
2023static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2024{
2025 /* First calcualte the latency in ns */
2026 u32 mc_latency = 2000; /* 2000 ns. */
2027 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2028 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2029 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2030 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2031 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2032 (wm->num_heads * cursor_line_pair_return_time);
2033 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2034 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2035 fixed20_12 a, b, c;
2036
2037 if (wm->num_heads == 0)
2038 return 0;
2039
2040 a.full = dfixed_const(2);
2041 b.full = dfixed_const(1);
2042 if ((wm->vsc.full > a.full) ||
2043 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2044 (wm->vtaps >= 5) ||
2045 ((wm->vsc.full >= a.full) && wm->interlaced))
2046 max_src_lines_per_dst_line = 4;
2047 else
2048 max_src_lines_per_dst_line = 2;
2049
2050 a.full = dfixed_const(available_bandwidth);
2051 b.full = dfixed_const(wm->num_heads);
2052 a.full = dfixed_div(a, b);
2053
2054 b.full = dfixed_const(1000);
2055 c.full = dfixed_const(wm->disp_clk);
2056 b.full = dfixed_div(c, b);
2057 c.full = dfixed_const(wm->bytes_per_pixel);
2058 b.full = dfixed_mul(b, c);
2059
2060 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2061
2062 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2063 b.full = dfixed_const(1000);
2064 c.full = dfixed_const(lb_fill_bw);
2065 b.full = dfixed_div(c, b);
2066 a.full = dfixed_div(a, b);
2067 line_fill_time = dfixed_trunc(a);
2068
2069 if (line_fill_time < wm->active_time)
2070 return latency;
2071 else
2072 return latency + (line_fill_time - wm->active_time);
2073
2074}
2075
2076static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2077{
2078 if (evergreen_average_bandwidth(wm) <=
2079 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2080 return true;
2081 else
2082 return false;
2083};
2084
2085static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2086{
2087 if (evergreen_average_bandwidth(wm) <=
2088 (evergreen_available_bandwidth(wm) / wm->num_heads))
2089 return true;
2090 else
2091 return false;
2092};
2093
2094static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2095{
2096 u32 lb_partitions = wm->lb_size / wm->src_width;
2097 u32 line_time = wm->active_time + wm->blank_time;
2098 u32 latency_tolerant_lines;
2099 u32 latency_hiding;
2100 fixed20_12 a;
2101
2102 a.full = dfixed_const(1);
2103 if (wm->vsc.full > a.full)
2104 latency_tolerant_lines = 1;
2105 else {
2106 if (lb_partitions <= (wm->vtaps + 1))
2107 latency_tolerant_lines = 1;
2108 else
2109 latency_tolerant_lines = 2;
2110 }
2111
2112 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2113
2114 if (evergreen_latency_watermark(wm) <= latency_hiding)
2115 return true;
2116 else
2117 return false;
2118}
2119
2120static void evergreen_program_watermarks(struct radeon_device *rdev,
2121 struct radeon_crtc *radeon_crtc,
2122 u32 lb_size, u32 num_heads)
2123{
2124 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2125 struct evergreen_wm_params wm_low, wm_high;
2126 u32 dram_channels;
f9d9c362
AD		 2127 u32 pixel_period;
2128 u32 line_time = 0;
2129 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2130 u32 priority_a_mark = 0, priority_b_mark = 0;
2131 u32 priority_a_cnt = PRIORITY_OFF;
2132 u32 priority_b_cnt = PRIORITY_OFF;
2133 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2134 u32 tmp, arb_control3;
2135 fixed20_12 a, b, c;
2136
2137 if (radeon_crtc->base.enabled && num_heads && mode) {
2138 pixel_period = 1000000 / (u32)mode->clock;
2139 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2140 priority_a_cnt = 0;
2141 priority_b_cnt = 0;
cf0cfdd7
AD		 2142 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2143
2144 /* watermark for high clocks */
2145 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2146 wm_high.yclk =
2147 radeon_dpm_get_mclk(rdev, false) * 10;
2148 wm_high.sclk =
2149 radeon_dpm_get_sclk(rdev, false) * 10;
2150 } else {
2151 wm_high.yclk = rdev->pm.current_mclk * 10;
2152 wm_high.sclk = rdev->pm.current_sclk * 10;
2153 }
f9d9c362 2154
cf0cfdd7
AD		 2155 wm_high.disp_clk = mode->clock;
2156 wm_high.src_width = mode->crtc_hdisplay;
2157 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2158 wm_high.blank_time = line_time - wm_high.active_time;
2159 wm_high.interlaced = false;
f9d9c362 2160 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2161 wm_high.interlaced = true;
2162 wm_high.vsc = radeon_crtc->vsc;
2163 wm_high.vtaps = 1;
f9d9c362 2164 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2165 wm_high.vtaps = 2;
2166 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2167 wm_high.lb_size = lb_size;
2168 wm_high.dram_channels = dram_channels;
2169 wm_high.num_heads = num_heads;
2170
2171 /* watermark for low clocks */
2172 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2173 wm_low.yclk =
2174 radeon_dpm_get_mclk(rdev, true) * 10;
2175 wm_low.sclk =
2176 radeon_dpm_get_sclk(rdev, true) * 10;
2177 } else {
2178 wm_low.yclk = rdev->pm.current_mclk * 10;
2179 wm_low.sclk = rdev->pm.current_sclk * 10;
2180 }
2181
2182 wm_low.disp_clk = mode->clock;
2183 wm_low.src_width = mode->crtc_hdisplay;
2184 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2185 wm_low.blank_time = line_time - wm_low.active_time;
2186 wm_low.interlaced = false;
2187 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2188 wm_low.interlaced = true;
2189 wm_low.vsc = radeon_crtc->vsc;
2190 wm_low.vtaps = 1;
2191 if (radeon_crtc->rmx_type != RMX_OFF)
2192 wm_low.vtaps = 2;
2193 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2194 wm_low.lb_size = lb_size;
2195 wm_low.dram_channels = dram_channels;
2196 wm_low.num_heads = num_heads;
f9d9c362
AD		 2197
2198 /* set for high clocks */
cf0cfdd7 2199 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2200 /* set for low clocks */
cf0cfdd7 2201 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2202
2203 /* possibly force display priority to high */
2204 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2205 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2206 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2207 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2208 (rdev->disp_priority == 2)) {
cf0cfdd7 2209 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2210 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2211 }
2212 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2213 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2214 !evergreen_check_latency_hiding(&wm_low) ||
2215 (rdev->disp_priority == 2)) {
2216 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2217 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2218 }
2219
2220 a.full = dfixed_const(1000);
2221 b.full = dfixed_const(mode->clock);
2222 b.full = dfixed_div(b, a);
2223 c.full = dfixed_const(latency_watermark_a);
2224 c.full = dfixed_mul(c, b);
2225 c.full = dfixed_mul(c, radeon_crtc->hsc);
2226 c.full = dfixed_div(c, a);
2227 a.full = dfixed_const(16);
2228 c.full = dfixed_div(c, a);
2229 priority_a_mark = dfixed_trunc(c);
2230 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2231
2232 a.full = dfixed_const(1000);
2233 b.full = dfixed_const(mode->clock);
2234 b.full = dfixed_div(b, a);
2235 c.full = dfixed_const(latency_watermark_b);
2236 c.full = dfixed_mul(c, b);
2237 c.full = dfixed_mul(c, radeon_crtc->hsc);
2238 c.full = dfixed_div(c, a);
2239 a.full = dfixed_const(16);
2240 c.full = dfixed_div(c, a);
2241 priority_b_mark = dfixed_trunc(c);
2242 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2243 }
2244
2245 /* select wm A */
2246 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2247 tmp = arb_control3;
2248 tmp &= ~LATENCY_WATERMARK_MASK(3);
2249 tmp |= LATENCY_WATERMARK_MASK(1);
2250 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2251 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2252 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2253 LATENCY_HIGH_WATERMARK(line_time)));
2254 /* select wm B */
2255 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2256 tmp &= ~LATENCY_WATERMARK_MASK(3);
2257 tmp |= LATENCY_WATERMARK_MASK(2);
2258 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2259 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2260 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2261 LATENCY_HIGH_WATERMARK(line_time)));
2262 /* restore original selection */
2263 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2264
2265 /* write the priority marks */
2266 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2267 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2268
2269}
2270
377edc8b
AD		 2271/**
2272 * evergreen_bandwidth_update - update display watermarks callback.
2273 *
2274 * @rdev: radeon_device pointer
2275 *
2276 * Update the display watermarks based on the requested mode(s)
2277 * (evergreen+).
2278 */
0ca2ab52 2279void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2280{
f9d9c362
AD		 2281 struct drm_display_mode *mode0 = NULL;
2282 struct drm_display_mode *mode1 = NULL;
2283 u32 num_heads = 0, lb_size;
2284 int i;
2285
2286 radeon_update_display_priority(rdev);
2287
2288 for (i = 0; i < rdev->num_crtc; i++) {
2289 if (rdev->mode_info.crtcs[i]->base.enabled)
2290 num_heads++;
2291 }
2292 for (i = 0; i < rdev->num_crtc; i += 2) {
2293 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2294 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2295 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2296 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2297 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2298 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2299 }
bcc1c2a1
AD		 2300}
2301
377edc8b
AD		 2302/**
2303 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2304 *
2305 * @rdev: radeon_device pointer
2306 *
2307 * Wait for the MC (memory controller) to be idle.
2308 * (evergreen+).
2309 * Returns 0 if the MC is idle, -1 if not.
2310 */
b9952a8a 2311int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2312{
2313 unsigned i;
2314 u32 tmp;
2315
2316 for (i = 0; i < rdev->usec_timeout; i++) {
2317 /* read MC_STATUS */
2318 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2319 if (!tmp)
2320 return 0;
2321 udelay(1);
2322 }
2323 return -1;
2324}
2325
2326/*
2327 * GART
2328 */
0fcdb61e
AD		 2329void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2330{
2331 unsigned i;
2332 u32 tmp;
2333
6f2f48a9
AD		 2334 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2335
0fcdb61e
AD		 2336 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2337 for (i = 0; i < rdev->usec_timeout; i++) {
2338 /* read MC_STATUS */
2339 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2340 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2341 if (tmp == 2) {
2342 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2343 return;
2344 }
2345 if (tmp) {
2346 return;
2347 }
2348 udelay(1);
2349 }
2350}
2351
1109ca09 2352static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2353{
2354 u32 tmp;
0fcdb61e 2355 int r;
bcc1c2a1 2356
c9a1be96 2357 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2358 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2359 return -EINVAL;
2360 }
2361 r = radeon_gart_table_vram_pin(rdev);
2362 if (r)
2363 return r;
82568565 2364 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2365 /* Setup L2 cache */
2366 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2367 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2368 EFFECTIVE_L2_QUEUE_SIZE(7));
2369 WREG32(VM_L2_CNTL2, 0);
2370 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2371 /* Setup TLB control */
2372 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2373 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2374 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2375 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2376 if (rdev->flags & RADEON_IS_IGP) {
2377 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2378 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2379 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2380 } else {
2381 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2382 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2383 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2384 if ((rdev->family == CHIP_JUNIPER) ||
2385 (rdev->family == CHIP_CYPRESS) ||
2386 (rdev->family == CHIP_HEMLOCK) ||
2387 (rdev->family == CHIP_BARTS))
2388 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2389 }
bcc1c2a1
AD		 2390 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2391 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2392 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2393 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2394 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2395 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2396 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2397 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2398 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2399 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2400 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2401 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2402
0fcdb61e 2403 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2404 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2405 (unsigned)(rdev->mc.gtt_size >> 20),
2406 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2407 rdev->gart.ready = true;
2408 return 0;
2409}
2410
1109ca09 2411static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2412{
2413 u32 tmp;
bcc1c2a1
AD		 2414
2415 /* Disable all tables */
0fcdb61e
AD		 2416 WREG32(VM_CONTEXT0_CNTL, 0);
2417 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2418
2419 /* Setup L2 cache */
2420 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2421 EFFECTIVE_L2_QUEUE_SIZE(7));
2422 WREG32(VM_L2_CNTL2, 0);
2423 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2424 /* Setup TLB control */
2425 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2426 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2430 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2431 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2432 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2433 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2434}
2435
1109ca09 2436static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2437{
2438 evergreen_pcie_gart_disable(rdev);
2439 radeon_gart_table_vram_free(rdev);
2440 radeon_gart_fini(rdev);
2441}
2442
2443
1109ca09 2444static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2445{
2446 u32 tmp;
bcc1c2a1
AD		 2447
2448 /* Setup L2 cache */
2449 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2450 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2451 EFFECTIVE_L2_QUEUE_SIZE(7));
2452 WREG32(VM_L2_CNTL2, 0);
2453 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2454 /* Setup TLB control */
2455 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2456 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2457 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2458 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2459 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2462 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2463 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2464 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2465 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2466 WREG32(VM_CONTEXT0_CNTL, 0);
2467 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2468}
2469
b9952a8a 2470void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2471{
62444b74
AD		 2472 u32 crtc_enabled, tmp, frame_count, blackout;
2473 int i, j;
2474
5153550a
AD		 2475 if (!ASIC_IS_NODCE(rdev)) {
2476 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2477 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2478
5153550a
AD		 2479 /* disable VGA render */
2480 WREG32(VGA_RENDER_CONTROL, 0);
2481 }
62444b74
AD		 2482 /* blank the display controllers */
2483 for (i = 0; i < rdev->num_crtc; i++) {
2484 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2485 if (crtc_enabled) {
2486 save->crtc_enabled[i] = true;
2487 if (ASIC_IS_DCE6(rdev)) {
2488 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2489 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2490 radeon_wait_for_vblank(rdev, i);
abf1457b 2491 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2492 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2493 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2494 }
2495 } else {
2496 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2497 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2498 radeon_wait_for_vblank(rdev, i);
abf1457b 2499 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2500 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2501 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2502 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2503 }
2504 }
2505 /* wait for the next frame */
2506 frame_count = radeon_get_vblank_counter(rdev, i);
2507 for (j = 0; j < rdev->usec_timeout; j++) {
2508 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2509 break;
2510 udelay(1);
2511 }
abf1457b
AD		 2512
2513 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2514 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2515 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2516 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2517 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2518 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2519 save->crtc_enabled[i] = false;
2520 /* ***** */
804cc4a0
AD		 2521 } else {
2522 save->crtc_enabled[i] = false;
62444b74 2523 }
18007401 2524 }
bcc1c2a1 2525
62444b74
AD		 2526 radeon_mc_wait_for_idle(rdev);
2527
2528 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2529 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2530 /* Block CPU access */
2531 WREG32(BIF_FB_EN, 0);
2532 /* blackout the MC */
2533 blackout &= ~BLACKOUT_MODE_MASK;
2534 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2535 }
ed39fadd
AD		 2536 /* wait for the MC to settle */
2537 udelay(100);
968c0166
AD		 2538
2539 /* lock double buffered regs */
2540 for (i = 0; i < rdev->num_crtc; i++) {
2541 if (save->crtc_enabled[i]) {
2542 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2543 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2544 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2545 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2546 }
2547 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2548 if (!(tmp & 1)) {
2549 tmp |= 1;
2550 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2551 }
2552 }
2553 }
bcc1c2a1
AD		 2554}
2555
b9952a8a 2556void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2557{
62444b74
AD		 2558 u32 tmp, frame_count;
2559 int i, j;
18007401 2560
62444b74
AD		 2561 /* update crtc base addresses */
2562 for (i = 0; i < rdev->num_crtc; i++) {
2563 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2564 upper_32_bits(rdev->mc.vram_start));
62444b74 2565 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2566 upper_32_bits(rdev->mc.vram_start));
62444b74 2567 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2568 (u32)rdev->mc.vram_start);
62444b74 2569 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2570 (u32)rdev->mc.vram_start);
2571 }
5153550a
AD		 2572
2573 if (!ASIC_IS_NODCE(rdev)) {
2574 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2575 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2576 }
62444b74 2577
968c0166
AD		 2578 /* unlock regs and wait for update */
2579 for (i = 0; i < rdev->num_crtc; i++) {
2580 if (save->crtc_enabled[i]) {
2581 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2582 if ((tmp & 0x3) != 0) {
2583 tmp &= ~0x3;
2584 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2585 }
2586 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2587 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2588 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2589 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2590 }
2591 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2592 if (tmp & 1) {
2593 tmp &= ~1;
2594 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2595 }
2596 for (j = 0; j < rdev->usec_timeout; j++) {
2597 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2598 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2599 break;
2600 udelay(1);
2601 }
2602 }
2603 }
2604
62444b74
AD		 2605 /* unblackout the MC */
2606 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2607 tmp &= ~BLACKOUT_MODE_MASK;
2608 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2609 /* allow CPU access */
2610 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2611
2612 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2613 if (save->crtc_enabled[i]) {
62444b74
AD		 2614 if (ASIC_IS_DCE6(rdev)) {
2615 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2616 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2617 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2618 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2619 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2620 } else {
2621 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2622 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2623 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2624 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2625 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2626 }
2627 /* wait for the next frame */
2628 frame_count = radeon_get_vblank_counter(rdev, i);
2629 for (j = 0; j < rdev->usec_timeout; j++) {
2630 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2631 break;
2632 udelay(1);
2633 }
2634 }
2635 }
5153550a
AD		 2636 if (!ASIC_IS_NODCE(rdev)) {
2637 /* Unlock vga access */
2638 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2639 mdelay(1);
2640 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2641 }
bcc1c2a1
AD		 2642}
2643
755d819e 2644void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2645{
2646 struct evergreen_mc_save save;
2647 u32 tmp;
2648 int i, j;
2649
2650 /* Initialize HDP */
2651 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2652 WREG32((0x2c14 + j), 0x00000000);
2653 WREG32((0x2c18 + j), 0x00000000);
2654 WREG32((0x2c1c + j), 0x00000000);
2655 WREG32((0x2c20 + j), 0x00000000);
2656 WREG32((0x2c24 + j), 0x00000000);
2657 }
2658 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2659
2660 evergreen_mc_stop(rdev, &save);
2661 if (evergreen_mc_wait_for_idle(rdev)) {
2662 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2663 }
2664 /* Lockout access through VGA aperture*/
2665 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2666 /* Update configuration */
2667 if (rdev->flags & RADEON_IS_AGP) {
2668 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2669 /* VRAM before AGP */
2670 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2671 rdev->mc.vram_start >> 12);
2672 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2673 rdev->mc.gtt_end >> 12);
2674 } else {
2675 /* VRAM after AGP */
2676 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2677 rdev->mc.gtt_start >> 12);
2678 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2679 rdev->mc.vram_end >> 12);
2680 }
2681 } else {
2682 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2683 rdev->mc.vram_start >> 12);
2684 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2685 rdev->mc.vram_end >> 12);
2686 }
3b9832f6 2687 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2688 /* llano/ontario only */
2689 if ((rdev->family == CHIP_PALM) ||
2690 (rdev->family == CHIP_SUMO) ||
2691 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2692 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2693 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2694 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2695 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2696 }
bcc1c2a1
AD		 2697 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2698 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2699 WREG32(MC_VM_FB_LOCATION, tmp);
2700 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2701 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2702 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2703 if (rdev->flags & RADEON_IS_AGP) {
2704 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2705 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2706 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2707 } else {
2708 WREG32(MC_VM_AGP_BASE, 0);
2709 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2710 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2711 }
2712 if (evergreen_mc_wait_for_idle(rdev)) {
2713 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2714 }
2715 evergreen_mc_resume(rdev, &save);
2716 /* we need to own VRAM, so turn off the VGA renderer here
2717 * to stop it overwriting our objects */
2718 rv515_vga_render_disable(rdev);
2719}
2720
bcc1c2a1
AD		 2721/*
2722 * CP.
2723 */
12920591
AD		 2724void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2725{
876dc9f3 2726 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2727 u32 next_rptr;
7b1f2485 2728
12920591 2729 /* set to DX10/11 mode */
e32eb50d
CK		 2730 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2731 radeon_ring_write(ring, 1);
45df6803
CK		 2732
2733 if (ring->rptr_save_reg) {
89d35807 2734 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2735 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2736 radeon_ring_write(ring, ((ring->rptr_save_reg -
2737 PACKET3_SET_CONFIG_REG_START) >> 2));
2738 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2739 } else if (rdev->wb.enabled) {
2740 next_rptr = ring->wptr + 5 + 4;
2741 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2742 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2743 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2744 radeon_ring_write(ring, next_rptr);
2745 radeon_ring_write(ring, 0);
45df6803
CK		 2746 }
2747
e32eb50d
CK		 2748 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2749 radeon_ring_write(ring,
0f234f5f
AD		 2750#ifdef __BIG_ENDIAN
2751 (2 << 0) |
2752#endif
2753 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2754 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2755 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2756}
2757
bcc1c2a1
AD		 2758
2759static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2760{
fe251e2f
AD		 2761 const __be32 *fw_data;
2762 int i;
2763
2764 if (!rdev->me_fw || !rdev->pfp_fw)
2765 return -EINVAL;
bcc1c2a1 2766
fe251e2f 2767 r700_cp_stop(rdev);
0f234f5f
AD		 2768 WREG32(CP_RB_CNTL,
2769#ifdef __BIG_ENDIAN
2770 BUF_SWAP_32BIT |
2771#endif
2772 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2773
2774 fw_data = (const __be32 *)rdev->pfp_fw->data;
2775 WREG32(CP_PFP_UCODE_ADDR, 0);
2776 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2777 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2778 WREG32(CP_PFP_UCODE_ADDR, 0);
2779
2780 fw_data = (const __be32 *)rdev->me_fw->data;
2781 WREG32(CP_ME_RAM_WADDR, 0);
2782 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2783 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2784
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2786 WREG32(CP_ME_RAM_WADDR, 0);
2787 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2788 return 0;
2789}
2790
7e7b41d2
AD		 2791static int evergreen_cp_start(struct radeon_device *rdev)
2792{
e32eb50d 2793 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2794 int r, i;
7e7b41d2
AD		 2795 uint32_t cp_me;
2796
e32eb50d 2797 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2798 if (r) {
2799 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2800 return r;
2801 }
e32eb50d
CK		 2802 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2803 radeon_ring_write(ring, 0x1);
2804 radeon_ring_write(ring, 0x0);
2805 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2806 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2807 radeon_ring_write(ring, 0);
2808 radeon_ring_write(ring, 0);
2809 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2810
2811 cp_me = 0xff;
2812 WREG32(CP_ME_CNTL, cp_me);
2813
e32eb50d 2814 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2815 if (r) {
2816 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2817 return r;
2818 }
2281a378
AD		 2819
2820 /* setup clear context state */
e32eb50d
CK		 2821 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2822 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2823
2824 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2825 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2826
e32eb50d
CK		 2827 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2829
2830 /* set clear context state */
e32eb50d
CK		 2831 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2832 radeon_ring_write(ring, 0);
2281a378
AD		 2833
2834 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2835 radeon_ring_write(ring, 0xc0026f00);
2836 radeon_ring_write(ring, 0x00000000);
2837 radeon_ring_write(ring, 0x00000000);
2838 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2839
2840 /* Clear consts */
e32eb50d
CK		 2841 radeon_ring_write(ring, 0xc0036f00);
2842 radeon_ring_write(ring, 0x00000bc4);
2843 radeon_ring_write(ring, 0xffffffff);
2844 radeon_ring_write(ring, 0xffffffff);
2845 radeon_ring_write(ring, 0xffffffff);
2281a378 2846
e32eb50d
CK		 2847 radeon_ring_write(ring, 0xc0026900);
2848 radeon_ring_write(ring, 0x00000316);
2849 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2850 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2851
e32eb50d 2852 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2853
2854 return 0;
2855}
2856
1109ca09 2857static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2858{
e32eb50d 2859 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2860 u32 tmp;
2861 u32 rb_bufsz;
2862 int r;
2863
2864 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2865 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2866 SOFT_RESET_PA |
2867 SOFT_RESET_SH |
2868 SOFT_RESET_VGT |
a49a50da 2869 SOFT_RESET_SPI |
fe251e2f
AD		 2870 SOFT_RESET_SX));
2871 RREG32(GRBM_SOFT_RESET);
2872 mdelay(15);
2873 WREG32(GRBM_SOFT_RESET, 0);
2874 RREG32(GRBM_SOFT_RESET);
2875
2876 /* Set ring buffer size */
e32eb50d 2877 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2878 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2879#ifdef __BIG_ENDIAN
2880 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2881#endif
fe251e2f 2882 WREG32(CP_RB_CNTL, tmp);
15d3332f 2883 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2884 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2885
2886 /* Set the write pointer delay */
2887 WREG32(CP_RB_WPTR_DELAY, 0);
2888
2889 /* Initialize the ring buffer's read and write pointers */
2890 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2891 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2892 ring->wptr = 0;
2893 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2894
48fc7f7e 2895 /* set the wb address whether it's enabled or not */
0f234f5f 2896 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2897 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2898 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2899 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2900
2901 if (rdev->wb.enabled)
2902 WREG32(SCRATCH_UMSK, 0xff);
2903 else {
2904 tmp |= RB_NO_UPDATE;
2905 WREG32(SCRATCH_UMSK, 0);
2906 }
2907
fe251e2f
AD		 2908 mdelay(1);
2909 WREG32(CP_RB_CNTL, tmp);
2910
e32eb50d 2911 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2912 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2913
e32eb50d 2914 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2915
7e7b41d2 2916 evergreen_cp_start(rdev);
e32eb50d 2917 ring->ready = true;
f712812e 2918 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2919 if (r) {
e32eb50d 2920 ring->ready = false;
fe251e2f
AD		 2921 return r;
2922 }
2923 return 0;
2924}
bcc1c2a1
AD		 2925
2926/*
2927 * Core functions
2928 */
bcc1c2a1
AD		 2929static void evergreen_gpu_init(struct radeon_device *rdev)
2930{
416a2bd2 2931 u32 gb_addr_config;
32fcdbf4 2932 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2933 u32 sx_debug_1;
2934 u32 smx_dc_ctl0;
2935 u32 sq_config;
2936 u32 sq_lds_resource_mgmt;
2937 u32 sq_gpr_resource_mgmt_1;
2938 u32 sq_gpr_resource_mgmt_2;
2939 u32 sq_gpr_resource_mgmt_3;
2940 u32 sq_thread_resource_mgmt;
2941 u32 sq_thread_resource_mgmt_2;
2942 u32 sq_stack_resource_mgmt_1;
2943 u32 sq_stack_resource_mgmt_2;
2944 u32 sq_stack_resource_mgmt_3;
2945 u32 vgt_cache_invalidation;
f25a5c63 2946 u32 hdp_host_path_cntl, tmp;
416a2bd2 2947 u32 disabled_rb_mask;
32fcdbf4
AD		 2948 int i, j, num_shader_engines, ps_thread_count;
2949
2950 switch (rdev->family) {
2951 case CHIP_CYPRESS:
2952 case CHIP_HEMLOCK:
2953 rdev->config.evergreen.num_ses = 2;
2954 rdev->config.evergreen.max_pipes = 4;
2955 rdev->config.evergreen.max_tile_pipes = 8;
2956 rdev->config.evergreen.max_simds = 10;
2957 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2958 rdev->config.evergreen.max_gprs = 256;
2959 rdev->config.evergreen.max_threads = 248;
2960 rdev->config.evergreen.max_gs_threads = 32;
2961 rdev->config.evergreen.max_stack_entries = 512;
2962 rdev->config.evergreen.sx_num_of_sets = 4;
2963 rdev->config.evergreen.sx_max_export_size = 256;
2964 rdev->config.evergreen.sx_max_export_pos_size = 64;
2965 rdev->config.evergreen.sx_max_export_smx_size = 192;
2966 rdev->config.evergreen.max_hw_contexts = 8;
2967 rdev->config.evergreen.sq_num_cf_insts = 2;
2968
2969 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2970 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2971 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2972 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2973 break;
2974 case CHIP_JUNIPER:
2975 rdev->config.evergreen.num_ses = 1;
2976 rdev->config.evergreen.max_pipes = 4;
2977 rdev->config.evergreen.max_tile_pipes = 4;
2978 rdev->config.evergreen.max_simds = 10;
2979 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2980 rdev->config.evergreen.max_gprs = 256;
2981 rdev->config.evergreen.max_threads = 248;
2982 rdev->config.evergreen.max_gs_threads = 32;
2983 rdev->config.evergreen.max_stack_entries = 512;
2984 rdev->config.evergreen.sx_num_of_sets = 4;
2985 rdev->config.evergreen.sx_max_export_size = 256;
2986 rdev->config.evergreen.sx_max_export_pos_size = 64;
2987 rdev->config.evergreen.sx_max_export_smx_size = 192;
2988 rdev->config.evergreen.max_hw_contexts = 8;
2989 rdev->config.evergreen.sq_num_cf_insts = 2;
2990
2991 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2992 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2993 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2994 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2995 break;
2996 case CHIP_REDWOOD:
2997 rdev->config.evergreen.num_ses = 1;
2998 rdev->config.evergreen.max_pipes = 4;
2999 rdev->config.evergreen.max_tile_pipes = 4;
3000 rdev->config.evergreen.max_simds = 5;
3001 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3002 rdev->config.evergreen.max_gprs = 256;
3003 rdev->config.evergreen.max_threads = 248;
3004 rdev->config.evergreen.max_gs_threads = 32;
3005 rdev->config.evergreen.max_stack_entries = 256;
3006 rdev->config.evergreen.sx_num_of_sets = 4;
3007 rdev->config.evergreen.sx_max_export_size = 256;
3008 rdev->config.evergreen.sx_max_export_pos_size = 64;
3009 rdev->config.evergreen.sx_max_export_smx_size = 192;
3010 rdev->config.evergreen.max_hw_contexts = 8;
3011 rdev->config.evergreen.sq_num_cf_insts = 2;
3012
3013 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3014 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3015 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3016 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3017 break;
3018 case CHIP_CEDAR:
3019 default:
3020 rdev->config.evergreen.num_ses = 1;
3021 rdev->config.evergreen.max_pipes = 2;
3022 rdev->config.evergreen.max_tile_pipes = 2;
3023 rdev->config.evergreen.max_simds = 2;
3024 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3025 rdev->config.evergreen.max_gprs = 256;
3026 rdev->config.evergreen.max_threads = 192;
3027 rdev->config.evergreen.max_gs_threads = 16;
3028 rdev->config.evergreen.max_stack_entries = 256;
3029 rdev->config.evergreen.sx_num_of_sets = 4;
3030 rdev->config.evergreen.sx_max_export_size = 128;
3031 rdev->config.evergreen.sx_max_export_pos_size = 32;
3032 rdev->config.evergreen.sx_max_export_smx_size = 96;
3033 rdev->config.evergreen.max_hw_contexts = 4;
3034 rdev->config.evergreen.sq_num_cf_insts = 1;
3035
d5e455e4
AD		 3036 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3037 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3038 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3039 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3040 break;
3041 case CHIP_PALM:
3042 rdev->config.evergreen.num_ses = 1;
3043 rdev->config.evergreen.max_pipes = 2;
3044 rdev->config.evergreen.max_tile_pipes = 2;
3045 rdev->config.evergreen.max_simds = 2;
3046 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3047 rdev->config.evergreen.max_gprs = 256;
3048 rdev->config.evergreen.max_threads = 192;
3049 rdev->config.evergreen.max_gs_threads = 16;
3050 rdev->config.evergreen.max_stack_entries = 256;
3051 rdev->config.evergreen.sx_num_of_sets = 4;
3052 rdev->config.evergreen.sx_max_export_size = 128;
3053 rdev->config.evergreen.sx_max_export_pos_size = 32;
3054 rdev->config.evergreen.sx_max_export_smx_size = 96;
3055 rdev->config.evergreen.max_hw_contexts = 4;
3056 rdev->config.evergreen.sq_num_cf_insts = 1;
3057
d5c5a72f
AD		 3058 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3059 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3060 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3061 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3062 break;
3063 case CHIP_SUMO:
3064 rdev->config.evergreen.num_ses = 1;
3065 rdev->config.evergreen.max_pipes = 4;
bd25f078 3066 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3067 if (rdev->pdev->device == 0x9648)
3068 rdev->config.evergreen.max_simds = 3;
3069 else if ((rdev->pdev->device == 0x9647) ||
3070 (rdev->pdev->device == 0x964a))
3071 rdev->config.evergreen.max_simds = 4;
3072 else
3073 rdev->config.evergreen.max_simds = 5;
3074 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3075 rdev->config.evergreen.max_gprs = 256;
3076 rdev->config.evergreen.max_threads = 248;
3077 rdev->config.evergreen.max_gs_threads = 32;
3078 rdev->config.evergreen.max_stack_entries = 256;
3079 rdev->config.evergreen.sx_num_of_sets = 4;
3080 rdev->config.evergreen.sx_max_export_size = 256;
3081 rdev->config.evergreen.sx_max_export_pos_size = 64;
3082 rdev->config.evergreen.sx_max_export_smx_size = 192;
3083 rdev->config.evergreen.max_hw_contexts = 8;
3084 rdev->config.evergreen.sq_num_cf_insts = 2;
3085
3086 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3087 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3088 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3089 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3090 break;
3091 case CHIP_SUMO2:
3092 rdev->config.evergreen.num_ses = 1;
3093 rdev->config.evergreen.max_pipes = 4;
3094 rdev->config.evergreen.max_tile_pipes = 4;
3095 rdev->config.evergreen.max_simds = 2;
3096 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3097 rdev->config.evergreen.max_gprs = 256;
3098 rdev->config.evergreen.max_threads = 248;
3099 rdev->config.evergreen.max_gs_threads = 32;
3100 rdev->config.evergreen.max_stack_entries = 512;
3101 rdev->config.evergreen.sx_num_of_sets = 4;
3102 rdev->config.evergreen.sx_max_export_size = 256;
3103 rdev->config.evergreen.sx_max_export_pos_size = 64;
3104 rdev->config.evergreen.sx_max_export_smx_size = 192;
3105 rdev->config.evergreen.max_hw_contexts = 8;
3106 rdev->config.evergreen.sq_num_cf_insts = 2;
3107
adb68fa2
AD		 3108 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3109 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3111 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3112 break;
3113 case CHIP_BARTS:
3114 rdev->config.evergreen.num_ses = 2;
3115 rdev->config.evergreen.max_pipes = 4;
3116 rdev->config.evergreen.max_tile_pipes = 8;
3117 rdev->config.evergreen.max_simds = 7;
3118 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3119 rdev->config.evergreen.max_gprs = 256;
3120 rdev->config.evergreen.max_threads = 248;
3121 rdev->config.evergreen.max_gs_threads = 32;
3122 rdev->config.evergreen.max_stack_entries = 512;
3123 rdev->config.evergreen.sx_num_of_sets = 4;
3124 rdev->config.evergreen.sx_max_export_size = 256;
3125 rdev->config.evergreen.sx_max_export_pos_size = 64;
3126 rdev->config.evergreen.sx_max_export_smx_size = 192;
3127 rdev->config.evergreen.max_hw_contexts = 8;
3128 rdev->config.evergreen.sq_num_cf_insts = 2;
3129
3130 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3131 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3132 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3133 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3134 break;
3135 case CHIP_TURKS:
3136 rdev->config.evergreen.num_ses = 1;
3137 rdev->config.evergreen.max_pipes = 4;
3138 rdev->config.evergreen.max_tile_pipes = 4;
3139 rdev->config.evergreen.max_simds = 6;
3140 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3141 rdev->config.evergreen.max_gprs = 256;
3142 rdev->config.evergreen.max_threads = 248;
3143 rdev->config.evergreen.max_gs_threads = 32;
3144 rdev->config.evergreen.max_stack_entries = 256;
3145 rdev->config.evergreen.sx_num_of_sets = 4;
3146 rdev->config.evergreen.sx_max_export_size = 256;
3147 rdev->config.evergreen.sx_max_export_pos_size = 64;
3148 rdev->config.evergreen.sx_max_export_smx_size = 192;
3149 rdev->config.evergreen.max_hw_contexts = 8;
3150 rdev->config.evergreen.sq_num_cf_insts = 2;
3151
3152 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3153 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3154 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3155 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3156 break;
3157 case CHIP_CAICOS:
3158 rdev->config.evergreen.num_ses = 1;
bd25f078 3159 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3160 rdev->config.evergreen.max_tile_pipes = 2;
3161 rdev->config.evergreen.max_simds = 2;
3162 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 192;
3165 rdev->config.evergreen.max_gs_threads = 16;
3166 rdev->config.evergreen.max_stack_entries = 256;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 128;
3169 rdev->config.evergreen.sx_max_export_pos_size = 32;
3170 rdev->config.evergreen.sx_max_export_smx_size = 96;
3171 rdev->config.evergreen.max_hw_contexts = 4;
3172 rdev->config.evergreen.sq_num_cf_insts = 1;
3173
32fcdbf4
AD		 3174 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3177 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3178 break;
3179 }
3180
3181 /* Initialize HDP */
3182 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3183 WREG32((0x2c14 + j), 0x00000000);
3184 WREG32((0x2c18 + j), 0x00000000);
3185 WREG32((0x2c1c + j), 0x00000000);
3186 WREG32((0x2c20 + j), 0x00000000);
3187 WREG32((0x2c24 + j), 0x00000000);
3188 }
3189
3190 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3191
d054ac16
AD		 3192 evergreen_fix_pci_max_read_req_size(rdev);
3193
32fcdbf4 3194 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3195 if ((rdev->family == CHIP_PALM) ||
3196 (rdev->family == CHIP_SUMO) ||
3197 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3198 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3199 else
3200 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3201
1aa52bd3
AD		 3202 /* setup tiling info dword. gb_addr_config is not adequate since it does
3203 * not have bank info, so create a custom tiling dword.
3204 * bits 3:0 num_pipes
3205 * bits 7:4 num_banks
3206 * bits 11:8 group_size
3207 * bits 15:12 row_size
3208 */
3209 rdev->config.evergreen.tile_config = 0;
3210 switch (rdev->config.evergreen.max_tile_pipes) {
3211 case 1:
3212 default:
3213 rdev->config.evergreen.tile_config |= (0 << 0);
3214 break;
3215 case 2:
3216 rdev->config.evergreen.tile_config |= (1 << 0);
3217 break;
3218 case 4:
3219 rdev->config.evergreen.tile_config |= (2 << 0);
3220 break;
3221 case 8:
3222 rdev->config.evergreen.tile_config |= (3 << 0);
3223 break;
3224 }
d698a34d 3225 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3226 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3227 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3228 else {
c8d15edc
AD		 3229 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3230 case 0: /* four banks */
29d65406 3231 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3232 break;
3233 case 1: /* eight banks */
3234 rdev->config.evergreen.tile_config |= 1 << 4;
3235 break;
3236 case 2: /* sixteen banks */
3237 default:
3238 rdev->config.evergreen.tile_config |= 2 << 4;
3239 break;
3240 }
29d65406 3241 }
416a2bd2 3242 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3243 rdev->config.evergreen.tile_config |=
3244 ((gb_addr_config & 0x30000000) >> 28) << 12;
3245
416a2bd2 3246 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3247
416a2bd2
AD		 3248 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3249 u32 efuse_straps_4;
3250 u32 efuse_straps_3;
32fcdbf4 3251
ff82bbc4
AD		 3252 efuse_straps_4 = RREG32_RCU(0x204);
3253 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3254 tmp = (((efuse_straps_4 & 0xf) << 4) |
3255 ((efuse_straps_3 & 0xf0000000) >> 28));
3256 } else {
3257 tmp = 0;
3258 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3259 u32 rb_disable_bitmap;
3260
3261 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3262 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3263 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3264 tmp <<= 4;
3265 tmp |= rb_disable_bitmap;
32fcdbf4 3266 }
416a2bd2
AD		 3267 }
3268 /* enabled rb are just the one not disabled :) */
3269 disabled_rb_mask = tmp;
cedb655a
AD		 3270 tmp = 0;
3271 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3272 tmp |= (1 << i);
3273 /* if all the backends are disabled, fix it up here */
3274 if ((disabled_rb_mask & tmp) == tmp) {
3275 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3276 disabled_rb_mask &= ~(1 << i);
3277 }
32fcdbf4 3278
416a2bd2
AD		 3279 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3280 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3281
416a2bd2
AD		 3282 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3283 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3284 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3285 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3286 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3287 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3288 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3289
f7eb9730
AD		 3290 if ((rdev->config.evergreen.max_backends == 1) &&
3291 (rdev->flags & RADEON_IS_IGP)) {
3292 if ((disabled_rb_mask & 3) == 1) {
3293 /* RB0 disabled, RB1 enabled */
3294 tmp = 0x11111111;
3295 } else {
3296 /* RB1 disabled, RB0 enabled */
3297 tmp = 0x00000000;
3298 }
3299 } else {
3300 tmp = gb_addr_config & NUM_PIPES_MASK;
3301 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3302 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3303 }
416a2bd2 3304 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3305
3306 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3307 WREG32(CGTS_TCC_DISABLE, 0);
3308 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3309 WREG32(CGTS_USER_TCC_DISABLE, 0);
3310
3311 /* set HW defaults for 3D engine */
3312 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3313 ROQ_IB2_START(0x2b)));
3314
3315 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3316
3317 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3318 SYNC_GRADIENT |
3319 SYNC_WALKER |
3320 SYNC_ALIGNER));
3321
3322 sx_debug_1 = RREG32(SX_DEBUG_1);
3323 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3324 WREG32(SX_DEBUG_1, sx_debug_1);
3325
3326
3327 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3328 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3329 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3330 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3331
b866d133
AD		 3332 if (rdev->family <= CHIP_SUMO2)
3333 WREG32(SMX_SAR_CTL0, 0x00010000);
3334
32fcdbf4
AD		 3335 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3336 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3337 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3338
3339 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3340 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3341 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3342
3343 WREG32(VGT_NUM_INSTANCES, 1);
3344 WREG32(SPI_CONFIG_CNTL, 0);
3345 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3346 WREG32(CP_PERFMON_CNTL, 0);
3347
3348 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3349 FETCH_FIFO_HIWATER(0x4) |
3350 DONE_FIFO_HIWATER(0xe0) |
3351 ALU_UPDATE_FIFO_HIWATER(0x8)));
3352
3353 sq_config = RREG32(SQ_CONFIG);
3354 sq_config &= ~(PS_PRIO(3) |
3355 VS_PRIO(3) |
3356 GS_PRIO(3) |
3357 ES_PRIO(3));
3358 sq_config |= (VC_ENABLE |
3359 EXPORT_SRC_C |
3360 PS_PRIO(0) |
3361 VS_PRIO(1) |
3362 GS_PRIO(2) |
3363 ES_PRIO(3));
3364
d5e455e4
AD		 3365 switch (rdev->family) {
3366 case CHIP_CEDAR:
3367 case CHIP_PALM:
d5c5a72f
AD		 3368 case CHIP_SUMO:
3369 case CHIP_SUMO2:
adb68fa2 3370 case CHIP_CAICOS:
32fcdbf4
AD		 3371 /* no vertex cache */
3372 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3373 break;
3374 default:
3375 break;
3376 }
32fcdbf4
AD		 3377
3378 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3379
3380 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3381 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3382 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3383 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3384 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3385 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3386 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3387
d5e455e4
AD		 3388 switch (rdev->family) {
3389 case CHIP_CEDAR:
3390 case CHIP_PALM:
d5c5a72f
AD		 3391 case CHIP_SUMO:
3392 case CHIP_SUMO2:
32fcdbf4 3393 ps_thread_count = 96;
d5e455e4
AD		 3394 break;
3395 default:
32fcdbf4 3396 ps_thread_count = 128;
d5e455e4
AD		 3397 break;
3398 }
32fcdbf4
AD		 3399
3400 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3401 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3402 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3403 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3404 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3405 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3406
3407 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3408 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3409 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3410 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3411 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3412 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3413
3414 WREG32(SQ_CONFIG, sq_config);
3415 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3416 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3417 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3418 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3419 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3420 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3421 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3422 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3423 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3424 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3425
3426 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3427 FORCE_EOV_MAX_REZ_CNT(255)));
3428
d5e455e4
AD		 3429 switch (rdev->family) {
3430 case CHIP_CEDAR:
3431 case CHIP_PALM:
d5c5a72f
AD		 3432 case CHIP_SUMO:
3433 case CHIP_SUMO2:
adb68fa2 3434 case CHIP_CAICOS:
32fcdbf4 3435 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3436 break;
3437 default:
32fcdbf4 3438 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3439 break;
3440 }
32fcdbf4
AD		 3441 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3442 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3443
3444 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3445 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3446 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3447
60a4a3e0
AD		 3448 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3449 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3450
32fcdbf4
AD		 3451 WREG32(CB_PERF_CTR0_SEL_0, 0);
3452 WREG32(CB_PERF_CTR0_SEL_1, 0);
3453 WREG32(CB_PERF_CTR1_SEL_0, 0);
3454 WREG32(CB_PERF_CTR1_SEL_1, 0);
3455 WREG32(CB_PERF_CTR2_SEL_0, 0);
3456 WREG32(CB_PERF_CTR2_SEL_1, 0);
3457 WREG32(CB_PERF_CTR3_SEL_0, 0);
3458 WREG32(CB_PERF_CTR3_SEL_1, 0);
3459
60a4a3e0
AD		 3460 /* clear render buffer base addresses */
3461 WREG32(CB_COLOR0_BASE, 0);
3462 WREG32(CB_COLOR1_BASE, 0);
3463 WREG32(CB_COLOR2_BASE, 0);
3464 WREG32(CB_COLOR3_BASE, 0);
3465 WREG32(CB_COLOR4_BASE, 0);
3466 WREG32(CB_COLOR5_BASE, 0);
3467 WREG32(CB_COLOR6_BASE, 0);
3468 WREG32(CB_COLOR7_BASE, 0);
3469 WREG32(CB_COLOR8_BASE, 0);
3470 WREG32(CB_COLOR9_BASE, 0);
3471 WREG32(CB_COLOR10_BASE, 0);
3472 WREG32(CB_COLOR11_BASE, 0);
3473
3474 /* set the shader const cache sizes to 0 */
3475 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3476 WREG32(i, 0);
3477 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3478 WREG32(i, 0);
3479
f25a5c63
AD		 3480 tmp = RREG32(HDP_MISC_CNTL);
3481 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3482 WREG32(HDP_MISC_CNTL, tmp);
3483
32fcdbf4
AD		 3484 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3485 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3486
3487 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3488
3489 udelay(50);
3490
bcc1c2a1
AD		 3491}
3492
3493int evergreen_mc_init(struct radeon_device *rdev)
3494{
bcc1c2a1
AD		 3495 u32 tmp;
3496 int chansize, numchan;
bcc1c2a1
AD		 3497
3498 /* Get VRAM informations */
3499 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3500 if ((rdev->family == CHIP_PALM) ||
3501 (rdev->family == CHIP_SUMO) ||
3502 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3503 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3504 else
3505 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3506 if (tmp & CHANSIZE_OVERRIDE) {
3507 chansize = 16;
3508 } else if (tmp & CHANSIZE_MASK) {
3509 chansize = 64;
3510 } else {
3511 chansize = 32;
3512 }
3513 tmp = RREG32(MC_SHARED_CHMAP);
3514 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3515 case 0:
3516 default:
3517 numchan = 1;
3518 break;
3519 case 1:
3520 numchan = 2;
3521 break;
3522 case 2:
3523 numchan = 4;
3524 break;
3525 case 3:
3526 numchan = 8;
3527 break;
3528 }
3529 rdev->mc.vram_width = numchan * chansize;
3530 /* Could aper size report 0 ? */
01d73a69
JC		 3531 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3532 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3533 /* Setup GPU memory space */
05b3ef69
AD		 3534 if ((rdev->family == CHIP_PALM) ||
3535 (rdev->family == CHIP_SUMO) ||
3536 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3537 /* size in bytes on fusion */
3538 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3539 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3540 } else {
05b3ef69 3541 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3542 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3543 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3544 }
51e5fcd3 3545 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3546 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3547 radeon_update_bandwidth_info(rdev);
3548
bcc1c2a1
AD		 3549 return 0;
3550}
d594e46a 3551
187e3593 3552void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3553{
64c56e8c 3554 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3555 RREG32(GRBM_STATUS));
64c56e8c 3556 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3557 RREG32(GRBM_STATUS_SE0));
64c56e8c 3558 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3559 RREG32(GRBM_STATUS_SE1));
64c56e8c 3560 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3561 RREG32(SRBM_STATUS));
a65a4369
AD		 3562 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3563 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3564 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3565 RREG32(CP_STALLED_STAT1));
3566 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3567 RREG32(CP_STALLED_STAT2));
3568 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3569 RREG32(CP_BUSY_STAT));
3570 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3571 RREG32(CP_STAT));
eaaa6983
JG		 3572 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3573 RREG32(DMA_STATUS_REG));
168757ea
AD		 3574 if (rdev->family >= CHIP_CAYMAN) {
3575 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3576 RREG32(DMA_STATUS_REG + 0x800));
3577 }
0ecebb9e
AD		 3578}
3579
168757ea 3580bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3581{
a65a4369
AD		 3582 u32 crtc_hung = 0;
3583 u32 crtc_status[6];
3584 u32 i, j, tmp;
3585
3586 for (i = 0; i < rdev->num_crtc; i++) {
3587 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3588 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3589 crtc_hung |= (1 << i);
3590 }
3591 }
3592
3593 for (j = 0; j < 10; j++) {
3594 for (i = 0; i < rdev->num_crtc; i++) {
3595 if (crtc_hung & (1 << i)) {
3596 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3597 if (tmp != crtc_status[i])
3598 crtc_hung &= ~(1 << i);
3599 }
3600 }
3601 if (crtc_hung == 0)
3602 return false;
3603 udelay(100);
3604 }
3605
3606 return true;
3607}
3608
3609static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3610{
3611 u32 reset_mask = 0;
b7630473 3612 u32 tmp;
0ecebb9e 3613
a65a4369
AD		 3614 /* GRBM_STATUS */
3615 tmp = RREG32(GRBM_STATUS);
3616 if (tmp & (PA_BUSY | SC_BUSY |
3617 SH_BUSY | SX_BUSY |
3618 TA_BUSY | VGT_BUSY |
3619 DB_BUSY | CB_BUSY |
3620 SPI_BUSY | VGT_BUSY_NO_DMA))
3621 reset_mask |= RADEON_RESET_GFX;
3622
3623 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3624 CP_BUSY | CP_COHERENCY_BUSY))
3625 reset_mask |= RADEON_RESET_CP;
3626
3627 if (tmp & GRBM_EE_BUSY)
3628 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3629
a65a4369
AD		 3630 /* DMA_STATUS_REG */
3631 tmp = RREG32(DMA_STATUS_REG);
3632 if (!(tmp & DMA_IDLE))
3633 reset_mask |= RADEON_RESET_DMA;
3634
3635 /* SRBM_STATUS2 */
3636 tmp = RREG32(SRBM_STATUS2);
3637 if (tmp & DMA_BUSY)
3638 reset_mask |= RADEON_RESET_DMA;
3639
3640 /* SRBM_STATUS */
3641 tmp = RREG32(SRBM_STATUS);
3642 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3643 reset_mask |= RADEON_RESET_RLC;
3644
3645 if (tmp & IH_BUSY)
3646 reset_mask |= RADEON_RESET_IH;
3647
3648 if (tmp & SEM_BUSY)
3649 reset_mask |= RADEON_RESET_SEM;
3650
3651 if (tmp & GRBM_RQ_PENDING)
3652 reset_mask |= RADEON_RESET_GRBM;
3653
3654 if (tmp & VMC_BUSY)
3655 reset_mask |= RADEON_RESET_VMC;
3656
3657 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3658 MCC_BUSY | MCD_BUSY))
3659 reset_mask |= RADEON_RESET_MC;
3660
3661 if (evergreen_is_display_hung(rdev))
3662 reset_mask |= RADEON_RESET_DISPLAY;
3663
3664 /* VM_L2_STATUS */
3665 tmp = RREG32(VM_L2_STATUS);
3666 if (tmp & L2_BUSY)
3667 reset_mask |= RADEON_RESET_VMC;
3668
d808fc88
AD		 3669 /* Skip MC reset as it's mostly likely not hung, just busy */
3670 if (reset_mask & RADEON_RESET_MC) {
3671 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3672 reset_mask &= ~RADEON_RESET_MC;
3673 }
3674
a65a4369
AD		 3675 return reset_mask;
3676}
3677
3678static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3679{
3680 struct evergreen_mc_save save;
3681 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3682 u32 tmp;
19fc42ed 3683
0ecebb9e 3684 if (reset_mask == 0)
a65a4369 3685 return;
0ecebb9e
AD		 3686
3687 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3688
b7630473
AD		 3689 evergreen_print_gpu_status_regs(rdev);
3690
b7630473
AD		 3691 /* Disable CP parsing/prefetching */
3692 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3693
3694 if (reset_mask & RADEON_RESET_DMA) {
3695 /* Disable DMA */
3696 tmp = RREG32(DMA_RB_CNTL);
3697 tmp &= ~DMA_RB_ENABLE;
3698 WREG32(DMA_RB_CNTL, tmp);
3699 }
3700
b21b6e7a
AD		 3701 udelay(50);
3702
3703 evergreen_mc_stop(rdev, &save);
3704 if (evergreen_mc_wait_for_idle(rdev)) {
3705 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3706 }
3707
b7630473
AD		 3708 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3709 grbm_soft_reset |= SOFT_RESET_DB |
3710 SOFT_RESET_CB |
3711 SOFT_RESET_PA |
3712 SOFT_RESET_SC |
3713 SOFT_RESET_SPI |
3714 SOFT_RESET_SX |
3715 SOFT_RESET_SH |
3716 SOFT_RESET_TC |
3717 SOFT_RESET_TA |
3718 SOFT_RESET_VC |
3719 SOFT_RESET_VGT;
3720 }
3721
3722 if (reset_mask & RADEON_RESET_CP) {
3723 grbm_soft_reset |= SOFT_RESET_CP |
3724 SOFT_RESET_VGT;
3725
3726 srbm_soft_reset |= SOFT_RESET_GRBM;
3727 }
0ecebb9e
AD		 3728
3729 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3730 srbm_soft_reset |= SOFT_RESET_DMA;
3731
a65a4369
AD		 3732 if (reset_mask & RADEON_RESET_DISPLAY)
3733 srbm_soft_reset |= SOFT_RESET_DC;
3734
3735 if (reset_mask & RADEON_RESET_RLC)
3736 srbm_soft_reset |= SOFT_RESET_RLC;
3737
3738 if (reset_mask & RADEON_RESET_SEM)
3739 srbm_soft_reset |= SOFT_RESET_SEM;
3740
3741 if (reset_mask & RADEON_RESET_IH)
3742 srbm_soft_reset |= SOFT_RESET_IH;
3743
3744 if (reset_mask & RADEON_RESET_GRBM)
3745 srbm_soft_reset |= SOFT_RESET_GRBM;
3746
3747 if (reset_mask & RADEON_RESET_VMC)
3748 srbm_soft_reset |= SOFT_RESET_VMC;
3749
24178ec4
AD		 3750 if (!(rdev->flags & RADEON_IS_IGP)) {
3751 if (reset_mask & RADEON_RESET_MC)
3752 srbm_soft_reset |= SOFT_RESET_MC;
3753 }
a65a4369 3754
b7630473
AD		 3755 if (grbm_soft_reset) {
3756 tmp = RREG32(GRBM_SOFT_RESET);
3757 tmp |= grbm_soft_reset;
3758 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3759 WREG32(GRBM_SOFT_RESET, tmp);
3760 tmp = RREG32(GRBM_SOFT_RESET);
3761
3762 udelay(50);
3763
3764 tmp &= ~grbm_soft_reset;
3765 WREG32(GRBM_SOFT_RESET, tmp);
3766 tmp = RREG32(GRBM_SOFT_RESET);
3767 }
3768
3769 if (srbm_soft_reset) {
3770 tmp = RREG32(SRBM_SOFT_RESET);
3771 tmp |= srbm_soft_reset;
3772 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3773 WREG32(SRBM_SOFT_RESET, tmp);
3774 tmp = RREG32(SRBM_SOFT_RESET);
3775
3776 udelay(50);
3777
3778 tmp &= ~srbm_soft_reset;
3779 WREG32(SRBM_SOFT_RESET, tmp);
3780 tmp = RREG32(SRBM_SOFT_RESET);
3781 }
0ecebb9e
AD		 3782
3783 /* Wait a little for things to settle down */
3784 udelay(50);
3785
747943ea 3786 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3787 udelay(50);
3788
b7630473 3789 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3790}
3791
a2d07b74 3792int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3793{
a65a4369
AD		 3794 u32 reset_mask;
3795
3796 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3797
3798 if (reset_mask)
3799 r600_set_bios_scratch_engine_hung(rdev, true);
3800
3801 evergreen_gpu_soft_reset(rdev, reset_mask);
3802
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (!reset_mask)
3806 r600_set_bios_scratch_engine_hung(rdev, false);
3807
3808 return 0;
747943ea
AD		 3809}
3810
123bc183
AD		 3811/**
3812 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3813 *
3814 * @rdev: radeon_device pointer
3815 * @ring: radeon_ring structure holding ring information
3816 *
3817 * Check if the GFX engine is locked up.
3818 * Returns true if the engine appears to be locked up, false if not.
3819 */
3820bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3821{
3822 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3823
3824 if (!(reset_mask & (RADEON_RESET_GFX |
3825 RADEON_RESET_COMPUTE |
3826 RADEON_RESET_CP))) {
3827 radeon_ring_lockup_update(ring);
3828 return false;
3829 }
3830 /* force CP activities */
3831 radeon_ring_force_activity(rdev, ring);
3832 return radeon_ring_test_lockup(rdev, ring);
3833}
3834
3835/**
3836 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3837 *
3838 * @rdev: radeon_device pointer
3839 * @ring: radeon_ring structure holding ring information
3840 *
3841 * Check if the async DMA engine is locked up.
3842 * Returns true if the engine appears to be locked up, false if not.
3843 */
3844bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3845{
3846 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3847
3848 if (!(reset_mask & RADEON_RESET_DMA)) {
3849 radeon_ring_lockup_update(ring);
3850 return false;
3851 }
3852 /* force ring activities */
3853 radeon_ring_force_activity(rdev, ring);
3854 return radeon_ring_test_lockup(rdev, ring);
3855}
3856
2948f5e6
AD		 3857/*
3858 * RLC
3859 */
3860#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3861#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3862
3863void sumo_rlc_fini(struct radeon_device *rdev)
3864{
3865 int r;
3866
3867 /* save restore block */
3868 if (rdev->rlc.save_restore_obj) {
3869 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3870 if (unlikely(r != 0))
3871 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3872 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3873 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3874
3875 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3876 rdev->rlc.save_restore_obj = NULL;
3877 }
3878
3879 /* clear state block */
3880 if (rdev->rlc.clear_state_obj) {
3881 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3882 if (unlikely(r != 0))
3883 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3884 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3885 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3886
3887 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3888 rdev->rlc.clear_state_obj = NULL;
3889 }
3890}
3891
3892int sumo_rlc_init(struct radeon_device *rdev)
3893{
3894 u32 *src_ptr;
3895 volatile u32 *dst_ptr;
3896 u32 dws, data, i, j, k, reg_num;
3897 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3898 u64 reg_list_mc_addr;
3899 struct cs_section_def *cs_data;
3900 int r;
3901
3902 src_ptr = rdev->rlc.reg_list;
3903 dws = rdev->rlc.reg_list_size;
3904 cs_data = rdev->rlc.cs_data;
3905
3906 /* save restore block */
3907 if (rdev->rlc.save_restore_obj == NULL) {
3908 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3909 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3910 if (r) {
3911 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3912 return r;
3913 }
3914 }
3915
3916 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3917 if (unlikely(r != 0)) {
3918 sumo_rlc_fini(rdev);
3919 return r;
3920 }
3921 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3922 &rdev->rlc.save_restore_gpu_addr);
3923 if (r) {
3924 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3925 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3926 sumo_rlc_fini(rdev);
3927 return r;
3928 }
3929 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3930 if (r) {
3931 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3932 sumo_rlc_fini(rdev);
3933 return r;
3934 }
3935 /* write the sr buffer */
3936 dst_ptr = rdev->rlc.sr_ptr;
3937 /* format:
3938 * dw0: (reg2 << 16) | reg1
3939 * dw1: reg1 save space
3940 * dw2: reg2 save space
3941 */
3942 for (i = 0; i < dws; i++) {
3943 data = src_ptr[i] >> 2;
3944 i++;
3945 if (i < dws)
3946 data |= (src_ptr[i] >> 2) << 16;
3947 j = (((i - 1) * 3) / 2);
3948 dst_ptr[j] = data;
3949 }
3950 j = ((i * 3) / 2);
3951 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3952
3953 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3954 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3955
3956 /* clear state block */
3957 reg_list_num = 0;
3958 dws = 0;
3959 for (i = 0; cs_data[i].section != NULL; i++) {
3960 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3961 reg_list_num++;
3962 dws += cs_data[i].section[j].reg_count;
3963 }
3964 }
3965 reg_list_blk_index = (3 * reg_list_num + 2);
3966 dws += reg_list_blk_index;
3967
3968 if (rdev->rlc.clear_state_obj == NULL) {
3969 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3970 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3971 if (r) {
3972 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3973 sumo_rlc_fini(rdev);
3974 return r;
3975 }
3976 }
3977 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3978 if (unlikely(r != 0)) {
3979 sumo_rlc_fini(rdev);
3980 return r;
3981 }
3982 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3983 &rdev->rlc.clear_state_gpu_addr);
3984 if (r) {
3985
3986 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3987 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3988 sumo_rlc_fini(rdev);
3989 return r;
3990 }
3991 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3992 if (r) {
3993 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3994 sumo_rlc_fini(rdev);
3995 return r;
3996 }
3997 /* set up the cs buffer */
3998 dst_ptr = rdev->rlc.cs_ptr;
3999 reg_list_hdr_blk_index = 0;
4000 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4001 data = upper_32_bits(reg_list_mc_addr);
4002 dst_ptr[reg_list_hdr_blk_index] = data;
4003 reg_list_hdr_blk_index++;
4004 for (i = 0; cs_data[i].section != NULL; i++) {
4005 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4006 reg_num = cs_data[i].section[j].reg_count;
4007 data = reg_list_mc_addr & 0xffffffff;
4008 dst_ptr[reg_list_hdr_blk_index] = data;
4009 reg_list_hdr_blk_index++;
4010
4011 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4012 dst_ptr[reg_list_hdr_blk_index] = data;
4013 reg_list_hdr_blk_index++;
4014
4015 data = 0x08000000 | (reg_num * 4);
4016 dst_ptr[reg_list_hdr_blk_index] = data;
4017 reg_list_hdr_blk_index++;
4018
4019 for (k = 0; k < reg_num; k++) {
4020 data = cs_data[i].section[j].extent[k];
4021 dst_ptr[reg_list_blk_index + k] = data;
4022 }
4023 reg_list_mc_addr += reg_num * 4;
4024 reg_list_blk_index += reg_num;
4025 }
4026 }
4027 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4028
4029 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4030 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4031
4032 return 0;
4033}
4034
4035static void evergreen_rlc_start(struct radeon_device *rdev)
4036{
4037 if (rdev->flags & RADEON_IS_IGP)
4038 WREG32(RLC_CNTL, RLC_ENABLE | GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC);
4039 else
4040 WREG32(RLC_CNTL, RLC_ENABLE);
4041}
4042
4043int evergreen_rlc_resume(struct radeon_device *rdev)
4044{
4045 u32 i;
4046 const __be32 *fw_data;
4047
4048 if (!rdev->rlc_fw)
4049 return -EINVAL;
4050
4051 r600_rlc_stop(rdev);
4052
4053 WREG32(RLC_HB_CNTL, 0);
4054
4055 if (rdev->flags & RADEON_IS_IGP) {
4056 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4057 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4058 } else {
4059 WREG32(RLC_HB_BASE, 0);
4060 WREG32(RLC_HB_RPTR, 0);
4061 WREG32(RLC_HB_WPTR, 0);
4062 }
4063 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4064 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4065 WREG32(RLC_MC_CNTL, 0);
4066 WREG32(RLC_UCODE_CNTL, 0);
4067
4068 fw_data = (const __be32 *)rdev->rlc_fw->data;
4069 if (rdev->family >= CHIP_ARUBA) {
4070 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4071 WREG32(RLC_UCODE_ADDR, i);
4072 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4073 }
4074 } else if (rdev->family >= CHIP_CAYMAN) {
4075 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4076 WREG32(RLC_UCODE_ADDR, i);
4077 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4078 }
4079 } else {
4080 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4081 WREG32(RLC_UCODE_ADDR, i);
4082 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4083 }
4084 }
4085 WREG32(RLC_UCODE_ADDR, 0);
4086
4087 evergreen_rlc_start(rdev);
4088
4089 return 0;
4090}
4091
45f9a39b
AD		 4092/* Interrupts */
4093
4094u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4095{
46437057 4096 if (crtc >= rdev->num_crtc)
45f9a39b 4097 return 0;
46437057
AD		 4098 else
4099 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4100}
4101
4102void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4103{
4104 u32 tmp;
4105
1b37078b
AD		 4106 if (rdev->family >= CHIP_CAYMAN) {
4107 cayman_cp_int_cntl_setup(rdev, 0,
4108 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4109 cayman_cp_int_cntl_setup(rdev, 1, 0);
4110 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4111 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4112 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4113 } else
4114 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4115 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4116 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4117 WREG32(GRBM_INT_CNTL, 0);
4118 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4119 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4120 if (rdev->num_crtc >= 4) {
18007401
AD		 4121 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4122 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4123 }
4124 if (rdev->num_crtc >= 6) {
18007401
AD		 4125 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4126 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4127 }
45f9a39b
AD		 4128
4129 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4130 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4131 if (rdev->num_crtc >= 4) {
18007401
AD		 4132 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4133 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4134 }
4135 if (rdev->num_crtc >= 6) {
18007401
AD		 4136 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4137 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4138 }
45f9a39b 4139
05b3ef69
AD		 4140 /* only one DAC on DCE6 */
4141 if (!ASIC_IS_DCE6(rdev))
4142 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4143 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4144
4145 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4146 WREG32(DC_HPD1_INT_CONTROL, tmp);
4147 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4148 WREG32(DC_HPD2_INT_CONTROL, tmp);
4149 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4150 WREG32(DC_HPD3_INT_CONTROL, tmp);
4151 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4152 WREG32(DC_HPD4_INT_CONTROL, tmp);
4153 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4154 WREG32(DC_HPD5_INT_CONTROL, tmp);
4155 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4156 WREG32(DC_HPD6_INT_CONTROL, tmp);
4157
4158}
4159
4160int evergreen_irq_set(struct radeon_device *rdev)
4161{
4162 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4163 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4164 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4165 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4166 u32 grbm_int_cntl = 0;
6f34be50 4167 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4168 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4169 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4170 u32 thermal_int = 0;
45f9a39b
AD		 4171
4172 if (!rdev->irq.installed) {
fce7d61b 4173 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4174 return -EINVAL;
4175 }
4176 /* don't enable anything if the ih is disabled */
4177 if (!rdev->ih.enabled) {
4178 r600_disable_interrupts(rdev);
4179 /* force the active interrupt state to all disabled */
4180 evergreen_disable_interrupt_state(rdev);
4181 return 0;
4182 }
4183
4184 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4185 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4186 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4187 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4188 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4189 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4190 if (rdev->family == CHIP_ARUBA)
4191 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4192 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4193 else
4194 thermal_int = RREG32(CG_THERMAL_INT) &
4195 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4196
f122c610
AD		 4197 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4198 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4199 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4200 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4201 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4202 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4203
233d1ad5
AD		 4204 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4205
1b37078b
AD		 4206 if (rdev->family >= CHIP_CAYMAN) {
4207 /* enable CP interrupts on all rings */
736fc37f 4208 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4209 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4210 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4211 }
736fc37f 4212 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4213 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4214 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4215 }
736fc37f 4216 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4217 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4218 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4219 }
4220 } else {
736fc37f 4221 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4222 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4223 cp_int_cntl |= RB_INT_ENABLE;
4224 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4225 }
45f9a39b 4226 }
1b37078b 4227
233d1ad5
AD		 4228 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4229 DRM_DEBUG("r600_irq_set: sw int dma\n");
4230 dma_cntl |= TRAP_ENABLE;
4231 }
4232
f60cbd11
AD		 4233 if (rdev->family >= CHIP_CAYMAN) {
4234 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4235 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4236 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4237 dma_cntl1 |= TRAP_ENABLE;
4238 }
4239 }
4240
dc50ba7f
AD		 4241 if (rdev->irq.dpm_thermal) {
4242 DRM_DEBUG("dpm thermal\n");
4243 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4244 }
4245
6f34be50 4246 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4247 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4248 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4249 crtc1 |= VBLANK_INT_MASK;
4250 }
6f34be50 4251 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4252 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4253 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4254 crtc2 |= VBLANK_INT_MASK;
4255 }
6f34be50 4256 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4257 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4258 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4259 crtc3 |= VBLANK_INT_MASK;
4260 }
6f34be50 4261 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4262 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4263 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4264 crtc4 |= VBLANK_INT_MASK;
4265 }
6f34be50 4266 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4267 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4268 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4269 crtc5 |= VBLANK_INT_MASK;
4270 }
6f34be50 4271 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4272 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4273 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4274 crtc6 |= VBLANK_INT_MASK;
4275 }
4276 if (rdev->irq.hpd[0]) {
4277 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4278 hpd1 |= DC_HPDx_INT_EN;
4279 }
4280 if (rdev->irq.hpd[1]) {
4281 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4282 hpd2 |= DC_HPDx_INT_EN;
4283 }
4284 if (rdev->irq.hpd[2]) {
4285 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4286 hpd3 |= DC_HPDx_INT_EN;
4287 }
4288 if (rdev->irq.hpd[3]) {
4289 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4290 hpd4 |= DC_HPDx_INT_EN;
4291 }
4292 if (rdev->irq.hpd[4]) {
4293 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4294 hpd5 |= DC_HPDx_INT_EN;
4295 }
4296 if (rdev->irq.hpd[5]) {
4297 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4298 hpd6 |= DC_HPDx_INT_EN;
4299 }
f122c610
AD		 4300 if (rdev->irq.afmt[0]) {
4301 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4302 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4303 }
4304 if (rdev->irq.afmt[1]) {
4305 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4306 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4307 }
4308 if (rdev->irq.afmt[2]) {
4309 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4310 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4311 }
4312 if (rdev->irq.afmt[3]) {
4313 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4314 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4315 }
4316 if (rdev->irq.afmt[4]) {
4317 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4318 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4319 }
4320 if (rdev->irq.afmt[5]) {
4321 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4322 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4323 }
45f9a39b 4324
1b37078b
AD		 4325 if (rdev->family >= CHIP_CAYMAN) {
4326 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4327 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4328 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4329 } else
4330 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4331
4332 WREG32(DMA_CNTL, dma_cntl);
4333
f60cbd11
AD		 4334 if (rdev->family >= CHIP_CAYMAN)
4335 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4336
2031f77c 4337 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4338
4339 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4340 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4341 if (rdev->num_crtc >= 4) {
18007401
AD		 4342 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4343 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4344 }
4345 if (rdev->num_crtc >= 6) {
18007401
AD		 4346 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4347 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4348 }
45f9a39b 4349
6f34be50
AD		 4350 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4351 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4352 if (rdev->num_crtc >= 4) {
4353 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4354 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4355 }
4356 if (rdev->num_crtc >= 6) {
4357 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4358 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4359 }
6f34be50 4360
45f9a39b
AD		 4361 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4362 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4363 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4364 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4365 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4366 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4367 if (rdev->family == CHIP_ARUBA)
4368 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4369 else
4370 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4371
f122c610
AD		 4372 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4373 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4374 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4375 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4376 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4377 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4378
bcc1c2a1
AD		 4379 return 0;
4380}
4381
cbdd4501 4382static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4383{
4384 u32 tmp;
4385
6f34be50
AD		 4386 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4387 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4388 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4389 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4390 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4391 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4392 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4393 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4394 if (rdev->num_crtc >= 4) {
4395 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4396 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4397 }
4398 if (rdev->num_crtc >= 6) {
4399 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4400 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4401 }
6f34be50 4402
f122c610
AD		 4403 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4404 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4405 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4406 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4407 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4408 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4409
6f34be50
AD		 4410 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4411 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4412 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4413 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4414 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4415 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4416 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4417 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4418 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4419 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4420 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4421 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4422
b7eff394
AD		 4423 if (rdev->num_crtc >= 4) {
4424 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4425 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4426 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4427 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4428 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4429 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4430 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4431 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4432 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4433 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4434 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4435 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4436 }
4437
4438 if (rdev->num_crtc >= 6) {
4439 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4440 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4441 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4442 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4443 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4444 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4445 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4446 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4447 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4448 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4449 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4450 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4451 }
45f9a39b 4452
6f34be50 4453 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4454 tmp = RREG32(DC_HPD1_INT_CONTROL);
4455 tmp |= DC_HPDx_INT_ACK;
4456 WREG32(DC_HPD1_INT_CONTROL, tmp);
4457 }
6f34be50 4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4459 tmp = RREG32(DC_HPD2_INT_CONTROL);
4460 tmp |= DC_HPDx_INT_ACK;
4461 WREG32(DC_HPD2_INT_CONTROL, tmp);
4462 }
6f34be50 4463 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4464 tmp = RREG32(DC_HPD3_INT_CONTROL);
4465 tmp |= DC_HPDx_INT_ACK;
4466 WREG32(DC_HPD3_INT_CONTROL, tmp);
4467 }
6f34be50 4468 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4469 tmp = RREG32(DC_HPD4_INT_CONTROL);
4470 tmp |= DC_HPDx_INT_ACK;
4471 WREG32(DC_HPD4_INT_CONTROL, tmp);
4472 }
6f34be50 4473 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4474 tmp = RREG32(DC_HPD5_INT_CONTROL);
4475 tmp |= DC_HPDx_INT_ACK;
4476 WREG32(DC_HPD5_INT_CONTROL, tmp);
4477 }
6f34be50 4478 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4479 tmp = RREG32(DC_HPD5_INT_CONTROL);
4480 tmp |= DC_HPDx_INT_ACK;
4481 WREG32(DC_HPD6_INT_CONTROL, tmp);
4482 }
f122c610
AD		 4483 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4484 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4485 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4486 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4487 }
4488 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4489 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4490 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4491 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4492 }
4493 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4494 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4495 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4496 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4497 }
4498 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4499 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4500 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4501 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4502 }
4503 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4504 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4505 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4506 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4507 }
4508 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4509 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4510 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4511 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4512 }
45f9a39b
AD		 4513}
4514
1109ca09 4515static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4516{
45f9a39b
AD		 4517 r600_disable_interrupts(rdev);
4518 /* Wait and acknowledge irq */
4519 mdelay(1);
6f34be50 4520 evergreen_irq_ack(rdev);
45f9a39b
AD		 4521 evergreen_disable_interrupt_state(rdev);
4522}
4523
755d819e 4524void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4525{
4526 evergreen_irq_disable(rdev);
4527 r600_rlc_stop(rdev);
4528}
4529
cbdd4501 4530static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4531{
4532 u32 wptr, tmp;
4533
724c80e1 4534 if (rdev->wb.enabled)
204ae24d 4535 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4536 else
4537 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4538
4539 if (wptr & RB_OVERFLOW) {
4540 /* When a ring buffer overflow happen start parsing interrupt
4541 * from the last not overwritten vector (wptr + 16). Hopefully
4542 * this should allow us to catchup.
4543 */
4544 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4545 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4546 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4547 tmp = RREG32(IH_RB_CNTL);
4548 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4549 WREG32(IH_RB_CNTL, tmp);
4550 }
4551 return (wptr & rdev->ih.ptr_mask);
4552}
4553
4554int evergreen_irq_process(struct radeon_device *rdev)
4555{
682f1a54
DA		 4556 u32 wptr;
4557 u32 rptr;
45f9a39b
AD		 4558 u32 src_id, src_data;
4559 u32 ring_index;
45f9a39b 4560 bool queue_hotplug = false;
f122c610 4561 bool queue_hdmi = false;
dc50ba7f 4562 bool queue_thermal = false;
45f9a39b 4563
682f1a54 4564 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4565 return IRQ_NONE;
4566
682f1a54 4567 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4568
4569restart_ih:
4570 /* is somebody else already processing irqs? */
4571 if (atomic_xchg(&rdev->ih.lock, 1))
4572 return IRQ_NONE;
4573
682f1a54
DA		 4574 rptr = rdev->ih.rptr;
4575 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4576
964f6645
BH		 4577 /* Order reading of wptr vs. reading of IH ring data */
4578 rmb();
4579
45f9a39b 4580 /* display interrupts */
6f34be50 4581 evergreen_irq_ack(rdev);
45f9a39b 4582
45f9a39b
AD		 4583 while (rptr != wptr) {
4584 /* wptr/rptr are in bytes! */
4585 ring_index = rptr / 4;
0f234f5f
AD		 4586 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4587 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4588
4589 switch (src_id) {
4590 case 1: /* D1 vblank/vline */
4591 switch (src_data) {
4592 case 0: /* D1 vblank */
6f34be50 4593 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4594 if (rdev->irq.crtc_vblank_int[0]) {
4595 drm_handle_vblank(rdev->ddev, 0);
4596 rdev->pm.vblank_sync = true;
4597 wake_up(&rdev->irq.vblank_queue);
4598 }
736fc37f 4599 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4600 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4601 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4602 DRM_DEBUG("IH: D1 vblank\n");
4603 }
4604 break;
4605 case 1: /* D1 vline */
6f34be50
AD		 4606 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4607 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4608 DRM_DEBUG("IH: D1 vline\n");
4609 }
4610 break;
4611 default:
4612 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4613 break;
4614 }
4615 break;
4616 case 2: /* D2 vblank/vline */
4617 switch (src_data) {
4618 case 0: /* D2 vblank */
6f34be50 4619 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4620 if (rdev->irq.crtc_vblank_int[1]) {
4621 drm_handle_vblank(rdev->ddev, 1);
4622 rdev->pm.vblank_sync = true;
4623 wake_up(&rdev->irq.vblank_queue);
4624 }
736fc37f 4625 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4626 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4627 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4628 DRM_DEBUG("IH: D2 vblank\n");
4629 }
4630 break;
4631 case 1: /* D2 vline */
6f34be50
AD		 4632 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4633 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4634 DRM_DEBUG("IH: D2 vline\n");
4635 }
4636 break;
4637 default:
4638 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4639 break;
4640 }
4641 break;
4642 case 3: /* D3 vblank/vline */
4643 switch (src_data) {
4644 case 0: /* D3 vblank */
6f34be50
AD		 4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4646 if (rdev->irq.crtc_vblank_int[2]) {
4647 drm_handle_vblank(rdev->ddev, 2);
4648 rdev->pm.vblank_sync = true;
4649 wake_up(&rdev->irq.vblank_queue);
4650 }
736fc37f 4651 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4652 radeon_crtc_handle_flip(rdev, 2);
4653 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4654 DRM_DEBUG("IH: D3 vblank\n");
4655 }
4656 break;
4657 case 1: /* D3 vline */
6f34be50
AD		 4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4659 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4660 DRM_DEBUG("IH: D3 vline\n");
4661 }
4662 break;
4663 default:
4664 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4665 break;
4666 }
4667 break;
4668 case 4: /* D4 vblank/vline */
4669 switch (src_data) {
4670 case 0: /* D4 vblank */
6f34be50
AD		 4671 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4672 if (rdev->irq.crtc_vblank_int[3]) {
4673 drm_handle_vblank(rdev->ddev, 3);
4674 rdev->pm.vblank_sync = true;
4675 wake_up(&rdev->irq.vblank_queue);
4676 }
736fc37f 4677 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4678 radeon_crtc_handle_flip(rdev, 3);
4679 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4680 DRM_DEBUG("IH: D4 vblank\n");
4681 }
4682 break;
4683 case 1: /* D4 vline */
6f34be50
AD		 4684 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4685 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4686 DRM_DEBUG("IH: D4 vline\n");
4687 }
4688 break;
4689 default:
4690 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4691 break;
4692 }
4693 break;
4694 case 5: /* D5 vblank/vline */
4695 switch (src_data) {
4696 case 0: /* D5 vblank */
6f34be50
AD		 4697 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4698 if (rdev->irq.crtc_vblank_int[4]) {
4699 drm_handle_vblank(rdev->ddev, 4);
4700 rdev->pm.vblank_sync = true;
4701 wake_up(&rdev->irq.vblank_queue);
4702 }
736fc37f 4703 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4704 radeon_crtc_handle_flip(rdev, 4);
4705 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4706 DRM_DEBUG("IH: D5 vblank\n");
4707 }
4708 break;
4709 case 1: /* D5 vline */
6f34be50
AD		 4710 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4711 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4712 DRM_DEBUG("IH: D5 vline\n");
4713 }
4714 break;
4715 default:
4716 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4717 break;
4718 }
4719 break;
4720 case 6: /* D6 vblank/vline */
4721 switch (src_data) {
4722 case 0: /* D6 vblank */
6f34be50
AD		 4723 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4724 if (rdev->irq.crtc_vblank_int[5]) {
4725 drm_handle_vblank(rdev->ddev, 5);
4726 rdev->pm.vblank_sync = true;
4727 wake_up(&rdev->irq.vblank_queue);
4728 }
736fc37f 4729 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4730 radeon_crtc_handle_flip(rdev, 5);
4731 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4732 DRM_DEBUG("IH: D6 vblank\n");
4733 }
4734 break;
4735 case 1: /* D6 vline */
6f34be50
AD		 4736 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4737 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4738 DRM_DEBUG("IH: D6 vline\n");
4739 }
4740 break;
4741 default:
4742 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4743 break;
4744 }
4745 break;
4746 case 42: /* HPD hotplug */
4747 switch (src_data) {
4748 case 0:
6f34be50
AD		 4749 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4750 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4751 queue_hotplug = true;
4752 DRM_DEBUG("IH: HPD1\n");
4753 }
4754 break;
4755 case 1:
6f34be50
AD		 4756 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4757 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4758 queue_hotplug = true;
4759 DRM_DEBUG("IH: HPD2\n");
4760 }
4761 break;
4762 case 2:
6f34be50
AD		 4763 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4764 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4765 queue_hotplug = true;
4766 DRM_DEBUG("IH: HPD3\n");
4767 }
4768 break;
4769 case 3:
6f34be50
AD		 4770 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4771 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4772 queue_hotplug = true;
4773 DRM_DEBUG("IH: HPD4\n");
4774 }
4775 break;
4776 case 4:
6f34be50
AD		 4777 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4778 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4779 queue_hotplug = true;
4780 DRM_DEBUG("IH: HPD5\n");
4781 }
4782 break;
4783 case 5:
6f34be50
AD		 4784 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4785 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4786 queue_hotplug = true;
4787 DRM_DEBUG("IH: HPD6\n");
4788 }
4789 break;
4790 default:
4791 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4792 break;
4793 }
4794 break;
f122c610
AD		 4795 case 44: /* hdmi */
4796 switch (src_data) {
4797 case 0:
4798 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4799 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4800 queue_hdmi = true;
4801 DRM_DEBUG("IH: HDMI0\n");
4802 }
4803 break;
4804 case 1:
4805 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4806 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4807 queue_hdmi = true;
4808 DRM_DEBUG("IH: HDMI1\n");
4809 }
4810 break;
4811 case 2:
4812 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4813 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4814 queue_hdmi = true;
4815 DRM_DEBUG("IH: HDMI2\n");
4816 }
4817 break;
4818 case 3:
4819 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4820 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4821 queue_hdmi = true;
4822 DRM_DEBUG("IH: HDMI3\n");
4823 }
4824 break;
4825 case 4:
4826 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4827 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4828 queue_hdmi = true;
4829 DRM_DEBUG("IH: HDMI4\n");
4830 }
4831 break;
4832 case 5:
4833 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4834 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4835 queue_hdmi = true;
4836 DRM_DEBUG("IH: HDMI5\n");
4837 }
4838 break;
4839 default:
4840 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4841 break;
4842 }
f2ba57b5
CK		 4843 case 124: /* UVD */
4844 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4845 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4846 break;
ae133a11
CK		 4847 case 146:
4848 case 147:
4849 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4850 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4851 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4852 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4853 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4854 /* reset addr and status */
4855 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4856 break;
45f9a39b
AD		 4857 case 176: /* CP_INT in ring buffer */
4858 case 177: /* CP_INT in IB1 */
4859 case 178: /* CP_INT in IB2 */
4860 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4861 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4862 break;
4863 case 181: /* CP EOP event */
4864 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4865 if (rdev->family >= CHIP_CAYMAN) {
4866 switch (src_data) {
4867 case 0:
4868 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869 break;
4870 case 1:
4871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872 break;
4873 case 2:
4874 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875 break;
4876 }
4877 } else
4878 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4879 break;
233d1ad5
AD		 4880 case 224: /* DMA trap event */
4881 DRM_DEBUG("IH: DMA trap\n");
4882 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883 break;
dc50ba7f
AD		 4884 case 230: /* thermal low to high */
4885 DRM_DEBUG("IH: thermal low to high\n");
4886 rdev->pm.dpm.thermal.high_to_low = false;
4887 queue_thermal = true;
4888 break;
4889 case 231: /* thermal high to low */
4890 DRM_DEBUG("IH: thermal high to low\n");
4891 rdev->pm.dpm.thermal.high_to_low = true;
4892 queue_thermal = true;
4893 break;
2031f77c 4894 case 233: /* GUI IDLE */
303c805c 4895 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4896 break;
f60cbd11
AD		 4897 case 244: /* DMA trap event */
4898 if (rdev->family >= CHIP_CAYMAN) {
4899 DRM_DEBUG("IH: DMA1 trap\n");
4900 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901 }
4902 break;
45f9a39b
AD		 4903 default:
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905 break;
4906 }
4907
4908 /* wptr/rptr are in bytes! */
4909 rptr += 16;
4910 rptr &= rdev->ih.ptr_mask;
4911 }
45f9a39b 4912 if (queue_hotplug)
32c87fca 4913 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4914 if (queue_hdmi)
4915 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 4916 if (queue_thermal && rdev->pm.dpm_enabled)
4917 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 4918 rdev->ih.rptr = rptr;
4919 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4920 atomic_set(&rdev->ih.lock, 0);
4921
4922 /* make sure wptr hasn't changed while processing */
4923 wptr = evergreen_get_ih_wptr(rdev);
4924 if (wptr != rptr)
4925 goto restart_ih;
4926
45f9a39b
AD		 4927 return IRQ_HANDLED;
4928}
4929
233d1ad5
AD		 4930/**
4931 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4932 *
4933 * @rdev: radeon_device pointer
4934 * @fence: radeon fence object
4935 *
4936 * Add a DMA fence packet to the ring to write
4937 * the fence seq number and DMA trap packet to generate
4938 * an interrupt if needed (evergreen-SI).
4939 */
4940void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4941 struct radeon_fence *fence)
4942{
4943 struct radeon_ring *ring = &rdev->ring[fence->ring];
4944 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4945 /* write the fence */
0fcb6155 4946 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4947 radeon_ring_write(ring, addr & 0xfffffffc);
4948 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4949 radeon_ring_write(ring, fence->seq);
4950 /* generate an interrupt */
0fcb6155 4951 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4952 /* flush HDP */
0fcb6155 4953 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4954 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4955 radeon_ring_write(ring, 1);
4956}
4957
4958/**
4959 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4960 *
4961 * @rdev: radeon_device pointer
4962 * @ib: IB object to schedule
4963 *
4964 * Schedule an IB in the DMA ring (evergreen).
4965 */
4966void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4967 struct radeon_ib *ib)
4968{
4969 struct radeon_ring *ring = &rdev->ring[ib->ring];
4970
4971 if (rdev->wb.enabled) {
4972 u32 next_rptr = ring->wptr + 4;
4973 while ((next_rptr & 7) != 5)
4974 next_rptr++;
4975 next_rptr += 3;
0fcb6155 4976 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 4977 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4978 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4979 radeon_ring_write(ring, next_rptr);
4980 }
4981
4982 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4983 * Pad as necessary with NOPs.
4984 */
4985 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 4986 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4987 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 4988 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4989 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4990
4991}
4992
4993/**
4994 * evergreen_copy_dma - copy pages using the DMA engine
4995 *
4996 * @rdev: radeon_device pointer
4997 * @src_offset: src GPU address
4998 * @dst_offset: dst GPU address
4999 * @num_gpu_pages: number of GPU pages to xfer
5000 * @fence: radeon fence object
5001 *
5002 * Copy GPU paging using the DMA engine (evergreen-cayman).
5003 * Used by the radeon ttm implementation to move pages if
5004 * registered as the asic copy callback.
5005 */
5006int evergreen_copy_dma(struct radeon_device *rdev,
5007 uint64_t src_offset, uint64_t dst_offset,
5008 unsigned num_gpu_pages,
5009 struct radeon_fence **fence)
5010{
5011 struct radeon_semaphore *sem = NULL;
5012 int ring_index = rdev->asic->copy.dma_ring_index;
5013 struct radeon_ring *ring = &rdev->ring[ring_index];
5014 u32 size_in_dw, cur_size_in_dw;
5015 int i, num_loops;
5016 int r = 0;
5017
5018 r = radeon_semaphore_create(rdev, &sem);
5019 if (r) {
5020 DRM_ERROR("radeon: moving bo (%d).\n", r);
5021 return r;
5022 }
5023
5024 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5025 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5026 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5027 if (r) {
5028 DRM_ERROR("radeon: moving bo (%d).\n", r);
5029 radeon_semaphore_free(rdev, &sem, NULL);
5030 return r;
5031 }
5032
5033 if (radeon_fence_need_sync(*fence, ring->idx)) {
5034 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5035 ring->idx);
5036 radeon_fence_note_sync(*fence, ring->idx);
5037 } else {
5038 radeon_semaphore_free(rdev, &sem, NULL);
5039 }
5040
5041 for (i = 0; i < num_loops; i++) {
5042 cur_size_in_dw = size_in_dw;
5043 if (cur_size_in_dw > 0xFFFFF)
5044 cur_size_in_dw = 0xFFFFF;
5045 size_in_dw -= cur_size_in_dw;
0fcb6155 5046 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 5047 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5048 radeon_ring_write(ring, src_offset & 0xfffffffc);
5049 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5050 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5051 src_offset += cur_size_in_dw * 4;
5052 dst_offset += cur_size_in_dw * 4;
5053 }
5054
5055 r = radeon_fence_emit(rdev, fence, ring->idx);
5056 if (r) {
5057 radeon_ring_unlock_undo(rdev, ring);
5058 return r;
5059 }
5060
5061 radeon_ring_unlock_commit(rdev, ring);
5062 radeon_semaphore_free(rdev, &sem, *fence);
5063
5064 return r;
5065}
5066
bcc1c2a1
AD		 5067static int evergreen_startup(struct radeon_device *rdev)
5068{
f2ba57b5 5069 struct radeon_ring *ring;
bcc1c2a1
AD		 5070 int r;
5071
9e46a48d 5072 /* enable pcie gen2 link */
cd54033a 5073 evergreen_pcie_gen2_enable(rdev);
9e46a48d 5074
0af62b01
AD		 5075 if (ASIC_IS_DCE5(rdev)) {
5076 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5077 r = ni_init_microcode(rdev);
5078 if (r) {
5079 DRM_ERROR("Failed to load firmware!\n");
5080 return r;
5081 }
5082 }
755d819e 5083 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5084 if (r) {
0af62b01 5085 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5086 return r;
5087 }
0af62b01
AD		 5088 } else {
5089 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5090 r = r600_init_microcode(rdev);
5091 if (r) {
5092 DRM_ERROR("Failed to load firmware!\n");
5093 return r;
5094 }
5095 }
bcc1c2a1 5096 }
fe251e2f 5097
16cdf04d
AD		 5098 r = r600_vram_scratch_init(rdev);
5099 if (r)
5100 return r;
5101
bcc1c2a1 5102 evergreen_mc_program(rdev);
bcc1c2a1 5103 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5104 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5105 } else {
5106 r = evergreen_pcie_gart_enable(rdev);
5107 if (r)
5108 return r;
5109 }
bcc1c2a1 5110 evergreen_gpu_init(rdev);
bcc1c2a1 5111
d7ccd8fc 5112 r = evergreen_blit_init(rdev);
bcc1c2a1 5113 if (r) {
fb3d9e97 5114 r600_blit_fini(rdev);
27cd7769 5115 rdev->asic->copy.copy = NULL;
d7ccd8fc 5116 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 5117 }
5118
2948f5e6
AD		 5119 /* allocate rlc buffers */
5120 if (rdev->flags & RADEON_IS_IGP) {
5121 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5122 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5123 rdev->rlc.cs_data = evergreen_cs_data;
5124 r = sumo_rlc_init(rdev);
5125 if (r) {
5126 DRM_ERROR("Failed to init rlc BOs!\n");
5127 return r;
5128 }
5129 }
5130
724c80e1
AD		 5131 /* allocate wb buffer */
5132 r = radeon_wb_init(rdev);
5133 if (r)
5134 return r;
5135
30eb77f4
JG		 5136 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5137 if (r) {
5138 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5139 return r;
5140 }
5141
233d1ad5
AD		 5142 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5143 if (r) {
5144 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5145 return r;
5146 }
5147
f2ba57b5
CK		 5148 r = rv770_uvd_resume(rdev);
5149 if (!r) {
5150 r = radeon_fence_driver_start_ring(rdev,
5151 R600_RING_TYPE_UVD_INDEX);
5152 if (r)
5153 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5154 }
5155
5156 if (r)
5157 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5158
bcc1c2a1 5159 /* Enable IRQ */
e49f3959
AH		 5160 if (!rdev->irq.installed) {
5161 r = radeon_irq_kms_init(rdev);
5162 if (r)
5163 return r;
5164 }
5165
bcc1c2a1
AD		 5166 r = r600_irq_init(rdev);
5167 if (r) {
5168 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5169 radeon_irq_kms_fini(rdev);
5170 return r;
5171 }
45f9a39b 5172 evergreen_irq_set(rdev);
bcc1c2a1 5173
f2ba57b5 5174 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5175 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 5176 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5177 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 5178 if (r)
5179 return r;
233d1ad5
AD		 5180
5181 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5182 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5183 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 5184 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5185 if (r)
5186 return r;
5187
bcc1c2a1
AD		 5188 r = evergreen_cp_load_microcode(rdev);
5189 if (r)
5190 return r;
fe251e2f 5191 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5192 if (r)
5193 return r;
5194 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5195 if (r)
5196 return r;
fe251e2f 5197
f2ba57b5
CK		 5198 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5199 if (ring->ring_size) {
5200 r = radeon_ring_init(rdev, ring, ring->ring_size,
5201 R600_WB_UVD_RPTR_OFFSET,
5202 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5203 0, 0xfffff, RADEON_CP_PACKET2);
5204 if (!r)
5205 r = r600_uvd_init(rdev);
5206
5207 if (r)
5208 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5209 }
5210
2898c348
CK		 5211 r = radeon_ib_pool_init(rdev);
5212 if (r) {
5213 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5214 return r;
2898c348 5215 }
b15ba512 5216
69d2ae57
RM		 5217 r = r600_audio_init(rdev);
5218 if (r) {
5219 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5220 return r;
5221 }
5222
bcc1c2a1
AD		 5223 return 0;
5224}
5225
5226int evergreen_resume(struct radeon_device *rdev)
5227{
5228 int r;
5229
86f5c9ed
AD		 5230 /* reset the asic, the gfx blocks are often in a bad state
5231 * after the driver is unloaded or after a resume
5232 */
5233 if (radeon_asic_reset(rdev))
5234 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5235 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5236 * posting will perform necessary task to bring back GPU into good
5237 * shape.
5238 */
5239 /* post card */
5240 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5241
d4788db3
AD		 5242 /* init golden registers */
5243 evergreen_init_golden_registers(rdev);
5244
b15ba512 5245 rdev->accel_working = true;
bcc1c2a1
AD		 5246 r = evergreen_startup(rdev);
5247 if (r) {
755d819e 5248 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5249 rdev->accel_working = false;
bcc1c2a1
AD		 5250 return r;
5251 }
fe251e2f 5252
bcc1c2a1
AD		 5253 return r;
5254
5255}
5256
5257int evergreen_suspend(struct radeon_device *rdev)
5258{
69d2ae57 5259 r600_audio_fini(rdev);
f2ba57b5 5260 radeon_uvd_suspend(rdev);
bcc1c2a1 5261 r700_cp_stop(rdev);
233d1ad5 5262 r600_dma_stop(rdev);
f2ba57b5 5263 r600_uvd_rbc_stop(rdev);
45f9a39b 5264 evergreen_irq_suspend(rdev);
724c80e1 5265 radeon_wb_disable(rdev);
bcc1c2a1 5266 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5267
5268 return 0;
5269}
5270
bcc1c2a1
AD		 5271/* Plan is to move initialization in that function and use
5272 * helper function so that radeon_device_init pretty much
5273 * do nothing more than calling asic specific function. This
5274 * should also allow to remove a bunch of callback function
5275 * like vram_info.
5276 */
5277int evergreen_init(struct radeon_device *rdev)
5278{
5279 int r;
5280
bcc1c2a1
AD		 5281 /* Read BIOS */
5282 if (!radeon_get_bios(rdev)) {
5283 if (ASIC_IS_AVIVO(rdev))
5284 return -EINVAL;
5285 }
5286 /* Must be an ATOMBIOS */
5287 if (!rdev->is_atom_bios) {
755d819e 5288 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5289 return -EINVAL;
5290 }
5291 r = radeon_atombios_init(rdev);
5292 if (r)
5293 return r;
86f5c9ed
AD		 5294 /* reset the asic, the gfx blocks are often in a bad state
5295 * after the driver is unloaded or after a resume
5296 */
5297 if (radeon_asic_reset(rdev))
5298 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5299 /* Post card if necessary */
fd909c37 5300 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5301 if (!rdev->bios) {
5302 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5303 return -EINVAL;
5304 }
5305 DRM_INFO("GPU not posted. posting now...\n");
5306 atom_asic_init(rdev->mode_info.atom_context);
5307 }
d4788db3
AD		 5308 /* init golden registers */
5309 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5310 /* Initialize scratch registers */
5311 r600_scratch_init(rdev);
5312 /* Initialize surface registers */
5313 radeon_surface_init(rdev);
5314 /* Initialize clocks */
5315 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5316 /* Fence driver */
5317 r = radeon_fence_driver_init(rdev);
5318 if (r)
5319 return r;
d594e46a
JG		 5320 /* initialize AGP */
5321 if (rdev->flags & RADEON_IS_AGP) {
5322 r = radeon_agp_init(rdev);
5323 if (r)
5324 radeon_agp_disable(rdev);
5325 }
5326 /* initialize memory controller */
bcc1c2a1
AD		 5327 r = evergreen_mc_init(rdev);
5328 if (r)
5329 return r;
5330 /* Memory manager */
5331 r = radeon_bo_init(rdev);
5332 if (r)
5333 return r;
45f9a39b 5334
e32eb50d
CK		 5335 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5336 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5337
233d1ad5
AD		 5338 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5339 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5340
f2ba57b5
CK		 5341 r = radeon_uvd_init(rdev);
5342 if (!r) {
5343 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5344 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5345 4096);
5346 }
5347
bcc1c2a1
AD		 5348 rdev->ih.ring_obj = NULL;
5349 r600_ih_ring_init(rdev, 64 * 1024);
5350
5351 r = r600_pcie_gart_init(rdev);
5352 if (r)
5353 return r;
0fcdb61e 5354
148a03bc 5355 rdev->accel_working = true;
bcc1c2a1
AD		 5356 r = evergreen_startup(rdev);
5357 if (r) {
fe251e2f
AD		 5358 dev_err(rdev->dev, "disabling GPU acceleration\n");
5359 r700_cp_fini(rdev);
233d1ad5 5360 r600_dma_fini(rdev);
fe251e2f 5361 r600_irq_fini(rdev);
2948f5e6
AD		 5362 if (rdev->flags & RADEON_IS_IGP)
5363 sumo_rlc_fini(rdev);
724c80e1 5364 radeon_wb_fini(rdev);
2898c348 5365 radeon_ib_pool_fini(rdev);
fe251e2f 5366 radeon_irq_kms_fini(rdev);
0fcdb61e 5367 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5368 rdev->accel_working = false;
5369 }
77e00f2e
AD		 5370
5371 /* Don't start up if the MC ucode is missing on BTC parts.
5372 * The default clocks and voltages before the MC ucode
5373 * is loaded are not suffient for advanced operations.
5374 */
5375 if (ASIC_IS_DCE5(rdev)) {
5376 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5377 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5378 return -EINVAL;
5379 }
5380 }
5381
bcc1c2a1
AD		 5382 return 0;
5383}
5384
5385void evergreen_fini(struct radeon_device *rdev)
5386{
69d2ae57 5387 r600_audio_fini(rdev);
fb3d9e97 5388 r600_blit_fini(rdev);
45f9a39b 5389 r700_cp_fini(rdev);
233d1ad5 5390 r600_dma_fini(rdev);
bcc1c2a1 5391 r600_irq_fini(rdev);
2948f5e6
AD		 5392 if (rdev->flags & RADEON_IS_IGP)
5393 sumo_rlc_fini(rdev);
724c80e1 5394 radeon_wb_fini(rdev);
2898c348 5395 radeon_ib_pool_fini(rdev);
bcc1c2a1 5396 radeon_irq_kms_fini(rdev);
bcc1c2a1 5397 evergreen_pcie_gart_fini(rdev);
f2ba57b5 5398 radeon_uvd_fini(rdev);
16cdf04d 5399 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5400 radeon_gem_fini(rdev);
5401 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5402 radeon_agp_fini(rdev);
5403 radeon_bo_fini(rdev);
5404 radeon_atombios_fini(rdev);
5405 kfree(rdev->bios);
5406 rdev->bios = NULL;
bcc1c2a1 5407}
9e46a48d 5408
b07759bf 5409void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5410{
7e0e4196 5411 u32 link_width_cntl, speed_cntl;
9e46a48d 5412
d42dd579
AD		 5413 if (radeon_pcie_gen2 == 0)
5414 return;
5415
9e46a48d
AD		 5416 if (rdev->flags & RADEON_IS_IGP)
5417 return;
5418
5419 if (!(rdev->flags & RADEON_IS_PCIE))
5420 return;
5421
5422 /* x2 cards have a special sequence */
5423 if (ASIC_IS_X2(rdev))
5424 return;
5425
7e0e4196
KSS		 5426 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5427 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5428 return;
5429
492d2b61 5430 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5431 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5432 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5433 return;
5434 }
5435
197bbb3d
DA		 5436 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5437
9e46a48d
AD		 5438 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5439 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5440
492d2b61 5441 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5442 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5443 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5444
492d2b61 5445 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5446 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5447 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5448
492d2b61 5449 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5450 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5451 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5452
492d2b61 5453 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5454 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5455 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5456
492d2b61 5457 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5458 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5459 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5460
5461 } else {
492d2b61 5462 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5463 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5464 if (1)
5465 link_width_cntl |= LC_UPCONFIGURE_DIS;
5466 else
5467 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5468 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5469 }
5470}
Linux kernel - Deliverables
This page took 0.542933 seconds and 5 git commands to generate.