projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: retain user selected performance level across state changes
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
bcc1c2a1 149
d4788db3
AD		 150static const u32 evergreen_golden_registers[] =
151{
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5cc, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194};
195
196static const u32 evergreen_golden_registers2[] =
197{
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212};
213
214static const u32 cypress_mgcg_init[] =
215{
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365};
366
367static const u32 redwood_mgcg_init[] =
368{
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437};
438
439static const u32 cedar_golden_registers[] =
440{
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5cc, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480};
481
482static const u32 cedar_mgcg_init[] =
483{
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534};
535
536static const u32 juniper_mgcg_init[] =
537{
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632};
633
634static const u32 supersumo_golden_registers[] =
635{
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5cc, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707};
708
709static const u32 sumo_golden_registers[] =
710{
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716};
717
718static const u32 wrestler_golden_registers[] =
719{
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5cc, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769};
770
771static const u32 barts_golden_registers[] =
772{
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818};
819
820static const u32 turks_golden_registers[] =
821{
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869};
870
871static const u32 caicos_golden_registers[] =
872{
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920};
921
922static void evergreen_init_golden_registers(struct radeon_device *rdev)
923{
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006}
1007
285484e2
JG		 1008void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011{
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037}
1038
23d33ba3
AD		 1039static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041{
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061}
1062
1063int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064{
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084}
1085
a8b4925c
AD		 1086int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087{
1088 /* start off with something large */
facd112d 1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1090 int r;
1091
4ed10835
CK		 1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
facd112d
CK		 1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
a8b4925c
AD		 1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
facd112d 1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
facd112d 1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
facd112d 1140 if (fb_div < 307200)
a8b4925c
AD		 1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
facd112d 1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173}
1174
d054ac16
AD		 1175void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176{
1177 u16 ctl, v;
32195aec 1178 int err;
d054ac16 1179
32195aec 1180 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1181 if (err)
1182 return;
1183
1184 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1185
1186 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1187 * to avoid hangs or perfomance issues
1188 */
1189 if ((v == 0) || (v == 6) || (v == 7)) {
1190 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1191 ctl |= (2 << 12);
32195aec 1192 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1193 }
1194}
1195
134b480f
AD		 1196void dce4_program_fmt(struct drm_encoder *encoder)
1197{
1198 struct drm_device *dev = encoder->dev;
1199 struct radeon_device *rdev = dev->dev_private;
1200 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1201 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1202 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1203 int bpc = 0;
1204 u32 tmp = 0;
6214bb74 1205 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1206
6214bb74
AD		 1207 if (connector) {
1208 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1209 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1210 dither = radeon_connector->dither;
1211 }
134b480f
AD		 1212
1213 /* LVDS/eDP FMT is set up by atom */
1214 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1215 return;
1216
1217 /* not needed for analog */
1218 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1219 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1220 return;
1221
1222 if (bpc == 0)
1223 return;
1224
1225 switch (bpc) {
1226 case 6:
6214bb74 1227 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1228 /* XXX sort out optimal dither settings */
1229 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1230 FMT_SPATIAL_DITHER_EN);
1231 else
1232 tmp |= FMT_TRUNCATE_EN;
1233 break;
1234 case 8:
6214bb74 1235 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1236 /* XXX sort out optimal dither settings */
1237 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1238 FMT_RGB_RANDOM_ENABLE |
1239 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1240 else
1241 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1242 break;
1243 case 10:
1244 default:
1245 /* not needed */
1246 break;
1247 }
1248
1249 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1250}
1251
10257a6d
AD		 1252static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1253{
1254 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1255 return true;
1256 else
1257 return false;
1258}
1259
1260static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1261{
1262 u32 pos1, pos2;
1263
1264 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1265 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1266
1267 if (pos1 != pos2)
1268 return true;
1269 else
1270 return false;
1271}
1272
377edc8b
AD		 1273/**
1274 * dce4_wait_for_vblank - vblank wait asic callback.
1275 *
1276 * @rdev: radeon_device pointer
1277 * @crtc: crtc to wait for vblank on
1278 *
1279 * Wait for vblank on the requested crtc (evergreen+).
1280 */
3ae19b75
AD		 1281void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1282{
10257a6d 1283 unsigned i = 0;
3ae19b75 1284
4a15903d
AD		 1285 if (crtc >= rdev->num_crtc)
1286 return;
1287
10257a6d
AD		 1288 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1289 return;
1290
1291 /* depending on when we hit vblank, we may be close to active; if so,
1292 * wait for another frame.
1293 */
1294 while (dce4_is_in_vblank(rdev, crtc)) {
1295 if (i++ % 100 == 0) {
1296 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1297 break;
3ae19b75 1298 }
10257a6d
AD		 1299 }
1300
1301 while (!dce4_is_in_vblank(rdev, crtc)) {
1302 if (i++ % 100 == 0) {
1303 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1304 break;
3ae19b75
AD		 1305 }
1306 }
1307}
1308
377edc8b
AD		 1309/**
1310 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1311 *
1312 * @rdev: radeon_device pointer
1313 * @crtc: crtc to prepare for pageflip on
1314 *
1315 * Pre-pageflip callback (evergreen+).
1316 * Enables the pageflip irq (vblank irq).
1317 */
6f34be50
AD		 1318void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1319{
6f34be50
AD		 1320 /* enable the pflip int */
1321 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1322}
1323
377edc8b
AD		 1324/**
1325 * evergreen_post_page_flip - pos-pageflip callback.
1326 *
1327 * @rdev: radeon_device pointer
1328 * @crtc: crtc to cleanup pageflip on
1329 *
1330 * Post-pageflip callback (evergreen+).
1331 * Disables the pageflip irq (vblank irq).
1332 */
6f34be50
AD		 1333void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1334{
1335 /* disable the pflip int */
1336 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1337}
1338
377edc8b
AD		 1339/**
1340 * evergreen_page_flip - pageflip callback.
1341 *
1342 * @rdev: radeon_device pointer
1343 * @crtc_id: crtc to cleanup pageflip on
1344 * @crtc_base: new address of the crtc (GPU MC address)
1345 *
1346 * Does the actual pageflip (evergreen+).
1347 * During vblank we take the crtc lock and wait for the update_pending
1348 * bit to go high, when it does, we release the lock, and allow the
1349 * double buffered update to take place.
1350 * Returns the current update pending status.
1351 */
6f34be50
AD		 1352u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1353{
1354 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1355 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1356 int i;
6f34be50
AD		 1357
1358 /* Lock the graphics update lock */
1359 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1360 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1361
1362 /* update the scanout addresses */
1363 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1364 upper_32_bits(crtc_base));
1365 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1366 (u32)crtc_base);
1367
1368 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1369 upper_32_bits(crtc_base));
1370 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1371 (u32)crtc_base);
1372
1373 /* Wait for update_pending to go high. */
f6496479
AD		 1374 for (i = 0; i < rdev->usec_timeout; i++) {
1375 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1376 break;
1377 udelay(1);
1378 }
6f34be50
AD		 1379 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1380
1381 /* Unlock the lock, so double-buffering can take place inside vblank */
1382 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1383 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1384
1385 /* Return current update_pending status: */
1386 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1387}
1388
21a8122a 1389/* get temperature in millidegrees */
20d391d7 1390int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1391{
1c88d74f
AD		 1392 u32 temp, toffset;
1393 int actual_temp = 0;
67b3f823
AD		 1394
1395 if (rdev->family == CHIP_JUNIPER) {
1396 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1397 TOFFSET_SHIFT;
1398 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1399 TS0_ADC_DOUT_SHIFT;
1400
1401 if (toffset & 0x100)
1402 actual_temp = temp / 2 - (0x200 - toffset);
1403 else
1404 actual_temp = temp / 2 + toffset;
1405
1406 actual_temp = actual_temp * 1000;
1407
1408 } else {
1409 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1410 ASIC_T_SHIFT;
1411
1412 if (temp & 0x400)
1413 actual_temp = -256;
1414 else if (temp & 0x200)
1415 actual_temp = 255;
1416 else if (temp & 0x100) {
1417 actual_temp = temp & 0x1ff;
1418 actual_temp |= ~0x1ff;
1419 } else
1420 actual_temp = temp & 0xff;
1421
1422 actual_temp = (actual_temp * 1000) / 2;
1423 }
21a8122a 1424
67b3f823 1425 return actual_temp;
21a8122a
AD		 1426}
1427
20d391d7 1428int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1429{
1430 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1431 int actual_temp = temp - 49;
e33df25f
AD		 1432
1433 return actual_temp * 1000;
1434}
1435
377edc8b
AD		 1436/**
1437 * sumo_pm_init_profile - Initialize power profiles callback.
1438 *
1439 * @rdev: radeon_device pointer
1440 *
1441 * Initialize the power states used in profile mode
1442 * (sumo, trinity, SI).
1443 * Used for profile mode only.
1444 */
a4c9e2ee
AD		 1445void sumo_pm_init_profile(struct radeon_device *rdev)
1446{
1447 int idx;
1448
1449 /* default */
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1451 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1452 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1453 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1454
1455 /* low,mid sh/mh */
1456 if (rdev->flags & RADEON_IS_MOBILITY)
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1458 else
1459 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1464 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1465
1466 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1469 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1470
1471 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1474 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1475
1476 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1477 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1479 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1480
1481 /* high sh/mh */
1482 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1483 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1487 rdev->pm.power_state[idx].num_clock_modes - 1;
1488
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1491 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1492 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1493 rdev->pm.power_state[idx].num_clock_modes - 1;
1494}
1495
27810fb2
AD		 1496/**
1497 * btc_pm_init_profile - Initialize power profiles callback.
1498 *
1499 * @rdev: radeon_device pointer
1500 *
1501 * Initialize the power states used in profile mode
1502 * (BTC, cayman).
1503 * Used for profile mode only.
1504 */
1505void btc_pm_init_profile(struct radeon_device *rdev)
1506{
1507 int idx;
1508
1509 /* default */
1510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1511 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1512 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1513 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1514 /* starting with BTC, there is one state that is used for both
1515 * MH and SH. Difference is that we always use the high clock index for
1516 * mclk.
1517 */
1518 if (rdev->flags & RADEON_IS_MOBILITY)
1519 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1520 else
1521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1522 /* low sh */
1523 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1525 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1527 /* mid sh */
1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1531 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1532 /* high sh */
1533 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1536 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1537 /* low mh */
1538 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1539 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1540 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1541 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1542 /* mid mh */
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1545 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1546 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1547 /* high mh */
1548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1551 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1552}
1553
377edc8b
AD		 1554/**
1555 * evergreen_pm_misc - set additional pm hw parameters callback.
1556 *
1557 * @rdev: radeon_device pointer
1558 *
1559 * Set non-clock parameters associated with a power state
1560 * (voltage, etc.) (evergreen+).
1561 */
49e02b73
AD		 1562void evergreen_pm_misc(struct radeon_device *rdev)
1563{
a081a9d6
RM		 1564 int req_ps_idx = rdev->pm.requested_power_state_index;
1565 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1566 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1567 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1568
2feea49a 1569 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1570 /* 0xff0x are flags rather then an actual voltage */
1571 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1572 return;
2feea49a 1573 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1574 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1575 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1576 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1577 }
7ae764b1
AD		 1578
1579 /* starting with BTC, there is one state that is used for both
1580 * MH and SH. Difference is that we always use the high clock index for
1581 * mclk and vddci.
1582 */
1583 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1584 (rdev->family >= CHIP_BARTS) &&
1585 rdev->pm.active_crtc_count &&
1586 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1587 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1588 voltage = &rdev->pm.power_state[req_ps_idx].
1589 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1590
c6cf7777
AD		 1591 /* 0xff0x are flags rather then an actual voltage */
1592 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1593 return;
2feea49a
AD		 1594 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1595 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1596 rdev->pm.current_vddci = voltage->vddci;
1597 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1598 }
1599 }
49e02b73
AD		 1600}
1601
377edc8b
AD		 1602/**
1603 * evergreen_pm_prepare - pre-power state change callback.
1604 *
1605 * @rdev: radeon_device pointer
1606 *
1607 * Prepare for a power state change (evergreen+).
1608 */
49e02b73
AD		 1609void evergreen_pm_prepare(struct radeon_device *rdev)
1610{
1611 struct drm_device *ddev = rdev->ddev;
1612 struct drm_crtc *crtc;
1613 struct radeon_crtc *radeon_crtc;
1614 u32 tmp;
1615
1616 /* disable any active CRTCs */
1617 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1618 radeon_crtc = to_radeon_crtc(crtc);
1619 if (radeon_crtc->enabled) {
1620 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1621 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1622 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1623 }
1624 }
1625}
1626
377edc8b
AD		 1627/**
1628 * evergreen_pm_finish - post-power state change callback.
1629 *
1630 * @rdev: radeon_device pointer
1631 *
1632 * Clean up after a power state change (evergreen+).
1633 */
49e02b73
AD		 1634void evergreen_pm_finish(struct radeon_device *rdev)
1635{
1636 struct drm_device *ddev = rdev->ddev;
1637 struct drm_crtc *crtc;
1638 struct radeon_crtc *radeon_crtc;
1639 u32 tmp;
1640
1641 /* enable any active CRTCs */
1642 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1643 radeon_crtc = to_radeon_crtc(crtc);
1644 if (radeon_crtc->enabled) {
1645 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1646 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1647 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1648 }
1649 }
1650}
1651
377edc8b
AD		 1652/**
1653 * evergreen_hpd_sense - hpd sense callback.
1654 *
1655 * @rdev: radeon_device pointer
1656 * @hpd: hpd (hotplug detect) pin
1657 *
1658 * Checks if a digital monitor is connected (evergreen+).
1659 * Returns true if connected, false if not connected.
1660 */
bcc1c2a1
AD		 1661bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1662{
1663 bool connected = false;
0ca2ab52
AD		 1664
1665 switch (hpd) {
1666 case RADEON_HPD_1:
1667 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1668 connected = true;
1669 break;
1670 case RADEON_HPD_2:
1671 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1672 connected = true;
1673 break;
1674 case RADEON_HPD_3:
1675 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1676 connected = true;
1677 break;
1678 case RADEON_HPD_4:
1679 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1680 connected = true;
1681 break;
1682 case RADEON_HPD_5:
1683 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1684 connected = true;
1685 break;
1686 case RADEON_HPD_6:
1687 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1688 connected = true;
1689 break;
1690 default:
1691 break;
1692 }
1693
bcc1c2a1
AD		 1694 return connected;
1695}
1696
377edc8b
AD		 1697/**
1698 * evergreen_hpd_set_polarity - hpd set polarity callback.
1699 *
1700 * @rdev: radeon_device pointer
1701 * @hpd: hpd (hotplug detect) pin
1702 *
1703 * Set the polarity of the hpd pin (evergreen+).
1704 */
bcc1c2a1
AD		 1705void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1706 enum radeon_hpd_id hpd)
1707{
0ca2ab52
AD		 1708 u32 tmp;
1709 bool connected = evergreen_hpd_sense(rdev, hpd);
1710
1711 switch (hpd) {
1712 case RADEON_HPD_1:
1713 tmp = RREG32(DC_HPD1_INT_CONTROL);
1714 if (connected)
1715 tmp &= ~DC_HPDx_INT_POLARITY;
1716 else
1717 tmp |= DC_HPDx_INT_POLARITY;
1718 WREG32(DC_HPD1_INT_CONTROL, tmp);
1719 break;
1720 case RADEON_HPD_2:
1721 tmp = RREG32(DC_HPD2_INT_CONTROL);
1722 if (connected)
1723 tmp &= ~DC_HPDx_INT_POLARITY;
1724 else
1725 tmp |= DC_HPDx_INT_POLARITY;
1726 WREG32(DC_HPD2_INT_CONTROL, tmp);
1727 break;
1728 case RADEON_HPD_3:
1729 tmp = RREG32(DC_HPD3_INT_CONTROL);
1730 if (connected)
1731 tmp &= ~DC_HPDx_INT_POLARITY;
1732 else
1733 tmp |= DC_HPDx_INT_POLARITY;
1734 WREG32(DC_HPD3_INT_CONTROL, tmp);
1735 break;
1736 case RADEON_HPD_4:
1737 tmp = RREG32(DC_HPD4_INT_CONTROL);
1738 if (connected)
1739 tmp &= ~DC_HPDx_INT_POLARITY;
1740 else
1741 tmp |= DC_HPDx_INT_POLARITY;
1742 WREG32(DC_HPD4_INT_CONTROL, tmp);
1743 break;
1744 case RADEON_HPD_5:
1745 tmp = RREG32(DC_HPD5_INT_CONTROL);
1746 if (connected)
1747 tmp &= ~DC_HPDx_INT_POLARITY;
1748 else
1749 tmp |= DC_HPDx_INT_POLARITY;
1750 WREG32(DC_HPD5_INT_CONTROL, tmp);
1751 break;
1752 case RADEON_HPD_6:
1753 tmp = RREG32(DC_HPD6_INT_CONTROL);
1754 if (connected)
1755 tmp &= ~DC_HPDx_INT_POLARITY;
1756 else
1757 tmp |= DC_HPDx_INT_POLARITY;
1758 WREG32(DC_HPD6_INT_CONTROL, tmp);
1759 break;
1760 default:
1761 break;
1762 }
bcc1c2a1
AD		 1763}
1764
377edc8b
AD		 1765/**
1766 * evergreen_hpd_init - hpd setup callback.
1767 *
1768 * @rdev: radeon_device pointer
1769 *
1770 * Setup the hpd pins used by the card (evergreen+).
1771 * Enable the pin, set the polarity, and enable the hpd interrupts.
1772 */
bcc1c2a1
AD		 1773void evergreen_hpd_init(struct radeon_device *rdev)
1774{
0ca2ab52
AD		 1775 struct drm_device *dev = rdev->ddev;
1776 struct drm_connector *connector;
fb98257a 1777 unsigned enabled = 0;
0ca2ab52
AD		 1778 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1779 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1780
0ca2ab52
AD		 1781 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1782 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1783
1784 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1785 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1786 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1787 * aux dp channel on imac and help (but not completely fix)
1788 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1789 * also avoid interrupt storms during dpms.
1790 */
1791 continue;
1792 }
0ca2ab52
AD		 1793 switch (radeon_connector->hpd.hpd) {
1794 case RADEON_HPD_1:
1795 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1796 break;
1797 case RADEON_HPD_2:
1798 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1799 break;
1800 case RADEON_HPD_3:
1801 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1802 break;
1803 case RADEON_HPD_4:
1804 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1805 break;
1806 case RADEON_HPD_5:
1807 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1808 break;
1809 case RADEON_HPD_6:
1810 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1811 break;
1812 default:
1813 break;
1814 }
64912e99 1815 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1816 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1817 }
fb98257a 1818 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1819}
1820
377edc8b
AD		 1821/**
1822 * evergreen_hpd_fini - hpd tear down callback.
1823 *
1824 * @rdev: radeon_device pointer
1825 *
1826 * Tear down the hpd pins used by the card (evergreen+).
1827 * Disable the hpd interrupts.
1828 */
0ca2ab52 1829void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1830{
0ca2ab52
AD		 1831 struct drm_device *dev = rdev->ddev;
1832 struct drm_connector *connector;
fb98257a 1833 unsigned disabled = 0;
0ca2ab52
AD		 1834
1835 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1836 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1837 switch (radeon_connector->hpd.hpd) {
1838 case RADEON_HPD_1:
1839 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1840 break;
1841 case RADEON_HPD_2:
1842 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1843 break;
1844 case RADEON_HPD_3:
1845 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1846 break;
1847 case RADEON_HPD_4:
1848 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1849 break;
1850 case RADEON_HPD_5:
1851 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1852 break;
1853 case RADEON_HPD_6:
1854 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1855 break;
1856 default:
1857 break;
1858 }
fb98257a 1859 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1860 }
fb98257a 1861 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1862}
1863
f9d9c362
AD		 1864/* watermark setup */
1865
1866static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1867 struct radeon_crtc *radeon_crtc,
1868 struct drm_display_mode *mode,
1869 struct drm_display_mode *other_mode)
1870{
0b31e023
AD		 1871 u32 tmp, buffer_alloc, i;
1872 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1873 /*
1874 * Line Buffer Setup
1875 * There are 3 line buffers, each one shared by 2 display controllers.
1876 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1877 * the display controllers. The paritioning is done via one of four
1878 * preset allocations specified in bits 2:0:
1879 * first display controller
1880 * 0 - first half of lb (3840 * 2)
1881 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1882 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1883 * 3 - first 1/4 of lb (1920 * 2)
1884 * second display controller
1885 * 4 - second half of lb (3840 * 2)
1886 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1887 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1888 * 7 - last 1/4 of lb (1920 * 2)
1889 */
12dfc843
AD		 1890 /* this can get tricky if we have two large displays on a paired group
1891 * of crtcs. Ideally for multiple large displays we'd assign them to
1892 * non-linked crtcs for maximum line buffer allocation.
1893 */
1894 if (radeon_crtc->base.enabled && mode) {
0b31e023 1895 if (other_mode) {
f9d9c362 1896 tmp = 0; /* 1/2 */
0b31e023
AD		 1897 buffer_alloc = 1;
1898 } else {
12dfc843 1899 tmp = 2; /* whole */
0b31e023
AD		 1900 buffer_alloc = 2;
1901 }
1902 } else {
12dfc843 1903 tmp = 0;
0b31e023
AD		 1904 buffer_alloc = 0;
1905 }
f9d9c362
AD		 1906
1907 /* second controller of the pair uses second half of the lb */
1908 if (radeon_crtc->crtc_id % 2)
1909 tmp += 4;
1910 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1911
0b31e023
AD		 1912 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1913 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1914 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1915 for (i = 0; i < rdev->usec_timeout; i++) {
1916 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1917 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1918 break;
1919 udelay(1);
1920 }
1921 }
1922
12dfc843
AD		 1923 if (radeon_crtc->base.enabled && mode) {
1924 switch (tmp) {
1925 case 0:
1926 case 4:
1927 default:
1928 if (ASIC_IS_DCE5(rdev))
1929 return 4096 * 2;
1930 else
1931 return 3840 * 2;
1932 case 1:
1933 case 5:
1934 if (ASIC_IS_DCE5(rdev))
1935 return 6144 * 2;
1936 else
1937 return 5760 * 2;
1938 case 2:
1939 case 6:
1940 if (ASIC_IS_DCE5(rdev))
1941 return 8192 * 2;
1942 else
1943 return 7680 * 2;
1944 case 3:
1945 case 7:
1946 if (ASIC_IS_DCE5(rdev))
1947 return 2048 * 2;
1948 else
1949 return 1920 * 2;
1950 }
f9d9c362 1951 }
12dfc843
AD		 1952
1953 /* controller not enabled, so no lb used */
1954 return 0;
f9d9c362
AD		 1955}
1956
ca7db22b 1957u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1958{
1959 u32 tmp = RREG32(MC_SHARED_CHMAP);
1960
1961 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1962 case 0:
1963 default:
1964 return 1;
1965 case 1:
1966 return 2;
1967 case 2:
1968 return 4;
1969 case 3:
1970 return 8;
1971 }
1972}
1973
1974struct evergreen_wm_params {
1975 u32 dram_channels; /* number of dram channels */
1976 u32 yclk; /* bandwidth per dram data pin in kHz */
1977 u32 sclk; /* engine clock in kHz */
1978 u32 disp_clk; /* display clock in kHz */
1979 u32 src_width; /* viewport width */
1980 u32 active_time; /* active display time in ns */
1981 u32 blank_time; /* blank time in ns */
1982 bool interlaced; /* mode is interlaced */
1983 fixed20_12 vsc; /* vertical scale ratio */
1984 u32 num_heads; /* number of active crtcs */
1985 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1986 u32 lb_size; /* line buffer allocated to pipe */
1987 u32 vtaps; /* vertical scaler taps */
1988};
1989
1990static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1991{
1992 /* Calculate DRAM Bandwidth and the part allocated to display. */
1993 fixed20_12 dram_efficiency; /* 0.7 */
1994 fixed20_12 yclk, dram_channels, bandwidth;
1995 fixed20_12 a;
1996
1997 a.full = dfixed_const(1000);
1998 yclk.full = dfixed_const(wm->yclk);
1999 yclk.full = dfixed_div(yclk, a);
2000 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2001 a.full = dfixed_const(10);
2002 dram_efficiency.full = dfixed_const(7);
2003 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2004 bandwidth.full = dfixed_mul(dram_channels, yclk);
2005 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2006
2007 return dfixed_trunc(bandwidth);
2008}
2009
2010static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2011{
2012 /* Calculate DRAM Bandwidth and the part allocated to display. */
2013 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2014 fixed20_12 yclk, dram_channels, bandwidth;
2015 fixed20_12 a;
2016
2017 a.full = dfixed_const(1000);
2018 yclk.full = dfixed_const(wm->yclk);
2019 yclk.full = dfixed_div(yclk, a);
2020 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2021 a.full = dfixed_const(10);
2022 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2023 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2024 bandwidth.full = dfixed_mul(dram_channels, yclk);
2025 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2026
2027 return dfixed_trunc(bandwidth);
2028}
2029
2030static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2031{
2032 /* Calculate the display Data return Bandwidth */
2033 fixed20_12 return_efficiency; /* 0.8 */
2034 fixed20_12 sclk, bandwidth;
2035 fixed20_12 a;
2036
2037 a.full = dfixed_const(1000);
2038 sclk.full = dfixed_const(wm->sclk);
2039 sclk.full = dfixed_div(sclk, a);
2040 a.full = dfixed_const(10);
2041 return_efficiency.full = dfixed_const(8);
2042 return_efficiency.full = dfixed_div(return_efficiency, a);
2043 a.full = dfixed_const(32);
2044 bandwidth.full = dfixed_mul(a, sclk);
2045 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2046
2047 return dfixed_trunc(bandwidth);
2048}
2049
2050static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2051{
2052 /* Calculate the DMIF Request Bandwidth */
2053 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2054 fixed20_12 disp_clk, bandwidth;
2055 fixed20_12 a;
2056
2057 a.full = dfixed_const(1000);
2058 disp_clk.full = dfixed_const(wm->disp_clk);
2059 disp_clk.full = dfixed_div(disp_clk, a);
2060 a.full = dfixed_const(10);
2061 disp_clk_request_efficiency.full = dfixed_const(8);
2062 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2063 a.full = dfixed_const(32);
2064 bandwidth.full = dfixed_mul(a, disp_clk);
2065 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2066
2067 return dfixed_trunc(bandwidth);
2068}
2069
2070static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2071{
2072 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2073 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2074 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2075 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2076
2077 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2078}
2079
2080static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2081{
2082 /* Calculate the display mode Average Bandwidth
2083 * DisplayMode should contain the source and destination dimensions,
2084 * timing, etc.
2085 */
2086 fixed20_12 bpp;
2087 fixed20_12 line_time;
2088 fixed20_12 src_width;
2089 fixed20_12 bandwidth;
2090 fixed20_12 a;
2091
2092 a.full = dfixed_const(1000);
2093 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2094 line_time.full = dfixed_div(line_time, a);
2095 bpp.full = dfixed_const(wm->bytes_per_pixel);
2096 src_width.full = dfixed_const(wm->src_width);
2097 bandwidth.full = dfixed_mul(src_width, bpp);
2098 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2099 bandwidth.full = dfixed_div(bandwidth, line_time);
2100
2101 return dfixed_trunc(bandwidth);
2102}
2103
2104static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2105{
2106 /* First calcualte the latency in ns */
2107 u32 mc_latency = 2000; /* 2000 ns. */
2108 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2109 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2110 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2111 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2112 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2113 (wm->num_heads * cursor_line_pair_return_time);
2114 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2115 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2116 fixed20_12 a, b, c;
2117
2118 if (wm->num_heads == 0)
2119 return 0;
2120
2121 a.full = dfixed_const(2);
2122 b.full = dfixed_const(1);
2123 if ((wm->vsc.full > a.full) ||
2124 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2125 (wm->vtaps >= 5) ||
2126 ((wm->vsc.full >= a.full) && wm->interlaced))
2127 max_src_lines_per_dst_line = 4;
2128 else
2129 max_src_lines_per_dst_line = 2;
2130
2131 a.full = dfixed_const(available_bandwidth);
2132 b.full = dfixed_const(wm->num_heads);
2133 a.full = dfixed_div(a, b);
2134
2135 b.full = dfixed_const(1000);
2136 c.full = dfixed_const(wm->disp_clk);
2137 b.full = dfixed_div(c, b);
2138 c.full = dfixed_const(wm->bytes_per_pixel);
2139 b.full = dfixed_mul(b, c);
2140
2141 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2142
2143 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2144 b.full = dfixed_const(1000);
2145 c.full = dfixed_const(lb_fill_bw);
2146 b.full = dfixed_div(c, b);
2147 a.full = dfixed_div(a, b);
2148 line_fill_time = dfixed_trunc(a);
2149
2150 if (line_fill_time < wm->active_time)
2151 return latency;
2152 else
2153 return latency + (line_fill_time - wm->active_time);
2154
2155}
2156
2157static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2158{
2159 if (evergreen_average_bandwidth(wm) <=
2160 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2161 return true;
2162 else
2163 return false;
2164};
2165
2166static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2167{
2168 if (evergreen_average_bandwidth(wm) <=
2169 (evergreen_available_bandwidth(wm) / wm->num_heads))
2170 return true;
2171 else
2172 return false;
2173};
2174
2175static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2176{
2177 u32 lb_partitions = wm->lb_size / wm->src_width;
2178 u32 line_time = wm->active_time + wm->blank_time;
2179 u32 latency_tolerant_lines;
2180 u32 latency_hiding;
2181 fixed20_12 a;
2182
2183 a.full = dfixed_const(1);
2184 if (wm->vsc.full > a.full)
2185 latency_tolerant_lines = 1;
2186 else {
2187 if (lb_partitions <= (wm->vtaps + 1))
2188 latency_tolerant_lines = 1;
2189 else
2190 latency_tolerant_lines = 2;
2191 }
2192
2193 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2194
2195 if (evergreen_latency_watermark(wm) <= latency_hiding)
2196 return true;
2197 else
2198 return false;
2199}
2200
2201static void evergreen_program_watermarks(struct radeon_device *rdev,
2202 struct radeon_crtc *radeon_crtc,
2203 u32 lb_size, u32 num_heads)
2204{
2205 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2206 struct evergreen_wm_params wm_low, wm_high;
2207 u32 dram_channels;
f9d9c362
AD		 2208 u32 pixel_period;
2209 u32 line_time = 0;
2210 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2211 u32 priority_a_mark = 0, priority_b_mark = 0;
2212 u32 priority_a_cnt = PRIORITY_OFF;
2213 u32 priority_b_cnt = PRIORITY_OFF;
2214 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2215 u32 tmp, arb_control3;
2216 fixed20_12 a, b, c;
2217
2218 if (radeon_crtc->base.enabled && num_heads && mode) {
2219 pixel_period = 1000000 / (u32)mode->clock;
2220 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2221 priority_a_cnt = 0;
2222 priority_b_cnt = 0;
cf0cfdd7
AD		 2223 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2224
2225 /* watermark for high clocks */
2226 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2227 wm_high.yclk =
2228 radeon_dpm_get_mclk(rdev, false) * 10;
2229 wm_high.sclk =
2230 radeon_dpm_get_sclk(rdev, false) * 10;
2231 } else {
2232 wm_high.yclk = rdev->pm.current_mclk * 10;
2233 wm_high.sclk = rdev->pm.current_sclk * 10;
2234 }
f9d9c362 2235
cf0cfdd7
AD		 2236 wm_high.disp_clk = mode->clock;
2237 wm_high.src_width = mode->crtc_hdisplay;
2238 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2239 wm_high.blank_time = line_time - wm_high.active_time;
2240 wm_high.interlaced = false;
f9d9c362 2241 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2242 wm_high.interlaced = true;
2243 wm_high.vsc = radeon_crtc->vsc;
2244 wm_high.vtaps = 1;
f9d9c362 2245 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2246 wm_high.vtaps = 2;
2247 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2248 wm_high.lb_size = lb_size;
2249 wm_high.dram_channels = dram_channels;
2250 wm_high.num_heads = num_heads;
2251
2252 /* watermark for low clocks */
2253 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2254 wm_low.yclk =
2255 radeon_dpm_get_mclk(rdev, true) * 10;
2256 wm_low.sclk =
2257 radeon_dpm_get_sclk(rdev, true) * 10;
2258 } else {
2259 wm_low.yclk = rdev->pm.current_mclk * 10;
2260 wm_low.sclk = rdev->pm.current_sclk * 10;
2261 }
2262
2263 wm_low.disp_clk = mode->clock;
2264 wm_low.src_width = mode->crtc_hdisplay;
2265 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2266 wm_low.blank_time = line_time - wm_low.active_time;
2267 wm_low.interlaced = false;
2268 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2269 wm_low.interlaced = true;
2270 wm_low.vsc = radeon_crtc->vsc;
2271 wm_low.vtaps = 1;
2272 if (radeon_crtc->rmx_type != RMX_OFF)
2273 wm_low.vtaps = 2;
2274 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2275 wm_low.lb_size = lb_size;
2276 wm_low.dram_channels = dram_channels;
2277 wm_low.num_heads = num_heads;
f9d9c362
AD		 2278
2279 /* set for high clocks */
cf0cfdd7 2280 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2281 /* set for low clocks */
cf0cfdd7 2282 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2283
2284 /* possibly force display priority to high */
2285 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2286 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2287 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2288 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2289 (rdev->disp_priority == 2)) {
cf0cfdd7 2290 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2291 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2292 }
2293 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2294 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2295 !evergreen_check_latency_hiding(&wm_low) ||
2296 (rdev->disp_priority == 2)) {
2297 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2298 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2299 }
2300
2301 a.full = dfixed_const(1000);
2302 b.full = dfixed_const(mode->clock);
2303 b.full = dfixed_div(b, a);
2304 c.full = dfixed_const(latency_watermark_a);
2305 c.full = dfixed_mul(c, b);
2306 c.full = dfixed_mul(c, radeon_crtc->hsc);
2307 c.full = dfixed_div(c, a);
2308 a.full = dfixed_const(16);
2309 c.full = dfixed_div(c, a);
2310 priority_a_mark = dfixed_trunc(c);
2311 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2312
2313 a.full = dfixed_const(1000);
2314 b.full = dfixed_const(mode->clock);
2315 b.full = dfixed_div(b, a);
2316 c.full = dfixed_const(latency_watermark_b);
2317 c.full = dfixed_mul(c, b);
2318 c.full = dfixed_mul(c, radeon_crtc->hsc);
2319 c.full = dfixed_div(c, a);
2320 a.full = dfixed_const(16);
2321 c.full = dfixed_div(c, a);
2322 priority_b_mark = dfixed_trunc(c);
2323 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2324 }
2325
2326 /* select wm A */
2327 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2328 tmp = arb_control3;
2329 tmp &= ~LATENCY_WATERMARK_MASK(3);
2330 tmp |= LATENCY_WATERMARK_MASK(1);
2331 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2332 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2333 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2334 LATENCY_HIGH_WATERMARK(line_time)));
2335 /* select wm B */
2336 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2337 tmp &= ~LATENCY_WATERMARK_MASK(3);
2338 tmp |= LATENCY_WATERMARK_MASK(2);
2339 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2340 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2341 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2342 LATENCY_HIGH_WATERMARK(line_time)));
2343 /* restore original selection */
2344 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2345
2346 /* write the priority marks */
2347 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2348 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2349
7178d2a6
AD		 2350 /* save values for DPM */
2351 radeon_crtc->line_time = line_time;
2352 radeon_crtc->wm_high = latency_watermark_a;
2353 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2354}
2355
377edc8b
AD		 2356/**
2357 * evergreen_bandwidth_update - update display watermarks callback.
2358 *
2359 * @rdev: radeon_device pointer
2360 *
2361 * Update the display watermarks based on the requested mode(s)
2362 * (evergreen+).
2363 */
0ca2ab52 2364void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2365{
f9d9c362
AD		 2366 struct drm_display_mode *mode0 = NULL;
2367 struct drm_display_mode *mode1 = NULL;
2368 u32 num_heads = 0, lb_size;
2369 int i;
2370
2371 radeon_update_display_priority(rdev);
2372
2373 for (i = 0; i < rdev->num_crtc; i++) {
2374 if (rdev->mode_info.crtcs[i]->base.enabled)
2375 num_heads++;
2376 }
2377 for (i = 0; i < rdev->num_crtc; i += 2) {
2378 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2379 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2380 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2381 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2382 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2383 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2384 }
bcc1c2a1
AD		 2385}
2386
377edc8b
AD		 2387/**
2388 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2389 *
2390 * @rdev: radeon_device pointer
2391 *
2392 * Wait for the MC (memory controller) to be idle.
2393 * (evergreen+).
2394 * Returns 0 if the MC is idle, -1 if not.
2395 */
b9952a8a 2396int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2397{
2398 unsigned i;
2399 u32 tmp;
2400
2401 for (i = 0; i < rdev->usec_timeout; i++) {
2402 /* read MC_STATUS */
2403 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2404 if (!tmp)
2405 return 0;
2406 udelay(1);
2407 }
2408 return -1;
2409}
2410
2411/*
2412 * GART
2413 */
0fcdb61e
AD		 2414void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2415{
2416 unsigned i;
2417 u32 tmp;
2418
6f2f48a9
AD		 2419 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2420
0fcdb61e
AD		 2421 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2422 for (i = 0; i < rdev->usec_timeout; i++) {
2423 /* read MC_STATUS */
2424 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2425 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2426 if (tmp == 2) {
2427 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2428 return;
2429 }
2430 if (tmp) {
2431 return;
2432 }
2433 udelay(1);
2434 }
2435}
2436
1109ca09 2437static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2438{
2439 u32 tmp;
0fcdb61e 2440 int r;
bcc1c2a1 2441
c9a1be96 2442 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2443 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2444 return -EINVAL;
2445 }
2446 r = radeon_gart_table_vram_pin(rdev);
2447 if (r)
2448 return r;
82568565 2449 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2450 /* Setup L2 cache */
2451 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2452 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2453 EFFECTIVE_L2_QUEUE_SIZE(7));
2454 WREG32(VM_L2_CNTL2, 0);
2455 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2456 /* Setup TLB control */
2457 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2458 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2459 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2460 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2461 if (rdev->flags & RADEON_IS_IGP) {
2462 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2463 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2464 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2465 } else {
2466 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2469 if ((rdev->family == CHIP_JUNIPER) ||
2470 (rdev->family == CHIP_CYPRESS) ||
2471 (rdev->family == CHIP_HEMLOCK) ||
2472 (rdev->family == CHIP_BARTS))
2473 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2474 }
bcc1c2a1
AD		 2475 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2476 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2477 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2478 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2479 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2480 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2481 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2482 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2483 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2484 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2485 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2486 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2487
0fcdb61e 2488 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2489 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2490 (unsigned)(rdev->mc.gtt_size >> 20),
2491 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2492 rdev->gart.ready = true;
2493 return 0;
2494}
2495
1109ca09 2496static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2497{
2498 u32 tmp;
bcc1c2a1
AD		 2499
2500 /* Disable all tables */
0fcdb61e
AD		 2501 WREG32(VM_CONTEXT0_CNTL, 0);
2502 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2503
2504 /* Setup L2 cache */
2505 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2506 EFFECTIVE_L2_QUEUE_SIZE(7));
2507 WREG32(VM_L2_CNTL2, 0);
2508 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2509 /* Setup TLB control */
2510 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2511 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2512 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2513 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2515 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2516 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2517 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2518 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2519}
2520
1109ca09 2521static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2522{
2523 evergreen_pcie_gart_disable(rdev);
2524 radeon_gart_table_vram_free(rdev);
2525 radeon_gart_fini(rdev);
2526}
2527
2528
1109ca09 2529static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2530{
2531 u32 tmp;
bcc1c2a1
AD		 2532
2533 /* Setup L2 cache */
2534 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2535 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2536 EFFECTIVE_L2_QUEUE_SIZE(7));
2537 WREG32(VM_L2_CNTL2, 0);
2538 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2539 /* Setup TLB control */
2540 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2541 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2542 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2543 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2544 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2545 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2546 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2547 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2548 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2549 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2550 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2551 WREG32(VM_CONTEXT0_CNTL, 0);
2552 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2553}
2554
b9952a8a 2555void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2556{
62444b74
AD		 2557 u32 crtc_enabled, tmp, frame_count, blackout;
2558 int i, j;
2559
5153550a
AD		 2560 if (!ASIC_IS_NODCE(rdev)) {
2561 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2562 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2563
5153550a
AD		 2564 /* disable VGA render */
2565 WREG32(VGA_RENDER_CONTROL, 0);
2566 }
62444b74
AD		 2567 /* blank the display controllers */
2568 for (i = 0; i < rdev->num_crtc; i++) {
2569 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2570 if (crtc_enabled) {
2571 save->crtc_enabled[i] = true;
2572 if (ASIC_IS_DCE6(rdev)) {
2573 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2574 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2575 radeon_wait_for_vblank(rdev, i);
abf1457b 2576 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2577 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2578 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2579 }
2580 } else {
2581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2582 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2583 radeon_wait_for_vblank(rdev, i);
abf1457b 2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2585 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2586 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2587 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2588 }
2589 }
2590 /* wait for the next frame */
2591 frame_count = radeon_get_vblank_counter(rdev, i);
2592 for (j = 0; j < rdev->usec_timeout; j++) {
2593 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2594 break;
2595 udelay(1);
2596 }
abf1457b
AD		 2597
2598 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2599 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2600 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2601 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2602 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2603 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2604 save->crtc_enabled[i] = false;
2605 /* ***** */
804cc4a0
AD		 2606 } else {
2607 save->crtc_enabled[i] = false;
62444b74 2608 }
18007401 2609 }
bcc1c2a1 2610
62444b74
AD		 2611 radeon_mc_wait_for_idle(rdev);
2612
2613 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2614 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2615 /* Block CPU access */
2616 WREG32(BIF_FB_EN, 0);
2617 /* blackout the MC */
2618 blackout &= ~BLACKOUT_MODE_MASK;
2619 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2620 }
ed39fadd
AD		 2621 /* wait for the MC to settle */
2622 udelay(100);
968c0166
AD		 2623
2624 /* lock double buffered regs */
2625 for (i = 0; i < rdev->num_crtc; i++) {
2626 if (save->crtc_enabled[i]) {
2627 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2628 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2629 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2630 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2631 }
2632 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2633 if (!(tmp & 1)) {
2634 tmp |= 1;
2635 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2636 }
2637 }
2638 }
bcc1c2a1
AD		 2639}
2640
b9952a8a 2641void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2642{
62444b74
AD		 2643 u32 tmp, frame_count;
2644 int i, j;
18007401 2645
62444b74
AD		 2646 /* update crtc base addresses */
2647 for (i = 0; i < rdev->num_crtc; i++) {
2648 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2649 upper_32_bits(rdev->mc.vram_start));
62444b74 2650 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2651 upper_32_bits(rdev->mc.vram_start));
62444b74 2652 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2653 (u32)rdev->mc.vram_start);
62444b74 2654 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2655 (u32)rdev->mc.vram_start);
2656 }
5153550a
AD		 2657
2658 if (!ASIC_IS_NODCE(rdev)) {
2659 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2660 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2661 }
62444b74 2662
968c0166
AD		 2663 /* unlock regs and wait for update */
2664 for (i = 0; i < rdev->num_crtc; i++) {
2665 if (save->crtc_enabled[i]) {
2666 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2667 if ((tmp & 0x3) != 0) {
2668 tmp &= ~0x3;
2669 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2670 }
2671 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2672 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2673 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2674 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2675 }
2676 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2677 if (tmp & 1) {
2678 tmp &= ~1;
2679 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2680 }
2681 for (j = 0; j < rdev->usec_timeout; j++) {
2682 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2683 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2684 break;
2685 udelay(1);
2686 }
2687 }
2688 }
2689
62444b74
AD		 2690 /* unblackout the MC */
2691 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2692 tmp &= ~BLACKOUT_MODE_MASK;
2693 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2694 /* allow CPU access */
2695 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2696
2697 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2698 if (save->crtc_enabled[i]) {
62444b74
AD		 2699 if (ASIC_IS_DCE6(rdev)) {
2700 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2701 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2702 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2703 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2704 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2705 } else {
2706 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2707 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2708 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2709 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2710 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2711 }
2712 /* wait for the next frame */
2713 frame_count = radeon_get_vblank_counter(rdev, i);
2714 for (j = 0; j < rdev->usec_timeout; j++) {
2715 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2716 break;
2717 udelay(1);
2718 }
2719 }
2720 }
5153550a
AD		 2721 if (!ASIC_IS_NODCE(rdev)) {
2722 /* Unlock vga access */
2723 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2724 mdelay(1);
2725 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2726 }
bcc1c2a1
AD		 2727}
2728
755d819e 2729void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2730{
2731 struct evergreen_mc_save save;
2732 u32 tmp;
2733 int i, j;
2734
2735 /* Initialize HDP */
2736 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2737 WREG32((0x2c14 + j), 0x00000000);
2738 WREG32((0x2c18 + j), 0x00000000);
2739 WREG32((0x2c1c + j), 0x00000000);
2740 WREG32((0x2c20 + j), 0x00000000);
2741 WREG32((0x2c24 + j), 0x00000000);
2742 }
2743 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2744
2745 evergreen_mc_stop(rdev, &save);
2746 if (evergreen_mc_wait_for_idle(rdev)) {
2747 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2748 }
2749 /* Lockout access through VGA aperture*/
2750 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2751 /* Update configuration */
2752 if (rdev->flags & RADEON_IS_AGP) {
2753 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2754 /* VRAM before AGP */
2755 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2756 rdev->mc.vram_start >> 12);
2757 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2758 rdev->mc.gtt_end >> 12);
2759 } else {
2760 /* VRAM after AGP */
2761 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2762 rdev->mc.gtt_start >> 12);
2763 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2764 rdev->mc.vram_end >> 12);
2765 }
2766 } else {
2767 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2768 rdev->mc.vram_start >> 12);
2769 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2770 rdev->mc.vram_end >> 12);
2771 }
3b9832f6 2772 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2773 /* llano/ontario only */
2774 if ((rdev->family == CHIP_PALM) ||
2775 (rdev->family == CHIP_SUMO) ||
2776 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2777 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2778 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2779 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2780 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2781 }
bcc1c2a1
AD		 2782 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2783 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2784 WREG32(MC_VM_FB_LOCATION, tmp);
2785 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2786 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2787 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2788 if (rdev->flags & RADEON_IS_AGP) {
2789 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2790 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2791 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2792 } else {
2793 WREG32(MC_VM_AGP_BASE, 0);
2794 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2795 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2796 }
2797 if (evergreen_mc_wait_for_idle(rdev)) {
2798 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2799 }
2800 evergreen_mc_resume(rdev, &save);
2801 /* we need to own VRAM, so turn off the VGA renderer here
2802 * to stop it overwriting our objects */
2803 rv515_vga_render_disable(rdev);
2804}
2805
bcc1c2a1
AD		 2806/*
2807 * CP.
2808 */
12920591
AD		 2809void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2810{
876dc9f3 2811 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2812 u32 next_rptr;
7b1f2485 2813
12920591 2814 /* set to DX10/11 mode */
e32eb50d
CK		 2815 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2816 radeon_ring_write(ring, 1);
45df6803
CK		 2817
2818 if (ring->rptr_save_reg) {
89d35807 2819 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2820 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2821 radeon_ring_write(ring, ((ring->rptr_save_reg -
2822 PACKET3_SET_CONFIG_REG_START) >> 2));
2823 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2824 } else if (rdev->wb.enabled) {
2825 next_rptr = ring->wptr + 5 + 4;
2826 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2827 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2828 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2829 radeon_ring_write(ring, next_rptr);
2830 radeon_ring_write(ring, 0);
45df6803
CK		 2831 }
2832
e32eb50d
CK		 2833 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2834 radeon_ring_write(ring,
0f234f5f
AD		 2835#ifdef __BIG_ENDIAN
2836 (2 << 0) |
2837#endif
2838 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2839 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2840 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2841}
2842
bcc1c2a1
AD		 2843
2844static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2845{
fe251e2f
AD		 2846 const __be32 *fw_data;
2847 int i;
2848
2849 if (!rdev->me_fw || !rdev->pfp_fw)
2850 return -EINVAL;
bcc1c2a1 2851
fe251e2f 2852 r700_cp_stop(rdev);
0f234f5f
AD		 2853 WREG32(CP_RB_CNTL,
2854#ifdef __BIG_ENDIAN
2855 BUF_SWAP_32BIT |
2856#endif
2857 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2858
2859 fw_data = (const __be32 *)rdev->pfp_fw->data;
2860 WREG32(CP_PFP_UCODE_ADDR, 0);
2861 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2862 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2863 WREG32(CP_PFP_UCODE_ADDR, 0);
2864
2865 fw_data = (const __be32 *)rdev->me_fw->data;
2866 WREG32(CP_ME_RAM_WADDR, 0);
2867 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2868 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2869
2870 WREG32(CP_PFP_UCODE_ADDR, 0);
2871 WREG32(CP_ME_RAM_WADDR, 0);
2872 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2873 return 0;
2874}
2875
7e7b41d2
AD		 2876static int evergreen_cp_start(struct radeon_device *rdev)
2877{
e32eb50d 2878 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2879 int r, i;
7e7b41d2
AD		 2880 uint32_t cp_me;
2881
e32eb50d 2882 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2883 if (r) {
2884 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2885 return r;
2886 }
e32eb50d
CK		 2887 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2888 radeon_ring_write(ring, 0x1);
2889 radeon_ring_write(ring, 0x0);
2890 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2891 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2892 radeon_ring_write(ring, 0);
2893 radeon_ring_write(ring, 0);
2894 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2895
2896 cp_me = 0xff;
2897 WREG32(CP_ME_CNTL, cp_me);
2898
e32eb50d 2899 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2900 if (r) {
2901 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2902 return r;
2903 }
2281a378
AD		 2904
2905 /* setup clear context state */
e32eb50d
CK		 2906 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2907 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2908
2909 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2910 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2911
e32eb50d
CK		 2912 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2913 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2914
2915 /* set clear context state */
e32eb50d
CK		 2916 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2917 radeon_ring_write(ring, 0);
2281a378
AD		 2918
2919 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2920 radeon_ring_write(ring, 0xc0026f00);
2921 radeon_ring_write(ring, 0x00000000);
2922 radeon_ring_write(ring, 0x00000000);
2923 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2924
2925 /* Clear consts */
e32eb50d
CK		 2926 radeon_ring_write(ring, 0xc0036f00);
2927 radeon_ring_write(ring, 0x00000bc4);
2928 radeon_ring_write(ring, 0xffffffff);
2929 radeon_ring_write(ring, 0xffffffff);
2930 radeon_ring_write(ring, 0xffffffff);
2281a378 2931
e32eb50d
CK		 2932 radeon_ring_write(ring, 0xc0026900);
2933 radeon_ring_write(ring, 0x00000316);
2934 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2935 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2936
e32eb50d 2937 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2938
2939 return 0;
2940}
2941
1109ca09 2942static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2943{
e32eb50d 2944 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2945 u32 tmp;
2946 u32 rb_bufsz;
2947 int r;
2948
2949 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2950 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2951 SOFT_RESET_PA |
2952 SOFT_RESET_SH |
2953 SOFT_RESET_VGT |
a49a50da 2954 SOFT_RESET_SPI |
fe251e2f
AD		 2955 SOFT_RESET_SX));
2956 RREG32(GRBM_SOFT_RESET);
2957 mdelay(15);
2958 WREG32(GRBM_SOFT_RESET, 0);
2959 RREG32(GRBM_SOFT_RESET);
2960
2961 /* Set ring buffer size */
b72a8925
DV		 2962 rb_bufsz = order_base_2(ring->ring_size / 8);
2963 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2964#ifdef __BIG_ENDIAN
2965 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2966#endif
fe251e2f 2967 WREG32(CP_RB_CNTL, tmp);
15d3332f 2968 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2969 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2970
2971 /* Set the write pointer delay */
2972 WREG32(CP_RB_WPTR_DELAY, 0);
2973
2974 /* Initialize the ring buffer's read and write pointers */
2975 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2976 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2977 ring->wptr = 0;
2978 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2979
48fc7f7e 2980 /* set the wb address whether it's enabled or not */
0f234f5f 2981 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2982 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2983 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2984 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2985
2986 if (rdev->wb.enabled)
2987 WREG32(SCRATCH_UMSK, 0xff);
2988 else {
2989 tmp |= RB_NO_UPDATE;
2990 WREG32(SCRATCH_UMSK, 0);
2991 }
2992
fe251e2f
AD		 2993 mdelay(1);
2994 WREG32(CP_RB_CNTL, tmp);
2995
e32eb50d 2996 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2997 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2998
e32eb50d 2999 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 3000
7e7b41d2 3001 evergreen_cp_start(rdev);
e32eb50d 3002 ring->ready = true;
f712812e 3003 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3004 if (r) {
e32eb50d 3005 ring->ready = false;
fe251e2f
AD		 3006 return r;
3007 }
3008 return 0;
3009}
bcc1c2a1
AD		 3010
3011/*
3012 * Core functions
3013 */
bcc1c2a1
AD		 3014static void evergreen_gpu_init(struct radeon_device *rdev)
3015{
416a2bd2 3016 u32 gb_addr_config;
32fcdbf4 3017 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3018 u32 sx_debug_1;
3019 u32 smx_dc_ctl0;
3020 u32 sq_config;
3021 u32 sq_lds_resource_mgmt;
3022 u32 sq_gpr_resource_mgmt_1;
3023 u32 sq_gpr_resource_mgmt_2;
3024 u32 sq_gpr_resource_mgmt_3;
3025 u32 sq_thread_resource_mgmt;
3026 u32 sq_thread_resource_mgmt_2;
3027 u32 sq_stack_resource_mgmt_1;
3028 u32 sq_stack_resource_mgmt_2;
3029 u32 sq_stack_resource_mgmt_3;
3030 u32 vgt_cache_invalidation;
f25a5c63 3031 u32 hdp_host_path_cntl, tmp;
416a2bd2 3032 u32 disabled_rb_mask;
32fcdbf4
AD		 3033 int i, j, num_shader_engines, ps_thread_count;
3034
3035 switch (rdev->family) {
3036 case CHIP_CYPRESS:
3037 case CHIP_HEMLOCK:
3038 rdev->config.evergreen.num_ses = 2;
3039 rdev->config.evergreen.max_pipes = 4;
3040 rdev->config.evergreen.max_tile_pipes = 8;
3041 rdev->config.evergreen.max_simds = 10;
3042 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3043 rdev->config.evergreen.max_gprs = 256;
3044 rdev->config.evergreen.max_threads = 248;
3045 rdev->config.evergreen.max_gs_threads = 32;
3046 rdev->config.evergreen.max_stack_entries = 512;
3047 rdev->config.evergreen.sx_num_of_sets = 4;
3048 rdev->config.evergreen.sx_max_export_size = 256;
3049 rdev->config.evergreen.sx_max_export_pos_size = 64;
3050 rdev->config.evergreen.sx_max_export_smx_size = 192;
3051 rdev->config.evergreen.max_hw_contexts = 8;
3052 rdev->config.evergreen.sq_num_cf_insts = 2;
3053
3054 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3055 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3056 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3057 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3058 break;
3059 case CHIP_JUNIPER:
3060 rdev->config.evergreen.num_ses = 1;
3061 rdev->config.evergreen.max_pipes = 4;
3062 rdev->config.evergreen.max_tile_pipes = 4;
3063 rdev->config.evergreen.max_simds = 10;
3064 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3065 rdev->config.evergreen.max_gprs = 256;
3066 rdev->config.evergreen.max_threads = 248;
3067 rdev->config.evergreen.max_gs_threads = 32;
3068 rdev->config.evergreen.max_stack_entries = 512;
3069 rdev->config.evergreen.sx_num_of_sets = 4;
3070 rdev->config.evergreen.sx_max_export_size = 256;
3071 rdev->config.evergreen.sx_max_export_pos_size = 64;
3072 rdev->config.evergreen.sx_max_export_smx_size = 192;
3073 rdev->config.evergreen.max_hw_contexts = 8;
3074 rdev->config.evergreen.sq_num_cf_insts = 2;
3075
3076 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3077 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3078 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3079 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3080 break;
3081 case CHIP_REDWOOD:
3082 rdev->config.evergreen.num_ses = 1;
3083 rdev->config.evergreen.max_pipes = 4;
3084 rdev->config.evergreen.max_tile_pipes = 4;
3085 rdev->config.evergreen.max_simds = 5;
3086 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3087 rdev->config.evergreen.max_gprs = 256;
3088 rdev->config.evergreen.max_threads = 248;
3089 rdev->config.evergreen.max_gs_threads = 32;
3090 rdev->config.evergreen.max_stack_entries = 256;
3091 rdev->config.evergreen.sx_num_of_sets = 4;
3092 rdev->config.evergreen.sx_max_export_size = 256;
3093 rdev->config.evergreen.sx_max_export_pos_size = 64;
3094 rdev->config.evergreen.sx_max_export_smx_size = 192;
3095 rdev->config.evergreen.max_hw_contexts = 8;
3096 rdev->config.evergreen.sq_num_cf_insts = 2;
3097
3098 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3099 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3100 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3101 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3102 break;
3103 case CHIP_CEDAR:
3104 default:
3105 rdev->config.evergreen.num_ses = 1;
3106 rdev->config.evergreen.max_pipes = 2;
3107 rdev->config.evergreen.max_tile_pipes = 2;
3108 rdev->config.evergreen.max_simds = 2;
3109 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3110 rdev->config.evergreen.max_gprs = 256;
3111 rdev->config.evergreen.max_threads = 192;
3112 rdev->config.evergreen.max_gs_threads = 16;
3113 rdev->config.evergreen.max_stack_entries = 256;
3114 rdev->config.evergreen.sx_num_of_sets = 4;
3115 rdev->config.evergreen.sx_max_export_size = 128;
3116 rdev->config.evergreen.sx_max_export_pos_size = 32;
3117 rdev->config.evergreen.sx_max_export_smx_size = 96;
3118 rdev->config.evergreen.max_hw_contexts = 4;
3119 rdev->config.evergreen.sq_num_cf_insts = 1;
3120
d5e455e4
AD		 3121 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3122 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3123 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3124 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3125 break;
3126 case CHIP_PALM:
3127 rdev->config.evergreen.num_ses = 1;
3128 rdev->config.evergreen.max_pipes = 2;
3129 rdev->config.evergreen.max_tile_pipes = 2;
3130 rdev->config.evergreen.max_simds = 2;
3131 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3132 rdev->config.evergreen.max_gprs = 256;
3133 rdev->config.evergreen.max_threads = 192;
3134 rdev->config.evergreen.max_gs_threads = 16;
3135 rdev->config.evergreen.max_stack_entries = 256;
3136 rdev->config.evergreen.sx_num_of_sets = 4;
3137 rdev->config.evergreen.sx_max_export_size = 128;
3138 rdev->config.evergreen.sx_max_export_pos_size = 32;
3139 rdev->config.evergreen.sx_max_export_smx_size = 96;
3140 rdev->config.evergreen.max_hw_contexts = 4;
3141 rdev->config.evergreen.sq_num_cf_insts = 1;
3142
d5c5a72f
AD		 3143 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3144 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3145 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3146 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3147 break;
3148 case CHIP_SUMO:
3149 rdev->config.evergreen.num_ses = 1;
3150 rdev->config.evergreen.max_pipes = 4;
bd25f078 3151 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3152 if (rdev->pdev->device == 0x9648)
3153 rdev->config.evergreen.max_simds = 3;
3154 else if ((rdev->pdev->device == 0x9647) ||
3155 (rdev->pdev->device == 0x964a))
3156 rdev->config.evergreen.max_simds = 4;
3157 else
3158 rdev->config.evergreen.max_simds = 5;
3159 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3160 rdev->config.evergreen.max_gprs = 256;
3161 rdev->config.evergreen.max_threads = 248;
3162 rdev->config.evergreen.max_gs_threads = 32;
3163 rdev->config.evergreen.max_stack_entries = 256;
3164 rdev->config.evergreen.sx_num_of_sets = 4;
3165 rdev->config.evergreen.sx_max_export_size = 256;
3166 rdev->config.evergreen.sx_max_export_pos_size = 64;
3167 rdev->config.evergreen.sx_max_export_smx_size = 192;
3168 rdev->config.evergreen.max_hw_contexts = 8;
3169 rdev->config.evergreen.sq_num_cf_insts = 2;
3170
3171 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3172 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3173 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3174 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3175 break;
3176 case CHIP_SUMO2:
3177 rdev->config.evergreen.num_ses = 1;
3178 rdev->config.evergreen.max_pipes = 4;
3179 rdev->config.evergreen.max_tile_pipes = 4;
3180 rdev->config.evergreen.max_simds = 2;
3181 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3182 rdev->config.evergreen.max_gprs = 256;
3183 rdev->config.evergreen.max_threads = 248;
3184 rdev->config.evergreen.max_gs_threads = 32;
3185 rdev->config.evergreen.max_stack_entries = 512;
3186 rdev->config.evergreen.sx_num_of_sets = 4;
3187 rdev->config.evergreen.sx_max_export_size = 256;
3188 rdev->config.evergreen.sx_max_export_pos_size = 64;
3189 rdev->config.evergreen.sx_max_export_smx_size = 192;
3190 rdev->config.evergreen.max_hw_contexts = 8;
3191 rdev->config.evergreen.sq_num_cf_insts = 2;
3192
adb68fa2
AD		 3193 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3194 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3195 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3196 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3197 break;
3198 case CHIP_BARTS:
3199 rdev->config.evergreen.num_ses = 2;
3200 rdev->config.evergreen.max_pipes = 4;
3201 rdev->config.evergreen.max_tile_pipes = 8;
3202 rdev->config.evergreen.max_simds = 7;
3203 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3204 rdev->config.evergreen.max_gprs = 256;
3205 rdev->config.evergreen.max_threads = 248;
3206 rdev->config.evergreen.max_gs_threads = 32;
3207 rdev->config.evergreen.max_stack_entries = 512;
3208 rdev->config.evergreen.sx_num_of_sets = 4;
3209 rdev->config.evergreen.sx_max_export_size = 256;
3210 rdev->config.evergreen.sx_max_export_pos_size = 64;
3211 rdev->config.evergreen.sx_max_export_smx_size = 192;
3212 rdev->config.evergreen.max_hw_contexts = 8;
3213 rdev->config.evergreen.sq_num_cf_insts = 2;
3214
3215 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3216 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3217 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3218 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3219 break;
3220 case CHIP_TURKS:
3221 rdev->config.evergreen.num_ses = 1;
3222 rdev->config.evergreen.max_pipes = 4;
3223 rdev->config.evergreen.max_tile_pipes = 4;
3224 rdev->config.evergreen.max_simds = 6;
3225 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3226 rdev->config.evergreen.max_gprs = 256;
3227 rdev->config.evergreen.max_threads = 248;
3228 rdev->config.evergreen.max_gs_threads = 32;
3229 rdev->config.evergreen.max_stack_entries = 256;
3230 rdev->config.evergreen.sx_num_of_sets = 4;
3231 rdev->config.evergreen.sx_max_export_size = 256;
3232 rdev->config.evergreen.sx_max_export_pos_size = 64;
3233 rdev->config.evergreen.sx_max_export_smx_size = 192;
3234 rdev->config.evergreen.max_hw_contexts = 8;
3235 rdev->config.evergreen.sq_num_cf_insts = 2;
3236
3237 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3238 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3240 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3241 break;
3242 case CHIP_CAICOS:
3243 rdev->config.evergreen.num_ses = 1;
bd25f078 3244 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3245 rdev->config.evergreen.max_tile_pipes = 2;
3246 rdev->config.evergreen.max_simds = 2;
3247 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3248 rdev->config.evergreen.max_gprs = 256;
3249 rdev->config.evergreen.max_threads = 192;
3250 rdev->config.evergreen.max_gs_threads = 16;
3251 rdev->config.evergreen.max_stack_entries = 256;
3252 rdev->config.evergreen.sx_num_of_sets = 4;
3253 rdev->config.evergreen.sx_max_export_size = 128;
3254 rdev->config.evergreen.sx_max_export_pos_size = 32;
3255 rdev->config.evergreen.sx_max_export_smx_size = 96;
3256 rdev->config.evergreen.max_hw_contexts = 4;
3257 rdev->config.evergreen.sq_num_cf_insts = 1;
3258
32fcdbf4
AD		 3259 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3260 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3261 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3262 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3263 break;
3264 }
3265
3266 /* Initialize HDP */
3267 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3268 WREG32((0x2c14 + j), 0x00000000);
3269 WREG32((0x2c18 + j), 0x00000000);
3270 WREG32((0x2c1c + j), 0x00000000);
3271 WREG32((0x2c20 + j), 0x00000000);
3272 WREG32((0x2c24 + j), 0x00000000);
3273 }
3274
3275 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3276
d054ac16
AD		 3277 evergreen_fix_pci_max_read_req_size(rdev);
3278
32fcdbf4 3279 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3280 if ((rdev->family == CHIP_PALM) ||
3281 (rdev->family == CHIP_SUMO) ||
3282 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3283 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3284 else
3285 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3286
1aa52bd3
AD		 3287 /* setup tiling info dword. gb_addr_config is not adequate since it does
3288 * not have bank info, so create a custom tiling dword.
3289 * bits 3:0 num_pipes
3290 * bits 7:4 num_banks
3291 * bits 11:8 group_size
3292 * bits 15:12 row_size
3293 */
3294 rdev->config.evergreen.tile_config = 0;
3295 switch (rdev->config.evergreen.max_tile_pipes) {
3296 case 1:
3297 default:
3298 rdev->config.evergreen.tile_config |= (0 << 0);
3299 break;
3300 case 2:
3301 rdev->config.evergreen.tile_config |= (1 << 0);
3302 break;
3303 case 4:
3304 rdev->config.evergreen.tile_config |= (2 << 0);
3305 break;
3306 case 8:
3307 rdev->config.evergreen.tile_config |= (3 << 0);
3308 break;
3309 }
d698a34d 3310 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3311 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3312 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3313 else {
c8d15edc
AD		 3314 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3315 case 0: /* four banks */
29d65406 3316 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3317 break;
3318 case 1: /* eight banks */
3319 rdev->config.evergreen.tile_config |= 1 << 4;
3320 break;
3321 case 2: /* sixteen banks */
3322 default:
3323 rdev->config.evergreen.tile_config |= 2 << 4;
3324 break;
3325 }
29d65406 3326 }
416a2bd2 3327 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3328 rdev->config.evergreen.tile_config |=
3329 ((gb_addr_config & 0x30000000) >> 28) << 12;
3330
416a2bd2 3331 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3332
416a2bd2
AD		 3333 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3334 u32 efuse_straps_4;
3335 u32 efuse_straps_3;
32fcdbf4 3336
ff82bbc4
AD		 3337 efuse_straps_4 = RREG32_RCU(0x204);
3338 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3339 tmp = (((efuse_straps_4 & 0xf) << 4) |
3340 ((efuse_straps_3 & 0xf0000000) >> 28));
3341 } else {
3342 tmp = 0;
3343 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3344 u32 rb_disable_bitmap;
3345
3346 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3347 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3348 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3349 tmp <<= 4;
3350 tmp |= rb_disable_bitmap;
32fcdbf4 3351 }
416a2bd2
AD		 3352 }
3353 /* enabled rb are just the one not disabled :) */
3354 disabled_rb_mask = tmp;
cedb655a
AD		 3355 tmp = 0;
3356 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3357 tmp |= (1 << i);
3358 /* if all the backends are disabled, fix it up here */
3359 if ((disabled_rb_mask & tmp) == tmp) {
3360 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3361 disabled_rb_mask &= ~(1 << i);
3362 }
32fcdbf4 3363
416a2bd2
AD		 3364 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3365 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3366
416a2bd2
AD		 3367 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3368 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3369 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3370 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3371 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3372 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3373 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3374
f7eb9730
AD		 3375 if ((rdev->config.evergreen.max_backends == 1) &&
3376 (rdev->flags & RADEON_IS_IGP)) {
3377 if ((disabled_rb_mask & 3) == 1) {
3378 /* RB0 disabled, RB1 enabled */
3379 tmp = 0x11111111;
3380 } else {
3381 /* RB1 disabled, RB0 enabled */
3382 tmp = 0x00000000;
3383 }
3384 } else {
3385 tmp = gb_addr_config & NUM_PIPES_MASK;
3386 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3387 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3388 }
416a2bd2 3389 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3390
3391 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3392 WREG32(CGTS_TCC_DISABLE, 0);
3393 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3394 WREG32(CGTS_USER_TCC_DISABLE, 0);
3395
3396 /* set HW defaults for 3D engine */
3397 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3398 ROQ_IB2_START(0x2b)));
3399
3400 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3401
3402 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3403 SYNC_GRADIENT |
3404 SYNC_WALKER |
3405 SYNC_ALIGNER));
3406
3407 sx_debug_1 = RREG32(SX_DEBUG_1);
3408 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3409 WREG32(SX_DEBUG_1, sx_debug_1);
3410
3411
3412 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3413 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3414 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3415 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3416
b866d133
AD		 3417 if (rdev->family <= CHIP_SUMO2)
3418 WREG32(SMX_SAR_CTL0, 0x00010000);
3419
32fcdbf4
AD		 3420 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3421 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3422 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3423
3424 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3425 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3426 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3427
3428 WREG32(VGT_NUM_INSTANCES, 1);
3429 WREG32(SPI_CONFIG_CNTL, 0);
3430 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3431 WREG32(CP_PERFMON_CNTL, 0);
3432
3433 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3434 FETCH_FIFO_HIWATER(0x4) |
3435 DONE_FIFO_HIWATER(0xe0) |
3436 ALU_UPDATE_FIFO_HIWATER(0x8)));
3437
3438 sq_config = RREG32(SQ_CONFIG);
3439 sq_config &= ~(PS_PRIO(3) |
3440 VS_PRIO(3) |
3441 GS_PRIO(3) |
3442 ES_PRIO(3));
3443 sq_config |= (VC_ENABLE |
3444 EXPORT_SRC_C |
3445 PS_PRIO(0) |
3446 VS_PRIO(1) |
3447 GS_PRIO(2) |
3448 ES_PRIO(3));
3449
d5e455e4
AD		 3450 switch (rdev->family) {
3451 case CHIP_CEDAR:
3452 case CHIP_PALM:
d5c5a72f
AD		 3453 case CHIP_SUMO:
3454 case CHIP_SUMO2:
adb68fa2 3455 case CHIP_CAICOS:
32fcdbf4
AD		 3456 /* no vertex cache */
3457 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3458 break;
3459 default:
3460 break;
3461 }
32fcdbf4
AD		 3462
3463 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3464
3465 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3466 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3467 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3468 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3469 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3470 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3471 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3472
d5e455e4
AD		 3473 switch (rdev->family) {
3474 case CHIP_CEDAR:
3475 case CHIP_PALM:
d5c5a72f
AD		 3476 case CHIP_SUMO:
3477 case CHIP_SUMO2:
32fcdbf4 3478 ps_thread_count = 96;
d5e455e4
AD		 3479 break;
3480 default:
32fcdbf4 3481 ps_thread_count = 128;
d5e455e4
AD		 3482 break;
3483 }
32fcdbf4
AD		 3484
3485 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3486 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3487 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3488 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3489 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3490 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3491
3492 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3493 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3494 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3495 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3496 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3497 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3498
3499 WREG32(SQ_CONFIG, sq_config);
3500 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3501 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3502 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3503 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3504 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3505 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3506 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3507 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3508 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3509 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3510
3511 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3512 FORCE_EOV_MAX_REZ_CNT(255)));
3513
d5e455e4
AD		 3514 switch (rdev->family) {
3515 case CHIP_CEDAR:
3516 case CHIP_PALM:
d5c5a72f
AD		 3517 case CHIP_SUMO:
3518 case CHIP_SUMO2:
adb68fa2 3519 case CHIP_CAICOS:
32fcdbf4 3520 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3521 break;
3522 default:
32fcdbf4 3523 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3524 break;
3525 }
32fcdbf4
AD		 3526 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3527 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3528
3529 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3530 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3531 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3532
60a4a3e0
AD		 3533 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3534 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3535
32fcdbf4
AD		 3536 WREG32(CB_PERF_CTR0_SEL_0, 0);
3537 WREG32(CB_PERF_CTR0_SEL_1, 0);
3538 WREG32(CB_PERF_CTR1_SEL_0, 0);
3539 WREG32(CB_PERF_CTR1_SEL_1, 0);
3540 WREG32(CB_PERF_CTR2_SEL_0, 0);
3541 WREG32(CB_PERF_CTR2_SEL_1, 0);
3542 WREG32(CB_PERF_CTR3_SEL_0, 0);
3543 WREG32(CB_PERF_CTR3_SEL_1, 0);
3544
60a4a3e0
AD		 3545 /* clear render buffer base addresses */
3546 WREG32(CB_COLOR0_BASE, 0);
3547 WREG32(CB_COLOR1_BASE, 0);
3548 WREG32(CB_COLOR2_BASE, 0);
3549 WREG32(CB_COLOR3_BASE, 0);
3550 WREG32(CB_COLOR4_BASE, 0);
3551 WREG32(CB_COLOR5_BASE, 0);
3552 WREG32(CB_COLOR6_BASE, 0);
3553 WREG32(CB_COLOR7_BASE, 0);
3554 WREG32(CB_COLOR8_BASE, 0);
3555 WREG32(CB_COLOR9_BASE, 0);
3556 WREG32(CB_COLOR10_BASE, 0);
3557 WREG32(CB_COLOR11_BASE, 0);
3558
3559 /* set the shader const cache sizes to 0 */
3560 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3561 WREG32(i, 0);
3562 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3563 WREG32(i, 0);
3564
f25a5c63
AD		 3565 tmp = RREG32(HDP_MISC_CNTL);
3566 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3567 WREG32(HDP_MISC_CNTL, tmp);
3568
32fcdbf4
AD		 3569 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3570 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3571
3572 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3573
3574 udelay(50);
3575
bcc1c2a1
AD		 3576}
3577
3578int evergreen_mc_init(struct radeon_device *rdev)
3579{
bcc1c2a1
AD		 3580 u32 tmp;
3581 int chansize, numchan;
bcc1c2a1
AD		 3582
3583 /* Get VRAM informations */
3584 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3585 if ((rdev->family == CHIP_PALM) ||
3586 (rdev->family == CHIP_SUMO) ||
3587 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3588 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3589 else
3590 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3591 if (tmp & CHANSIZE_OVERRIDE) {
3592 chansize = 16;
3593 } else if (tmp & CHANSIZE_MASK) {
3594 chansize = 64;
3595 } else {
3596 chansize = 32;
3597 }
3598 tmp = RREG32(MC_SHARED_CHMAP);
3599 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3600 case 0:
3601 default:
3602 numchan = 1;
3603 break;
3604 case 1:
3605 numchan = 2;
3606 break;
3607 case 2:
3608 numchan = 4;
3609 break;
3610 case 3:
3611 numchan = 8;
3612 break;
3613 }
3614 rdev->mc.vram_width = numchan * chansize;
3615 /* Could aper size report 0 ? */
01d73a69
JC		 3616 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3617 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3618 /* Setup GPU memory space */
05b3ef69
AD		 3619 if ((rdev->family == CHIP_PALM) ||
3620 (rdev->family == CHIP_SUMO) ||
3621 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3622 /* size in bytes on fusion */
3623 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3624 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3625 } else {
05b3ef69 3626 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3627 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3628 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3629 }
51e5fcd3 3630 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3631 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3632 radeon_update_bandwidth_info(rdev);
3633
bcc1c2a1
AD		 3634 return 0;
3635}
d594e46a 3636
187e3593 3637void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3638{
64c56e8c 3639 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3640 RREG32(GRBM_STATUS));
64c56e8c 3641 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3642 RREG32(GRBM_STATUS_SE0));
64c56e8c 3643 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3644 RREG32(GRBM_STATUS_SE1));
64c56e8c 3645 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3646 RREG32(SRBM_STATUS));
a65a4369
AD		 3647 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3648 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3649 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3650 RREG32(CP_STALLED_STAT1));
3651 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3652 RREG32(CP_STALLED_STAT2));
3653 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3654 RREG32(CP_BUSY_STAT));
3655 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3656 RREG32(CP_STAT));
eaaa6983
JG		 3657 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3658 RREG32(DMA_STATUS_REG));
168757ea
AD		 3659 if (rdev->family >= CHIP_CAYMAN) {
3660 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3661 RREG32(DMA_STATUS_REG + 0x800));
3662 }
0ecebb9e
AD		 3663}
3664
168757ea 3665bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3666{
a65a4369
AD		 3667 u32 crtc_hung = 0;
3668 u32 crtc_status[6];
3669 u32 i, j, tmp;
3670
3671 for (i = 0; i < rdev->num_crtc; i++) {
3672 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3673 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3674 crtc_hung |= (1 << i);
3675 }
3676 }
3677
3678 for (j = 0; j < 10; j++) {
3679 for (i = 0; i < rdev->num_crtc; i++) {
3680 if (crtc_hung & (1 << i)) {
3681 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3682 if (tmp != crtc_status[i])
3683 crtc_hung &= ~(1 << i);
3684 }
3685 }
3686 if (crtc_hung == 0)
3687 return false;
3688 udelay(100);
3689 }
3690
3691 return true;
3692}
3693
2483b4ea 3694u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3695{
3696 u32 reset_mask = 0;
b7630473 3697 u32 tmp;
0ecebb9e 3698
a65a4369
AD		 3699 /* GRBM_STATUS */
3700 tmp = RREG32(GRBM_STATUS);
3701 if (tmp & (PA_BUSY | SC_BUSY |
3702 SH_BUSY | SX_BUSY |
3703 TA_BUSY | VGT_BUSY |
3704 DB_BUSY | CB_BUSY |
3705 SPI_BUSY | VGT_BUSY_NO_DMA))
3706 reset_mask |= RADEON_RESET_GFX;
3707
3708 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3709 CP_BUSY | CP_COHERENCY_BUSY))
3710 reset_mask |= RADEON_RESET_CP;
3711
3712 if (tmp & GRBM_EE_BUSY)
3713 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3714
a65a4369
AD		 3715 /* DMA_STATUS_REG */
3716 tmp = RREG32(DMA_STATUS_REG);
3717 if (!(tmp & DMA_IDLE))
3718 reset_mask |= RADEON_RESET_DMA;
3719
3720 /* SRBM_STATUS2 */
3721 tmp = RREG32(SRBM_STATUS2);
3722 if (tmp & DMA_BUSY)
3723 reset_mask |= RADEON_RESET_DMA;
3724
3725 /* SRBM_STATUS */
3726 tmp = RREG32(SRBM_STATUS);
3727 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3728 reset_mask |= RADEON_RESET_RLC;
3729
3730 if (tmp & IH_BUSY)
3731 reset_mask |= RADEON_RESET_IH;
3732
3733 if (tmp & SEM_BUSY)
3734 reset_mask |= RADEON_RESET_SEM;
3735
3736 if (tmp & GRBM_RQ_PENDING)
3737 reset_mask |= RADEON_RESET_GRBM;
3738
3739 if (tmp & VMC_BUSY)
3740 reset_mask |= RADEON_RESET_VMC;
3741
3742 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3743 MCC_BUSY | MCD_BUSY))
3744 reset_mask |= RADEON_RESET_MC;
3745
3746 if (evergreen_is_display_hung(rdev))
3747 reset_mask |= RADEON_RESET_DISPLAY;
3748
3749 /* VM_L2_STATUS */
3750 tmp = RREG32(VM_L2_STATUS);
3751 if (tmp & L2_BUSY)
3752 reset_mask |= RADEON_RESET_VMC;
3753
d808fc88
AD		 3754 /* Skip MC reset as it's mostly likely not hung, just busy */
3755 if (reset_mask & RADEON_RESET_MC) {
3756 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3757 reset_mask &= ~RADEON_RESET_MC;
3758 }
3759
a65a4369
AD		 3760 return reset_mask;
3761}
3762
3763static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3764{
3765 struct evergreen_mc_save save;
3766 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3767 u32 tmp;
19fc42ed 3768
0ecebb9e 3769 if (reset_mask == 0)
a65a4369 3770 return;
0ecebb9e
AD		 3771
3772 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3773
b7630473
AD		 3774 evergreen_print_gpu_status_regs(rdev);
3775
b7630473
AD		 3776 /* Disable CP parsing/prefetching */
3777 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3778
3779 if (reset_mask & RADEON_RESET_DMA) {
3780 /* Disable DMA */
3781 tmp = RREG32(DMA_RB_CNTL);
3782 tmp &= ~DMA_RB_ENABLE;
3783 WREG32(DMA_RB_CNTL, tmp);
3784 }
3785
b21b6e7a
AD		 3786 udelay(50);
3787
3788 evergreen_mc_stop(rdev, &save);
3789 if (evergreen_mc_wait_for_idle(rdev)) {
3790 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3791 }
3792
b7630473
AD		 3793 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3794 grbm_soft_reset |= SOFT_RESET_DB |
3795 SOFT_RESET_CB |
3796 SOFT_RESET_PA |
3797 SOFT_RESET_SC |
3798 SOFT_RESET_SPI |
3799 SOFT_RESET_SX |
3800 SOFT_RESET_SH |
3801 SOFT_RESET_TC |
3802 SOFT_RESET_TA |
3803 SOFT_RESET_VC |
3804 SOFT_RESET_VGT;
3805 }
3806
3807 if (reset_mask & RADEON_RESET_CP) {
3808 grbm_soft_reset |= SOFT_RESET_CP |
3809 SOFT_RESET_VGT;
3810
3811 srbm_soft_reset |= SOFT_RESET_GRBM;
3812 }
0ecebb9e
AD		 3813
3814 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3815 srbm_soft_reset |= SOFT_RESET_DMA;
3816
a65a4369
AD		 3817 if (reset_mask & RADEON_RESET_DISPLAY)
3818 srbm_soft_reset |= SOFT_RESET_DC;
3819
3820 if (reset_mask & RADEON_RESET_RLC)
3821 srbm_soft_reset |= SOFT_RESET_RLC;
3822
3823 if (reset_mask & RADEON_RESET_SEM)
3824 srbm_soft_reset |= SOFT_RESET_SEM;
3825
3826 if (reset_mask & RADEON_RESET_IH)
3827 srbm_soft_reset |= SOFT_RESET_IH;
3828
3829 if (reset_mask & RADEON_RESET_GRBM)
3830 srbm_soft_reset |= SOFT_RESET_GRBM;
3831
3832 if (reset_mask & RADEON_RESET_VMC)
3833 srbm_soft_reset |= SOFT_RESET_VMC;
3834
24178ec4
AD		 3835 if (!(rdev->flags & RADEON_IS_IGP)) {
3836 if (reset_mask & RADEON_RESET_MC)
3837 srbm_soft_reset |= SOFT_RESET_MC;
3838 }
a65a4369 3839
b7630473
AD		 3840 if (grbm_soft_reset) {
3841 tmp = RREG32(GRBM_SOFT_RESET);
3842 tmp |= grbm_soft_reset;
3843 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3844 WREG32(GRBM_SOFT_RESET, tmp);
3845 tmp = RREG32(GRBM_SOFT_RESET);
3846
3847 udelay(50);
3848
3849 tmp &= ~grbm_soft_reset;
3850 WREG32(GRBM_SOFT_RESET, tmp);
3851 tmp = RREG32(GRBM_SOFT_RESET);
3852 }
3853
3854 if (srbm_soft_reset) {
3855 tmp = RREG32(SRBM_SOFT_RESET);
3856 tmp |= srbm_soft_reset;
3857 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3858 WREG32(SRBM_SOFT_RESET, tmp);
3859 tmp = RREG32(SRBM_SOFT_RESET);
3860
3861 udelay(50);
3862
3863 tmp &= ~srbm_soft_reset;
3864 WREG32(SRBM_SOFT_RESET, tmp);
3865 tmp = RREG32(SRBM_SOFT_RESET);
3866 }
0ecebb9e
AD		 3867
3868 /* Wait a little for things to settle down */
3869 udelay(50);
3870
747943ea 3871 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3872 udelay(50);
3873
b7630473 3874 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3875}
3876
a2d07b74 3877int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3878{
a65a4369
AD		 3879 u32 reset_mask;
3880
3881 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3882
3883 if (reset_mask)
3884 r600_set_bios_scratch_engine_hung(rdev, true);
3885
3886 evergreen_gpu_soft_reset(rdev, reset_mask);
3887
3888 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3889
3890 if (!reset_mask)
3891 r600_set_bios_scratch_engine_hung(rdev, false);
3892
3893 return 0;
747943ea
AD		 3894}
3895
123bc183
AD		 3896/**
3897 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3898 *
3899 * @rdev: radeon_device pointer
3900 * @ring: radeon_ring structure holding ring information
3901 *
3902 * Check if the GFX engine is locked up.
3903 * Returns true if the engine appears to be locked up, false if not.
3904 */
3905bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3906{
3907 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3908
3909 if (!(reset_mask & (RADEON_RESET_GFX |
3910 RADEON_RESET_COMPUTE |
3911 RADEON_RESET_CP))) {
3912 radeon_ring_lockup_update(ring);
3913 return false;
3914 }
3915 /* force CP activities */
3916 radeon_ring_force_activity(rdev, ring);
3917 return radeon_ring_test_lockup(rdev, ring);
3918}
3919
2948f5e6
AD		 3920/*
3921 * RLC
3922 */
3923#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3924#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3925
3926void sumo_rlc_fini(struct radeon_device *rdev)
3927{
3928 int r;
3929
3930 /* save restore block */
3931 if (rdev->rlc.save_restore_obj) {
3932 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3933 if (unlikely(r != 0))
3934 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3935 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3936 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3937
3938 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3939 rdev->rlc.save_restore_obj = NULL;
3940 }
3941
3942 /* clear state block */
3943 if (rdev->rlc.clear_state_obj) {
3944 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3945 if (unlikely(r != 0))
3946 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3947 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3948 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3949
3950 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3951 rdev->rlc.clear_state_obj = NULL;
3952 }
22c775ce
AD		 3953
3954 /* clear state block */
3955 if (rdev->rlc.cp_table_obj) {
3956 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3957 if (unlikely(r != 0))
3958 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3959 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3960 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3961
3962 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3963 rdev->rlc.cp_table_obj = NULL;
3964 }
2948f5e6
AD		 3965}
3966
22c775ce
AD		 3967#define CP_ME_TABLE_SIZE 96
3968
2948f5e6
AD		 3969int sumo_rlc_init(struct radeon_device *rdev)
3970{
1fd11777 3971 const u32 *src_ptr;
2948f5e6
AD		 3972 volatile u32 *dst_ptr;
3973 u32 dws, data, i, j, k, reg_num;
59a82d0e 3974 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 3975 u64 reg_list_mc_addr;
1fd11777 3976 const struct cs_section_def *cs_data;
2948f5e6
AD		 3977 int r;
3978
3979 src_ptr = rdev->rlc.reg_list;
3980 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 3981 if (rdev->family >= CHIP_BONAIRE) {
3982 dws += (5 * 16) + 48 + 48 + 64;
3983 }
2948f5e6
AD		 3984 cs_data = rdev->rlc.cs_data;
3985
10b7ca7e
AD		 3986 if (src_ptr) {
3987 /* save restore block */
3988 if (rdev->rlc.save_restore_obj == NULL) {
3989 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3990 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3991 if (r) {
3992 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3993 return r;
3994 }
3995 }
3996
3997 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3998 if (unlikely(r != 0)) {
3999 sumo_rlc_fini(rdev);
4000 return r;
4001 }
4002 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4003 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4004 if (r) {
10b7ca7e
AD		 4005 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4006 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4007 sumo_rlc_fini(rdev);
2948f5e6
AD		 4008 return r;
4009 }
2948f5e6 4010
10b7ca7e
AD		 4011 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4012 if (r) {
4013 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4014 sumo_rlc_fini(rdev);
4015 return r;
4016 }
4017 /* write the sr buffer */
4018 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4019 if (rdev->family >= CHIP_TAHITI) {
4020 /* SI */
59a82d0e 4021 for (i = 0; i < rdev->rlc.reg_list_size; i++)
1fd11777
AD		 4022 dst_ptr[i] = src_ptr[i];
4023 } else {
4024 /* ON/LN/TN */
4025 /* format:
4026 * dw0: (reg2 << 16) | reg1
4027 * dw1: reg1 save space
4028 * dw2: reg2 save space
4029 */
4030 for (i = 0; i < dws; i++) {
4031 data = src_ptr[i] >> 2;
4032 i++;
4033 if (i < dws)
4034 data |= (src_ptr[i] >> 2) << 16;
4035 j = (((i - 1) * 3) / 2);
4036 dst_ptr[j] = data;
4037 }
4038 j = ((i * 3) / 2);
4039 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
10b7ca7e 4040 }
10b7ca7e 4041 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4042 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4043 }
2948f5e6 4044
10b7ca7e
AD		 4045 if (cs_data) {
4046 /* clear state block */
a0f38609
AD		 4047 if (rdev->family >= CHIP_BONAIRE) {
4048 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4049 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4050 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4051 dws = rdev->rlc.clear_state_size + (256 / 4);
4052 } else {
4053 reg_list_num = 0;
4054 dws = 0;
4055 for (i = 0; cs_data[i].section != NULL; i++) {
4056 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4057 reg_list_num++;
4058 dws += cs_data[i].section[j].reg_count;
4059 }
10b7ca7e 4060 }
59a82d0e
AD		 4061 reg_list_blk_index = (3 * reg_list_num + 2);
4062 dws += reg_list_blk_index;
4063 rdev->rlc.clear_state_size = dws;
2948f5e6 4064 }
2948f5e6 4065
10b7ca7e 4066 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4067 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
10b7ca7e
AD		 4068 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4069 if (r) {
4070 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4071 sumo_rlc_fini(rdev);
4072 return r;
4073 }
4074 }
4075 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4076 if (unlikely(r != 0)) {
4077 sumo_rlc_fini(rdev);
4078 return r;
4079 }
4080 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4081 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4082 if (r) {
10b7ca7e
AD		 4083 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4084 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4085 sumo_rlc_fini(rdev);
4086 return r;
4087 }
2948f5e6 4088
10b7ca7e
AD		 4089 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4090 if (r) {
4091 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4092 sumo_rlc_fini(rdev);
4093 return r;
4094 }
4095 /* set up the cs buffer */
4096 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4097 if (rdev->family >= CHIP_BONAIRE) {
4098 cik_get_csb_buffer(rdev, dst_ptr);
4099 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4100 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4101 dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
4102 dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
4103 dst_ptr[2] = rdev->rlc.clear_state_size;
4104 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4105 } else {
4106 reg_list_hdr_blk_index = 0;
4107 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4108 data = upper_32_bits(reg_list_mc_addr);
2948f5e6
AD		 4109 dst_ptr[reg_list_hdr_blk_index] = data;
4110 reg_list_hdr_blk_index++;
59a82d0e
AD		 4111 for (i = 0; cs_data[i].section != NULL; i++) {
4112 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4113 reg_num = cs_data[i].section[j].reg_count;
4114 data = reg_list_mc_addr & 0xffffffff;
4115 dst_ptr[reg_list_hdr_blk_index] = data;
4116 reg_list_hdr_blk_index++;
4117
4118 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4119 dst_ptr[reg_list_hdr_blk_index] = data;
4120 reg_list_hdr_blk_index++;
4121
4122 data = 0x08000000 | (reg_num * 4);
4123 dst_ptr[reg_list_hdr_blk_index] = data;
4124 reg_list_hdr_blk_index++;
4125
4126 for (k = 0; k < reg_num; k++) {
4127 data = cs_data[i].section[j].extent[k];
4128 dst_ptr[reg_list_blk_index + k] = data;
4129 }
4130 reg_list_mc_addr += reg_num * 4;
4131 reg_list_blk_index += reg_num;
10b7ca7e 4132 }
2948f5e6 4133 }
59a82d0e 4134 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
2948f5e6 4135 }
10b7ca7e
AD		 4136 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4137 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4138 }
2948f5e6 4139
22c775ce
AD		 4140 if (rdev->rlc.cp_table_size) {
4141 if (rdev->rlc.cp_table_obj == NULL) {
4142 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4143 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4144 if (r) {
4145 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4146 sumo_rlc_fini(rdev);
4147 return r;
4148 }
4149 }
4150
4151 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4152 if (unlikely(r != 0)) {
4153 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4154 sumo_rlc_fini(rdev);
4155 return r;
4156 }
4157 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4158 &rdev->rlc.cp_table_gpu_addr);
4159 if (r) {
4160 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4161 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4162 sumo_rlc_fini(rdev);
4163 return r;
4164 }
4165 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4166 if (r) {
4167 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4168 sumo_rlc_fini(rdev);
4169 return r;
4170 }
4171
4172 cik_init_cp_pg_table(rdev);
4173
4174 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4175 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4176
4177 }
2948f5e6
AD		 4178
4179 return 0;
4180}
4181
4182static void evergreen_rlc_start(struct radeon_device *rdev)
4183{
8ba10463
AD		 4184 u32 mask = RLC_ENABLE;
4185
4186 if (rdev->flags & RADEON_IS_IGP) {
4187 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4188 }
4189
4190 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4191}
4192
4193int evergreen_rlc_resume(struct radeon_device *rdev)
4194{
4195 u32 i;
4196 const __be32 *fw_data;
4197
4198 if (!rdev->rlc_fw)
4199 return -EINVAL;
4200
4201 r600_rlc_stop(rdev);
4202
4203 WREG32(RLC_HB_CNTL, 0);
4204
4205 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4206 if (rdev->family == CHIP_ARUBA) {
4207 u32 always_on_bitmap =
4208 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4209 /* find out the number of active simds */
4210 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4211 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4212 tmp = hweight32(~tmp);
4213 if (tmp == rdev->config.cayman.max_simds_per_se) {
4214 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4215 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4216 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4217 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4218 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4219 }
4220 } else {
4221 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4222 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4223 }
2948f5e6
AD		 4224 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4225 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4226 } else {
4227 WREG32(RLC_HB_BASE, 0);
4228 WREG32(RLC_HB_RPTR, 0);
4229 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4230 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4231 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4232 }
2948f5e6
AD		 4233 WREG32(RLC_MC_CNTL, 0);
4234 WREG32(RLC_UCODE_CNTL, 0);
4235
4236 fw_data = (const __be32 *)rdev->rlc_fw->data;
4237 if (rdev->family >= CHIP_ARUBA) {
4238 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4239 WREG32(RLC_UCODE_ADDR, i);
4240 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4241 }
4242 } else if (rdev->family >= CHIP_CAYMAN) {
4243 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4244 WREG32(RLC_UCODE_ADDR, i);
4245 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4246 }
4247 } else {
4248 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4249 WREG32(RLC_UCODE_ADDR, i);
4250 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4251 }
4252 }
4253 WREG32(RLC_UCODE_ADDR, 0);
4254
4255 evergreen_rlc_start(rdev);
4256
4257 return 0;
4258}
4259
45f9a39b
AD		 4260/* Interrupts */
4261
4262u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4263{
46437057 4264 if (crtc >= rdev->num_crtc)
45f9a39b 4265 return 0;
46437057
AD		 4266 else
4267 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4268}
4269
4270void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4271{
4272 u32 tmp;
4273
1b37078b
AD		 4274 if (rdev->family >= CHIP_CAYMAN) {
4275 cayman_cp_int_cntl_setup(rdev, 0,
4276 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4277 cayman_cp_int_cntl_setup(rdev, 1, 0);
4278 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4279 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4280 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4281 } else
4282 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4283 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4284 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4285 WREG32(GRBM_INT_CNTL, 0);
4286 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4287 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4288 if (rdev->num_crtc >= 4) {
18007401
AD		 4289 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4290 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4291 }
4292 if (rdev->num_crtc >= 6) {
18007401
AD		 4293 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4294 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4295 }
45f9a39b
AD		 4296
4297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4298 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4299 if (rdev->num_crtc >= 4) {
18007401
AD		 4300 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4301 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4302 }
4303 if (rdev->num_crtc >= 6) {
18007401
AD		 4304 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4305 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4306 }
45f9a39b 4307
05b3ef69
AD		 4308 /* only one DAC on DCE6 */
4309 if (!ASIC_IS_DCE6(rdev))
4310 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4311 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4312
4313 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4314 WREG32(DC_HPD1_INT_CONTROL, tmp);
4315 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4316 WREG32(DC_HPD2_INT_CONTROL, tmp);
4317 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4318 WREG32(DC_HPD3_INT_CONTROL, tmp);
4319 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4320 WREG32(DC_HPD4_INT_CONTROL, tmp);
4321 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4322 WREG32(DC_HPD5_INT_CONTROL, tmp);
4323 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4324 WREG32(DC_HPD6_INT_CONTROL, tmp);
4325
4326}
4327
4328int evergreen_irq_set(struct radeon_device *rdev)
4329{
4330 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4331 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4332 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4333 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4334 u32 grbm_int_cntl = 0;
6f34be50 4335 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4336 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4337 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4338 u32 thermal_int = 0;
45f9a39b
AD		 4339
4340 if (!rdev->irq.installed) {
fce7d61b 4341 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4342 return -EINVAL;
4343 }
4344 /* don't enable anything if the ih is disabled */
4345 if (!rdev->ih.enabled) {
4346 r600_disable_interrupts(rdev);
4347 /* force the active interrupt state to all disabled */
4348 evergreen_disable_interrupt_state(rdev);
4349 return 0;
4350 }
4351
4352 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4353 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4354 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4355 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4356 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4357 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4358 if (rdev->family == CHIP_ARUBA)
4359 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4360 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4361 else
4362 thermal_int = RREG32(CG_THERMAL_INT) &
4363 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4364
f122c610
AD		 4365 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4366 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4367 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4368 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4369 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4370 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4371
233d1ad5
AD		 4372 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4373
1b37078b
AD		 4374 if (rdev->family >= CHIP_CAYMAN) {
4375 /* enable CP interrupts on all rings */
736fc37f 4376 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4377 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4378 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4379 }
736fc37f 4380 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4381 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4382 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4383 }
736fc37f 4384 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4385 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4386 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4387 }
4388 } else {
736fc37f 4389 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4390 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4391 cp_int_cntl |= RB_INT_ENABLE;
4392 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4393 }
45f9a39b 4394 }
1b37078b 4395
233d1ad5
AD		 4396 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4397 DRM_DEBUG("r600_irq_set: sw int dma\n");
4398 dma_cntl |= TRAP_ENABLE;
4399 }
4400
f60cbd11
AD		 4401 if (rdev->family >= CHIP_CAYMAN) {
4402 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4403 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4404 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4405 dma_cntl1 |= TRAP_ENABLE;
4406 }
4407 }
4408
dc50ba7f
AD		 4409 if (rdev->irq.dpm_thermal) {
4410 DRM_DEBUG("dpm thermal\n");
4411 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4412 }
4413
6f34be50 4414 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4415 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4416 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4417 crtc1 |= VBLANK_INT_MASK;
4418 }
6f34be50 4419 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4420 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4421 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4422 crtc2 |= VBLANK_INT_MASK;
4423 }
6f34be50 4424 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4425 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4426 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4427 crtc3 |= VBLANK_INT_MASK;
4428 }
6f34be50 4429 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4430 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4431 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4432 crtc4 |= VBLANK_INT_MASK;
4433 }
6f34be50 4434 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4435 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4436 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4437 crtc5 |= VBLANK_INT_MASK;
4438 }
6f34be50 4439 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4440 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4441 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4442 crtc6 |= VBLANK_INT_MASK;
4443 }
4444 if (rdev->irq.hpd[0]) {
4445 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4446 hpd1 |= DC_HPDx_INT_EN;
4447 }
4448 if (rdev->irq.hpd[1]) {
4449 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4450 hpd2 |= DC_HPDx_INT_EN;
4451 }
4452 if (rdev->irq.hpd[2]) {
4453 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4454 hpd3 |= DC_HPDx_INT_EN;
4455 }
4456 if (rdev->irq.hpd[3]) {
4457 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4458 hpd4 |= DC_HPDx_INT_EN;
4459 }
4460 if (rdev->irq.hpd[4]) {
4461 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4462 hpd5 |= DC_HPDx_INT_EN;
4463 }
4464 if (rdev->irq.hpd[5]) {
4465 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4466 hpd6 |= DC_HPDx_INT_EN;
4467 }
f122c610
AD		 4468 if (rdev->irq.afmt[0]) {
4469 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4470 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4471 }
4472 if (rdev->irq.afmt[1]) {
4473 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4474 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4475 }
4476 if (rdev->irq.afmt[2]) {
4477 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4478 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4479 }
4480 if (rdev->irq.afmt[3]) {
4481 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4482 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4483 }
4484 if (rdev->irq.afmt[4]) {
4485 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4486 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4487 }
4488 if (rdev->irq.afmt[5]) {
4489 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4490 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4491 }
45f9a39b 4492
1b37078b
AD		 4493 if (rdev->family >= CHIP_CAYMAN) {
4494 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4495 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4496 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4497 } else
4498 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4499
4500 WREG32(DMA_CNTL, dma_cntl);
4501
f60cbd11
AD		 4502 if (rdev->family >= CHIP_CAYMAN)
4503 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4504
2031f77c 4505 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4506
4507 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4508 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4509 if (rdev->num_crtc >= 4) {
18007401
AD		 4510 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4511 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4512 }
4513 if (rdev->num_crtc >= 6) {
18007401
AD		 4514 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4515 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4516 }
45f9a39b 4517
6f34be50
AD		 4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4520 if (rdev->num_crtc >= 4) {
4521 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4523 }
4524 if (rdev->num_crtc >= 6) {
4525 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4526 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4527 }
6f34be50 4528
45f9a39b
AD		 4529 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4530 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4531 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4532 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4533 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4534 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4535 if (rdev->family == CHIP_ARUBA)
4536 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4537 else
4538 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4539
f122c610
AD		 4540 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4541 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4542 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4543 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4544 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4545 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4546
bcc1c2a1
AD		 4547 return 0;
4548}
4549
cbdd4501 4550static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4551{
4552 u32 tmp;
4553
6f34be50
AD		 4554 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4555 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4556 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4557 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4558 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4559 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4560 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4561 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4562 if (rdev->num_crtc >= 4) {
4563 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4564 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4565 }
4566 if (rdev->num_crtc >= 6) {
4567 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4568 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4569 }
6f34be50 4570
f122c610
AD		 4571 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4572 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4573 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4574 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4575 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4576 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4577
6f34be50
AD		 4578 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4579 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4580 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4581 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4582 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4583 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4584 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4585 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4586 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4587 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4588 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4589 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4590
b7eff394
AD		 4591 if (rdev->num_crtc >= 4) {
4592 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4593 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4594 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4595 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4596 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4597 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4598 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4599 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4600 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4601 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4602 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4603 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4604 }
4605
4606 if (rdev->num_crtc >= 6) {
4607 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4608 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4609 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4610 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4611 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4612 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4613 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4614 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4615 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4616 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4617 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4618 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4619 }
45f9a39b 4620
6f34be50 4621 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4622 tmp = RREG32(DC_HPD1_INT_CONTROL);
4623 tmp |= DC_HPDx_INT_ACK;
4624 WREG32(DC_HPD1_INT_CONTROL, tmp);
4625 }
6f34be50 4626 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4627 tmp = RREG32(DC_HPD2_INT_CONTROL);
4628 tmp |= DC_HPDx_INT_ACK;
4629 WREG32(DC_HPD2_INT_CONTROL, tmp);
4630 }
6f34be50 4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4632 tmp = RREG32(DC_HPD3_INT_CONTROL);
4633 tmp |= DC_HPDx_INT_ACK;
4634 WREG32(DC_HPD3_INT_CONTROL, tmp);
4635 }
6f34be50 4636 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4637 tmp = RREG32(DC_HPD4_INT_CONTROL);
4638 tmp |= DC_HPDx_INT_ACK;
4639 WREG32(DC_HPD4_INT_CONTROL, tmp);
4640 }
6f34be50 4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4642 tmp = RREG32(DC_HPD5_INT_CONTROL);
4643 tmp |= DC_HPDx_INT_ACK;
4644 WREG32(DC_HPD5_INT_CONTROL, tmp);
4645 }
6f34be50 4646 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4647 tmp = RREG32(DC_HPD5_INT_CONTROL);
4648 tmp |= DC_HPDx_INT_ACK;
4649 WREG32(DC_HPD6_INT_CONTROL, tmp);
4650 }
f122c610
AD		 4651 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4652 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4653 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4654 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4655 }
4656 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4657 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4658 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4659 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4660 }
4661 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4662 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4663 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4664 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4665 }
4666 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4667 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4668 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4669 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4670 }
4671 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4672 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4673 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4674 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4675 }
4676 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4677 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4678 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4679 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4680 }
45f9a39b
AD		 4681}
4682
1109ca09 4683static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4684{
45f9a39b
AD		 4685 r600_disable_interrupts(rdev);
4686 /* Wait and acknowledge irq */
4687 mdelay(1);
6f34be50 4688 evergreen_irq_ack(rdev);
45f9a39b
AD		 4689 evergreen_disable_interrupt_state(rdev);
4690}
4691
755d819e 4692void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4693{
4694 evergreen_irq_disable(rdev);
4695 r600_rlc_stop(rdev);
4696}
4697
cbdd4501 4698static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4699{
4700 u32 wptr, tmp;
4701
724c80e1 4702 if (rdev->wb.enabled)
204ae24d 4703 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4704 else
4705 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4706
4707 if (wptr & RB_OVERFLOW) {
4708 /* When a ring buffer overflow happen start parsing interrupt
4709 * from the last not overwritten vector (wptr + 16). Hopefully
4710 * this should allow us to catchup.
4711 */
4712 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4713 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4714 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4715 tmp = RREG32(IH_RB_CNTL);
4716 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4717 WREG32(IH_RB_CNTL, tmp);
4718 }
4719 return (wptr & rdev->ih.ptr_mask);
4720}
4721
4722int evergreen_irq_process(struct radeon_device *rdev)
4723{
682f1a54
DA		 4724 u32 wptr;
4725 u32 rptr;
45f9a39b
AD		 4726 u32 src_id, src_data;
4727 u32 ring_index;
45f9a39b 4728 bool queue_hotplug = false;
f122c610 4729 bool queue_hdmi = false;
dc50ba7f 4730 bool queue_thermal = false;
54e2e49c 4731 u32 status, addr;
45f9a39b 4732
682f1a54 4733 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4734 return IRQ_NONE;
4735
682f1a54 4736 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4737
4738restart_ih:
4739 /* is somebody else already processing irqs? */
4740 if (atomic_xchg(&rdev->ih.lock, 1))
4741 return IRQ_NONE;
4742
682f1a54
DA		 4743 rptr = rdev->ih.rptr;
4744 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4745
964f6645
BH		 4746 /* Order reading of wptr vs. reading of IH ring data */
4747 rmb();
4748
45f9a39b 4749 /* display interrupts */
6f34be50 4750 evergreen_irq_ack(rdev);
45f9a39b 4751
45f9a39b
AD		 4752 while (rptr != wptr) {
4753 /* wptr/rptr are in bytes! */
4754 ring_index = rptr / 4;
0f234f5f
AD		 4755 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4756 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4757
4758 switch (src_id) {
4759 case 1: /* D1 vblank/vline */
4760 switch (src_data) {
4761 case 0: /* D1 vblank */
6f34be50 4762 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4763 if (rdev->irq.crtc_vblank_int[0]) {
4764 drm_handle_vblank(rdev->ddev, 0);
4765 rdev->pm.vblank_sync = true;
4766 wake_up(&rdev->irq.vblank_queue);
4767 }
736fc37f 4768 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4769 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4770 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4771 DRM_DEBUG("IH: D1 vblank\n");
4772 }
4773 break;
4774 case 1: /* D1 vline */
6f34be50
AD		 4775 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4776 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4777 DRM_DEBUG("IH: D1 vline\n");
4778 }
4779 break;
4780 default:
4781 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4782 break;
4783 }
4784 break;
4785 case 2: /* D2 vblank/vline */
4786 switch (src_data) {
4787 case 0: /* D2 vblank */
6f34be50 4788 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4789 if (rdev->irq.crtc_vblank_int[1]) {
4790 drm_handle_vblank(rdev->ddev, 1);
4791 rdev->pm.vblank_sync = true;
4792 wake_up(&rdev->irq.vblank_queue);
4793 }
736fc37f 4794 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4795 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4796 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4797 DRM_DEBUG("IH: D2 vblank\n");
4798 }
4799 break;
4800 case 1: /* D2 vline */
6f34be50
AD		 4801 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4802 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4803 DRM_DEBUG("IH: D2 vline\n");
4804 }
4805 break;
4806 default:
4807 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4808 break;
4809 }
4810 break;
4811 case 3: /* D3 vblank/vline */
4812 switch (src_data) {
4813 case 0: /* D3 vblank */
6f34be50
AD		 4814 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4815 if (rdev->irq.crtc_vblank_int[2]) {
4816 drm_handle_vblank(rdev->ddev, 2);
4817 rdev->pm.vblank_sync = true;
4818 wake_up(&rdev->irq.vblank_queue);
4819 }
736fc37f 4820 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4821 radeon_crtc_handle_flip(rdev, 2);
4822 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4823 DRM_DEBUG("IH: D3 vblank\n");
4824 }
4825 break;
4826 case 1: /* D3 vline */
6f34be50
AD		 4827 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4828 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4829 DRM_DEBUG("IH: D3 vline\n");
4830 }
4831 break;
4832 default:
4833 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4834 break;
4835 }
4836 break;
4837 case 4: /* D4 vblank/vline */
4838 switch (src_data) {
4839 case 0: /* D4 vblank */
6f34be50
AD		 4840 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4841 if (rdev->irq.crtc_vblank_int[3]) {
4842 drm_handle_vblank(rdev->ddev, 3);
4843 rdev->pm.vblank_sync = true;
4844 wake_up(&rdev->irq.vblank_queue);
4845 }
736fc37f 4846 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4847 radeon_crtc_handle_flip(rdev, 3);
4848 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4849 DRM_DEBUG("IH: D4 vblank\n");
4850 }
4851 break;
4852 case 1: /* D4 vline */
6f34be50
AD		 4853 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4854 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4855 DRM_DEBUG("IH: D4 vline\n");
4856 }
4857 break;
4858 default:
4859 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4860 break;
4861 }
4862 break;
4863 case 5: /* D5 vblank/vline */
4864 switch (src_data) {
4865 case 0: /* D5 vblank */
6f34be50
AD		 4866 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4867 if (rdev->irq.crtc_vblank_int[4]) {
4868 drm_handle_vblank(rdev->ddev, 4);
4869 rdev->pm.vblank_sync = true;
4870 wake_up(&rdev->irq.vblank_queue);
4871 }
736fc37f 4872 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4873 radeon_crtc_handle_flip(rdev, 4);
4874 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4875 DRM_DEBUG("IH: D5 vblank\n");
4876 }
4877 break;
4878 case 1: /* D5 vline */
6f34be50
AD		 4879 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4880 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4881 DRM_DEBUG("IH: D5 vline\n");
4882 }
4883 break;
4884 default:
4885 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4886 break;
4887 }
4888 break;
4889 case 6: /* D6 vblank/vline */
4890 switch (src_data) {
4891 case 0: /* D6 vblank */
6f34be50
AD		 4892 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4893 if (rdev->irq.crtc_vblank_int[5]) {
4894 drm_handle_vblank(rdev->ddev, 5);
4895 rdev->pm.vblank_sync = true;
4896 wake_up(&rdev->irq.vblank_queue);
4897 }
736fc37f 4898 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4899 radeon_crtc_handle_flip(rdev, 5);
4900 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4901 DRM_DEBUG("IH: D6 vblank\n");
4902 }
4903 break;
4904 case 1: /* D6 vline */
6f34be50
AD		 4905 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4906 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4907 DRM_DEBUG("IH: D6 vline\n");
4908 }
4909 break;
4910 default:
4911 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4912 break;
4913 }
4914 break;
4915 case 42: /* HPD hotplug */
4916 switch (src_data) {
4917 case 0:
6f34be50
AD		 4918 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4919 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4920 queue_hotplug = true;
4921 DRM_DEBUG("IH: HPD1\n");
4922 }
4923 break;
4924 case 1:
6f34be50
AD		 4925 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4926 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4927 queue_hotplug = true;
4928 DRM_DEBUG("IH: HPD2\n");
4929 }
4930 break;
4931 case 2:
6f34be50
AD		 4932 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4933 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4934 queue_hotplug = true;
4935 DRM_DEBUG("IH: HPD3\n");
4936 }
4937 break;
4938 case 3:
6f34be50
AD		 4939 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4940 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4941 queue_hotplug = true;
4942 DRM_DEBUG("IH: HPD4\n");
4943 }
4944 break;
4945 case 4:
6f34be50
AD		 4946 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4947 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4948 queue_hotplug = true;
4949 DRM_DEBUG("IH: HPD5\n");
4950 }
4951 break;
4952 case 5:
6f34be50
AD		 4953 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4954 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4955 queue_hotplug = true;
4956 DRM_DEBUG("IH: HPD6\n");
4957 }
4958 break;
4959 default:
4960 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4961 break;
4962 }
4963 break;
f122c610
AD		 4964 case 44: /* hdmi */
4965 switch (src_data) {
4966 case 0:
4967 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4968 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4969 queue_hdmi = true;
4970 DRM_DEBUG("IH: HDMI0\n");
4971 }
4972 break;
4973 case 1:
4974 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4975 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4976 queue_hdmi = true;
4977 DRM_DEBUG("IH: HDMI1\n");
4978 }
4979 break;
4980 case 2:
4981 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4982 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4983 queue_hdmi = true;
4984 DRM_DEBUG("IH: HDMI2\n");
4985 }
4986 break;
4987 case 3:
4988 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4989 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4990 queue_hdmi = true;
4991 DRM_DEBUG("IH: HDMI3\n");
4992 }
4993 break;
4994 case 4:
4995 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4996 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4997 queue_hdmi = true;
4998 DRM_DEBUG("IH: HDMI4\n");
4999 }
5000 break;
5001 case 5:
5002 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5003 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5004 queue_hdmi = true;
5005 DRM_DEBUG("IH: HDMI5\n");
5006 }
5007 break;
5008 default:
5009 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5010 break;
5011 }
f2ba57b5
CK		 5012 case 124: /* UVD */
5013 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5014 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5015 break;
ae133a11
CK		 5016 case 146:
5017 case 147:
54e2e49c
AD		 5018 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5019 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 5020 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5021 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5022 addr);
ae133a11 5023 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5024 status);
5025 cayman_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 5026 /* reset addr and status */
5027 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5028 break;
45f9a39b
AD		 5029 case 176: /* CP_INT in ring buffer */
5030 case 177: /* CP_INT in IB1 */
5031 case 178: /* CP_INT in IB2 */
5032 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5033 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5034 break;
5035 case 181: /* CP EOP event */
5036 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5037 if (rdev->family >= CHIP_CAYMAN) {
5038 switch (src_data) {
5039 case 0:
5040 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5041 break;
5042 case 1:
5043 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5044 break;
5045 case 2:
5046 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5047 break;
5048 }
5049 } else
5050 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5051 break;
233d1ad5
AD		 5052 case 224: /* DMA trap event */
5053 DRM_DEBUG("IH: DMA trap\n");
5054 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5055 break;
dc50ba7f
AD		 5056 case 230: /* thermal low to high */
5057 DRM_DEBUG("IH: thermal low to high\n");
5058 rdev->pm.dpm.thermal.high_to_low = false;
5059 queue_thermal = true;
5060 break;
5061 case 231: /* thermal high to low */
5062 DRM_DEBUG("IH: thermal high to low\n");
5063 rdev->pm.dpm.thermal.high_to_low = true;
5064 queue_thermal = true;
5065 break;
2031f77c 5066 case 233: /* GUI IDLE */
303c805c 5067 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5068 break;
f60cbd11
AD		 5069 case 244: /* DMA trap event */
5070 if (rdev->family >= CHIP_CAYMAN) {
5071 DRM_DEBUG("IH: DMA1 trap\n");
5072 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5073 }
5074 break;
45f9a39b
AD		 5075 default:
5076 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5077 break;
5078 }
5079
5080 /* wptr/rptr are in bytes! */
5081 rptr += 16;
5082 rptr &= rdev->ih.ptr_mask;
5083 }
45f9a39b 5084 if (queue_hotplug)
32c87fca 5085 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5086 if (queue_hdmi)
5087 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5088 if (queue_thermal && rdev->pm.dpm_enabled)
5089 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 5090 rdev->ih.rptr = rptr;
5091 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5092 atomic_set(&rdev->ih.lock, 0);
5093
5094 /* make sure wptr hasn't changed while processing */
5095 wptr = evergreen_get_ih_wptr(rdev);
5096 if (wptr != rptr)
5097 goto restart_ih;
5098
45f9a39b
AD		 5099 return IRQ_HANDLED;
5100}
5101
bcc1c2a1
AD		 5102static int evergreen_startup(struct radeon_device *rdev)
5103{
f2ba57b5 5104 struct radeon_ring *ring;
bcc1c2a1
AD		 5105 int r;
5106
9e46a48d 5107 /* enable pcie gen2 link */
cd54033a 5108 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5109 /* enable aspm */
5110 evergreen_program_aspm(rdev);
9e46a48d 5111
e5903d39
AD		 5112 /* scratch needs to be initialized before MC */
5113 r = r600_vram_scratch_init(rdev);
5114 if (r)
5115 return r;
5116
6fab3feb
AD		 5117 evergreen_mc_program(rdev);
5118
0af62b01
AD		 5119 if (ASIC_IS_DCE5(rdev)) {
5120 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5121 r = ni_init_microcode(rdev);
5122 if (r) {
5123 DRM_ERROR("Failed to load firmware!\n");
5124 return r;
5125 }
5126 }
755d819e 5127 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5128 if (r) {
0af62b01 5129 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5130 return r;
5131 }
0af62b01
AD		 5132 } else {
5133 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5134 r = r600_init_microcode(rdev);
5135 if (r) {
5136 DRM_ERROR("Failed to load firmware!\n");
5137 return r;
5138 }
5139 }
bcc1c2a1 5140 }
fe251e2f 5141
bcc1c2a1 5142 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5143 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5144 } else {
5145 r = evergreen_pcie_gart_enable(rdev);
5146 if (r)
5147 return r;
5148 }
bcc1c2a1 5149 evergreen_gpu_init(rdev);
bcc1c2a1 5150
2948f5e6
AD		 5151 /* allocate rlc buffers */
5152 if (rdev->flags & RADEON_IS_IGP) {
5153 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5154 rdev->rlc.reg_list_size =
5155 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5156 rdev->rlc.cs_data = evergreen_cs_data;
5157 r = sumo_rlc_init(rdev);
5158 if (r) {
5159 DRM_ERROR("Failed to init rlc BOs!\n");
5160 return r;
5161 }
5162 }
5163
724c80e1
AD		 5164 /* allocate wb buffer */
5165 r = radeon_wb_init(rdev);
5166 if (r)
5167 return r;
5168
30eb77f4
JG		 5169 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5170 if (r) {
5171 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5172 return r;
5173 }
5174
233d1ad5
AD		 5175 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5176 if (r) {
5177 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5178 return r;
5179 }
5180
e409b128 5181 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5182 if (!r) {
5183 r = radeon_fence_driver_start_ring(rdev,
5184 R600_RING_TYPE_UVD_INDEX);
5185 if (r)
5186 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5187 }
5188
5189 if (r)
5190 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5191
bcc1c2a1 5192 /* Enable IRQ */
e49f3959
AH		 5193 if (!rdev->irq.installed) {
5194 r = radeon_irq_kms_init(rdev);
5195 if (r)
5196 return r;
5197 }
5198
bcc1c2a1
AD		 5199 r = r600_irq_init(rdev);
5200 if (r) {
5201 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5202 radeon_irq_kms_fini(rdev);
5203 return r;
5204 }
45f9a39b 5205 evergreen_irq_set(rdev);
bcc1c2a1 5206
f2ba57b5 5207 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5208 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a 5209 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2e1e6dad 5210 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5211 if (r)
5212 return r;
233d1ad5
AD		 5213
5214 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5215 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5216 DMA_RB_RPTR, DMA_RB_WPTR,
2e1e6dad 5217 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5218 if (r)
5219 return r;
5220
bcc1c2a1
AD		 5221 r = evergreen_cp_load_microcode(rdev);
5222 if (r)
5223 return r;
fe251e2f 5224 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5225 if (r)
5226 return r;
5227 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5228 if (r)
5229 return r;
fe251e2f 5230
f2ba57b5
CK		 5231 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5232 if (ring->ring_size) {
02c9f7fa 5233 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
f2ba57b5 5234 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 5235 RADEON_CP_PACKET2);
f2ba57b5 5236 if (!r)
e409b128 5237 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5238
5239 if (r)
5240 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5241 }
5242
2898c348
CK		 5243 r = radeon_ib_pool_init(rdev);
5244 if (r) {
5245 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5246 return r;
2898c348 5247 }
b15ba512 5248
69d2ae57
RM		 5249 r = r600_audio_init(rdev);
5250 if (r) {
5251 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5252 return r;
5253 }
5254
bcc1c2a1
AD		 5255 return 0;
5256}
5257
5258int evergreen_resume(struct radeon_device *rdev)
5259{
5260 int r;
5261
86f5c9ed
AD		 5262 /* reset the asic, the gfx blocks are often in a bad state
5263 * after the driver is unloaded or after a resume
5264 */
5265 if (radeon_asic_reset(rdev))
5266 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5267 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5268 * posting will perform necessary task to bring back GPU into good
5269 * shape.
5270 */
5271 /* post card */
5272 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5273
d4788db3
AD		 5274 /* init golden registers */
5275 evergreen_init_golden_registers(rdev);
5276
b15ba512 5277 rdev->accel_working = true;
bcc1c2a1
AD		 5278 r = evergreen_startup(rdev);
5279 if (r) {
755d819e 5280 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5281 rdev->accel_working = false;
bcc1c2a1
AD		 5282 return r;
5283 }
fe251e2f 5284
bcc1c2a1
AD		 5285 return r;
5286
5287}
5288
5289int evergreen_suspend(struct radeon_device *rdev)
5290{
69d2ae57 5291 r600_audio_fini(rdev);
e409b128 5292 uvd_v1_0_fini(rdev);
f2ba57b5 5293 radeon_uvd_suspend(rdev);
bcc1c2a1 5294 r700_cp_stop(rdev);
233d1ad5 5295 r600_dma_stop(rdev);
45f9a39b 5296 evergreen_irq_suspend(rdev);
724c80e1 5297 radeon_wb_disable(rdev);
bcc1c2a1 5298 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5299
5300 return 0;
5301}
5302
bcc1c2a1
AD		 5303/* Plan is to move initialization in that function and use
5304 * helper function so that radeon_device_init pretty much
5305 * do nothing more than calling asic specific function. This
5306 * should also allow to remove a bunch of callback function
5307 * like vram_info.
5308 */
5309int evergreen_init(struct radeon_device *rdev)
5310{
5311 int r;
5312
bcc1c2a1
AD		 5313 /* Read BIOS */
5314 if (!radeon_get_bios(rdev)) {
5315 if (ASIC_IS_AVIVO(rdev))
5316 return -EINVAL;
5317 }
5318 /* Must be an ATOMBIOS */
5319 if (!rdev->is_atom_bios) {
755d819e 5320 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5321 return -EINVAL;
5322 }
5323 r = radeon_atombios_init(rdev);
5324 if (r)
5325 return r;
86f5c9ed
AD		 5326 /* reset the asic, the gfx blocks are often in a bad state
5327 * after the driver is unloaded or after a resume
5328 */
5329 if (radeon_asic_reset(rdev))
5330 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5331 /* Post card if necessary */
fd909c37 5332 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5333 if (!rdev->bios) {
5334 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5335 return -EINVAL;
5336 }
5337 DRM_INFO("GPU not posted. posting now...\n");
5338 atom_asic_init(rdev->mode_info.atom_context);
5339 }
d4788db3
AD		 5340 /* init golden registers */
5341 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5342 /* Initialize scratch registers */
5343 r600_scratch_init(rdev);
5344 /* Initialize surface registers */
5345 radeon_surface_init(rdev);
5346 /* Initialize clocks */
5347 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5348 /* Fence driver */
5349 r = radeon_fence_driver_init(rdev);
5350 if (r)
5351 return r;
d594e46a
JG		 5352 /* initialize AGP */
5353 if (rdev->flags & RADEON_IS_AGP) {
5354 r = radeon_agp_init(rdev);
5355 if (r)
5356 radeon_agp_disable(rdev);
5357 }
5358 /* initialize memory controller */
bcc1c2a1
AD		 5359 r = evergreen_mc_init(rdev);
5360 if (r)
5361 return r;
5362 /* Memory manager */
5363 r = radeon_bo_init(rdev);
5364 if (r)
5365 return r;
45f9a39b 5366
e32eb50d
CK		 5367 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5368 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5369
233d1ad5
AD		 5370 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5371 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5372
f2ba57b5
CK		 5373 r = radeon_uvd_init(rdev);
5374 if (!r) {
5375 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5376 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5377 4096);
5378 }
5379
bcc1c2a1
AD		 5380 rdev->ih.ring_obj = NULL;
5381 r600_ih_ring_init(rdev, 64 * 1024);
5382
5383 r = r600_pcie_gart_init(rdev);
5384 if (r)
5385 return r;
0fcdb61e 5386
148a03bc 5387 rdev->accel_working = true;
bcc1c2a1
AD		 5388 r = evergreen_startup(rdev);
5389 if (r) {
fe251e2f
AD		 5390 dev_err(rdev->dev, "disabling GPU acceleration\n");
5391 r700_cp_fini(rdev);
233d1ad5 5392 r600_dma_fini(rdev);
fe251e2f 5393 r600_irq_fini(rdev);
2948f5e6
AD		 5394 if (rdev->flags & RADEON_IS_IGP)
5395 sumo_rlc_fini(rdev);
724c80e1 5396 radeon_wb_fini(rdev);
2898c348 5397 radeon_ib_pool_fini(rdev);
fe251e2f 5398 radeon_irq_kms_fini(rdev);
0fcdb61e 5399 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5400 rdev->accel_working = false;
5401 }
77e00f2e
AD		 5402
5403 /* Don't start up if the MC ucode is missing on BTC parts.
5404 * The default clocks and voltages before the MC ucode
5405 * is loaded are not suffient for advanced operations.
5406 */
5407 if (ASIC_IS_DCE5(rdev)) {
5408 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5409 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5410 return -EINVAL;
5411 }
5412 }
5413
bcc1c2a1
AD		 5414 return 0;
5415}
5416
5417void evergreen_fini(struct radeon_device *rdev)
5418{
69d2ae57 5419 r600_audio_fini(rdev);
45f9a39b 5420 r700_cp_fini(rdev);
233d1ad5 5421 r600_dma_fini(rdev);
bcc1c2a1 5422 r600_irq_fini(rdev);
2948f5e6
AD		 5423 if (rdev->flags & RADEON_IS_IGP)
5424 sumo_rlc_fini(rdev);
724c80e1 5425 radeon_wb_fini(rdev);
2898c348 5426 radeon_ib_pool_fini(rdev);
bcc1c2a1 5427 radeon_irq_kms_fini(rdev);
bcc1c2a1 5428 evergreen_pcie_gart_fini(rdev);
e409b128 5429 uvd_v1_0_fini(rdev);
f2ba57b5 5430 radeon_uvd_fini(rdev);
16cdf04d 5431 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5432 radeon_gem_fini(rdev);
5433 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5434 radeon_agp_fini(rdev);
5435 radeon_bo_fini(rdev);
5436 radeon_atombios_fini(rdev);
5437 kfree(rdev->bios);
5438 rdev->bios = NULL;
bcc1c2a1 5439}
9e46a48d 5440
b07759bf 5441void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5442{
7e0e4196 5443 u32 link_width_cntl, speed_cntl;
9e46a48d 5444
d42dd579
AD		 5445 if (radeon_pcie_gen2 == 0)
5446 return;
5447
9e46a48d
AD		 5448 if (rdev->flags & RADEON_IS_IGP)
5449 return;
5450
5451 if (!(rdev->flags & RADEON_IS_PCIE))
5452 return;
5453
5454 /* x2 cards have a special sequence */
5455 if (ASIC_IS_X2(rdev))
5456 return;
5457
7e0e4196
KSS		 5458 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5459 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5460 return;
5461
492d2b61 5462 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5463 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5464 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5465 return;
5466 }
5467
197bbb3d
DA		 5468 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5469
9e46a48d
AD		 5470 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5471 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5472
492d2b61 5473 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5474 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5475 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5476
492d2b61 5477 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5478 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5479 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5480
492d2b61 5481 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5482 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5483 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5484
492d2b61 5485 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5486 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5487 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5488
492d2b61 5489 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5490 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5491 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5492
5493 } else {
492d2b61 5494 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5495 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5496 if (1)
5497 link_width_cntl |= LC_UPCONFIGURE_DIS;
5498 else
5499 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5500 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5501 }
5502}
f52382d7
AD		 5503
5504void evergreen_program_aspm(struct radeon_device *rdev)
5505{
5506 u32 data, orig;
5507 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5508 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5509 /* fusion_platform = true
5510 * if the system is a fusion system
5511 * (APU or DGPU in a fusion system).
5512 * todo: check if the system is a fusion platform.
5513 */
5514 bool fusion_platform = false;
5515
1294d4a3
AD		 5516 if (radeon_aspm == 0)
5517 return;
5518
f52382d7
AD		 5519 if (!(rdev->flags & RADEON_IS_PCIE))
5520 return;
5521
5522 switch (rdev->family) {
5523 case CHIP_CYPRESS:
5524 case CHIP_HEMLOCK:
5525 case CHIP_JUNIPER:
5526 case CHIP_REDWOOD:
5527 case CHIP_CEDAR:
5528 case CHIP_SUMO:
5529 case CHIP_SUMO2:
5530 case CHIP_PALM:
5531 case CHIP_ARUBA:
5532 disable_l0s = true;
5533 break;
5534 default:
5535 disable_l0s = false;
5536 break;
5537 }
5538
5539 if (rdev->flags & RADEON_IS_IGP)
5540 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5541
5542 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5543 if (fusion_platform)
5544 data &= ~MULTI_PIF;
5545 else
5546 data |= MULTI_PIF;
5547 if (data != orig)
5548 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5549
5550 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5551 if (fusion_platform)
5552 data &= ~MULTI_PIF;
5553 else
5554 data |= MULTI_PIF;
5555 if (data != orig)
5556 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5557
5558 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5559 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5560 if (!disable_l0s) {
5561 if (rdev->family >= CHIP_BARTS)
5562 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5563 else
5564 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5565 }
5566
5567 if (!disable_l1) {
5568 if (rdev->family >= CHIP_BARTS)
5569 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5570 else
5571 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5572
5573 if (!disable_plloff_in_l1) {
5574 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5575 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5576 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5577 if (data != orig)
5578 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5579
5580 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5581 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5582 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5583 if (data != orig)
5584 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5585
5586 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5587 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5588 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5589 if (data != orig)
5590 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5591
5592 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5593 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5594 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5595 if (data != orig)
5596 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5597
5598 if (rdev->family >= CHIP_BARTS) {
5599 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5600 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5601 data |= PLL_RAMP_UP_TIME_0(4);
5602 if (data != orig)
5603 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5604
5605 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5606 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5607 data |= PLL_RAMP_UP_TIME_1(4);
5608 if (data != orig)
5609 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5610
5611 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5612 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5613 data |= PLL_RAMP_UP_TIME_0(4);
5614 if (data != orig)
5615 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5616
5617 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5618 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5619 data |= PLL_RAMP_UP_TIME_1(4);
5620 if (data != orig)
5621 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5622 }
5623
5624 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5625 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5626 data |= LC_DYN_LANES_PWR_STATE(3);
5627 if (data != orig)
5628 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5629
5630 if (rdev->family >= CHIP_BARTS) {
5631 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5632 data &= ~LS2_EXIT_TIME_MASK;
5633 data |= LS2_EXIT_TIME(1);
5634 if (data != orig)
5635 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5636
5637 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5638 data &= ~LS2_EXIT_TIME_MASK;
5639 data |= LS2_EXIT_TIME(1);
5640 if (data != orig)
5641 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5642 }
5643 }
5644 }
5645
5646 /* evergreen parts only */
5647 if (rdev->family < CHIP_BARTS)
5648 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5649
5650 if (pcie_lc_cntl != pcie_lc_cntl_old)
5651 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5652}
Linux kernel - Deliverables
This page took 0.589978 seconds and 5 git commands to generate.