projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/cik: drop cg_update from dpm code
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
bcc1c2a1 149
d4788db3
AD		 150static const u32 evergreen_golden_registers[] =
151{
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5cc, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194};
195
196static const u32 evergreen_golden_registers2[] =
197{
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212};
213
214static const u32 cypress_mgcg_init[] =
215{
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365};
366
367static const u32 redwood_mgcg_init[] =
368{
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437};
438
439static const u32 cedar_golden_registers[] =
440{
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5cc, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480};
481
482static const u32 cedar_mgcg_init[] =
483{
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534};
535
536static const u32 juniper_mgcg_init[] =
537{
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632};
633
634static const u32 supersumo_golden_registers[] =
635{
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5cc, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707};
708
709static const u32 sumo_golden_registers[] =
710{
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716};
717
718static const u32 wrestler_golden_registers[] =
719{
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5cc, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769};
770
771static const u32 barts_golden_registers[] =
772{
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818};
819
820static const u32 turks_golden_registers[] =
821{
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869};
870
871static const u32 caicos_golden_registers[] =
872{
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920};
921
922static void evergreen_init_golden_registers(struct radeon_device *rdev)
923{
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006}
1007
285484e2
JG		 1008void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011{
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037}
1038
23d33ba3
AD		 1039static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041{
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061}
1062
1063int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064{
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084}
1085
a8b4925c
AD		 1086int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087{
1088 /* start off with something large */
facd112d 1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1090 int r;
1091
4ed10835
CK		 1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
facd112d
CK		 1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
a8b4925c
AD		 1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
facd112d 1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
facd112d 1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
facd112d 1140 if (fb_div < 307200)
a8b4925c
AD		 1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
facd112d 1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173}
1174
d054ac16
AD		 1175void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176{
c11592fe
YW		 1177 int readrq;
1178 u16 v;
d054ac16 1179
c11592fe
YW		 1180 readrq = pcie_get_readrq(rdev->pdev);
1181 v = ffs(readrq) - 8;
d054ac16
AD		 1182 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183 * to avoid hangs or perfomance issues
1184 */
c11592fe
YW		 1185 if ((v == 0) || (v == 6) || (v == 7))
1186 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1187}
1188
134b480f
AD		 1189void dce4_program_fmt(struct drm_encoder *encoder)
1190{
1191 struct drm_device *dev = encoder->dev;
1192 struct radeon_device *rdev = dev->dev_private;
1193 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1194 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1195 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1196 int bpc = 0;
1197 u32 tmp = 0;
6214bb74 1198 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1199
6214bb74
AD		 1200 if (connector) {
1201 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1202 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1203 dither = radeon_connector->dither;
1204 }
134b480f
AD		 1205
1206 /* LVDS/eDP FMT is set up by atom */
1207 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1208 return;
1209
1210 /* not needed for analog */
1211 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1212 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1213 return;
1214
1215 if (bpc == 0)
1216 return;
1217
1218 switch (bpc) {
1219 case 6:
6214bb74 1220 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1221 /* XXX sort out optimal dither settings */
1222 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1223 FMT_SPATIAL_DITHER_EN);
1224 else
1225 tmp |= FMT_TRUNCATE_EN;
1226 break;
1227 case 8:
6214bb74 1228 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1229 /* XXX sort out optimal dither settings */
1230 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1231 FMT_RGB_RANDOM_ENABLE |
1232 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1233 else
1234 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1235 break;
1236 case 10:
1237 default:
1238 /* not needed */
1239 break;
1240 }
1241
1242 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1243}
1244
10257a6d
AD		 1245static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1246{
1247 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1248 return true;
1249 else
1250 return false;
1251}
1252
1253static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1254{
1255 u32 pos1, pos2;
1256
1257 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1258 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259
1260 if (pos1 != pos2)
1261 return true;
1262 else
1263 return false;
1264}
1265
377edc8b
AD		 1266/**
1267 * dce4_wait_for_vblank - vblank wait asic callback.
1268 *
1269 * @rdev: radeon_device pointer
1270 * @crtc: crtc to wait for vblank on
1271 *
1272 * Wait for vblank on the requested crtc (evergreen+).
1273 */
3ae19b75
AD		 1274void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1275{
10257a6d 1276 unsigned i = 0;
3ae19b75 1277
4a15903d
AD		 1278 if (crtc >= rdev->num_crtc)
1279 return;
1280
10257a6d
AD		 1281 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1282 return;
1283
1284 /* depending on when we hit vblank, we may be close to active; if so,
1285 * wait for another frame.
1286 */
1287 while (dce4_is_in_vblank(rdev, crtc)) {
1288 if (i++ % 100 == 0) {
1289 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1290 break;
3ae19b75 1291 }
10257a6d
AD		 1292 }
1293
1294 while (!dce4_is_in_vblank(rdev, crtc)) {
1295 if (i++ % 100 == 0) {
1296 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1297 break;
3ae19b75
AD		 1298 }
1299 }
1300}
1301
377edc8b
AD		 1302/**
1303 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1304 *
1305 * @rdev: radeon_device pointer
1306 * @crtc: crtc to prepare for pageflip on
1307 *
1308 * Pre-pageflip callback (evergreen+).
1309 * Enables the pageflip irq (vblank irq).
1310 */
6f34be50
AD		 1311void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1312{
6f34be50
AD		 1313 /* enable the pflip int */
1314 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1315}
1316
377edc8b
AD		 1317/**
1318 * evergreen_post_page_flip - pos-pageflip callback.
1319 *
1320 * @rdev: radeon_device pointer
1321 * @crtc: crtc to cleanup pageflip on
1322 *
1323 * Post-pageflip callback (evergreen+).
1324 * Disables the pageflip irq (vblank irq).
1325 */
6f34be50
AD		 1326void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1327{
1328 /* disable the pflip int */
1329 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1330}
1331
377edc8b
AD		 1332/**
1333 * evergreen_page_flip - pageflip callback.
1334 *
1335 * @rdev: radeon_device pointer
1336 * @crtc_id: crtc to cleanup pageflip on
1337 * @crtc_base: new address of the crtc (GPU MC address)
1338 *
1339 * Does the actual pageflip (evergreen+).
1340 * During vblank we take the crtc lock and wait for the update_pending
1341 * bit to go high, when it does, we release the lock, and allow the
1342 * double buffered update to take place.
1343 * Returns the current update pending status.
1344 */
6f34be50
AD		 1345u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1346{
1347 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1348 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1349 int i;
6f34be50
AD		 1350
1351 /* Lock the graphics update lock */
1352 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1353 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1354
1355 /* update the scanout addresses */
1356 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1357 upper_32_bits(crtc_base));
1358 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1359 (u32)crtc_base);
1360
1361 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1362 upper_32_bits(crtc_base));
1363 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1364 (u32)crtc_base);
1365
1366 /* Wait for update_pending to go high. */
f6496479
AD		 1367 for (i = 0; i < rdev->usec_timeout; i++) {
1368 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1369 break;
1370 udelay(1);
1371 }
6f34be50
AD		 1372 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1373
1374 /* Unlock the lock, so double-buffering can take place inside vblank */
1375 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1376 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1377
1378 /* Return current update_pending status: */
1379 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1380}
1381
21a8122a 1382/* get temperature in millidegrees */
20d391d7 1383int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1384{
1c88d74f
AD		 1385 u32 temp, toffset;
1386 int actual_temp = 0;
67b3f823
AD		 1387
1388 if (rdev->family == CHIP_JUNIPER) {
1389 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1390 TOFFSET_SHIFT;
1391 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1392 TS0_ADC_DOUT_SHIFT;
1393
1394 if (toffset & 0x100)
1395 actual_temp = temp / 2 - (0x200 - toffset);
1396 else
1397 actual_temp = temp / 2 + toffset;
1398
1399 actual_temp = actual_temp * 1000;
1400
1401 } else {
1402 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1403 ASIC_T_SHIFT;
1404
1405 if (temp & 0x400)
1406 actual_temp = -256;
1407 else if (temp & 0x200)
1408 actual_temp = 255;
1409 else if (temp & 0x100) {
1410 actual_temp = temp & 0x1ff;
1411 actual_temp |= ~0x1ff;
1412 } else
1413 actual_temp = temp & 0xff;
1414
1415 actual_temp = (actual_temp * 1000) / 2;
1416 }
21a8122a 1417
67b3f823 1418 return actual_temp;
21a8122a
AD		 1419}
1420
20d391d7 1421int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1422{
1423 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1424 int actual_temp = temp - 49;
e33df25f
AD		 1425
1426 return actual_temp * 1000;
1427}
1428
377edc8b
AD		 1429/**
1430 * sumo_pm_init_profile - Initialize power profiles callback.
1431 *
1432 * @rdev: radeon_device pointer
1433 *
1434 * Initialize the power states used in profile mode
1435 * (sumo, trinity, SI).
1436 * Used for profile mode only.
1437 */
a4c9e2ee
AD		 1438void sumo_pm_init_profile(struct radeon_device *rdev)
1439{
1440 int idx;
1441
1442 /* default */
1443 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1444 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1445 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1447
1448 /* low,mid sh/mh */
1449 if (rdev->flags & RADEON_IS_MOBILITY)
1450 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1451 else
1452 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1453
1454 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1457 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1458
1459 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1463
1464 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1468
1469 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1473
1474 /* high sh/mh */
1475 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1476 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1477 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1479 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1480 rdev->pm.power_state[idx].num_clock_modes - 1;
1481
1482 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1486 rdev->pm.power_state[idx].num_clock_modes - 1;
1487}
1488
27810fb2
AD		 1489/**
1490 * btc_pm_init_profile - Initialize power profiles callback.
1491 *
1492 * @rdev: radeon_device pointer
1493 *
1494 * Initialize the power states used in profile mode
1495 * (BTC, cayman).
1496 * Used for profile mode only.
1497 */
1498void btc_pm_init_profile(struct radeon_device *rdev)
1499{
1500 int idx;
1501
1502 /* default */
1503 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1507 /* starting with BTC, there is one state that is used for both
1508 * MH and SH. Difference is that we always use the high clock index for
1509 * mclk.
1510 */
1511 if (rdev->flags & RADEON_IS_MOBILITY)
1512 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1513 else
1514 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1515 /* low sh */
1516 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1520 /* mid sh */
1521 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1525 /* high sh */
1526 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1530 /* low mh */
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535 /* mid mh */
1536 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1540 /* high mh */
1541 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1545}
1546
377edc8b
AD		 1547/**
1548 * evergreen_pm_misc - set additional pm hw parameters callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Set non-clock parameters associated with a power state
1553 * (voltage, etc.) (evergreen+).
1554 */
49e02b73
AD		 1555void evergreen_pm_misc(struct radeon_device *rdev)
1556{
a081a9d6
RM		 1557 int req_ps_idx = rdev->pm.requested_power_state_index;
1558 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1559 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1560 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1561
2feea49a 1562 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1563 /* 0xff0x are flags rather then an actual voltage */
1564 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1565 return;
2feea49a 1566 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1567 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1568 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1569 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1570 }
7ae764b1
AD		 1571
1572 /* starting with BTC, there is one state that is used for both
1573 * MH and SH. Difference is that we always use the high clock index for
1574 * mclk and vddci.
1575 */
1576 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1577 (rdev->family >= CHIP_BARTS) &&
1578 rdev->pm.active_crtc_count &&
1579 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1580 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1581 voltage = &rdev->pm.power_state[req_ps_idx].
1582 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1583
c6cf7777
AD		 1584 /* 0xff0x are flags rather then an actual voltage */
1585 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1586 return;
2feea49a
AD		 1587 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1588 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1589 rdev->pm.current_vddci = voltage->vddci;
1590 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1591 }
1592 }
49e02b73
AD		 1593}
1594
377edc8b
AD		 1595/**
1596 * evergreen_pm_prepare - pre-power state change callback.
1597 *
1598 * @rdev: radeon_device pointer
1599 *
1600 * Prepare for a power state change (evergreen+).
1601 */
49e02b73
AD		 1602void evergreen_pm_prepare(struct radeon_device *rdev)
1603{
1604 struct drm_device *ddev = rdev->ddev;
1605 struct drm_crtc *crtc;
1606 struct radeon_crtc *radeon_crtc;
1607 u32 tmp;
1608
1609 /* disable any active CRTCs */
1610 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1611 radeon_crtc = to_radeon_crtc(crtc);
1612 if (radeon_crtc->enabled) {
1613 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1614 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1615 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1616 }
1617 }
1618}
1619
377edc8b
AD		 1620/**
1621 * evergreen_pm_finish - post-power state change callback.
1622 *
1623 * @rdev: radeon_device pointer
1624 *
1625 * Clean up after a power state change (evergreen+).
1626 */
49e02b73
AD		 1627void evergreen_pm_finish(struct radeon_device *rdev)
1628{
1629 struct drm_device *ddev = rdev->ddev;
1630 struct drm_crtc *crtc;
1631 struct radeon_crtc *radeon_crtc;
1632 u32 tmp;
1633
1634 /* enable any active CRTCs */
1635 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1636 radeon_crtc = to_radeon_crtc(crtc);
1637 if (radeon_crtc->enabled) {
1638 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1639 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1640 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1641 }
1642 }
1643}
1644
377edc8b
AD		 1645/**
1646 * evergreen_hpd_sense - hpd sense callback.
1647 *
1648 * @rdev: radeon_device pointer
1649 * @hpd: hpd (hotplug detect) pin
1650 *
1651 * Checks if a digital monitor is connected (evergreen+).
1652 * Returns true if connected, false if not connected.
1653 */
bcc1c2a1
AD		 1654bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1655{
1656 bool connected = false;
0ca2ab52
AD		 1657
1658 switch (hpd) {
1659 case RADEON_HPD_1:
1660 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1661 connected = true;
1662 break;
1663 case RADEON_HPD_2:
1664 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
1666 break;
1667 case RADEON_HPD_3:
1668 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1669 connected = true;
1670 break;
1671 case RADEON_HPD_4:
1672 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1673 connected = true;
1674 break;
1675 case RADEON_HPD_5:
1676 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1677 connected = true;
1678 break;
1679 case RADEON_HPD_6:
1680 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1681 connected = true;
1682 break;
1683 default:
1684 break;
1685 }
1686
bcc1c2a1
AD		 1687 return connected;
1688}
1689
377edc8b
AD		 1690/**
1691 * evergreen_hpd_set_polarity - hpd set polarity callback.
1692 *
1693 * @rdev: radeon_device pointer
1694 * @hpd: hpd (hotplug detect) pin
1695 *
1696 * Set the polarity of the hpd pin (evergreen+).
1697 */
bcc1c2a1
AD		 1698void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1699 enum radeon_hpd_id hpd)
1700{
0ca2ab52
AD		 1701 u32 tmp;
1702 bool connected = evergreen_hpd_sense(rdev, hpd);
1703
1704 switch (hpd) {
1705 case RADEON_HPD_1:
1706 tmp = RREG32(DC_HPD1_INT_CONTROL);
1707 if (connected)
1708 tmp &= ~DC_HPDx_INT_POLARITY;
1709 else
1710 tmp |= DC_HPDx_INT_POLARITY;
1711 WREG32(DC_HPD1_INT_CONTROL, tmp);
1712 break;
1713 case RADEON_HPD_2:
1714 tmp = RREG32(DC_HPD2_INT_CONTROL);
1715 if (connected)
1716 tmp &= ~DC_HPDx_INT_POLARITY;
1717 else
1718 tmp |= DC_HPDx_INT_POLARITY;
1719 WREG32(DC_HPD2_INT_CONTROL, tmp);
1720 break;
1721 case RADEON_HPD_3:
1722 tmp = RREG32(DC_HPD3_INT_CONTROL);
1723 if (connected)
1724 tmp &= ~DC_HPDx_INT_POLARITY;
1725 else
1726 tmp |= DC_HPDx_INT_POLARITY;
1727 WREG32(DC_HPD3_INT_CONTROL, tmp);
1728 break;
1729 case RADEON_HPD_4:
1730 tmp = RREG32(DC_HPD4_INT_CONTROL);
1731 if (connected)
1732 tmp &= ~DC_HPDx_INT_POLARITY;
1733 else
1734 tmp |= DC_HPDx_INT_POLARITY;
1735 WREG32(DC_HPD4_INT_CONTROL, tmp);
1736 break;
1737 case RADEON_HPD_5:
1738 tmp = RREG32(DC_HPD5_INT_CONTROL);
1739 if (connected)
1740 tmp &= ~DC_HPDx_INT_POLARITY;
1741 else
1742 tmp |= DC_HPDx_INT_POLARITY;
1743 WREG32(DC_HPD5_INT_CONTROL, tmp);
1744 break;
1745 case RADEON_HPD_6:
1746 tmp = RREG32(DC_HPD6_INT_CONTROL);
1747 if (connected)
1748 tmp &= ~DC_HPDx_INT_POLARITY;
1749 else
1750 tmp |= DC_HPDx_INT_POLARITY;
1751 WREG32(DC_HPD6_INT_CONTROL, tmp);
1752 break;
1753 default:
1754 break;
1755 }
bcc1c2a1
AD		 1756}
1757
377edc8b
AD		 1758/**
1759 * evergreen_hpd_init - hpd setup callback.
1760 *
1761 * @rdev: radeon_device pointer
1762 *
1763 * Setup the hpd pins used by the card (evergreen+).
1764 * Enable the pin, set the polarity, and enable the hpd interrupts.
1765 */
bcc1c2a1
AD		 1766void evergreen_hpd_init(struct radeon_device *rdev)
1767{
0ca2ab52
AD		 1768 struct drm_device *dev = rdev->ddev;
1769 struct drm_connector *connector;
fb98257a 1770 unsigned enabled = 0;
0ca2ab52
AD		 1771 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1772 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1773
0ca2ab52
AD		 1774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1775 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1776
1777 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1778 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1779 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1780 * aux dp channel on imac and help (but not completely fix)
1781 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1782 * also avoid interrupt storms during dpms.
1783 */
1784 continue;
1785 }
0ca2ab52
AD		 1786 switch (radeon_connector->hpd.hpd) {
1787 case RADEON_HPD_1:
1788 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1789 break;
1790 case RADEON_HPD_2:
1791 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1792 break;
1793 case RADEON_HPD_3:
1794 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1795 break;
1796 case RADEON_HPD_4:
1797 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1798 break;
1799 case RADEON_HPD_5:
1800 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1801 break;
1802 case RADEON_HPD_6:
1803 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1804 break;
1805 default:
1806 break;
1807 }
64912e99 1808 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1809 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1810 }
fb98257a 1811 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1812}
1813
377edc8b
AD		 1814/**
1815 * evergreen_hpd_fini - hpd tear down callback.
1816 *
1817 * @rdev: radeon_device pointer
1818 *
1819 * Tear down the hpd pins used by the card (evergreen+).
1820 * Disable the hpd interrupts.
1821 */
0ca2ab52 1822void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1823{
0ca2ab52
AD		 1824 struct drm_device *dev = rdev->ddev;
1825 struct drm_connector *connector;
fb98257a 1826 unsigned disabled = 0;
0ca2ab52
AD		 1827
1828 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1829 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1830 switch (radeon_connector->hpd.hpd) {
1831 case RADEON_HPD_1:
1832 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1833 break;
1834 case RADEON_HPD_2:
1835 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1836 break;
1837 case RADEON_HPD_3:
1838 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1839 break;
1840 case RADEON_HPD_4:
1841 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1842 break;
1843 case RADEON_HPD_5:
1844 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1845 break;
1846 case RADEON_HPD_6:
1847 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1848 break;
1849 default:
1850 break;
1851 }
fb98257a 1852 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1853 }
fb98257a 1854 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1855}
1856
f9d9c362
AD		 1857/* watermark setup */
1858
1859static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1860 struct radeon_crtc *radeon_crtc,
1861 struct drm_display_mode *mode,
1862 struct drm_display_mode *other_mode)
1863{
0b31e023
AD		 1864 u32 tmp, buffer_alloc, i;
1865 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1866 /*
1867 * Line Buffer Setup
1868 * There are 3 line buffers, each one shared by 2 display controllers.
1869 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1870 * the display controllers. The paritioning is done via one of four
1871 * preset allocations specified in bits 2:0:
1872 * first display controller
1873 * 0 - first half of lb (3840 * 2)
1874 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1875 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1876 * 3 - first 1/4 of lb (1920 * 2)
1877 * second display controller
1878 * 4 - second half of lb (3840 * 2)
1879 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1880 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1881 * 7 - last 1/4 of lb (1920 * 2)
1882 */
12dfc843
AD		 1883 /* this can get tricky if we have two large displays on a paired group
1884 * of crtcs. Ideally for multiple large displays we'd assign them to
1885 * non-linked crtcs for maximum line buffer allocation.
1886 */
1887 if (radeon_crtc->base.enabled && mode) {
0b31e023 1888 if (other_mode) {
f9d9c362 1889 tmp = 0; /* 1/2 */
0b31e023
AD		 1890 buffer_alloc = 1;
1891 } else {
12dfc843 1892 tmp = 2; /* whole */
0b31e023
AD		 1893 buffer_alloc = 2;
1894 }
1895 } else {
12dfc843 1896 tmp = 0;
0b31e023
AD		 1897 buffer_alloc = 0;
1898 }
f9d9c362
AD		 1899
1900 /* second controller of the pair uses second half of the lb */
1901 if (radeon_crtc->crtc_id % 2)
1902 tmp += 4;
1903 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1904
0b31e023
AD		 1905 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1906 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1907 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1908 for (i = 0; i < rdev->usec_timeout; i++) {
1909 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1910 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1911 break;
1912 udelay(1);
1913 }
1914 }
1915
12dfc843
AD		 1916 if (radeon_crtc->base.enabled && mode) {
1917 switch (tmp) {
1918 case 0:
1919 case 4:
1920 default:
1921 if (ASIC_IS_DCE5(rdev))
1922 return 4096 * 2;
1923 else
1924 return 3840 * 2;
1925 case 1:
1926 case 5:
1927 if (ASIC_IS_DCE5(rdev))
1928 return 6144 * 2;
1929 else
1930 return 5760 * 2;
1931 case 2:
1932 case 6:
1933 if (ASIC_IS_DCE5(rdev))
1934 return 8192 * 2;
1935 else
1936 return 7680 * 2;
1937 case 3:
1938 case 7:
1939 if (ASIC_IS_DCE5(rdev))
1940 return 2048 * 2;
1941 else
1942 return 1920 * 2;
1943 }
f9d9c362 1944 }
12dfc843
AD		 1945
1946 /* controller not enabled, so no lb used */
1947 return 0;
f9d9c362
AD		 1948}
1949
ca7db22b 1950u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1951{
1952 u32 tmp = RREG32(MC_SHARED_CHMAP);
1953
1954 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1955 case 0:
1956 default:
1957 return 1;
1958 case 1:
1959 return 2;
1960 case 2:
1961 return 4;
1962 case 3:
1963 return 8;
1964 }
1965}
1966
1967struct evergreen_wm_params {
1968 u32 dram_channels; /* number of dram channels */
1969 u32 yclk; /* bandwidth per dram data pin in kHz */
1970 u32 sclk; /* engine clock in kHz */
1971 u32 disp_clk; /* display clock in kHz */
1972 u32 src_width; /* viewport width */
1973 u32 active_time; /* active display time in ns */
1974 u32 blank_time; /* blank time in ns */
1975 bool interlaced; /* mode is interlaced */
1976 fixed20_12 vsc; /* vertical scale ratio */
1977 u32 num_heads; /* number of active crtcs */
1978 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1979 u32 lb_size; /* line buffer allocated to pipe */
1980 u32 vtaps; /* vertical scaler taps */
1981};
1982
1983static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1984{
1985 /* Calculate DRAM Bandwidth and the part allocated to display. */
1986 fixed20_12 dram_efficiency; /* 0.7 */
1987 fixed20_12 yclk, dram_channels, bandwidth;
1988 fixed20_12 a;
1989
1990 a.full = dfixed_const(1000);
1991 yclk.full = dfixed_const(wm->yclk);
1992 yclk.full = dfixed_div(yclk, a);
1993 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1994 a.full = dfixed_const(10);
1995 dram_efficiency.full = dfixed_const(7);
1996 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1997 bandwidth.full = dfixed_mul(dram_channels, yclk);
1998 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1999
2000 return dfixed_trunc(bandwidth);
2001}
2002
2003static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2004{
2005 /* Calculate DRAM Bandwidth and the part allocated to display. */
2006 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2007 fixed20_12 yclk, dram_channels, bandwidth;
2008 fixed20_12 a;
2009
2010 a.full = dfixed_const(1000);
2011 yclk.full = dfixed_const(wm->yclk);
2012 yclk.full = dfixed_div(yclk, a);
2013 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2014 a.full = dfixed_const(10);
2015 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2016 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2017 bandwidth.full = dfixed_mul(dram_channels, yclk);
2018 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2019
2020 return dfixed_trunc(bandwidth);
2021}
2022
2023static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2024{
2025 /* Calculate the display Data return Bandwidth */
2026 fixed20_12 return_efficiency; /* 0.8 */
2027 fixed20_12 sclk, bandwidth;
2028 fixed20_12 a;
2029
2030 a.full = dfixed_const(1000);
2031 sclk.full = dfixed_const(wm->sclk);
2032 sclk.full = dfixed_div(sclk, a);
2033 a.full = dfixed_const(10);
2034 return_efficiency.full = dfixed_const(8);
2035 return_efficiency.full = dfixed_div(return_efficiency, a);
2036 a.full = dfixed_const(32);
2037 bandwidth.full = dfixed_mul(a, sclk);
2038 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2039
2040 return dfixed_trunc(bandwidth);
2041}
2042
2043static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2044{
2045 /* Calculate the DMIF Request Bandwidth */
2046 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2047 fixed20_12 disp_clk, bandwidth;
2048 fixed20_12 a;
2049
2050 a.full = dfixed_const(1000);
2051 disp_clk.full = dfixed_const(wm->disp_clk);
2052 disp_clk.full = dfixed_div(disp_clk, a);
2053 a.full = dfixed_const(10);
2054 disp_clk_request_efficiency.full = dfixed_const(8);
2055 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2056 a.full = dfixed_const(32);
2057 bandwidth.full = dfixed_mul(a, disp_clk);
2058 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2059
2060 return dfixed_trunc(bandwidth);
2061}
2062
2063static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2064{
2065 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2066 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2067 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2068 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2069
2070 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2071}
2072
2073static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2074{
2075 /* Calculate the display mode Average Bandwidth
2076 * DisplayMode should contain the source and destination dimensions,
2077 * timing, etc.
2078 */
2079 fixed20_12 bpp;
2080 fixed20_12 line_time;
2081 fixed20_12 src_width;
2082 fixed20_12 bandwidth;
2083 fixed20_12 a;
2084
2085 a.full = dfixed_const(1000);
2086 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2087 line_time.full = dfixed_div(line_time, a);
2088 bpp.full = dfixed_const(wm->bytes_per_pixel);
2089 src_width.full = dfixed_const(wm->src_width);
2090 bandwidth.full = dfixed_mul(src_width, bpp);
2091 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2092 bandwidth.full = dfixed_div(bandwidth, line_time);
2093
2094 return dfixed_trunc(bandwidth);
2095}
2096
2097static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2098{
2099 /* First calcualte the latency in ns */
2100 u32 mc_latency = 2000; /* 2000 ns. */
2101 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2102 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2103 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2104 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2105 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2106 (wm->num_heads * cursor_line_pair_return_time);
2107 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2108 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2109 fixed20_12 a, b, c;
2110
2111 if (wm->num_heads == 0)
2112 return 0;
2113
2114 a.full = dfixed_const(2);
2115 b.full = dfixed_const(1);
2116 if ((wm->vsc.full > a.full) ||
2117 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2118 (wm->vtaps >= 5) ||
2119 ((wm->vsc.full >= a.full) && wm->interlaced))
2120 max_src_lines_per_dst_line = 4;
2121 else
2122 max_src_lines_per_dst_line = 2;
2123
2124 a.full = dfixed_const(available_bandwidth);
2125 b.full = dfixed_const(wm->num_heads);
2126 a.full = dfixed_div(a, b);
2127
2128 b.full = dfixed_const(1000);
2129 c.full = dfixed_const(wm->disp_clk);
2130 b.full = dfixed_div(c, b);
2131 c.full = dfixed_const(wm->bytes_per_pixel);
2132 b.full = dfixed_mul(b, c);
2133
2134 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2135
2136 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2137 b.full = dfixed_const(1000);
2138 c.full = dfixed_const(lb_fill_bw);
2139 b.full = dfixed_div(c, b);
2140 a.full = dfixed_div(a, b);
2141 line_fill_time = dfixed_trunc(a);
2142
2143 if (line_fill_time < wm->active_time)
2144 return latency;
2145 else
2146 return latency + (line_fill_time - wm->active_time);
2147
2148}
2149
2150static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2151{
2152 if (evergreen_average_bandwidth(wm) <=
2153 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2154 return true;
2155 else
2156 return false;
2157};
2158
2159static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2160{
2161 if (evergreen_average_bandwidth(wm) <=
2162 (evergreen_available_bandwidth(wm) / wm->num_heads))
2163 return true;
2164 else
2165 return false;
2166};
2167
2168static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2169{
2170 u32 lb_partitions = wm->lb_size / wm->src_width;
2171 u32 line_time = wm->active_time + wm->blank_time;
2172 u32 latency_tolerant_lines;
2173 u32 latency_hiding;
2174 fixed20_12 a;
2175
2176 a.full = dfixed_const(1);
2177 if (wm->vsc.full > a.full)
2178 latency_tolerant_lines = 1;
2179 else {
2180 if (lb_partitions <= (wm->vtaps + 1))
2181 latency_tolerant_lines = 1;
2182 else
2183 latency_tolerant_lines = 2;
2184 }
2185
2186 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2187
2188 if (evergreen_latency_watermark(wm) <= latency_hiding)
2189 return true;
2190 else
2191 return false;
2192}
2193
2194static void evergreen_program_watermarks(struct radeon_device *rdev,
2195 struct radeon_crtc *radeon_crtc,
2196 u32 lb_size, u32 num_heads)
2197{
2198 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2199 struct evergreen_wm_params wm_low, wm_high;
2200 u32 dram_channels;
f9d9c362
AD		 2201 u32 pixel_period;
2202 u32 line_time = 0;
2203 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2204 u32 priority_a_mark = 0, priority_b_mark = 0;
2205 u32 priority_a_cnt = PRIORITY_OFF;
2206 u32 priority_b_cnt = PRIORITY_OFF;
2207 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2208 u32 tmp, arb_control3;
2209 fixed20_12 a, b, c;
2210
2211 if (radeon_crtc->base.enabled && num_heads && mode) {
2212 pixel_period = 1000000 / (u32)mode->clock;
2213 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2214 priority_a_cnt = 0;
2215 priority_b_cnt = 0;
cf0cfdd7
AD		 2216 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2217
2218 /* watermark for high clocks */
2219 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2220 wm_high.yclk =
2221 radeon_dpm_get_mclk(rdev, false) * 10;
2222 wm_high.sclk =
2223 radeon_dpm_get_sclk(rdev, false) * 10;
2224 } else {
2225 wm_high.yclk = rdev->pm.current_mclk * 10;
2226 wm_high.sclk = rdev->pm.current_sclk * 10;
2227 }
f9d9c362 2228
cf0cfdd7
AD		 2229 wm_high.disp_clk = mode->clock;
2230 wm_high.src_width = mode->crtc_hdisplay;
2231 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2232 wm_high.blank_time = line_time - wm_high.active_time;
2233 wm_high.interlaced = false;
f9d9c362 2234 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2235 wm_high.interlaced = true;
2236 wm_high.vsc = radeon_crtc->vsc;
2237 wm_high.vtaps = 1;
f9d9c362 2238 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2239 wm_high.vtaps = 2;
2240 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2241 wm_high.lb_size = lb_size;
2242 wm_high.dram_channels = dram_channels;
2243 wm_high.num_heads = num_heads;
2244
2245 /* watermark for low clocks */
2246 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2247 wm_low.yclk =
2248 radeon_dpm_get_mclk(rdev, true) * 10;
2249 wm_low.sclk =
2250 radeon_dpm_get_sclk(rdev, true) * 10;
2251 } else {
2252 wm_low.yclk = rdev->pm.current_mclk * 10;
2253 wm_low.sclk = rdev->pm.current_sclk * 10;
2254 }
2255
2256 wm_low.disp_clk = mode->clock;
2257 wm_low.src_width = mode->crtc_hdisplay;
2258 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2259 wm_low.blank_time = line_time - wm_low.active_time;
2260 wm_low.interlaced = false;
2261 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2262 wm_low.interlaced = true;
2263 wm_low.vsc = radeon_crtc->vsc;
2264 wm_low.vtaps = 1;
2265 if (radeon_crtc->rmx_type != RMX_OFF)
2266 wm_low.vtaps = 2;
2267 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2268 wm_low.lb_size = lb_size;
2269 wm_low.dram_channels = dram_channels;
2270 wm_low.num_heads = num_heads;
f9d9c362
AD		 2271
2272 /* set for high clocks */
cf0cfdd7 2273 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2274 /* set for low clocks */
cf0cfdd7 2275 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2276
2277 /* possibly force display priority to high */
2278 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2279 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2280 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2281 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2282 (rdev->disp_priority == 2)) {
cf0cfdd7 2283 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2284 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2285 }
2286 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2287 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2288 !evergreen_check_latency_hiding(&wm_low) ||
2289 (rdev->disp_priority == 2)) {
2290 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2291 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2292 }
2293
2294 a.full = dfixed_const(1000);
2295 b.full = dfixed_const(mode->clock);
2296 b.full = dfixed_div(b, a);
2297 c.full = dfixed_const(latency_watermark_a);
2298 c.full = dfixed_mul(c, b);
2299 c.full = dfixed_mul(c, radeon_crtc->hsc);
2300 c.full = dfixed_div(c, a);
2301 a.full = dfixed_const(16);
2302 c.full = dfixed_div(c, a);
2303 priority_a_mark = dfixed_trunc(c);
2304 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2305
2306 a.full = dfixed_const(1000);
2307 b.full = dfixed_const(mode->clock);
2308 b.full = dfixed_div(b, a);
2309 c.full = dfixed_const(latency_watermark_b);
2310 c.full = dfixed_mul(c, b);
2311 c.full = dfixed_mul(c, radeon_crtc->hsc);
2312 c.full = dfixed_div(c, a);
2313 a.full = dfixed_const(16);
2314 c.full = dfixed_div(c, a);
2315 priority_b_mark = dfixed_trunc(c);
2316 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2317 }
2318
2319 /* select wm A */
2320 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2321 tmp = arb_control3;
2322 tmp &= ~LATENCY_WATERMARK_MASK(3);
2323 tmp |= LATENCY_WATERMARK_MASK(1);
2324 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2325 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2326 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2327 LATENCY_HIGH_WATERMARK(line_time)));
2328 /* select wm B */
2329 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2330 tmp &= ~LATENCY_WATERMARK_MASK(3);
2331 tmp |= LATENCY_WATERMARK_MASK(2);
2332 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2333 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2334 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2335 LATENCY_HIGH_WATERMARK(line_time)));
2336 /* restore original selection */
2337 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2338
2339 /* write the priority marks */
2340 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2341 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2342
7178d2a6
AD		 2343 /* save values for DPM */
2344 radeon_crtc->line_time = line_time;
2345 radeon_crtc->wm_high = latency_watermark_a;
2346 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2347}
2348
377edc8b
AD		 2349/**
2350 * evergreen_bandwidth_update - update display watermarks callback.
2351 *
2352 * @rdev: radeon_device pointer
2353 *
2354 * Update the display watermarks based on the requested mode(s)
2355 * (evergreen+).
2356 */
0ca2ab52 2357void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2358{
f9d9c362
AD		 2359 struct drm_display_mode *mode0 = NULL;
2360 struct drm_display_mode *mode1 = NULL;
2361 u32 num_heads = 0, lb_size;
2362 int i;
2363
2364 radeon_update_display_priority(rdev);
2365
2366 for (i = 0; i < rdev->num_crtc; i++) {
2367 if (rdev->mode_info.crtcs[i]->base.enabled)
2368 num_heads++;
2369 }
2370 for (i = 0; i < rdev->num_crtc; i += 2) {
2371 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2372 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2373 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2374 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2375 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2376 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2377 }
bcc1c2a1
AD		 2378}
2379
377edc8b
AD		 2380/**
2381 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2382 *
2383 * @rdev: radeon_device pointer
2384 *
2385 * Wait for the MC (memory controller) to be idle.
2386 * (evergreen+).
2387 * Returns 0 if the MC is idle, -1 if not.
2388 */
b9952a8a 2389int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2390{
2391 unsigned i;
2392 u32 tmp;
2393
2394 for (i = 0; i < rdev->usec_timeout; i++) {
2395 /* read MC_STATUS */
2396 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2397 if (!tmp)
2398 return 0;
2399 udelay(1);
2400 }
2401 return -1;
2402}
2403
2404/*
2405 * GART
2406 */
0fcdb61e
AD		 2407void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2408{
2409 unsigned i;
2410 u32 tmp;
2411
6f2f48a9
AD		 2412 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2413
0fcdb61e
AD		 2414 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2415 for (i = 0; i < rdev->usec_timeout; i++) {
2416 /* read MC_STATUS */
2417 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2418 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2419 if (tmp == 2) {
2420 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2421 return;
2422 }
2423 if (tmp) {
2424 return;
2425 }
2426 udelay(1);
2427 }
2428}
2429
1109ca09 2430static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2431{
2432 u32 tmp;
0fcdb61e 2433 int r;
bcc1c2a1 2434
c9a1be96 2435 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2436 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2437 return -EINVAL;
2438 }
2439 r = radeon_gart_table_vram_pin(rdev);
2440 if (r)
2441 return r;
82568565 2442 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2443 /* Setup L2 cache */
2444 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2445 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2446 EFFECTIVE_L2_QUEUE_SIZE(7));
2447 WREG32(VM_L2_CNTL2, 0);
2448 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2449 /* Setup TLB control */
2450 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2451 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2452 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2453 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2454 if (rdev->flags & RADEON_IS_IGP) {
2455 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2456 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2457 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2458 } else {
2459 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2462 if ((rdev->family == CHIP_JUNIPER) ||
2463 (rdev->family == CHIP_CYPRESS) ||
2464 (rdev->family == CHIP_HEMLOCK) ||
2465 (rdev->family == CHIP_BARTS))
2466 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2467 }
bcc1c2a1
AD		 2468 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2472 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2473 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2474 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2475 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2476 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2477 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2478 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2479 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2480
0fcdb61e 2481 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2482 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2483 (unsigned)(rdev->mc.gtt_size >> 20),
2484 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2485 rdev->gart.ready = true;
2486 return 0;
2487}
2488
1109ca09 2489static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2490{
2491 u32 tmp;
bcc1c2a1
AD		 2492
2493 /* Disable all tables */
0fcdb61e
AD		 2494 WREG32(VM_CONTEXT0_CNTL, 0);
2495 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2496
2497 /* Setup L2 cache */
2498 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2499 EFFECTIVE_L2_QUEUE_SIZE(7));
2500 WREG32(VM_L2_CNTL2, 0);
2501 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2502 /* Setup TLB control */
2503 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2504 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2505 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2506 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2507 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2508 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2510 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2511 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2512}
2513
1109ca09 2514static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2515{
2516 evergreen_pcie_gart_disable(rdev);
2517 radeon_gart_table_vram_free(rdev);
2518 radeon_gart_fini(rdev);
2519}
2520
2521
1109ca09 2522static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2523{
2524 u32 tmp;
bcc1c2a1
AD		 2525
2526 /* Setup L2 cache */
2527 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2528 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2529 EFFECTIVE_L2_QUEUE_SIZE(7));
2530 WREG32(VM_L2_CNTL2, 0);
2531 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2532 /* Setup TLB control */
2533 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2534 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2535 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2536 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2537 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2538 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2539 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2540 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2541 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2542 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2543 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2544 WREG32(VM_CONTEXT0_CNTL, 0);
2545 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2546}
2547
b9952a8a 2548void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2549{
62444b74
AD		 2550 u32 crtc_enabled, tmp, frame_count, blackout;
2551 int i, j;
2552
5153550a
AD		 2553 if (!ASIC_IS_NODCE(rdev)) {
2554 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2555 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2556
5153550a
AD		 2557 /* disable VGA render */
2558 WREG32(VGA_RENDER_CONTROL, 0);
2559 }
62444b74
AD		 2560 /* blank the display controllers */
2561 for (i = 0; i < rdev->num_crtc; i++) {
2562 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2563 if (crtc_enabled) {
2564 save->crtc_enabled[i] = true;
2565 if (ASIC_IS_DCE6(rdev)) {
2566 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2567 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2568 radeon_wait_for_vblank(rdev, i);
abf1457b 2569 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2570 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2571 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2572 }
2573 } else {
2574 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2575 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2576 radeon_wait_for_vblank(rdev, i);
abf1457b 2577 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2578 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2579 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2580 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2581 }
2582 }
2583 /* wait for the next frame */
2584 frame_count = radeon_get_vblank_counter(rdev, i);
2585 for (j = 0; j < rdev->usec_timeout; j++) {
2586 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2587 break;
2588 udelay(1);
2589 }
abf1457b
AD		 2590
2591 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2592 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2593 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2594 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2595 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2596 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2597 save->crtc_enabled[i] = false;
2598 /* ***** */
804cc4a0
AD		 2599 } else {
2600 save->crtc_enabled[i] = false;
62444b74 2601 }
18007401 2602 }
bcc1c2a1 2603
62444b74
AD		 2604 radeon_mc_wait_for_idle(rdev);
2605
2606 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2607 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2608 /* Block CPU access */
2609 WREG32(BIF_FB_EN, 0);
2610 /* blackout the MC */
2611 blackout &= ~BLACKOUT_MODE_MASK;
2612 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2613 }
ed39fadd
AD		 2614 /* wait for the MC to settle */
2615 udelay(100);
968c0166
AD		 2616
2617 /* lock double buffered regs */
2618 for (i = 0; i < rdev->num_crtc; i++) {
2619 if (save->crtc_enabled[i]) {
2620 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2621 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2622 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2623 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2624 }
2625 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2626 if (!(tmp & 1)) {
2627 tmp |= 1;
2628 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2629 }
2630 }
2631 }
bcc1c2a1
AD		 2632}
2633
b9952a8a 2634void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2635{
62444b74
AD		 2636 u32 tmp, frame_count;
2637 int i, j;
18007401 2638
62444b74
AD		 2639 /* update crtc base addresses */
2640 for (i = 0; i < rdev->num_crtc; i++) {
2641 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2642 upper_32_bits(rdev->mc.vram_start));
62444b74 2643 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2644 upper_32_bits(rdev->mc.vram_start));
62444b74 2645 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2646 (u32)rdev->mc.vram_start);
62444b74 2647 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2648 (u32)rdev->mc.vram_start);
2649 }
5153550a
AD		 2650
2651 if (!ASIC_IS_NODCE(rdev)) {
2652 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2653 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2654 }
62444b74 2655
968c0166
AD		 2656 /* unlock regs and wait for update */
2657 for (i = 0; i < rdev->num_crtc; i++) {
2658 if (save->crtc_enabled[i]) {
2659 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2660 if ((tmp & 0x3) != 0) {
2661 tmp &= ~0x3;
2662 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2663 }
2664 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2665 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2666 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2667 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2668 }
2669 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2670 if (tmp & 1) {
2671 tmp &= ~1;
2672 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2673 }
2674 for (j = 0; j < rdev->usec_timeout; j++) {
2675 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2676 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2677 break;
2678 udelay(1);
2679 }
2680 }
2681 }
2682
62444b74
AD		 2683 /* unblackout the MC */
2684 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2685 tmp &= ~BLACKOUT_MODE_MASK;
2686 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2687 /* allow CPU access */
2688 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2689
2690 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2691 if (save->crtc_enabled[i]) {
62444b74
AD		 2692 if (ASIC_IS_DCE6(rdev)) {
2693 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2694 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2695 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2696 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2697 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2698 } else {
2699 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2700 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2701 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2702 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2703 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2704 }
2705 /* wait for the next frame */
2706 frame_count = radeon_get_vblank_counter(rdev, i);
2707 for (j = 0; j < rdev->usec_timeout; j++) {
2708 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2709 break;
2710 udelay(1);
2711 }
2712 }
2713 }
5153550a
AD		 2714 if (!ASIC_IS_NODCE(rdev)) {
2715 /* Unlock vga access */
2716 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2717 mdelay(1);
2718 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2719 }
bcc1c2a1
AD		 2720}
2721
755d819e 2722void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2723{
2724 struct evergreen_mc_save save;
2725 u32 tmp;
2726 int i, j;
2727
2728 /* Initialize HDP */
2729 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2730 WREG32((0x2c14 + j), 0x00000000);
2731 WREG32((0x2c18 + j), 0x00000000);
2732 WREG32((0x2c1c + j), 0x00000000);
2733 WREG32((0x2c20 + j), 0x00000000);
2734 WREG32((0x2c24 + j), 0x00000000);
2735 }
2736 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2737
2738 evergreen_mc_stop(rdev, &save);
2739 if (evergreen_mc_wait_for_idle(rdev)) {
2740 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2741 }
2742 /* Lockout access through VGA aperture*/
2743 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2744 /* Update configuration */
2745 if (rdev->flags & RADEON_IS_AGP) {
2746 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2747 /* VRAM before AGP */
2748 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2749 rdev->mc.vram_start >> 12);
2750 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2751 rdev->mc.gtt_end >> 12);
2752 } else {
2753 /* VRAM after AGP */
2754 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2755 rdev->mc.gtt_start >> 12);
2756 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2757 rdev->mc.vram_end >> 12);
2758 }
2759 } else {
2760 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2761 rdev->mc.vram_start >> 12);
2762 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2763 rdev->mc.vram_end >> 12);
2764 }
3b9832f6 2765 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2766 /* llano/ontario only */
2767 if ((rdev->family == CHIP_PALM) ||
2768 (rdev->family == CHIP_SUMO) ||
2769 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2770 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2771 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2772 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2773 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2774 }
bcc1c2a1
AD		 2775 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2776 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2777 WREG32(MC_VM_FB_LOCATION, tmp);
2778 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2779 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2780 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2781 if (rdev->flags & RADEON_IS_AGP) {
2782 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2783 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2784 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2785 } else {
2786 WREG32(MC_VM_AGP_BASE, 0);
2787 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2788 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2789 }
2790 if (evergreen_mc_wait_for_idle(rdev)) {
2791 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2792 }
2793 evergreen_mc_resume(rdev, &save);
2794 /* we need to own VRAM, so turn off the VGA renderer here
2795 * to stop it overwriting our objects */
2796 rv515_vga_render_disable(rdev);
2797}
2798
bcc1c2a1
AD		 2799/*
2800 * CP.
2801 */
12920591
AD		 2802void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2803{
876dc9f3 2804 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2805 u32 next_rptr;
7b1f2485 2806
12920591 2807 /* set to DX10/11 mode */
e32eb50d
CK		 2808 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2809 radeon_ring_write(ring, 1);
45df6803
CK		 2810
2811 if (ring->rptr_save_reg) {
89d35807 2812 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2813 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2814 radeon_ring_write(ring, ((ring->rptr_save_reg -
2815 PACKET3_SET_CONFIG_REG_START) >> 2));
2816 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2817 } else if (rdev->wb.enabled) {
2818 next_rptr = ring->wptr + 5 + 4;
2819 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2820 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2821 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2822 radeon_ring_write(ring, next_rptr);
2823 radeon_ring_write(ring, 0);
45df6803
CK		 2824 }
2825
e32eb50d
CK		 2826 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2827 radeon_ring_write(ring,
0f234f5f
AD		 2828#ifdef __BIG_ENDIAN
2829 (2 << 0) |
2830#endif
2831 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2832 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2833 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2834}
2835
bcc1c2a1
AD		 2836
2837static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2838{
fe251e2f
AD		 2839 const __be32 *fw_data;
2840 int i;
2841
2842 if (!rdev->me_fw || !rdev->pfp_fw)
2843 return -EINVAL;
bcc1c2a1 2844
fe251e2f 2845 r700_cp_stop(rdev);
0f234f5f
AD		 2846 WREG32(CP_RB_CNTL,
2847#ifdef __BIG_ENDIAN
2848 BUF_SWAP_32BIT |
2849#endif
2850 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2851
2852 fw_data = (const __be32 *)rdev->pfp_fw->data;
2853 WREG32(CP_PFP_UCODE_ADDR, 0);
2854 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2855 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2856 WREG32(CP_PFP_UCODE_ADDR, 0);
2857
2858 fw_data = (const __be32 *)rdev->me_fw->data;
2859 WREG32(CP_ME_RAM_WADDR, 0);
2860 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2861 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2862
2863 WREG32(CP_PFP_UCODE_ADDR, 0);
2864 WREG32(CP_ME_RAM_WADDR, 0);
2865 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2866 return 0;
2867}
2868
7e7b41d2
AD		 2869static int evergreen_cp_start(struct radeon_device *rdev)
2870{
e32eb50d 2871 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2872 int r, i;
7e7b41d2
AD		 2873 uint32_t cp_me;
2874
e32eb50d 2875 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2876 if (r) {
2877 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2878 return r;
2879 }
e32eb50d
CK		 2880 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2881 radeon_ring_write(ring, 0x1);
2882 radeon_ring_write(ring, 0x0);
2883 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2884 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2885 radeon_ring_write(ring, 0);
2886 radeon_ring_write(ring, 0);
2887 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2888
2889 cp_me = 0xff;
2890 WREG32(CP_ME_CNTL, cp_me);
2891
e32eb50d 2892 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2893 if (r) {
2894 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2895 return r;
2896 }
2281a378
AD		 2897
2898 /* setup clear context state */
e32eb50d
CK		 2899 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2900 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2901
2902 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2903 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2904
e32eb50d
CK		 2905 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2906 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2907
2908 /* set clear context state */
e32eb50d
CK		 2909 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2910 radeon_ring_write(ring, 0);
2281a378
AD		 2911
2912 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2913 radeon_ring_write(ring, 0xc0026f00);
2914 radeon_ring_write(ring, 0x00000000);
2915 radeon_ring_write(ring, 0x00000000);
2916 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2917
2918 /* Clear consts */
e32eb50d
CK		 2919 radeon_ring_write(ring, 0xc0036f00);
2920 radeon_ring_write(ring, 0x00000bc4);
2921 radeon_ring_write(ring, 0xffffffff);
2922 radeon_ring_write(ring, 0xffffffff);
2923 radeon_ring_write(ring, 0xffffffff);
2281a378 2924
e32eb50d
CK		 2925 radeon_ring_write(ring, 0xc0026900);
2926 radeon_ring_write(ring, 0x00000316);
2927 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2928 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2929
e32eb50d 2930 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2931
2932 return 0;
2933}
2934
1109ca09 2935static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2936{
e32eb50d 2937 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2938 u32 tmp;
2939 u32 rb_bufsz;
2940 int r;
2941
2942 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2943 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2944 SOFT_RESET_PA |
2945 SOFT_RESET_SH |
2946 SOFT_RESET_VGT |
a49a50da 2947 SOFT_RESET_SPI |
fe251e2f
AD		 2948 SOFT_RESET_SX));
2949 RREG32(GRBM_SOFT_RESET);
2950 mdelay(15);
2951 WREG32(GRBM_SOFT_RESET, 0);
2952 RREG32(GRBM_SOFT_RESET);
2953
2954 /* Set ring buffer size */
b72a8925
DV		 2955 rb_bufsz = order_base_2(ring->ring_size / 8);
2956 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2957#ifdef __BIG_ENDIAN
2958 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2959#endif
fe251e2f 2960 WREG32(CP_RB_CNTL, tmp);
15d3332f 2961 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2962 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2963
2964 /* Set the write pointer delay */
2965 WREG32(CP_RB_WPTR_DELAY, 0);
2966
2967 /* Initialize the ring buffer's read and write pointers */
2968 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2969 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2970 ring->wptr = 0;
2971 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2972
48fc7f7e 2973 /* set the wb address whether it's enabled or not */
0f234f5f 2974 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2975 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2976 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2977 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2978
2979 if (rdev->wb.enabled)
2980 WREG32(SCRATCH_UMSK, 0xff);
2981 else {
2982 tmp |= RB_NO_UPDATE;
2983 WREG32(SCRATCH_UMSK, 0);
2984 }
2985
fe251e2f
AD		 2986 mdelay(1);
2987 WREG32(CP_RB_CNTL, tmp);
2988
e32eb50d 2989 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2990 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2991
e32eb50d 2992 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2993
7e7b41d2 2994 evergreen_cp_start(rdev);
e32eb50d 2995 ring->ready = true;
f712812e 2996 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2997 if (r) {
e32eb50d 2998 ring->ready = false;
fe251e2f
AD		 2999 return r;
3000 }
3001 return 0;
3002}
bcc1c2a1
AD		 3003
3004/*
3005 * Core functions
3006 */
bcc1c2a1
AD		 3007static void evergreen_gpu_init(struct radeon_device *rdev)
3008{
416a2bd2 3009 u32 gb_addr_config;
32fcdbf4 3010 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3011 u32 sx_debug_1;
3012 u32 smx_dc_ctl0;
3013 u32 sq_config;
3014 u32 sq_lds_resource_mgmt;
3015 u32 sq_gpr_resource_mgmt_1;
3016 u32 sq_gpr_resource_mgmt_2;
3017 u32 sq_gpr_resource_mgmt_3;
3018 u32 sq_thread_resource_mgmt;
3019 u32 sq_thread_resource_mgmt_2;
3020 u32 sq_stack_resource_mgmt_1;
3021 u32 sq_stack_resource_mgmt_2;
3022 u32 sq_stack_resource_mgmt_3;
3023 u32 vgt_cache_invalidation;
f25a5c63 3024 u32 hdp_host_path_cntl, tmp;
416a2bd2 3025 u32 disabled_rb_mask;
32fcdbf4
AD		 3026 int i, j, num_shader_engines, ps_thread_count;
3027
3028 switch (rdev->family) {
3029 case CHIP_CYPRESS:
3030 case CHIP_HEMLOCK:
3031 rdev->config.evergreen.num_ses = 2;
3032 rdev->config.evergreen.max_pipes = 4;
3033 rdev->config.evergreen.max_tile_pipes = 8;
3034 rdev->config.evergreen.max_simds = 10;
3035 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3036 rdev->config.evergreen.max_gprs = 256;
3037 rdev->config.evergreen.max_threads = 248;
3038 rdev->config.evergreen.max_gs_threads = 32;
3039 rdev->config.evergreen.max_stack_entries = 512;
3040 rdev->config.evergreen.sx_num_of_sets = 4;
3041 rdev->config.evergreen.sx_max_export_size = 256;
3042 rdev->config.evergreen.sx_max_export_pos_size = 64;
3043 rdev->config.evergreen.sx_max_export_smx_size = 192;
3044 rdev->config.evergreen.max_hw_contexts = 8;
3045 rdev->config.evergreen.sq_num_cf_insts = 2;
3046
3047 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3048 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3049 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3050 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3051 break;
3052 case CHIP_JUNIPER:
3053 rdev->config.evergreen.num_ses = 1;
3054 rdev->config.evergreen.max_pipes = 4;
3055 rdev->config.evergreen.max_tile_pipes = 4;
3056 rdev->config.evergreen.max_simds = 10;
3057 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3058 rdev->config.evergreen.max_gprs = 256;
3059 rdev->config.evergreen.max_threads = 248;
3060 rdev->config.evergreen.max_gs_threads = 32;
3061 rdev->config.evergreen.max_stack_entries = 512;
3062 rdev->config.evergreen.sx_num_of_sets = 4;
3063 rdev->config.evergreen.sx_max_export_size = 256;
3064 rdev->config.evergreen.sx_max_export_pos_size = 64;
3065 rdev->config.evergreen.sx_max_export_smx_size = 192;
3066 rdev->config.evergreen.max_hw_contexts = 8;
3067 rdev->config.evergreen.sq_num_cf_insts = 2;
3068
3069 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3070 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3071 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3072 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3073 break;
3074 case CHIP_REDWOOD:
3075 rdev->config.evergreen.num_ses = 1;
3076 rdev->config.evergreen.max_pipes = 4;
3077 rdev->config.evergreen.max_tile_pipes = 4;
3078 rdev->config.evergreen.max_simds = 5;
3079 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3080 rdev->config.evergreen.max_gprs = 256;
3081 rdev->config.evergreen.max_threads = 248;
3082 rdev->config.evergreen.max_gs_threads = 32;
3083 rdev->config.evergreen.max_stack_entries = 256;
3084 rdev->config.evergreen.sx_num_of_sets = 4;
3085 rdev->config.evergreen.sx_max_export_size = 256;
3086 rdev->config.evergreen.sx_max_export_pos_size = 64;
3087 rdev->config.evergreen.sx_max_export_smx_size = 192;
3088 rdev->config.evergreen.max_hw_contexts = 8;
3089 rdev->config.evergreen.sq_num_cf_insts = 2;
3090
3091 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3092 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3093 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3094 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3095 break;
3096 case CHIP_CEDAR:
3097 default:
3098 rdev->config.evergreen.num_ses = 1;
3099 rdev->config.evergreen.max_pipes = 2;
3100 rdev->config.evergreen.max_tile_pipes = 2;
3101 rdev->config.evergreen.max_simds = 2;
3102 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3103 rdev->config.evergreen.max_gprs = 256;
3104 rdev->config.evergreen.max_threads = 192;
3105 rdev->config.evergreen.max_gs_threads = 16;
3106 rdev->config.evergreen.max_stack_entries = 256;
3107 rdev->config.evergreen.sx_num_of_sets = 4;
3108 rdev->config.evergreen.sx_max_export_size = 128;
3109 rdev->config.evergreen.sx_max_export_pos_size = 32;
3110 rdev->config.evergreen.sx_max_export_smx_size = 96;
3111 rdev->config.evergreen.max_hw_contexts = 4;
3112 rdev->config.evergreen.sq_num_cf_insts = 1;
3113
d5e455e4
AD		 3114 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3115 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3116 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3117 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3118 break;
3119 case CHIP_PALM:
3120 rdev->config.evergreen.num_ses = 1;
3121 rdev->config.evergreen.max_pipes = 2;
3122 rdev->config.evergreen.max_tile_pipes = 2;
3123 rdev->config.evergreen.max_simds = 2;
3124 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3125 rdev->config.evergreen.max_gprs = 256;
3126 rdev->config.evergreen.max_threads = 192;
3127 rdev->config.evergreen.max_gs_threads = 16;
3128 rdev->config.evergreen.max_stack_entries = 256;
3129 rdev->config.evergreen.sx_num_of_sets = 4;
3130 rdev->config.evergreen.sx_max_export_size = 128;
3131 rdev->config.evergreen.sx_max_export_pos_size = 32;
3132 rdev->config.evergreen.sx_max_export_smx_size = 96;
3133 rdev->config.evergreen.max_hw_contexts = 4;
3134 rdev->config.evergreen.sq_num_cf_insts = 1;
3135
d5c5a72f
AD		 3136 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3137 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3138 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3139 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3140 break;
3141 case CHIP_SUMO:
3142 rdev->config.evergreen.num_ses = 1;
3143 rdev->config.evergreen.max_pipes = 4;
bd25f078 3144 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3145 if (rdev->pdev->device == 0x9648)
3146 rdev->config.evergreen.max_simds = 3;
3147 else if ((rdev->pdev->device == 0x9647) ||
3148 (rdev->pdev->device == 0x964a))
3149 rdev->config.evergreen.max_simds = 4;
3150 else
3151 rdev->config.evergreen.max_simds = 5;
3152 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3153 rdev->config.evergreen.max_gprs = 256;
3154 rdev->config.evergreen.max_threads = 248;
3155 rdev->config.evergreen.max_gs_threads = 32;
3156 rdev->config.evergreen.max_stack_entries = 256;
3157 rdev->config.evergreen.sx_num_of_sets = 4;
3158 rdev->config.evergreen.sx_max_export_size = 256;
3159 rdev->config.evergreen.sx_max_export_pos_size = 64;
3160 rdev->config.evergreen.sx_max_export_smx_size = 192;
3161 rdev->config.evergreen.max_hw_contexts = 8;
3162 rdev->config.evergreen.sq_num_cf_insts = 2;
3163
3164 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3165 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3166 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3167 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3168 break;
3169 case CHIP_SUMO2:
3170 rdev->config.evergreen.num_ses = 1;
3171 rdev->config.evergreen.max_pipes = 4;
3172 rdev->config.evergreen.max_tile_pipes = 4;
3173 rdev->config.evergreen.max_simds = 2;
3174 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3175 rdev->config.evergreen.max_gprs = 256;
3176 rdev->config.evergreen.max_threads = 248;
3177 rdev->config.evergreen.max_gs_threads = 32;
3178 rdev->config.evergreen.max_stack_entries = 512;
3179 rdev->config.evergreen.sx_num_of_sets = 4;
3180 rdev->config.evergreen.sx_max_export_size = 256;
3181 rdev->config.evergreen.sx_max_export_pos_size = 64;
3182 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3183 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3184 rdev->config.evergreen.sq_num_cf_insts = 2;
3185
adb68fa2
AD		 3186 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3187 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3188 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3189 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3190 break;
3191 case CHIP_BARTS:
3192 rdev->config.evergreen.num_ses = 2;
3193 rdev->config.evergreen.max_pipes = 4;
3194 rdev->config.evergreen.max_tile_pipes = 8;
3195 rdev->config.evergreen.max_simds = 7;
3196 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3197 rdev->config.evergreen.max_gprs = 256;
3198 rdev->config.evergreen.max_threads = 248;
3199 rdev->config.evergreen.max_gs_threads = 32;
3200 rdev->config.evergreen.max_stack_entries = 512;
3201 rdev->config.evergreen.sx_num_of_sets = 4;
3202 rdev->config.evergreen.sx_max_export_size = 256;
3203 rdev->config.evergreen.sx_max_export_pos_size = 64;
3204 rdev->config.evergreen.sx_max_export_smx_size = 192;
3205 rdev->config.evergreen.max_hw_contexts = 8;
3206 rdev->config.evergreen.sq_num_cf_insts = 2;
3207
3208 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3209 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3210 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3211 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3212 break;
3213 case CHIP_TURKS:
3214 rdev->config.evergreen.num_ses = 1;
3215 rdev->config.evergreen.max_pipes = 4;
3216 rdev->config.evergreen.max_tile_pipes = 4;
3217 rdev->config.evergreen.max_simds = 6;
3218 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3219 rdev->config.evergreen.max_gprs = 256;
3220 rdev->config.evergreen.max_threads = 248;
3221 rdev->config.evergreen.max_gs_threads = 32;
3222 rdev->config.evergreen.max_stack_entries = 256;
3223 rdev->config.evergreen.sx_num_of_sets = 4;
3224 rdev->config.evergreen.sx_max_export_size = 256;
3225 rdev->config.evergreen.sx_max_export_pos_size = 64;
3226 rdev->config.evergreen.sx_max_export_smx_size = 192;
3227 rdev->config.evergreen.max_hw_contexts = 8;
3228 rdev->config.evergreen.sq_num_cf_insts = 2;
3229
3230 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3231 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3232 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3233 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3234 break;
3235 case CHIP_CAICOS:
3236 rdev->config.evergreen.num_ses = 1;
bd25f078 3237 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3238 rdev->config.evergreen.max_tile_pipes = 2;
3239 rdev->config.evergreen.max_simds = 2;
3240 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3241 rdev->config.evergreen.max_gprs = 256;
3242 rdev->config.evergreen.max_threads = 192;
3243 rdev->config.evergreen.max_gs_threads = 16;
3244 rdev->config.evergreen.max_stack_entries = 256;
3245 rdev->config.evergreen.sx_num_of_sets = 4;
3246 rdev->config.evergreen.sx_max_export_size = 128;
3247 rdev->config.evergreen.sx_max_export_pos_size = 32;
3248 rdev->config.evergreen.sx_max_export_smx_size = 96;
3249 rdev->config.evergreen.max_hw_contexts = 4;
3250 rdev->config.evergreen.sq_num_cf_insts = 1;
3251
32fcdbf4
AD		 3252 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3253 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3254 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3255 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3256 break;
3257 }
3258
3259 /* Initialize HDP */
3260 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3261 WREG32((0x2c14 + j), 0x00000000);
3262 WREG32((0x2c18 + j), 0x00000000);
3263 WREG32((0x2c1c + j), 0x00000000);
3264 WREG32((0x2c20 + j), 0x00000000);
3265 WREG32((0x2c24 + j), 0x00000000);
3266 }
3267
3268 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3269
d054ac16
AD		 3270 evergreen_fix_pci_max_read_req_size(rdev);
3271
32fcdbf4 3272 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3273 if ((rdev->family == CHIP_PALM) ||
3274 (rdev->family == CHIP_SUMO) ||
3275 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3276 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3277 else
3278 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3279
1aa52bd3
AD		 3280 /* setup tiling info dword. gb_addr_config is not adequate since it does
3281 * not have bank info, so create a custom tiling dword.
3282 * bits 3:0 num_pipes
3283 * bits 7:4 num_banks
3284 * bits 11:8 group_size
3285 * bits 15:12 row_size
3286 */
3287 rdev->config.evergreen.tile_config = 0;
3288 switch (rdev->config.evergreen.max_tile_pipes) {
3289 case 1:
3290 default:
3291 rdev->config.evergreen.tile_config |= (0 << 0);
3292 break;
3293 case 2:
3294 rdev->config.evergreen.tile_config |= (1 << 0);
3295 break;
3296 case 4:
3297 rdev->config.evergreen.tile_config |= (2 << 0);
3298 break;
3299 case 8:
3300 rdev->config.evergreen.tile_config |= (3 << 0);
3301 break;
3302 }
d698a34d 3303 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3304 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3305 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3306 else {
c8d15edc
AD		 3307 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3308 case 0: /* four banks */
29d65406 3309 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3310 break;
3311 case 1: /* eight banks */
3312 rdev->config.evergreen.tile_config |= 1 << 4;
3313 break;
3314 case 2: /* sixteen banks */
3315 default:
3316 rdev->config.evergreen.tile_config |= 2 << 4;
3317 break;
3318 }
29d65406 3319 }
416a2bd2 3320 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3321 rdev->config.evergreen.tile_config |=
3322 ((gb_addr_config & 0x30000000) >> 28) << 12;
3323
416a2bd2 3324 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3325
416a2bd2
AD		 3326 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3327 u32 efuse_straps_4;
3328 u32 efuse_straps_3;
32fcdbf4 3329
ff82bbc4
AD		 3330 efuse_straps_4 = RREG32_RCU(0x204);
3331 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3332 tmp = (((efuse_straps_4 & 0xf) << 4) |
3333 ((efuse_straps_3 & 0xf0000000) >> 28));
3334 } else {
3335 tmp = 0;
3336 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3337 u32 rb_disable_bitmap;
3338
3339 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3340 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3341 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3342 tmp <<= 4;
3343 tmp |= rb_disable_bitmap;
32fcdbf4 3344 }
416a2bd2
AD		 3345 }
3346 /* enabled rb are just the one not disabled :) */
3347 disabled_rb_mask = tmp;
cedb655a
AD		 3348 tmp = 0;
3349 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3350 tmp |= (1 << i);
3351 /* if all the backends are disabled, fix it up here */
3352 if ((disabled_rb_mask & tmp) == tmp) {
3353 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3354 disabled_rb_mask &= ~(1 << i);
3355 }
32fcdbf4 3356
416a2bd2
AD		 3357 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3358 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3359
416a2bd2
AD		 3360 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3361 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3362 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3363 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3364 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3365 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3366 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3367
f7eb9730
AD		 3368 if ((rdev->config.evergreen.max_backends == 1) &&
3369 (rdev->flags & RADEON_IS_IGP)) {
3370 if ((disabled_rb_mask & 3) == 1) {
3371 /* RB0 disabled, RB1 enabled */
3372 tmp = 0x11111111;
3373 } else {
3374 /* RB1 disabled, RB0 enabled */
3375 tmp = 0x00000000;
3376 }
3377 } else {
3378 tmp = gb_addr_config & NUM_PIPES_MASK;
3379 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3380 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3381 }
416a2bd2 3382 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3383
3384 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3385 WREG32(CGTS_TCC_DISABLE, 0);
3386 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3387 WREG32(CGTS_USER_TCC_DISABLE, 0);
3388
3389 /* set HW defaults for 3D engine */
3390 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3391 ROQ_IB2_START(0x2b)));
3392
3393 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3394
3395 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3396 SYNC_GRADIENT |
3397 SYNC_WALKER |
3398 SYNC_ALIGNER));
3399
3400 sx_debug_1 = RREG32(SX_DEBUG_1);
3401 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3402 WREG32(SX_DEBUG_1, sx_debug_1);
3403
3404
3405 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3406 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3407 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3408 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3409
b866d133
AD		 3410 if (rdev->family <= CHIP_SUMO2)
3411 WREG32(SMX_SAR_CTL0, 0x00010000);
3412
32fcdbf4
AD		 3413 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3414 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3415 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3416
3417 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3418 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3419 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3420
3421 WREG32(VGT_NUM_INSTANCES, 1);
3422 WREG32(SPI_CONFIG_CNTL, 0);
3423 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3424 WREG32(CP_PERFMON_CNTL, 0);
3425
3426 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3427 FETCH_FIFO_HIWATER(0x4) |
3428 DONE_FIFO_HIWATER(0xe0) |
3429 ALU_UPDATE_FIFO_HIWATER(0x8)));
3430
3431 sq_config = RREG32(SQ_CONFIG);
3432 sq_config &= ~(PS_PRIO(3) |
3433 VS_PRIO(3) |
3434 GS_PRIO(3) |
3435 ES_PRIO(3));
3436 sq_config |= (VC_ENABLE |
3437 EXPORT_SRC_C |
3438 PS_PRIO(0) |
3439 VS_PRIO(1) |
3440 GS_PRIO(2) |
3441 ES_PRIO(3));
3442
d5e455e4
AD		 3443 switch (rdev->family) {
3444 case CHIP_CEDAR:
3445 case CHIP_PALM:
d5c5a72f
AD		 3446 case CHIP_SUMO:
3447 case CHIP_SUMO2:
adb68fa2 3448 case CHIP_CAICOS:
32fcdbf4
AD		 3449 /* no vertex cache */
3450 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3451 break;
3452 default:
3453 break;
3454 }
32fcdbf4
AD		 3455
3456 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3457
3458 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3459 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3460 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3461 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3462 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3463 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3464 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3465
d5e455e4
AD		 3466 switch (rdev->family) {
3467 case CHIP_CEDAR:
3468 case CHIP_PALM:
d5c5a72f
AD		 3469 case CHIP_SUMO:
3470 case CHIP_SUMO2:
32fcdbf4 3471 ps_thread_count = 96;
d5e455e4
AD		 3472 break;
3473 default:
32fcdbf4 3474 ps_thread_count = 128;
d5e455e4
AD		 3475 break;
3476 }
32fcdbf4
AD		 3477
3478 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3479 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3480 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3481 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3482 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3483 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3484
3485 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3487 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3488 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3489 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3490 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3491
3492 WREG32(SQ_CONFIG, sq_config);
3493 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3494 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3495 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3496 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3497 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3498 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3499 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3500 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3501 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3502 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3503
3504 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3505 FORCE_EOV_MAX_REZ_CNT(255)));
3506
d5e455e4
AD		 3507 switch (rdev->family) {
3508 case CHIP_CEDAR:
3509 case CHIP_PALM:
d5c5a72f
AD		 3510 case CHIP_SUMO:
3511 case CHIP_SUMO2:
adb68fa2 3512 case CHIP_CAICOS:
32fcdbf4 3513 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3514 break;
3515 default:
32fcdbf4 3516 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3517 break;
3518 }
32fcdbf4
AD		 3519 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3520 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3521
3522 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3523 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3524 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3525
60a4a3e0
AD		 3526 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3527 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3528
32fcdbf4
AD		 3529 WREG32(CB_PERF_CTR0_SEL_0, 0);
3530 WREG32(CB_PERF_CTR0_SEL_1, 0);
3531 WREG32(CB_PERF_CTR1_SEL_0, 0);
3532 WREG32(CB_PERF_CTR1_SEL_1, 0);
3533 WREG32(CB_PERF_CTR2_SEL_0, 0);
3534 WREG32(CB_PERF_CTR2_SEL_1, 0);
3535 WREG32(CB_PERF_CTR3_SEL_0, 0);
3536 WREG32(CB_PERF_CTR3_SEL_1, 0);
3537
60a4a3e0
AD		 3538 /* clear render buffer base addresses */
3539 WREG32(CB_COLOR0_BASE, 0);
3540 WREG32(CB_COLOR1_BASE, 0);
3541 WREG32(CB_COLOR2_BASE, 0);
3542 WREG32(CB_COLOR3_BASE, 0);
3543 WREG32(CB_COLOR4_BASE, 0);
3544 WREG32(CB_COLOR5_BASE, 0);
3545 WREG32(CB_COLOR6_BASE, 0);
3546 WREG32(CB_COLOR7_BASE, 0);
3547 WREG32(CB_COLOR8_BASE, 0);
3548 WREG32(CB_COLOR9_BASE, 0);
3549 WREG32(CB_COLOR10_BASE, 0);
3550 WREG32(CB_COLOR11_BASE, 0);
3551
3552 /* set the shader const cache sizes to 0 */
3553 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3554 WREG32(i, 0);
3555 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3556 WREG32(i, 0);
3557
f25a5c63
AD		 3558 tmp = RREG32(HDP_MISC_CNTL);
3559 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3560 WREG32(HDP_MISC_CNTL, tmp);
3561
32fcdbf4
AD		 3562 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3563 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3564
3565 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3566
3567 udelay(50);
3568
bcc1c2a1
AD		 3569}
3570
3571int evergreen_mc_init(struct radeon_device *rdev)
3572{
bcc1c2a1
AD		 3573 u32 tmp;
3574 int chansize, numchan;
bcc1c2a1
AD		 3575
3576 /* Get VRAM informations */
3577 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3578 if ((rdev->family == CHIP_PALM) ||
3579 (rdev->family == CHIP_SUMO) ||
3580 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3581 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3582 else
3583 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3584 if (tmp & CHANSIZE_OVERRIDE) {
3585 chansize = 16;
3586 } else if (tmp & CHANSIZE_MASK) {
3587 chansize = 64;
3588 } else {
3589 chansize = 32;
3590 }
3591 tmp = RREG32(MC_SHARED_CHMAP);
3592 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3593 case 0:
3594 default:
3595 numchan = 1;
3596 break;
3597 case 1:
3598 numchan = 2;
3599 break;
3600 case 2:
3601 numchan = 4;
3602 break;
3603 case 3:
3604 numchan = 8;
3605 break;
3606 }
3607 rdev->mc.vram_width = numchan * chansize;
3608 /* Could aper size report 0 ? */
01d73a69
JC		 3609 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3610 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3611 /* Setup GPU memory space */
05b3ef69
AD		 3612 if ((rdev->family == CHIP_PALM) ||
3613 (rdev->family == CHIP_SUMO) ||
3614 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3615 /* size in bytes on fusion */
3616 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3617 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3618 } else {
05b3ef69 3619 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3620 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3621 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3622 }
51e5fcd3 3623 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3624 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3625 radeon_update_bandwidth_info(rdev);
3626
bcc1c2a1
AD		 3627 return 0;
3628}
d594e46a 3629
187e3593 3630void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3631{
64c56e8c 3632 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3633 RREG32(GRBM_STATUS));
64c56e8c 3634 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3635 RREG32(GRBM_STATUS_SE0));
64c56e8c 3636 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3637 RREG32(GRBM_STATUS_SE1));
64c56e8c 3638 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3639 RREG32(SRBM_STATUS));
a65a4369
AD		 3640 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3641 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3642 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3643 RREG32(CP_STALLED_STAT1));
3644 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3645 RREG32(CP_STALLED_STAT2));
3646 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3647 RREG32(CP_BUSY_STAT));
3648 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3649 RREG32(CP_STAT));
eaaa6983
JG		 3650 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3651 RREG32(DMA_STATUS_REG));
168757ea
AD		 3652 if (rdev->family >= CHIP_CAYMAN) {
3653 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3654 RREG32(DMA_STATUS_REG + 0x800));
3655 }
0ecebb9e
AD		 3656}
3657
168757ea 3658bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3659{
a65a4369
AD		 3660 u32 crtc_hung = 0;
3661 u32 crtc_status[6];
3662 u32 i, j, tmp;
3663
3664 for (i = 0; i < rdev->num_crtc; i++) {
3665 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3666 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3667 crtc_hung |= (1 << i);
3668 }
3669 }
3670
3671 for (j = 0; j < 10; j++) {
3672 for (i = 0; i < rdev->num_crtc; i++) {
3673 if (crtc_hung & (1 << i)) {
3674 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3675 if (tmp != crtc_status[i])
3676 crtc_hung &= ~(1 << i);
3677 }
3678 }
3679 if (crtc_hung == 0)
3680 return false;
3681 udelay(100);
3682 }
3683
3684 return true;
3685}
3686
2483b4ea 3687u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3688{
3689 u32 reset_mask = 0;
b7630473 3690 u32 tmp;
0ecebb9e 3691
a65a4369
AD		 3692 /* GRBM_STATUS */
3693 tmp = RREG32(GRBM_STATUS);
3694 if (tmp & (PA_BUSY | SC_BUSY |
3695 SH_BUSY | SX_BUSY |
3696 TA_BUSY | VGT_BUSY |
3697 DB_BUSY | CB_BUSY |
3698 SPI_BUSY | VGT_BUSY_NO_DMA))
3699 reset_mask |= RADEON_RESET_GFX;
3700
3701 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3702 CP_BUSY | CP_COHERENCY_BUSY))
3703 reset_mask |= RADEON_RESET_CP;
3704
3705 if (tmp & GRBM_EE_BUSY)
3706 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3707
a65a4369
AD		 3708 /* DMA_STATUS_REG */
3709 tmp = RREG32(DMA_STATUS_REG);
3710 if (!(tmp & DMA_IDLE))
3711 reset_mask |= RADEON_RESET_DMA;
3712
3713 /* SRBM_STATUS2 */
3714 tmp = RREG32(SRBM_STATUS2);
3715 if (tmp & DMA_BUSY)
3716 reset_mask |= RADEON_RESET_DMA;
3717
3718 /* SRBM_STATUS */
3719 tmp = RREG32(SRBM_STATUS);
3720 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3721 reset_mask |= RADEON_RESET_RLC;
3722
3723 if (tmp & IH_BUSY)
3724 reset_mask |= RADEON_RESET_IH;
3725
3726 if (tmp & SEM_BUSY)
3727 reset_mask |= RADEON_RESET_SEM;
3728
3729 if (tmp & GRBM_RQ_PENDING)
3730 reset_mask |= RADEON_RESET_GRBM;
3731
3732 if (tmp & VMC_BUSY)
3733 reset_mask |= RADEON_RESET_VMC;
3734
3735 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3736 MCC_BUSY | MCD_BUSY))
3737 reset_mask |= RADEON_RESET_MC;
3738
3739 if (evergreen_is_display_hung(rdev))
3740 reset_mask |= RADEON_RESET_DISPLAY;
3741
3742 /* VM_L2_STATUS */
3743 tmp = RREG32(VM_L2_STATUS);
3744 if (tmp & L2_BUSY)
3745 reset_mask |= RADEON_RESET_VMC;
3746
d808fc88
AD		 3747 /* Skip MC reset as it's mostly likely not hung, just busy */
3748 if (reset_mask & RADEON_RESET_MC) {
3749 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3750 reset_mask &= ~RADEON_RESET_MC;
3751 }
3752
a65a4369
AD		 3753 return reset_mask;
3754}
3755
3756static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3757{
3758 struct evergreen_mc_save save;
3759 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3760 u32 tmp;
19fc42ed 3761
0ecebb9e 3762 if (reset_mask == 0)
a65a4369 3763 return;
0ecebb9e
AD		 3764
3765 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3766
b7630473
AD		 3767 evergreen_print_gpu_status_regs(rdev);
3768
b7630473
AD		 3769 /* Disable CP parsing/prefetching */
3770 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3771
3772 if (reset_mask & RADEON_RESET_DMA) {
3773 /* Disable DMA */
3774 tmp = RREG32(DMA_RB_CNTL);
3775 tmp &= ~DMA_RB_ENABLE;
3776 WREG32(DMA_RB_CNTL, tmp);
3777 }
3778
b21b6e7a
AD		 3779 udelay(50);
3780
3781 evergreen_mc_stop(rdev, &save);
3782 if (evergreen_mc_wait_for_idle(rdev)) {
3783 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3784 }
3785
b7630473
AD		 3786 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3787 grbm_soft_reset |= SOFT_RESET_DB |
3788 SOFT_RESET_CB |
3789 SOFT_RESET_PA |
3790 SOFT_RESET_SC |
3791 SOFT_RESET_SPI |
3792 SOFT_RESET_SX |
3793 SOFT_RESET_SH |
3794 SOFT_RESET_TC |
3795 SOFT_RESET_TA |
3796 SOFT_RESET_VC |
3797 SOFT_RESET_VGT;
3798 }
3799
3800 if (reset_mask & RADEON_RESET_CP) {
3801 grbm_soft_reset |= SOFT_RESET_CP |
3802 SOFT_RESET_VGT;
3803
3804 srbm_soft_reset |= SOFT_RESET_GRBM;
3805 }
0ecebb9e
AD		 3806
3807 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3808 srbm_soft_reset |= SOFT_RESET_DMA;
3809
a65a4369
AD		 3810 if (reset_mask & RADEON_RESET_DISPLAY)
3811 srbm_soft_reset |= SOFT_RESET_DC;
3812
3813 if (reset_mask & RADEON_RESET_RLC)
3814 srbm_soft_reset |= SOFT_RESET_RLC;
3815
3816 if (reset_mask & RADEON_RESET_SEM)
3817 srbm_soft_reset |= SOFT_RESET_SEM;
3818
3819 if (reset_mask & RADEON_RESET_IH)
3820 srbm_soft_reset |= SOFT_RESET_IH;
3821
3822 if (reset_mask & RADEON_RESET_GRBM)
3823 srbm_soft_reset |= SOFT_RESET_GRBM;
3824
3825 if (reset_mask & RADEON_RESET_VMC)
3826 srbm_soft_reset |= SOFT_RESET_VMC;
3827
24178ec4
AD		 3828 if (!(rdev->flags & RADEON_IS_IGP)) {
3829 if (reset_mask & RADEON_RESET_MC)
3830 srbm_soft_reset |= SOFT_RESET_MC;
3831 }
a65a4369 3832
b7630473
AD		 3833 if (grbm_soft_reset) {
3834 tmp = RREG32(GRBM_SOFT_RESET);
3835 tmp |= grbm_soft_reset;
3836 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3837 WREG32(GRBM_SOFT_RESET, tmp);
3838 tmp = RREG32(GRBM_SOFT_RESET);
3839
3840 udelay(50);
3841
3842 tmp &= ~grbm_soft_reset;
3843 WREG32(GRBM_SOFT_RESET, tmp);
3844 tmp = RREG32(GRBM_SOFT_RESET);
3845 }
3846
3847 if (srbm_soft_reset) {
3848 tmp = RREG32(SRBM_SOFT_RESET);
3849 tmp |= srbm_soft_reset;
3850 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3851 WREG32(SRBM_SOFT_RESET, tmp);
3852 tmp = RREG32(SRBM_SOFT_RESET);
3853
3854 udelay(50);
3855
3856 tmp &= ~srbm_soft_reset;
3857 WREG32(SRBM_SOFT_RESET, tmp);
3858 tmp = RREG32(SRBM_SOFT_RESET);
3859 }
0ecebb9e
AD		 3860
3861 /* Wait a little for things to settle down */
3862 udelay(50);
3863
747943ea 3864 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3865 udelay(50);
3866
b7630473 3867 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3868}
3869
a2d07b74 3870int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3871{
a65a4369
AD		 3872 u32 reset_mask;
3873
3874 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3875
3876 if (reset_mask)
3877 r600_set_bios_scratch_engine_hung(rdev, true);
3878
3879 evergreen_gpu_soft_reset(rdev, reset_mask);
3880
3881 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3882
3883 if (!reset_mask)
3884 r600_set_bios_scratch_engine_hung(rdev, false);
3885
3886 return 0;
747943ea
AD		 3887}
3888
123bc183
AD		 3889/**
3890 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3891 *
3892 * @rdev: radeon_device pointer
3893 * @ring: radeon_ring structure holding ring information
3894 *
3895 * Check if the GFX engine is locked up.
3896 * Returns true if the engine appears to be locked up, false if not.
3897 */
3898bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3899{
3900 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3901
3902 if (!(reset_mask & (RADEON_RESET_GFX |
3903 RADEON_RESET_COMPUTE |
3904 RADEON_RESET_CP))) {
3905 radeon_ring_lockup_update(ring);
3906 return false;
3907 }
3908 /* force CP activities */
3909 radeon_ring_force_activity(rdev, ring);
3910 return radeon_ring_test_lockup(rdev, ring);
3911}
3912
2948f5e6
AD		 3913/*
3914 * RLC
3915 */
3916#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3917#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3918
3919void sumo_rlc_fini(struct radeon_device *rdev)
3920{
3921 int r;
3922
3923 /* save restore block */
3924 if (rdev->rlc.save_restore_obj) {
3925 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3926 if (unlikely(r != 0))
3927 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3928 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3929 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3930
3931 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3932 rdev->rlc.save_restore_obj = NULL;
3933 }
3934
3935 /* clear state block */
3936 if (rdev->rlc.clear_state_obj) {
3937 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3938 if (unlikely(r != 0))
3939 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3940 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3941 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3942
3943 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3944 rdev->rlc.clear_state_obj = NULL;
3945 }
22c775ce
AD		 3946
3947 /* clear state block */
3948 if (rdev->rlc.cp_table_obj) {
3949 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3950 if (unlikely(r != 0))
3951 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3952 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3953 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3954
3955 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3956 rdev->rlc.cp_table_obj = NULL;
3957 }
2948f5e6
AD		 3958}
3959
22c775ce
AD		 3960#define CP_ME_TABLE_SIZE 96
3961
2948f5e6
AD		 3962int sumo_rlc_init(struct radeon_device *rdev)
3963{
1fd11777 3964 const u32 *src_ptr;
2948f5e6
AD		 3965 volatile u32 *dst_ptr;
3966 u32 dws, data, i, j, k, reg_num;
59a82d0e 3967 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 3968 u64 reg_list_mc_addr;
1fd11777 3969 const struct cs_section_def *cs_data;
2948f5e6
AD		 3970 int r;
3971
3972 src_ptr = rdev->rlc.reg_list;
3973 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 3974 if (rdev->family >= CHIP_BONAIRE) {
3975 dws += (5 * 16) + 48 + 48 + 64;
3976 }
2948f5e6
AD		 3977 cs_data = rdev->rlc.cs_data;
3978
10b7ca7e
AD		 3979 if (src_ptr) {
3980 /* save restore block */
3981 if (rdev->rlc.save_restore_obj == NULL) {
3982 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3983 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3984 if (r) {
3985 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3986 return r;
3987 }
3988 }
3989
3990 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3991 if (unlikely(r != 0)) {
3992 sumo_rlc_fini(rdev);
3993 return r;
3994 }
3995 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3996 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 3997 if (r) {
10b7ca7e
AD		 3998 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3999 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4000 sumo_rlc_fini(rdev);
2948f5e6
AD		 4001 return r;
4002 }
2948f5e6 4003
10b7ca7e
AD		 4004 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4005 if (r) {
4006 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4007 sumo_rlc_fini(rdev);
4008 return r;
4009 }
4010 /* write the sr buffer */
4011 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4012 if (rdev->family >= CHIP_TAHITI) {
4013 /* SI */
59a82d0e 4014 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4015 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4016 } else {
4017 /* ON/LN/TN */
4018 /* format:
4019 * dw0: (reg2 << 16) | reg1
4020 * dw1: reg1 save space
4021 * dw2: reg2 save space
4022 */
4023 for (i = 0; i < dws; i++) {
4024 data = src_ptr[i] >> 2;
4025 i++;
4026 if (i < dws)
4027 data |= (src_ptr[i] >> 2) << 16;
4028 j = (((i - 1) * 3) / 2);
6ba81e53 4029 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4030 }
4031 j = ((i * 3) / 2);
6ba81e53 4032 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4033 }
10b7ca7e 4034 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4035 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4036 }
2948f5e6 4037
10b7ca7e
AD		 4038 if (cs_data) {
4039 /* clear state block */
a0f38609
AD		 4040 if (rdev->family >= CHIP_BONAIRE) {
4041 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4042 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4043 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4044 dws = rdev->rlc.clear_state_size + (256 / 4);
4045 } else {
4046 reg_list_num = 0;
4047 dws = 0;
4048 for (i = 0; cs_data[i].section != NULL; i++) {
4049 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4050 reg_list_num++;
4051 dws += cs_data[i].section[j].reg_count;
4052 }
10b7ca7e 4053 }
59a82d0e
AD		 4054 reg_list_blk_index = (3 * reg_list_num + 2);
4055 dws += reg_list_blk_index;
4056 rdev->rlc.clear_state_size = dws;
2948f5e6 4057 }
2948f5e6 4058
10b7ca7e 4059 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4060 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
10b7ca7e
AD		 4061 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4062 if (r) {
4063 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4064 sumo_rlc_fini(rdev);
4065 return r;
4066 }
4067 }
4068 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4069 if (unlikely(r != 0)) {
4070 sumo_rlc_fini(rdev);
4071 return r;
4072 }
4073 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4074 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4075 if (r) {
10b7ca7e
AD		 4076 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4077 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4078 sumo_rlc_fini(rdev);
4079 return r;
4080 }
2948f5e6 4081
10b7ca7e
AD		 4082 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4083 if (r) {
4084 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4085 sumo_rlc_fini(rdev);
4086 return r;
4087 }
4088 /* set up the cs buffer */
4089 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4090 if (rdev->family >= CHIP_BONAIRE) {
4091 cik_get_csb_buffer(rdev, dst_ptr);
4092 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4093 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4094 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4095 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4096 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4097 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4098 } else {
4099 reg_list_hdr_blk_index = 0;
4100 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4101 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4102 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4103 reg_list_hdr_blk_index++;
59a82d0e
AD		 4104 for (i = 0; cs_data[i].section != NULL; i++) {
4105 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4106 reg_num = cs_data[i].section[j].reg_count;
4107 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4108 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4109 reg_list_hdr_blk_index++;
4110
4111 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4112 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4113 reg_list_hdr_blk_index++;
4114
4115 data = 0x08000000 | (reg_num * 4);
6ba81e53 4116 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4117 reg_list_hdr_blk_index++;
4118
4119 for (k = 0; k < reg_num; k++) {
4120 data = cs_data[i].section[j].extent[k];
6ba81e53 4121 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4122 }
4123 reg_list_mc_addr += reg_num * 4;
4124 reg_list_blk_index += reg_num;
10b7ca7e 4125 }
2948f5e6 4126 }
6ba81e53 4127 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4128 }
10b7ca7e
AD		 4129 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4130 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4131 }
2948f5e6 4132
22c775ce
AD		 4133 if (rdev->rlc.cp_table_size) {
4134 if (rdev->rlc.cp_table_obj == NULL) {
4135 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4136 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4137 if (r) {
4138 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4139 sumo_rlc_fini(rdev);
4140 return r;
4141 }
4142 }
4143
4144 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4145 if (unlikely(r != 0)) {
4146 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4147 sumo_rlc_fini(rdev);
4148 return r;
4149 }
4150 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4151 &rdev->rlc.cp_table_gpu_addr);
4152 if (r) {
4153 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4154 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4155 sumo_rlc_fini(rdev);
4156 return r;
4157 }
4158 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4159 if (r) {
4160 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4161 sumo_rlc_fini(rdev);
4162 return r;
4163 }
4164
4165 cik_init_cp_pg_table(rdev);
4166
4167 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4168 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4169
4170 }
2948f5e6
AD		 4171
4172 return 0;
4173}
4174
4175static void evergreen_rlc_start(struct radeon_device *rdev)
4176{
8ba10463
AD		 4177 u32 mask = RLC_ENABLE;
4178
4179 if (rdev->flags & RADEON_IS_IGP) {
4180 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4181 }
4182
4183 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4184}
4185
4186int evergreen_rlc_resume(struct radeon_device *rdev)
4187{
4188 u32 i;
4189 const __be32 *fw_data;
4190
4191 if (!rdev->rlc_fw)
4192 return -EINVAL;
4193
4194 r600_rlc_stop(rdev);
4195
4196 WREG32(RLC_HB_CNTL, 0);
4197
4198 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4199 if (rdev->family == CHIP_ARUBA) {
4200 u32 always_on_bitmap =
4201 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4202 /* find out the number of active simds */
4203 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4204 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4205 tmp = hweight32(~tmp);
4206 if (tmp == rdev->config.cayman.max_simds_per_se) {
4207 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4208 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4209 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4210 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4211 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4212 }
4213 } else {
4214 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4215 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4216 }
2948f5e6
AD		 4217 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4218 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4219 } else {
4220 WREG32(RLC_HB_BASE, 0);
4221 WREG32(RLC_HB_RPTR, 0);
4222 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4223 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4224 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4225 }
2948f5e6
AD		 4226 WREG32(RLC_MC_CNTL, 0);
4227 WREG32(RLC_UCODE_CNTL, 0);
4228
4229 fw_data = (const __be32 *)rdev->rlc_fw->data;
4230 if (rdev->family >= CHIP_ARUBA) {
4231 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4232 WREG32(RLC_UCODE_ADDR, i);
4233 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4234 }
4235 } else if (rdev->family >= CHIP_CAYMAN) {
4236 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4237 WREG32(RLC_UCODE_ADDR, i);
4238 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4239 }
4240 } else {
4241 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4242 WREG32(RLC_UCODE_ADDR, i);
4243 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4244 }
4245 }
4246 WREG32(RLC_UCODE_ADDR, 0);
4247
4248 evergreen_rlc_start(rdev);
4249
4250 return 0;
4251}
4252
45f9a39b
AD		 4253/* Interrupts */
4254
4255u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4256{
46437057 4257 if (crtc >= rdev->num_crtc)
45f9a39b 4258 return 0;
46437057
AD		 4259 else
4260 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4261}
4262
4263void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4264{
4265 u32 tmp;
4266
1b37078b
AD		 4267 if (rdev->family >= CHIP_CAYMAN) {
4268 cayman_cp_int_cntl_setup(rdev, 0,
4269 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4270 cayman_cp_int_cntl_setup(rdev, 1, 0);
4271 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4272 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4273 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4274 } else
4275 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4276 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4277 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4278 WREG32(GRBM_INT_CNTL, 0);
4279 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4280 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4281 if (rdev->num_crtc >= 4) {
18007401
AD		 4282 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4283 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4284 }
4285 if (rdev->num_crtc >= 6) {
18007401
AD		 4286 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4287 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4288 }
45f9a39b
AD		 4289
4290 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4291 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4292 if (rdev->num_crtc >= 4) {
18007401
AD		 4293 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4294 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4295 }
4296 if (rdev->num_crtc >= 6) {
18007401
AD		 4297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4298 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4299 }
45f9a39b 4300
05b3ef69
AD		 4301 /* only one DAC on DCE6 */
4302 if (!ASIC_IS_DCE6(rdev))
4303 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4304 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4305
4306 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4307 WREG32(DC_HPD1_INT_CONTROL, tmp);
4308 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4309 WREG32(DC_HPD2_INT_CONTROL, tmp);
4310 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4311 WREG32(DC_HPD3_INT_CONTROL, tmp);
4312 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4313 WREG32(DC_HPD4_INT_CONTROL, tmp);
4314 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4315 WREG32(DC_HPD5_INT_CONTROL, tmp);
4316 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4317 WREG32(DC_HPD6_INT_CONTROL, tmp);
4318
4319}
4320
4321int evergreen_irq_set(struct radeon_device *rdev)
4322{
4323 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4324 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4325 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4326 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4327 u32 grbm_int_cntl = 0;
6f34be50 4328 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4329 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4330 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4331 u32 thermal_int = 0;
45f9a39b
AD		 4332
4333 if (!rdev->irq.installed) {
fce7d61b 4334 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4335 return -EINVAL;
4336 }
4337 /* don't enable anything if the ih is disabled */
4338 if (!rdev->ih.enabled) {
4339 r600_disable_interrupts(rdev);
4340 /* force the active interrupt state to all disabled */
4341 evergreen_disable_interrupt_state(rdev);
4342 return 0;
4343 }
4344
4345 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4346 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4347 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4348 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4349 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4350 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4351 if (rdev->family == CHIP_ARUBA)
4352 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4353 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4354 else
4355 thermal_int = RREG32(CG_THERMAL_INT) &
4356 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4357
f122c610
AD		 4358 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4359 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4360 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4361 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4362 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4363 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4364
233d1ad5
AD		 4365 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4366
1b37078b
AD		 4367 if (rdev->family >= CHIP_CAYMAN) {
4368 /* enable CP interrupts on all rings */
736fc37f 4369 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4370 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4371 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4372 }
736fc37f 4373 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4374 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4375 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4376 }
736fc37f 4377 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4378 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4379 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4380 }
4381 } else {
736fc37f 4382 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4383 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4384 cp_int_cntl |= RB_INT_ENABLE;
4385 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4386 }
45f9a39b 4387 }
1b37078b 4388
233d1ad5
AD		 4389 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4390 DRM_DEBUG("r600_irq_set: sw int dma\n");
4391 dma_cntl |= TRAP_ENABLE;
4392 }
4393
f60cbd11
AD		 4394 if (rdev->family >= CHIP_CAYMAN) {
4395 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4396 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4397 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4398 dma_cntl1 |= TRAP_ENABLE;
4399 }
4400 }
4401
dc50ba7f
AD		 4402 if (rdev->irq.dpm_thermal) {
4403 DRM_DEBUG("dpm thermal\n");
4404 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4405 }
4406
6f34be50 4407 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4408 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4409 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4410 crtc1 |= VBLANK_INT_MASK;
4411 }
6f34be50 4412 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4413 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4414 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4415 crtc2 |= VBLANK_INT_MASK;
4416 }
6f34be50 4417 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4418 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4419 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4420 crtc3 |= VBLANK_INT_MASK;
4421 }
6f34be50 4422 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4423 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4424 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4425 crtc4 |= VBLANK_INT_MASK;
4426 }
6f34be50 4427 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4428 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4429 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4430 crtc5 |= VBLANK_INT_MASK;
4431 }
6f34be50 4432 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4433 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4434 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4435 crtc6 |= VBLANK_INT_MASK;
4436 }
4437 if (rdev->irq.hpd[0]) {
4438 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4439 hpd1 |= DC_HPDx_INT_EN;
4440 }
4441 if (rdev->irq.hpd[1]) {
4442 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4443 hpd2 |= DC_HPDx_INT_EN;
4444 }
4445 if (rdev->irq.hpd[2]) {
4446 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4447 hpd3 |= DC_HPDx_INT_EN;
4448 }
4449 if (rdev->irq.hpd[3]) {
4450 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4451 hpd4 |= DC_HPDx_INT_EN;
4452 }
4453 if (rdev->irq.hpd[4]) {
4454 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4455 hpd5 |= DC_HPDx_INT_EN;
4456 }
4457 if (rdev->irq.hpd[5]) {
4458 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4459 hpd6 |= DC_HPDx_INT_EN;
4460 }
f122c610
AD		 4461 if (rdev->irq.afmt[0]) {
4462 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4463 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4464 }
4465 if (rdev->irq.afmt[1]) {
4466 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4467 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4468 }
4469 if (rdev->irq.afmt[2]) {
4470 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4471 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4472 }
4473 if (rdev->irq.afmt[3]) {
4474 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4475 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4476 }
4477 if (rdev->irq.afmt[4]) {
4478 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4479 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4480 }
4481 if (rdev->irq.afmt[5]) {
4482 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4483 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4484 }
45f9a39b 4485
1b37078b
AD		 4486 if (rdev->family >= CHIP_CAYMAN) {
4487 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4488 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4489 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4490 } else
4491 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4492
4493 WREG32(DMA_CNTL, dma_cntl);
4494
f60cbd11
AD		 4495 if (rdev->family >= CHIP_CAYMAN)
4496 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4497
2031f77c 4498 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4499
4500 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4501 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4502 if (rdev->num_crtc >= 4) {
18007401
AD		 4503 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4504 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4505 }
4506 if (rdev->num_crtc >= 6) {
18007401
AD		 4507 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4508 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4509 }
45f9a39b 4510
6f34be50
AD		 4511 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4512 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4513 if (rdev->num_crtc >= 4) {
4514 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4515 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4516 }
4517 if (rdev->num_crtc >= 6) {
4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4520 }
6f34be50 4521
45f9a39b
AD		 4522 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4523 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4524 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4525 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4526 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4527 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4528 if (rdev->family == CHIP_ARUBA)
4529 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4530 else
4531 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4532
f122c610
AD		 4533 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4534 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4535 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4536 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4537 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4538 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4539
bcc1c2a1
AD		 4540 return 0;
4541}
4542
cbdd4501 4543static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4544{
4545 u32 tmp;
4546
6f34be50
AD		 4547 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4548 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4549 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4550 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4551 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4552 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4553 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4554 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4555 if (rdev->num_crtc >= 4) {
4556 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4557 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4558 }
4559 if (rdev->num_crtc >= 6) {
4560 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4561 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4562 }
6f34be50 4563
f122c610
AD		 4564 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4565 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4566 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4567 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4568 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4569 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4570
6f34be50
AD		 4571 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4572 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4573 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4574 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4575 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4576 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4577 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4578 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4579 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4580 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4581 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4582 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4583
b7eff394
AD		 4584 if (rdev->num_crtc >= 4) {
4585 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4586 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4587 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4588 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4589 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4590 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4591 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4592 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4593 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4594 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4595 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4596 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4597 }
4598
4599 if (rdev->num_crtc >= 6) {
4600 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4601 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4602 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4604 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4605 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4606 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4607 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4608 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4609 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4610 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4611 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4612 }
45f9a39b 4613
6f34be50 4614 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4615 tmp = RREG32(DC_HPD1_INT_CONTROL);
4616 tmp |= DC_HPDx_INT_ACK;
4617 WREG32(DC_HPD1_INT_CONTROL, tmp);
4618 }
6f34be50 4619 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4620 tmp = RREG32(DC_HPD2_INT_CONTROL);
4621 tmp |= DC_HPDx_INT_ACK;
4622 WREG32(DC_HPD2_INT_CONTROL, tmp);
4623 }
6f34be50 4624 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4625 tmp = RREG32(DC_HPD3_INT_CONTROL);
4626 tmp |= DC_HPDx_INT_ACK;
4627 WREG32(DC_HPD3_INT_CONTROL, tmp);
4628 }
6f34be50 4629 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4630 tmp = RREG32(DC_HPD4_INT_CONTROL);
4631 tmp |= DC_HPDx_INT_ACK;
4632 WREG32(DC_HPD4_INT_CONTROL, tmp);
4633 }
6f34be50 4634 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4635 tmp = RREG32(DC_HPD5_INT_CONTROL);
4636 tmp |= DC_HPDx_INT_ACK;
4637 WREG32(DC_HPD5_INT_CONTROL, tmp);
4638 }
6f34be50 4639 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4640 tmp = RREG32(DC_HPD5_INT_CONTROL);
4641 tmp |= DC_HPDx_INT_ACK;
4642 WREG32(DC_HPD6_INT_CONTROL, tmp);
4643 }
f122c610
AD		 4644 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4645 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4646 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4647 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4648 }
4649 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4650 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4651 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4652 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4653 }
4654 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4655 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4656 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4657 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4658 }
4659 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4660 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4661 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4662 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4663 }
4664 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4665 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4666 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4667 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4668 }
4669 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4670 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4671 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4672 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4673 }
45f9a39b
AD		 4674}
4675
1109ca09 4676static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4677{
45f9a39b
AD		 4678 r600_disable_interrupts(rdev);
4679 /* Wait and acknowledge irq */
4680 mdelay(1);
6f34be50 4681 evergreen_irq_ack(rdev);
45f9a39b
AD		 4682 evergreen_disable_interrupt_state(rdev);
4683}
4684
755d819e 4685void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4686{
4687 evergreen_irq_disable(rdev);
4688 r600_rlc_stop(rdev);
4689}
4690
cbdd4501 4691static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4692{
4693 u32 wptr, tmp;
4694
724c80e1 4695 if (rdev->wb.enabled)
204ae24d 4696 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4697 else
4698 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4699
4700 if (wptr & RB_OVERFLOW) {
4701 /* When a ring buffer overflow happen start parsing interrupt
4702 * from the last not overwritten vector (wptr + 16). Hopefully
4703 * this should allow us to catchup.
4704 */
4705 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4706 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4707 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4708 tmp = RREG32(IH_RB_CNTL);
4709 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4710 WREG32(IH_RB_CNTL, tmp);
4711 }
4712 return (wptr & rdev->ih.ptr_mask);
4713}
4714
4715int evergreen_irq_process(struct radeon_device *rdev)
4716{
682f1a54
DA		 4717 u32 wptr;
4718 u32 rptr;
45f9a39b
AD		 4719 u32 src_id, src_data;
4720 u32 ring_index;
45f9a39b 4721 bool queue_hotplug = false;
f122c610 4722 bool queue_hdmi = false;
dc50ba7f 4723 bool queue_thermal = false;
54e2e49c 4724 u32 status, addr;
45f9a39b 4725
682f1a54 4726 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4727 return IRQ_NONE;
4728
682f1a54 4729 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4730
4731restart_ih:
4732 /* is somebody else already processing irqs? */
4733 if (atomic_xchg(&rdev->ih.lock, 1))
4734 return IRQ_NONE;
4735
682f1a54
DA		 4736 rptr = rdev->ih.rptr;
4737 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4738
964f6645
BH		 4739 /* Order reading of wptr vs. reading of IH ring data */
4740 rmb();
4741
45f9a39b 4742 /* display interrupts */
6f34be50 4743 evergreen_irq_ack(rdev);
45f9a39b 4744
45f9a39b
AD		 4745 while (rptr != wptr) {
4746 /* wptr/rptr are in bytes! */
4747 ring_index = rptr / 4;
0f234f5f
AD		 4748 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4749 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4750
4751 switch (src_id) {
4752 case 1: /* D1 vblank/vline */
4753 switch (src_data) {
4754 case 0: /* D1 vblank */
6f34be50 4755 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4756 if (rdev->irq.crtc_vblank_int[0]) {
4757 drm_handle_vblank(rdev->ddev, 0);
4758 rdev->pm.vblank_sync = true;
4759 wake_up(&rdev->irq.vblank_queue);
4760 }
736fc37f 4761 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4762 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4763 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4764 DRM_DEBUG("IH: D1 vblank\n");
4765 }
4766 break;
4767 case 1: /* D1 vline */
6f34be50
AD		 4768 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4769 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4770 DRM_DEBUG("IH: D1 vline\n");
4771 }
4772 break;
4773 default:
4774 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4775 break;
4776 }
4777 break;
4778 case 2: /* D2 vblank/vline */
4779 switch (src_data) {
4780 case 0: /* D2 vblank */
6f34be50 4781 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4782 if (rdev->irq.crtc_vblank_int[1]) {
4783 drm_handle_vblank(rdev->ddev, 1);
4784 rdev->pm.vblank_sync = true;
4785 wake_up(&rdev->irq.vblank_queue);
4786 }
736fc37f 4787 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4788 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4789 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4790 DRM_DEBUG("IH: D2 vblank\n");
4791 }
4792 break;
4793 case 1: /* D2 vline */
6f34be50
AD		 4794 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4795 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4796 DRM_DEBUG("IH: D2 vline\n");
4797 }
4798 break;
4799 default:
4800 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4801 break;
4802 }
4803 break;
4804 case 3: /* D3 vblank/vline */
4805 switch (src_data) {
4806 case 0: /* D3 vblank */
6f34be50
AD		 4807 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4808 if (rdev->irq.crtc_vblank_int[2]) {
4809 drm_handle_vblank(rdev->ddev, 2);
4810 rdev->pm.vblank_sync = true;
4811 wake_up(&rdev->irq.vblank_queue);
4812 }
736fc37f 4813 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4814 radeon_crtc_handle_flip(rdev, 2);
4815 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4816 DRM_DEBUG("IH: D3 vblank\n");
4817 }
4818 break;
4819 case 1: /* D3 vline */
6f34be50
AD		 4820 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4821 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4822 DRM_DEBUG("IH: D3 vline\n");
4823 }
4824 break;
4825 default:
4826 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4827 break;
4828 }
4829 break;
4830 case 4: /* D4 vblank/vline */
4831 switch (src_data) {
4832 case 0: /* D4 vblank */
6f34be50
AD		 4833 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4834 if (rdev->irq.crtc_vblank_int[3]) {
4835 drm_handle_vblank(rdev->ddev, 3);
4836 rdev->pm.vblank_sync = true;
4837 wake_up(&rdev->irq.vblank_queue);
4838 }
736fc37f 4839 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4840 radeon_crtc_handle_flip(rdev, 3);
4841 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4842 DRM_DEBUG("IH: D4 vblank\n");
4843 }
4844 break;
4845 case 1: /* D4 vline */
6f34be50
AD		 4846 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4847 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4848 DRM_DEBUG("IH: D4 vline\n");
4849 }
4850 break;
4851 default:
4852 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4853 break;
4854 }
4855 break;
4856 case 5: /* D5 vblank/vline */
4857 switch (src_data) {
4858 case 0: /* D5 vblank */
6f34be50
AD		 4859 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4860 if (rdev->irq.crtc_vblank_int[4]) {
4861 drm_handle_vblank(rdev->ddev, 4);
4862 rdev->pm.vblank_sync = true;
4863 wake_up(&rdev->irq.vblank_queue);
4864 }
736fc37f 4865 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4866 radeon_crtc_handle_flip(rdev, 4);
4867 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4868 DRM_DEBUG("IH: D5 vblank\n");
4869 }
4870 break;
4871 case 1: /* D5 vline */
6f34be50
AD		 4872 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4873 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4874 DRM_DEBUG("IH: D5 vline\n");
4875 }
4876 break;
4877 default:
4878 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4879 break;
4880 }
4881 break;
4882 case 6: /* D6 vblank/vline */
4883 switch (src_data) {
4884 case 0: /* D6 vblank */
6f34be50
AD		 4885 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4886 if (rdev->irq.crtc_vblank_int[5]) {
4887 drm_handle_vblank(rdev->ddev, 5);
4888 rdev->pm.vblank_sync = true;
4889 wake_up(&rdev->irq.vblank_queue);
4890 }
736fc37f 4891 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4892 radeon_crtc_handle_flip(rdev, 5);
4893 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4894 DRM_DEBUG("IH: D6 vblank\n");
4895 }
4896 break;
4897 case 1: /* D6 vline */
6f34be50
AD		 4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4899 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4900 DRM_DEBUG("IH: D6 vline\n");
4901 }
4902 break;
4903 default:
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905 break;
4906 }
4907 break;
4908 case 42: /* HPD hotplug */
4909 switch (src_data) {
4910 case 0:
6f34be50
AD		 4911 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4912 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4913 queue_hotplug = true;
4914 DRM_DEBUG("IH: HPD1\n");
4915 }
4916 break;
4917 case 1:
6f34be50
AD		 4918 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4919 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4920 queue_hotplug = true;
4921 DRM_DEBUG("IH: HPD2\n");
4922 }
4923 break;
4924 case 2:
6f34be50
AD		 4925 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4926 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4927 queue_hotplug = true;
4928 DRM_DEBUG("IH: HPD3\n");
4929 }
4930 break;
4931 case 3:
6f34be50
AD		 4932 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4933 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4934 queue_hotplug = true;
4935 DRM_DEBUG("IH: HPD4\n");
4936 }
4937 break;
4938 case 4:
6f34be50
AD		 4939 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4940 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4941 queue_hotplug = true;
4942 DRM_DEBUG("IH: HPD5\n");
4943 }
4944 break;
4945 case 5:
6f34be50
AD		 4946 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4947 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4948 queue_hotplug = true;
4949 DRM_DEBUG("IH: HPD6\n");
4950 }
4951 break;
4952 default:
4953 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4954 break;
4955 }
4956 break;
f122c610
AD		 4957 case 44: /* hdmi */
4958 switch (src_data) {
4959 case 0:
4960 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4961 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4962 queue_hdmi = true;
4963 DRM_DEBUG("IH: HDMI0\n");
4964 }
4965 break;
4966 case 1:
4967 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4968 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4969 queue_hdmi = true;
4970 DRM_DEBUG("IH: HDMI1\n");
4971 }
4972 break;
4973 case 2:
4974 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4975 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4976 queue_hdmi = true;
4977 DRM_DEBUG("IH: HDMI2\n");
4978 }
4979 break;
4980 case 3:
4981 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4982 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4983 queue_hdmi = true;
4984 DRM_DEBUG("IH: HDMI3\n");
4985 }
4986 break;
4987 case 4:
4988 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4989 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4990 queue_hdmi = true;
4991 DRM_DEBUG("IH: HDMI4\n");
4992 }
4993 break;
4994 case 5:
4995 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4996 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4997 queue_hdmi = true;
4998 DRM_DEBUG("IH: HDMI5\n");
4999 }
5000 break;
5001 default:
5002 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5003 break;
5004 }
f2ba57b5
CK		 5005 case 124: /* UVD */
5006 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5007 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5008 break;
ae133a11
CK		 5009 case 146:
5010 case 147:
54e2e49c
AD		 5011 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5012 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 5013 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5014 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5015 addr);
ae133a11 5016 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5017 status);
5018 cayman_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 5019 /* reset addr and status */
5020 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5021 break;
45f9a39b
AD		 5022 case 176: /* CP_INT in ring buffer */
5023 case 177: /* CP_INT in IB1 */
5024 case 178: /* CP_INT in IB2 */
5025 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5026 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5027 break;
5028 case 181: /* CP EOP event */
5029 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5030 if (rdev->family >= CHIP_CAYMAN) {
5031 switch (src_data) {
5032 case 0:
5033 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5034 break;
5035 case 1:
5036 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5037 break;
5038 case 2:
5039 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5040 break;
5041 }
5042 } else
5043 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5044 break;
233d1ad5
AD		 5045 case 224: /* DMA trap event */
5046 DRM_DEBUG("IH: DMA trap\n");
5047 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5048 break;
dc50ba7f
AD		 5049 case 230: /* thermal low to high */
5050 DRM_DEBUG("IH: thermal low to high\n");
5051 rdev->pm.dpm.thermal.high_to_low = false;
5052 queue_thermal = true;
5053 break;
5054 case 231: /* thermal high to low */
5055 DRM_DEBUG("IH: thermal high to low\n");
5056 rdev->pm.dpm.thermal.high_to_low = true;
5057 queue_thermal = true;
5058 break;
2031f77c 5059 case 233: /* GUI IDLE */
303c805c 5060 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5061 break;
f60cbd11
AD		 5062 case 244: /* DMA trap event */
5063 if (rdev->family >= CHIP_CAYMAN) {
5064 DRM_DEBUG("IH: DMA1 trap\n");
5065 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5066 }
5067 break;
45f9a39b
AD		 5068 default:
5069 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5070 break;
5071 }
5072
5073 /* wptr/rptr are in bytes! */
5074 rptr += 16;
5075 rptr &= rdev->ih.ptr_mask;
5076 }
45f9a39b 5077 if (queue_hotplug)
32c87fca 5078 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5079 if (queue_hdmi)
5080 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5081 if (queue_thermal && rdev->pm.dpm_enabled)
5082 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 5083 rdev->ih.rptr = rptr;
5084 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5085 atomic_set(&rdev->ih.lock, 0);
5086
5087 /* make sure wptr hasn't changed while processing */
5088 wptr = evergreen_get_ih_wptr(rdev);
5089 if (wptr != rptr)
5090 goto restart_ih;
5091
45f9a39b
AD		 5092 return IRQ_HANDLED;
5093}
5094
bcc1c2a1
AD		 5095static int evergreen_startup(struct radeon_device *rdev)
5096{
f2ba57b5 5097 struct radeon_ring *ring;
bcc1c2a1
AD		 5098 int r;
5099
9e46a48d 5100 /* enable pcie gen2 link */
cd54033a 5101 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5102 /* enable aspm */
5103 evergreen_program_aspm(rdev);
9e46a48d 5104
e5903d39
AD		 5105 /* scratch needs to be initialized before MC */
5106 r = r600_vram_scratch_init(rdev);
5107 if (r)
5108 return r;
5109
6fab3feb
AD		 5110 evergreen_mc_program(rdev);
5111
0af62b01
AD		 5112 if (ASIC_IS_DCE5(rdev)) {
5113 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5114 r = ni_init_microcode(rdev);
5115 if (r) {
5116 DRM_ERROR("Failed to load firmware!\n");
5117 return r;
5118 }
5119 }
755d819e 5120 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5121 if (r) {
0af62b01 5122 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5123 return r;
5124 }
0af62b01
AD		 5125 } else {
5126 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5127 r = r600_init_microcode(rdev);
5128 if (r) {
5129 DRM_ERROR("Failed to load firmware!\n");
5130 return r;
5131 }
5132 }
bcc1c2a1 5133 }
fe251e2f 5134
bcc1c2a1 5135 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5136 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5137 } else {
5138 r = evergreen_pcie_gart_enable(rdev);
5139 if (r)
5140 return r;
5141 }
bcc1c2a1 5142 evergreen_gpu_init(rdev);
bcc1c2a1 5143
2948f5e6
AD		 5144 /* allocate rlc buffers */
5145 if (rdev->flags & RADEON_IS_IGP) {
5146 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5147 rdev->rlc.reg_list_size =
5148 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5149 rdev->rlc.cs_data = evergreen_cs_data;
5150 r = sumo_rlc_init(rdev);
5151 if (r) {
5152 DRM_ERROR("Failed to init rlc BOs!\n");
5153 return r;
5154 }
5155 }
5156
724c80e1
AD		 5157 /* allocate wb buffer */
5158 r = radeon_wb_init(rdev);
5159 if (r)
5160 return r;
5161
30eb77f4
JG		 5162 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5163 if (r) {
5164 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5165 return r;
5166 }
5167
233d1ad5
AD		 5168 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5169 if (r) {
5170 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5171 return r;
5172 }
5173
e409b128 5174 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5175 if (!r) {
5176 r = radeon_fence_driver_start_ring(rdev,
5177 R600_RING_TYPE_UVD_INDEX);
5178 if (r)
5179 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5180 }
5181
5182 if (r)
5183 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5184
bcc1c2a1 5185 /* Enable IRQ */
e49f3959
AH		 5186 if (!rdev->irq.installed) {
5187 r = radeon_irq_kms_init(rdev);
5188 if (r)
5189 return r;
5190 }
5191
bcc1c2a1
AD		 5192 r = r600_irq_init(rdev);
5193 if (r) {
5194 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5195 radeon_irq_kms_fini(rdev);
5196 return r;
5197 }
45f9a39b 5198 evergreen_irq_set(rdev);
bcc1c2a1 5199
f2ba57b5 5200 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5201 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a 5202 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2e1e6dad 5203 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5204 if (r)
5205 return r;
233d1ad5
AD		 5206
5207 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5208 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5209 DMA_RB_RPTR, DMA_RB_WPTR,
2e1e6dad 5210 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5211 if (r)
5212 return r;
5213
bcc1c2a1
AD		 5214 r = evergreen_cp_load_microcode(rdev);
5215 if (r)
5216 return r;
fe251e2f 5217 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5218 if (r)
5219 return r;
5220 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5221 if (r)
5222 return r;
fe251e2f 5223
f2ba57b5
CK		 5224 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5225 if (ring->ring_size) {
02c9f7fa 5226 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
f2ba57b5 5227 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 5228 RADEON_CP_PACKET2);
f2ba57b5 5229 if (!r)
e409b128 5230 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5231
5232 if (r)
5233 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5234 }
5235
2898c348
CK		 5236 r = radeon_ib_pool_init(rdev);
5237 if (r) {
5238 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5239 return r;
2898c348 5240 }
b15ba512 5241
69d2ae57
RM		 5242 r = r600_audio_init(rdev);
5243 if (r) {
5244 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5245 return r;
5246 }
5247
bcc1c2a1
AD		 5248 return 0;
5249}
5250
5251int evergreen_resume(struct radeon_device *rdev)
5252{
5253 int r;
5254
86f5c9ed
AD		 5255 /* reset the asic, the gfx blocks are often in a bad state
5256 * after the driver is unloaded or after a resume
5257 */
5258 if (radeon_asic_reset(rdev))
5259 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5260 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5261 * posting will perform necessary task to bring back GPU into good
5262 * shape.
5263 */
5264 /* post card */
5265 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5266
d4788db3
AD		 5267 /* init golden registers */
5268 evergreen_init_golden_registers(rdev);
5269
b15ba512 5270 rdev->accel_working = true;
bcc1c2a1
AD		 5271 r = evergreen_startup(rdev);
5272 if (r) {
755d819e 5273 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5274 rdev->accel_working = false;
bcc1c2a1
AD		 5275 return r;
5276 }
fe251e2f 5277
bcc1c2a1
AD		 5278 return r;
5279
5280}
5281
5282int evergreen_suspend(struct radeon_device *rdev)
5283{
69d2ae57 5284 r600_audio_fini(rdev);
e409b128 5285 uvd_v1_0_fini(rdev);
f2ba57b5 5286 radeon_uvd_suspend(rdev);
bcc1c2a1 5287 r700_cp_stop(rdev);
233d1ad5 5288 r600_dma_stop(rdev);
45f9a39b 5289 evergreen_irq_suspend(rdev);
724c80e1 5290 radeon_wb_disable(rdev);
bcc1c2a1 5291 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5292
5293 return 0;
5294}
5295
bcc1c2a1
AD		 5296/* Plan is to move initialization in that function and use
5297 * helper function so that radeon_device_init pretty much
5298 * do nothing more than calling asic specific function. This
5299 * should also allow to remove a bunch of callback function
5300 * like vram_info.
5301 */
5302int evergreen_init(struct radeon_device *rdev)
5303{
5304 int r;
5305
bcc1c2a1
AD		 5306 /* Read BIOS */
5307 if (!radeon_get_bios(rdev)) {
5308 if (ASIC_IS_AVIVO(rdev))
5309 return -EINVAL;
5310 }
5311 /* Must be an ATOMBIOS */
5312 if (!rdev->is_atom_bios) {
755d819e 5313 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5314 return -EINVAL;
5315 }
5316 r = radeon_atombios_init(rdev);
5317 if (r)
5318 return r;
86f5c9ed
AD		 5319 /* reset the asic, the gfx blocks are often in a bad state
5320 * after the driver is unloaded or after a resume
5321 */
5322 if (radeon_asic_reset(rdev))
5323 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5324 /* Post card if necessary */
fd909c37 5325 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5326 if (!rdev->bios) {
5327 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5328 return -EINVAL;
5329 }
5330 DRM_INFO("GPU not posted. posting now...\n");
5331 atom_asic_init(rdev->mode_info.atom_context);
5332 }
d4788db3
AD		 5333 /* init golden registers */
5334 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5335 /* Initialize scratch registers */
5336 r600_scratch_init(rdev);
5337 /* Initialize surface registers */
5338 radeon_surface_init(rdev);
5339 /* Initialize clocks */
5340 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5341 /* Fence driver */
5342 r = radeon_fence_driver_init(rdev);
5343 if (r)
5344 return r;
d594e46a
JG		 5345 /* initialize AGP */
5346 if (rdev->flags & RADEON_IS_AGP) {
5347 r = radeon_agp_init(rdev);
5348 if (r)
5349 radeon_agp_disable(rdev);
5350 }
5351 /* initialize memory controller */
bcc1c2a1
AD		 5352 r = evergreen_mc_init(rdev);
5353 if (r)
5354 return r;
5355 /* Memory manager */
5356 r = radeon_bo_init(rdev);
5357 if (r)
5358 return r;
45f9a39b 5359
e32eb50d
CK		 5360 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5361 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5362
233d1ad5
AD		 5363 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5364 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5365
f2ba57b5
CK		 5366 r = radeon_uvd_init(rdev);
5367 if (!r) {
5368 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5369 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5370 4096);
5371 }
5372
bcc1c2a1
AD		 5373 rdev->ih.ring_obj = NULL;
5374 r600_ih_ring_init(rdev, 64 * 1024);
5375
5376 r = r600_pcie_gart_init(rdev);
5377 if (r)
5378 return r;
0fcdb61e 5379
148a03bc 5380 rdev->accel_working = true;
bcc1c2a1
AD		 5381 r = evergreen_startup(rdev);
5382 if (r) {
fe251e2f
AD		 5383 dev_err(rdev->dev, "disabling GPU acceleration\n");
5384 r700_cp_fini(rdev);
233d1ad5 5385 r600_dma_fini(rdev);
fe251e2f 5386 r600_irq_fini(rdev);
2948f5e6
AD		 5387 if (rdev->flags & RADEON_IS_IGP)
5388 sumo_rlc_fini(rdev);
724c80e1 5389 radeon_wb_fini(rdev);
2898c348 5390 radeon_ib_pool_fini(rdev);
fe251e2f 5391 radeon_irq_kms_fini(rdev);
0fcdb61e 5392 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5393 rdev->accel_working = false;
5394 }
77e00f2e
AD		 5395
5396 /* Don't start up if the MC ucode is missing on BTC parts.
5397 * The default clocks and voltages before the MC ucode
5398 * is loaded are not suffient for advanced operations.
5399 */
5400 if (ASIC_IS_DCE5(rdev)) {
5401 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5402 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5403 return -EINVAL;
5404 }
5405 }
5406
bcc1c2a1
AD		 5407 return 0;
5408}
5409
5410void evergreen_fini(struct radeon_device *rdev)
5411{
69d2ae57 5412 r600_audio_fini(rdev);
45f9a39b 5413 r700_cp_fini(rdev);
233d1ad5 5414 r600_dma_fini(rdev);
bcc1c2a1 5415 r600_irq_fini(rdev);
2948f5e6
AD		 5416 if (rdev->flags & RADEON_IS_IGP)
5417 sumo_rlc_fini(rdev);
724c80e1 5418 radeon_wb_fini(rdev);
2898c348 5419 radeon_ib_pool_fini(rdev);
bcc1c2a1 5420 radeon_irq_kms_fini(rdev);
bcc1c2a1 5421 evergreen_pcie_gart_fini(rdev);
e409b128 5422 uvd_v1_0_fini(rdev);
f2ba57b5 5423 radeon_uvd_fini(rdev);
16cdf04d 5424 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5425 radeon_gem_fini(rdev);
5426 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5427 radeon_agp_fini(rdev);
5428 radeon_bo_fini(rdev);
5429 radeon_atombios_fini(rdev);
5430 kfree(rdev->bios);
5431 rdev->bios = NULL;
bcc1c2a1 5432}
9e46a48d 5433
b07759bf 5434void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5435{
7e0e4196 5436 u32 link_width_cntl, speed_cntl;
9e46a48d 5437
d42dd579
AD		 5438 if (radeon_pcie_gen2 == 0)
5439 return;
5440
9e46a48d
AD		 5441 if (rdev->flags & RADEON_IS_IGP)
5442 return;
5443
5444 if (!(rdev->flags & RADEON_IS_PCIE))
5445 return;
5446
5447 /* x2 cards have a special sequence */
5448 if (ASIC_IS_X2(rdev))
5449 return;
5450
7e0e4196
KSS		 5451 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5452 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5453 return;
5454
492d2b61 5455 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5456 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5457 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5458 return;
5459 }
5460
197bbb3d
DA		 5461 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5462
9e46a48d
AD		 5463 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5464 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5465
492d2b61 5466 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5467 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5468 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5469
492d2b61 5470 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5471 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5472 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5473
492d2b61 5474 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5475 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5476 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5477
492d2b61 5478 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5479 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5480 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5481
492d2b61 5482 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5483 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5484 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5485
5486 } else {
492d2b61 5487 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5488 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5489 if (1)
5490 link_width_cntl |= LC_UPCONFIGURE_DIS;
5491 else
5492 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5493 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5494 }
5495}
f52382d7
AD		 5496
5497void evergreen_program_aspm(struct radeon_device *rdev)
5498{
5499 u32 data, orig;
5500 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5501 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5502 /* fusion_platform = true
5503 * if the system is a fusion system
5504 * (APU or DGPU in a fusion system).
5505 * todo: check if the system is a fusion platform.
5506 */
5507 bool fusion_platform = false;
5508
1294d4a3
AD		 5509 if (radeon_aspm == 0)
5510 return;
5511
f52382d7
AD		 5512 if (!(rdev->flags & RADEON_IS_PCIE))
5513 return;
5514
5515 switch (rdev->family) {
5516 case CHIP_CYPRESS:
5517 case CHIP_HEMLOCK:
5518 case CHIP_JUNIPER:
5519 case CHIP_REDWOOD:
5520 case CHIP_CEDAR:
5521 case CHIP_SUMO:
5522 case CHIP_SUMO2:
5523 case CHIP_PALM:
5524 case CHIP_ARUBA:
5525 disable_l0s = true;
5526 break;
5527 default:
5528 disable_l0s = false;
5529 break;
5530 }
5531
5532 if (rdev->flags & RADEON_IS_IGP)
5533 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5534
5535 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5536 if (fusion_platform)
5537 data &= ~MULTI_PIF;
5538 else
5539 data |= MULTI_PIF;
5540 if (data != orig)
5541 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5542
5543 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5544 if (fusion_platform)
5545 data &= ~MULTI_PIF;
5546 else
5547 data |= MULTI_PIF;
5548 if (data != orig)
5549 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5550
5551 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5552 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5553 if (!disable_l0s) {
5554 if (rdev->family >= CHIP_BARTS)
5555 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5556 else
5557 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5558 }
5559
5560 if (!disable_l1) {
5561 if (rdev->family >= CHIP_BARTS)
5562 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5563 else
5564 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5565
5566 if (!disable_plloff_in_l1) {
5567 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5568 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5569 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5570 if (data != orig)
5571 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5572
5573 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5574 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5575 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5576 if (data != orig)
5577 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5578
5579 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5580 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5581 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5582 if (data != orig)
5583 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5584
5585 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5586 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5587 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5588 if (data != orig)
5589 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5590
5591 if (rdev->family >= CHIP_BARTS) {
5592 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5593 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5594 data |= PLL_RAMP_UP_TIME_0(4);
5595 if (data != orig)
5596 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5597
5598 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5599 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5600 data |= PLL_RAMP_UP_TIME_1(4);
5601 if (data != orig)
5602 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5603
5604 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5605 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5606 data |= PLL_RAMP_UP_TIME_0(4);
5607 if (data != orig)
5608 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5609
5610 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5611 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5612 data |= PLL_RAMP_UP_TIME_1(4);
5613 if (data != orig)
5614 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5615 }
5616
5617 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5618 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5619 data |= LC_DYN_LANES_PWR_STATE(3);
5620 if (data != orig)
5621 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5622
5623 if (rdev->family >= CHIP_BARTS) {
5624 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5625 data &= ~LS2_EXIT_TIME_MASK;
5626 data |= LS2_EXIT_TIME(1);
5627 if (data != orig)
5628 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5629
5630 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5631 data &= ~LS2_EXIT_TIME_MASK;
5632 data |= LS2_EXIT_TIME(1);
5633 if (data != orig)
5634 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5635 }
5636 }
5637 }
5638
5639 /* evergreen parts only */
5640 if (rdev->family < CHIP_BARTS)
5641 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5642
5643 if (pcie_lc_cntl != pcie_lc_cntl_old)
5644 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5645}
Linux kernel - Deliverables
This page took 0.590174 seconds and 5 git commands to generate.