projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: clean up active vram sizing
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 149extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 150
d4788db3
AD		 151static const u32 evergreen_golden_registers[] =
152{
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
192 0x5cc, 0xffffffff, 0x00000001,
193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
195};
196
197static const u32 evergreen_golden_registers2[] =
198{
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
213};
214
215static const u32 cypress_mgcg_init[] =
216{
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
366};
367
368static const u32 redwood_mgcg_init[] =
369{
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
438};
439
440static const u32 cedar_golden_registers[] =
441{
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
479 0x5cc, 0xffffffff, 0x00000001,
480 0x9508, 0xffffffff, 0x00000002
481};
482
483static const u32 cedar_mgcg_init[] =
484{
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
535};
536
537static const u32 juniper_mgcg_init[] =
538{
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
633};
634
635static const u32 supersumo_golden_registers[] =
636{
637 0x5eb4, 0xffffffff, 0x00000002,
638 0x5cc, 0xffffffff, 0x00000001,
639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
708};
709
710static const u32 sumo_golden_registers[] =
711{
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
717};
718
719static const u32 wrestler_golden_registers[] =
720{
721 0x5eb4, 0xffffffff, 0x00000002,
722 0x5cc, 0xffffffff, 0x00000001,
723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
770};
771
772static const u32 barts_golden_registers[] =
773{
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
819};
820
821static const u32 turks_golden_registers[] =
822{
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
870};
871
872static const u32 caicos_golden_registers[] =
873{
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
921};
922
923static void evergreen_init_golden_registers(struct radeon_device *rdev)
924{
925 switch (rdev->family) {
926 case CHIP_CYPRESS:
927 case CHIP_HEMLOCK:
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
935 cypress_mgcg_init,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
937 break;
938 case CHIP_JUNIPER:
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
946 juniper_mgcg_init,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
948 break;
949 case CHIP_REDWOOD:
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
957 redwood_mgcg_init,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
959 break;
960 case CHIP_CEDAR:
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
968 cedar_mgcg_init,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
970 break;
971 case CHIP_PALM:
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
975 break;
976 case CHIP_SUMO:
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
980 break;
981 case CHIP_SUMO2:
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
988 break;
989 case CHIP_BARTS:
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
993 break;
994 case CHIP_TURKS:
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
998 break;
999 case CHIP_CAICOS:
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1003 break;
1004 default:
1005 break;
1006 }
1007}
1008
285484e2
JG		 1009void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1010 unsigned *bankh, unsigned *mtaspect,
1011 unsigned *tile_split)
1012{
1013 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1014 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1015 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1016 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1017 switch (*bankw) {
1018 default:
1019 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1020 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1021 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1022 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1023 }
1024 switch (*bankh) {
1025 default:
1026 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1027 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1028 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1029 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1030 }
1031 switch (*mtaspect) {
1032 default:
1033 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1034 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1035 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1036 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1037 }
1038}
1039
23d33ba3
AD		 1040static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1041 u32 cntl_reg, u32 status_reg)
1042{
1043 int r, i;
1044 struct atom_clock_dividers dividers;
1045
1046 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1047 clock, false, &dividers);
1048 if (r)
1049 return r;
1050
1051 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1052
1053 for (i = 0; i < 100; i++) {
1054 if (RREG32(status_reg) & DCLK_STATUS)
1055 break;
1056 mdelay(10);
1057 }
1058 if (i == 100)
1059 return -ETIMEDOUT;
1060
1061 return 0;
1062}
1063
1064int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1065{
1066 int r = 0;
1067 u32 cg_scratch = RREG32(CG_SCRATCH1);
1068
1069 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1070 if (r)
1071 goto done;
1072 cg_scratch &= 0xffff0000;
1073 cg_scratch |= vclk / 100; /* Mhz */
1074
1075 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1076 if (r)
1077 goto done;
1078 cg_scratch &= 0x0000ffff;
1079 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1080
1081done:
1082 WREG32(CG_SCRATCH1, cg_scratch);
1083
1084 return r;
1085}
1086
a8b4925c
AD		 1087int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1088{
1089 /* start off with something large */
facd112d 1090 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1091 int r;
1092
4ed10835
CK		 1093 /* bypass vclk and dclk with bclk */
1094 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1095 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1096 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1097
1098 /* put PLL in bypass mode */
1099 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1100
1101 if (!vclk || !dclk) {
1102 /* keep the Bypass mode, put PLL to sleep */
1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1104 return 0;
1105 }
1106
facd112d
CK		 1107 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1108 16384, 0x03FFFFFF, 0, 128, 5,
1109 &fb_div, &vclk_div, &dclk_div);
1110 if (r)
1111 return r;
a8b4925c
AD		 1112
1113 /* set VCO_MODE to 1 */
1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1115
1116 /* toggle UPLL_SLEEP to 1 then back to 0 */
1117 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1119
1120 /* deassert UPLL_RESET */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1122
1123 mdelay(1);
1124
facd112d 1125 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1126 if (r)
1127 return r;
1128
1129 /* assert UPLL_RESET again */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1131
1132 /* disable spread spectrum. */
1133 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1134
1135 /* set feedback divider */
facd112d 1136 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1137
1138 /* set ref divider to 0 */
1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1140
facd112d 1141 if (fb_div < 307200)
a8b4925c
AD		 1142 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1143 else
1144 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1145
1146 /* set PDIV_A and PDIV_B */
1147 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1148 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1149 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1150
1151 /* give the PLL some time to settle */
1152 mdelay(15);
1153
1154 /* deassert PLL_RESET */
1155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1156
1157 mdelay(15);
1158
1159 /* switch from bypass mode to normal mode */
1160 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1161
facd112d 1162 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1163 if (r)
1164 return r;
1165
1166 /* switch VCLK and DCLK selection */
1167 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1168 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1169 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1170
1171 mdelay(100);
1172
1173 return 0;
1174}
1175
d054ac16
AD		 1176void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1177{
c11592fe
YW		 1178 int readrq;
1179 u16 v;
d054ac16 1180
c11592fe
YW		 1181 readrq = pcie_get_readrq(rdev->pdev);
1182 v = ffs(readrq) - 8;
d054ac16
AD		 1183 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1184 * to avoid hangs or perfomance issues
1185 */
c11592fe
YW		 1186 if ((v == 0) || (v == 6) || (v == 7))
1187 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1188}
1189
134b480f
AD		 1190void dce4_program_fmt(struct drm_encoder *encoder)
1191{
1192 struct drm_device *dev = encoder->dev;
1193 struct radeon_device *rdev = dev->dev_private;
1194 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1195 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1196 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1197 int bpc = 0;
1198 u32 tmp = 0;
6214bb74 1199 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1200
6214bb74
AD		 1201 if (connector) {
1202 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1203 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1204 dither = radeon_connector->dither;
1205 }
134b480f
AD		 1206
1207 /* LVDS/eDP FMT is set up by atom */
1208 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1209 return;
1210
1211 /* not needed for analog */
1212 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1213 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1214 return;
1215
1216 if (bpc == 0)
1217 return;
1218
1219 switch (bpc) {
1220 case 6:
6214bb74 1221 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1222 /* XXX sort out optimal dither settings */
1223 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1224 FMT_SPATIAL_DITHER_EN);
1225 else
1226 tmp |= FMT_TRUNCATE_EN;
1227 break;
1228 case 8:
6214bb74 1229 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1230 /* XXX sort out optimal dither settings */
1231 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1232 FMT_RGB_RANDOM_ENABLE |
1233 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1234 else
1235 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1236 break;
1237 case 10:
1238 default:
1239 /* not needed */
1240 break;
1241 }
1242
1243 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1244}
1245
10257a6d
AD		 1246static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1247{
1248 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1249 return true;
1250 else
1251 return false;
1252}
1253
1254static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1255{
1256 u32 pos1, pos2;
1257
1258 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1260
1261 if (pos1 != pos2)
1262 return true;
1263 else
1264 return false;
1265}
1266
377edc8b
AD		 1267/**
1268 * dce4_wait_for_vblank - vblank wait asic callback.
1269 *
1270 * @rdev: radeon_device pointer
1271 * @crtc: crtc to wait for vblank on
1272 *
1273 * Wait for vblank on the requested crtc (evergreen+).
1274 */
3ae19b75
AD		 1275void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1276{
10257a6d 1277 unsigned i = 0;
3ae19b75 1278
4a15903d
AD		 1279 if (crtc >= rdev->num_crtc)
1280 return;
1281
10257a6d
AD		 1282 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1283 return;
1284
1285 /* depending on when we hit vblank, we may be close to active; if so,
1286 * wait for another frame.
1287 */
1288 while (dce4_is_in_vblank(rdev, crtc)) {
1289 if (i++ % 100 == 0) {
1290 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1291 break;
3ae19b75 1292 }
10257a6d
AD		 1293 }
1294
1295 while (!dce4_is_in_vblank(rdev, crtc)) {
1296 if (i++ % 100 == 0) {
1297 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1298 break;
3ae19b75
AD		 1299 }
1300 }
1301}
1302
377edc8b
AD		 1303/**
1304 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1305 *
1306 * @rdev: radeon_device pointer
1307 * @crtc: crtc to prepare for pageflip on
1308 *
1309 * Pre-pageflip callback (evergreen+).
1310 * Enables the pageflip irq (vblank irq).
1311 */
6f34be50
AD		 1312void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1313{
6f34be50
AD		 1314 /* enable the pflip int */
1315 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1316}
1317
377edc8b
AD		 1318/**
1319 * evergreen_post_page_flip - pos-pageflip callback.
1320 *
1321 * @rdev: radeon_device pointer
1322 * @crtc: crtc to cleanup pageflip on
1323 *
1324 * Post-pageflip callback (evergreen+).
1325 * Disables the pageflip irq (vblank irq).
1326 */
6f34be50
AD		 1327void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1328{
1329 /* disable the pflip int */
1330 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1331}
1332
377edc8b
AD		 1333/**
1334 * evergreen_page_flip - pageflip callback.
1335 *
1336 * @rdev: radeon_device pointer
1337 * @crtc_id: crtc to cleanup pageflip on
1338 * @crtc_base: new address of the crtc (GPU MC address)
1339 *
1340 * Does the actual pageflip (evergreen+).
1341 * During vblank we take the crtc lock and wait for the update_pending
1342 * bit to go high, when it does, we release the lock, and allow the
1343 * double buffered update to take place.
1344 * Returns the current update pending status.
1345 */
6f34be50
AD		 1346u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1347{
1348 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1349 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1350 int i;
6f34be50
AD		 1351
1352 /* Lock the graphics update lock */
1353 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1354 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1355
1356 /* update the scanout addresses */
1357 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1358 upper_32_bits(crtc_base));
1359 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1360 (u32)crtc_base);
1361
1362 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1363 upper_32_bits(crtc_base));
1364 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1365 (u32)crtc_base);
1366
1367 /* Wait for update_pending to go high. */
f6496479
AD		 1368 for (i = 0; i < rdev->usec_timeout; i++) {
1369 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1370 break;
1371 udelay(1);
1372 }
6f34be50
AD		 1373 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1374
1375 /* Unlock the lock, so double-buffering can take place inside vblank */
1376 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1377 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1378
1379 /* Return current update_pending status: */
1380 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1381}
1382
21a8122a 1383/* get temperature in millidegrees */
20d391d7 1384int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1385{
1c88d74f
AD		 1386 u32 temp, toffset;
1387 int actual_temp = 0;
67b3f823
AD		 1388
1389 if (rdev->family == CHIP_JUNIPER) {
1390 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1391 TOFFSET_SHIFT;
1392 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1393 TS0_ADC_DOUT_SHIFT;
1394
1395 if (toffset & 0x100)
1396 actual_temp = temp / 2 - (0x200 - toffset);
1397 else
1398 actual_temp = temp / 2 + toffset;
1399
1400 actual_temp = actual_temp * 1000;
1401
1402 } else {
1403 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1404 ASIC_T_SHIFT;
1405
1406 if (temp & 0x400)
1407 actual_temp = -256;
1408 else if (temp & 0x200)
1409 actual_temp = 255;
1410 else if (temp & 0x100) {
1411 actual_temp = temp & 0x1ff;
1412 actual_temp |= ~0x1ff;
1413 } else
1414 actual_temp = temp & 0xff;
1415
1416 actual_temp = (actual_temp * 1000) / 2;
1417 }
21a8122a 1418
67b3f823 1419 return actual_temp;
21a8122a
AD		 1420}
1421
20d391d7 1422int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1423{
1424 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1425 int actual_temp = temp - 49;
e33df25f
AD		 1426
1427 return actual_temp * 1000;
1428}
1429
377edc8b
AD		 1430/**
1431 * sumo_pm_init_profile - Initialize power profiles callback.
1432 *
1433 * @rdev: radeon_device pointer
1434 *
1435 * Initialize the power states used in profile mode
1436 * (sumo, trinity, SI).
1437 * Used for profile mode only.
1438 */
a4c9e2ee
AD		 1439void sumo_pm_init_profile(struct radeon_device *rdev)
1440{
1441 int idx;
1442
1443 /* default */
1444 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1445 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1448
1449 /* low,mid sh/mh */
1450 if (rdev->flags & RADEON_IS_MOBILITY)
1451 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1452 else
1453 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1454
1455 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1457 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1458 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1459
1460 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1464
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1469
1470 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1473 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1474
1475 /* high sh/mh */
1476 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1477 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1481 rdev->pm.power_state[idx].num_clock_modes - 1;
1482
1483 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1487 rdev->pm.power_state[idx].num_clock_modes - 1;
1488}
1489
27810fb2
AD		 1490/**
1491 * btc_pm_init_profile - Initialize power profiles callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Initialize the power states used in profile mode
1496 * (BTC, cayman).
1497 * Used for profile mode only.
1498 */
1499void btc_pm_init_profile(struct radeon_device *rdev)
1500{
1501 int idx;
1502
1503 /* default */
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1508 /* starting with BTC, there is one state that is used for both
1509 * MH and SH. Difference is that we always use the high clock index for
1510 * mclk.
1511 */
1512 if (rdev->flags & RADEON_IS_MOBILITY)
1513 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1514 else
1515 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1516 /* low sh */
1517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1521 /* mid sh */
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1526 /* high sh */
1527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1531 /* low mh */
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1536 /* mid mh */
1537 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1541 /* high mh */
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1546}
1547
377edc8b
AD		 1548/**
1549 * evergreen_pm_misc - set additional pm hw parameters callback.
1550 *
1551 * @rdev: radeon_device pointer
1552 *
1553 * Set non-clock parameters associated with a power state
1554 * (voltage, etc.) (evergreen+).
1555 */
49e02b73
AD		 1556void evergreen_pm_misc(struct radeon_device *rdev)
1557{
a081a9d6
RM		 1558 int req_ps_idx = rdev->pm.requested_power_state_index;
1559 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1560 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1561 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1562
2feea49a 1563 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1564 /* 0xff0x are flags rather then an actual voltage */
1565 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1566 return;
2feea49a 1567 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1568 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1569 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1570 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1571 }
7ae764b1
AD		 1572
1573 /* starting with BTC, there is one state that is used for both
1574 * MH and SH. Difference is that we always use the high clock index for
1575 * mclk and vddci.
1576 */
1577 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1578 (rdev->family >= CHIP_BARTS) &&
1579 rdev->pm.active_crtc_count &&
1580 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1581 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1582 voltage = &rdev->pm.power_state[req_ps_idx].
1583 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1584
c6cf7777
AD		 1585 /* 0xff0x are flags rather then an actual voltage */
1586 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1587 return;
2feea49a
AD		 1588 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1589 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1590 rdev->pm.current_vddci = voltage->vddci;
1591 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1592 }
1593 }
49e02b73
AD		 1594}
1595
377edc8b
AD		 1596/**
1597 * evergreen_pm_prepare - pre-power state change callback.
1598 *
1599 * @rdev: radeon_device pointer
1600 *
1601 * Prepare for a power state change (evergreen+).
1602 */
49e02b73
AD		 1603void evergreen_pm_prepare(struct radeon_device *rdev)
1604{
1605 struct drm_device *ddev = rdev->ddev;
1606 struct drm_crtc *crtc;
1607 struct radeon_crtc *radeon_crtc;
1608 u32 tmp;
1609
1610 /* disable any active CRTCs */
1611 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1612 radeon_crtc = to_radeon_crtc(crtc);
1613 if (radeon_crtc->enabled) {
1614 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1615 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1616 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1617 }
1618 }
1619}
1620
377edc8b
AD		 1621/**
1622 * evergreen_pm_finish - post-power state change callback.
1623 *
1624 * @rdev: radeon_device pointer
1625 *
1626 * Clean up after a power state change (evergreen+).
1627 */
49e02b73
AD		 1628void evergreen_pm_finish(struct radeon_device *rdev)
1629{
1630 struct drm_device *ddev = rdev->ddev;
1631 struct drm_crtc *crtc;
1632 struct radeon_crtc *radeon_crtc;
1633 u32 tmp;
1634
1635 /* enable any active CRTCs */
1636 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1637 radeon_crtc = to_radeon_crtc(crtc);
1638 if (radeon_crtc->enabled) {
1639 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1640 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1641 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1642 }
1643 }
1644}
1645
377edc8b
AD		 1646/**
1647 * evergreen_hpd_sense - hpd sense callback.
1648 *
1649 * @rdev: radeon_device pointer
1650 * @hpd: hpd (hotplug detect) pin
1651 *
1652 * Checks if a digital monitor is connected (evergreen+).
1653 * Returns true if connected, false if not connected.
1654 */
bcc1c2a1
AD		 1655bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1656{
1657 bool connected = false;
0ca2ab52
AD		 1658
1659 switch (hpd) {
1660 case RADEON_HPD_1:
1661 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1662 connected = true;
1663 break;
1664 case RADEON_HPD_2:
1665 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1666 connected = true;
1667 break;
1668 case RADEON_HPD_3:
1669 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1670 connected = true;
1671 break;
1672 case RADEON_HPD_4:
1673 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1674 connected = true;
1675 break;
1676 case RADEON_HPD_5:
1677 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1678 connected = true;
1679 break;
1680 case RADEON_HPD_6:
1681 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1682 connected = true;
1683 break;
1684 default:
1685 break;
1686 }
1687
bcc1c2a1
AD		 1688 return connected;
1689}
1690
377edc8b
AD		 1691/**
1692 * evergreen_hpd_set_polarity - hpd set polarity callback.
1693 *
1694 * @rdev: radeon_device pointer
1695 * @hpd: hpd (hotplug detect) pin
1696 *
1697 * Set the polarity of the hpd pin (evergreen+).
1698 */
bcc1c2a1
AD		 1699void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1700 enum radeon_hpd_id hpd)
1701{
0ca2ab52
AD		 1702 u32 tmp;
1703 bool connected = evergreen_hpd_sense(rdev, hpd);
1704
1705 switch (hpd) {
1706 case RADEON_HPD_1:
1707 tmp = RREG32(DC_HPD1_INT_CONTROL);
1708 if (connected)
1709 tmp &= ~DC_HPDx_INT_POLARITY;
1710 else
1711 tmp |= DC_HPDx_INT_POLARITY;
1712 WREG32(DC_HPD1_INT_CONTROL, tmp);
1713 break;
1714 case RADEON_HPD_2:
1715 tmp = RREG32(DC_HPD2_INT_CONTROL);
1716 if (connected)
1717 tmp &= ~DC_HPDx_INT_POLARITY;
1718 else
1719 tmp |= DC_HPDx_INT_POLARITY;
1720 WREG32(DC_HPD2_INT_CONTROL, tmp);
1721 break;
1722 case RADEON_HPD_3:
1723 tmp = RREG32(DC_HPD3_INT_CONTROL);
1724 if (connected)
1725 tmp &= ~DC_HPDx_INT_POLARITY;
1726 else
1727 tmp |= DC_HPDx_INT_POLARITY;
1728 WREG32(DC_HPD3_INT_CONTROL, tmp);
1729 break;
1730 case RADEON_HPD_4:
1731 tmp = RREG32(DC_HPD4_INT_CONTROL);
1732 if (connected)
1733 tmp &= ~DC_HPDx_INT_POLARITY;
1734 else
1735 tmp |= DC_HPDx_INT_POLARITY;
1736 WREG32(DC_HPD4_INT_CONTROL, tmp);
1737 break;
1738 case RADEON_HPD_5:
1739 tmp = RREG32(DC_HPD5_INT_CONTROL);
1740 if (connected)
1741 tmp &= ~DC_HPDx_INT_POLARITY;
1742 else
1743 tmp |= DC_HPDx_INT_POLARITY;
1744 WREG32(DC_HPD5_INT_CONTROL, tmp);
1745 break;
1746 case RADEON_HPD_6:
1747 tmp = RREG32(DC_HPD6_INT_CONTROL);
1748 if (connected)
1749 tmp &= ~DC_HPDx_INT_POLARITY;
1750 else
1751 tmp |= DC_HPDx_INT_POLARITY;
1752 WREG32(DC_HPD6_INT_CONTROL, tmp);
1753 break;
1754 default:
1755 break;
1756 }
bcc1c2a1
AD		 1757}
1758
377edc8b
AD		 1759/**
1760 * evergreen_hpd_init - hpd setup callback.
1761 *
1762 * @rdev: radeon_device pointer
1763 *
1764 * Setup the hpd pins used by the card (evergreen+).
1765 * Enable the pin, set the polarity, and enable the hpd interrupts.
1766 */
bcc1c2a1
AD		 1767void evergreen_hpd_init(struct radeon_device *rdev)
1768{
0ca2ab52
AD		 1769 struct drm_device *dev = rdev->ddev;
1770 struct drm_connector *connector;
fb98257a 1771 unsigned enabled = 0;
0ca2ab52
AD		 1772 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1773 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1774
0ca2ab52
AD		 1775 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1776 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1777
1778 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1779 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1780 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1781 * aux dp channel on imac and help (but not completely fix)
1782 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1783 * also avoid interrupt storms during dpms.
1784 */
1785 continue;
1786 }
0ca2ab52
AD		 1787 switch (radeon_connector->hpd.hpd) {
1788 case RADEON_HPD_1:
1789 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1790 break;
1791 case RADEON_HPD_2:
1792 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1793 break;
1794 case RADEON_HPD_3:
1795 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1796 break;
1797 case RADEON_HPD_4:
1798 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1799 break;
1800 case RADEON_HPD_5:
1801 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1802 break;
1803 case RADEON_HPD_6:
1804 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1805 break;
1806 default:
1807 break;
1808 }
64912e99 1809 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1810 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1811 }
fb98257a 1812 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1813}
1814
377edc8b
AD		 1815/**
1816 * evergreen_hpd_fini - hpd tear down callback.
1817 *
1818 * @rdev: radeon_device pointer
1819 *
1820 * Tear down the hpd pins used by the card (evergreen+).
1821 * Disable the hpd interrupts.
1822 */
0ca2ab52 1823void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1824{
0ca2ab52
AD		 1825 struct drm_device *dev = rdev->ddev;
1826 struct drm_connector *connector;
fb98257a 1827 unsigned disabled = 0;
0ca2ab52
AD		 1828
1829 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1830 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1831 switch (radeon_connector->hpd.hpd) {
1832 case RADEON_HPD_1:
1833 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1834 break;
1835 case RADEON_HPD_2:
1836 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1837 break;
1838 case RADEON_HPD_3:
1839 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1840 break;
1841 case RADEON_HPD_4:
1842 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1843 break;
1844 case RADEON_HPD_5:
1845 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1846 break;
1847 case RADEON_HPD_6:
1848 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1849 break;
1850 default:
1851 break;
1852 }
fb98257a 1853 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1854 }
fb98257a 1855 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1856}
1857
f9d9c362
AD		 1858/* watermark setup */
1859
1860static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1861 struct radeon_crtc *radeon_crtc,
1862 struct drm_display_mode *mode,
1863 struct drm_display_mode *other_mode)
1864{
0b31e023
AD		 1865 u32 tmp, buffer_alloc, i;
1866 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1867 /*
1868 * Line Buffer Setup
1869 * There are 3 line buffers, each one shared by 2 display controllers.
1870 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1871 * the display controllers. The paritioning is done via one of four
1872 * preset allocations specified in bits 2:0:
1873 * first display controller
1874 * 0 - first half of lb (3840 * 2)
1875 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1876 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1877 * 3 - first 1/4 of lb (1920 * 2)
1878 * second display controller
1879 * 4 - second half of lb (3840 * 2)
1880 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1881 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1882 * 7 - last 1/4 of lb (1920 * 2)
1883 */
12dfc843
AD		 1884 /* this can get tricky if we have two large displays on a paired group
1885 * of crtcs. Ideally for multiple large displays we'd assign them to
1886 * non-linked crtcs for maximum line buffer allocation.
1887 */
1888 if (radeon_crtc->base.enabled && mode) {
0b31e023 1889 if (other_mode) {
f9d9c362 1890 tmp = 0; /* 1/2 */
0b31e023
AD		 1891 buffer_alloc = 1;
1892 } else {
12dfc843 1893 tmp = 2; /* whole */
0b31e023
AD		 1894 buffer_alloc = 2;
1895 }
1896 } else {
12dfc843 1897 tmp = 0;
0b31e023
AD		 1898 buffer_alloc = 0;
1899 }
f9d9c362
AD		 1900
1901 /* second controller of the pair uses second half of the lb */
1902 if (radeon_crtc->crtc_id % 2)
1903 tmp += 4;
1904 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1905
0b31e023
AD		 1906 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1907 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1908 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1909 for (i = 0; i < rdev->usec_timeout; i++) {
1910 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1911 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1912 break;
1913 udelay(1);
1914 }
1915 }
1916
12dfc843
AD		 1917 if (radeon_crtc->base.enabled && mode) {
1918 switch (tmp) {
1919 case 0:
1920 case 4:
1921 default:
1922 if (ASIC_IS_DCE5(rdev))
1923 return 4096 * 2;
1924 else
1925 return 3840 * 2;
1926 case 1:
1927 case 5:
1928 if (ASIC_IS_DCE5(rdev))
1929 return 6144 * 2;
1930 else
1931 return 5760 * 2;
1932 case 2:
1933 case 6:
1934 if (ASIC_IS_DCE5(rdev))
1935 return 8192 * 2;
1936 else
1937 return 7680 * 2;
1938 case 3:
1939 case 7:
1940 if (ASIC_IS_DCE5(rdev))
1941 return 2048 * 2;
1942 else
1943 return 1920 * 2;
1944 }
f9d9c362 1945 }
12dfc843
AD		 1946
1947 /* controller not enabled, so no lb used */
1948 return 0;
f9d9c362
AD		 1949}
1950
ca7db22b 1951u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1952{
1953 u32 tmp = RREG32(MC_SHARED_CHMAP);
1954
1955 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1956 case 0:
1957 default:
1958 return 1;
1959 case 1:
1960 return 2;
1961 case 2:
1962 return 4;
1963 case 3:
1964 return 8;
1965 }
1966}
1967
1968struct evergreen_wm_params {
1969 u32 dram_channels; /* number of dram channels */
1970 u32 yclk; /* bandwidth per dram data pin in kHz */
1971 u32 sclk; /* engine clock in kHz */
1972 u32 disp_clk; /* display clock in kHz */
1973 u32 src_width; /* viewport width */
1974 u32 active_time; /* active display time in ns */
1975 u32 blank_time; /* blank time in ns */
1976 bool interlaced; /* mode is interlaced */
1977 fixed20_12 vsc; /* vertical scale ratio */
1978 u32 num_heads; /* number of active crtcs */
1979 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1980 u32 lb_size; /* line buffer allocated to pipe */
1981 u32 vtaps; /* vertical scaler taps */
1982};
1983
1984static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1985{
1986 /* Calculate DRAM Bandwidth and the part allocated to display. */
1987 fixed20_12 dram_efficiency; /* 0.7 */
1988 fixed20_12 yclk, dram_channels, bandwidth;
1989 fixed20_12 a;
1990
1991 a.full = dfixed_const(1000);
1992 yclk.full = dfixed_const(wm->yclk);
1993 yclk.full = dfixed_div(yclk, a);
1994 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1995 a.full = dfixed_const(10);
1996 dram_efficiency.full = dfixed_const(7);
1997 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1998 bandwidth.full = dfixed_mul(dram_channels, yclk);
1999 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2000
2001 return dfixed_trunc(bandwidth);
2002}
2003
2004static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2005{
2006 /* Calculate DRAM Bandwidth and the part allocated to display. */
2007 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2008 fixed20_12 yclk, dram_channels, bandwidth;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1000);
2012 yclk.full = dfixed_const(wm->yclk);
2013 yclk.full = dfixed_div(yclk, a);
2014 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2015 a.full = dfixed_const(10);
2016 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2017 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2018 bandwidth.full = dfixed_mul(dram_channels, yclk);
2019 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2020
2021 return dfixed_trunc(bandwidth);
2022}
2023
2024static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2025{
2026 /* Calculate the display Data return Bandwidth */
2027 fixed20_12 return_efficiency; /* 0.8 */
2028 fixed20_12 sclk, bandwidth;
2029 fixed20_12 a;
2030
2031 a.full = dfixed_const(1000);
2032 sclk.full = dfixed_const(wm->sclk);
2033 sclk.full = dfixed_div(sclk, a);
2034 a.full = dfixed_const(10);
2035 return_efficiency.full = dfixed_const(8);
2036 return_efficiency.full = dfixed_div(return_efficiency, a);
2037 a.full = dfixed_const(32);
2038 bandwidth.full = dfixed_mul(a, sclk);
2039 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2040
2041 return dfixed_trunc(bandwidth);
2042}
2043
2044static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2045{
2046 /* Calculate the DMIF Request Bandwidth */
2047 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2048 fixed20_12 disp_clk, bandwidth;
2049 fixed20_12 a;
2050
2051 a.full = dfixed_const(1000);
2052 disp_clk.full = dfixed_const(wm->disp_clk);
2053 disp_clk.full = dfixed_div(disp_clk, a);
2054 a.full = dfixed_const(10);
2055 disp_clk_request_efficiency.full = dfixed_const(8);
2056 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2057 a.full = dfixed_const(32);
2058 bandwidth.full = dfixed_mul(a, disp_clk);
2059 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2060
2061 return dfixed_trunc(bandwidth);
2062}
2063
2064static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2065{
2066 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2067 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2068 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2069 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2070
2071 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2072}
2073
2074static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2075{
2076 /* Calculate the display mode Average Bandwidth
2077 * DisplayMode should contain the source and destination dimensions,
2078 * timing, etc.
2079 */
2080 fixed20_12 bpp;
2081 fixed20_12 line_time;
2082 fixed20_12 src_width;
2083 fixed20_12 bandwidth;
2084 fixed20_12 a;
2085
2086 a.full = dfixed_const(1000);
2087 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2088 line_time.full = dfixed_div(line_time, a);
2089 bpp.full = dfixed_const(wm->bytes_per_pixel);
2090 src_width.full = dfixed_const(wm->src_width);
2091 bandwidth.full = dfixed_mul(src_width, bpp);
2092 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2093 bandwidth.full = dfixed_div(bandwidth, line_time);
2094
2095 return dfixed_trunc(bandwidth);
2096}
2097
2098static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2099{
2100 /* First calcualte the latency in ns */
2101 u32 mc_latency = 2000; /* 2000 ns. */
2102 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2103 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2104 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2105 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2106 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2107 (wm->num_heads * cursor_line_pair_return_time);
2108 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2109 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2110 fixed20_12 a, b, c;
2111
2112 if (wm->num_heads == 0)
2113 return 0;
2114
2115 a.full = dfixed_const(2);
2116 b.full = dfixed_const(1);
2117 if ((wm->vsc.full > a.full) ||
2118 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2119 (wm->vtaps >= 5) ||
2120 ((wm->vsc.full >= a.full) && wm->interlaced))
2121 max_src_lines_per_dst_line = 4;
2122 else
2123 max_src_lines_per_dst_line = 2;
2124
2125 a.full = dfixed_const(available_bandwidth);
2126 b.full = dfixed_const(wm->num_heads);
2127 a.full = dfixed_div(a, b);
2128
2129 b.full = dfixed_const(1000);
2130 c.full = dfixed_const(wm->disp_clk);
2131 b.full = dfixed_div(c, b);
2132 c.full = dfixed_const(wm->bytes_per_pixel);
2133 b.full = dfixed_mul(b, c);
2134
2135 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2136
2137 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2138 b.full = dfixed_const(1000);
2139 c.full = dfixed_const(lb_fill_bw);
2140 b.full = dfixed_div(c, b);
2141 a.full = dfixed_div(a, b);
2142 line_fill_time = dfixed_trunc(a);
2143
2144 if (line_fill_time < wm->active_time)
2145 return latency;
2146 else
2147 return latency + (line_fill_time - wm->active_time);
2148
2149}
2150
2151static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2152{
2153 if (evergreen_average_bandwidth(wm) <=
2154 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2155 return true;
2156 else
2157 return false;
2158};
2159
2160static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2161{
2162 if (evergreen_average_bandwidth(wm) <=
2163 (evergreen_available_bandwidth(wm) / wm->num_heads))
2164 return true;
2165 else
2166 return false;
2167};
2168
2169static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2170{
2171 u32 lb_partitions = wm->lb_size / wm->src_width;
2172 u32 line_time = wm->active_time + wm->blank_time;
2173 u32 latency_tolerant_lines;
2174 u32 latency_hiding;
2175 fixed20_12 a;
2176
2177 a.full = dfixed_const(1);
2178 if (wm->vsc.full > a.full)
2179 latency_tolerant_lines = 1;
2180 else {
2181 if (lb_partitions <= (wm->vtaps + 1))
2182 latency_tolerant_lines = 1;
2183 else
2184 latency_tolerant_lines = 2;
2185 }
2186
2187 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2188
2189 if (evergreen_latency_watermark(wm) <= latency_hiding)
2190 return true;
2191 else
2192 return false;
2193}
2194
2195static void evergreen_program_watermarks(struct radeon_device *rdev,
2196 struct radeon_crtc *radeon_crtc,
2197 u32 lb_size, u32 num_heads)
2198{
2199 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2200 struct evergreen_wm_params wm_low, wm_high;
2201 u32 dram_channels;
f9d9c362
AD		 2202 u32 pixel_period;
2203 u32 line_time = 0;
2204 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2205 u32 priority_a_mark = 0, priority_b_mark = 0;
2206 u32 priority_a_cnt = PRIORITY_OFF;
2207 u32 priority_b_cnt = PRIORITY_OFF;
2208 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2209 u32 tmp, arb_control3;
2210 fixed20_12 a, b, c;
2211
2212 if (radeon_crtc->base.enabled && num_heads && mode) {
2213 pixel_period = 1000000 / (u32)mode->clock;
2214 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2215 priority_a_cnt = 0;
2216 priority_b_cnt = 0;
cf0cfdd7
AD		 2217 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2218
2219 /* watermark for high clocks */
2220 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2221 wm_high.yclk =
2222 radeon_dpm_get_mclk(rdev, false) * 10;
2223 wm_high.sclk =
2224 radeon_dpm_get_sclk(rdev, false) * 10;
2225 } else {
2226 wm_high.yclk = rdev->pm.current_mclk * 10;
2227 wm_high.sclk = rdev->pm.current_sclk * 10;
2228 }
f9d9c362 2229
cf0cfdd7
AD		 2230 wm_high.disp_clk = mode->clock;
2231 wm_high.src_width = mode->crtc_hdisplay;
2232 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2233 wm_high.blank_time = line_time - wm_high.active_time;
2234 wm_high.interlaced = false;
f9d9c362 2235 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2236 wm_high.interlaced = true;
2237 wm_high.vsc = radeon_crtc->vsc;
2238 wm_high.vtaps = 1;
f9d9c362 2239 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2240 wm_high.vtaps = 2;
2241 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2242 wm_high.lb_size = lb_size;
2243 wm_high.dram_channels = dram_channels;
2244 wm_high.num_heads = num_heads;
2245
2246 /* watermark for low clocks */
2247 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2248 wm_low.yclk =
2249 radeon_dpm_get_mclk(rdev, true) * 10;
2250 wm_low.sclk =
2251 radeon_dpm_get_sclk(rdev, true) * 10;
2252 } else {
2253 wm_low.yclk = rdev->pm.current_mclk * 10;
2254 wm_low.sclk = rdev->pm.current_sclk * 10;
2255 }
2256
2257 wm_low.disp_clk = mode->clock;
2258 wm_low.src_width = mode->crtc_hdisplay;
2259 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2260 wm_low.blank_time = line_time - wm_low.active_time;
2261 wm_low.interlaced = false;
2262 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2263 wm_low.interlaced = true;
2264 wm_low.vsc = radeon_crtc->vsc;
2265 wm_low.vtaps = 1;
2266 if (radeon_crtc->rmx_type != RMX_OFF)
2267 wm_low.vtaps = 2;
2268 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2269 wm_low.lb_size = lb_size;
2270 wm_low.dram_channels = dram_channels;
2271 wm_low.num_heads = num_heads;
f9d9c362
AD		 2272
2273 /* set for high clocks */
cf0cfdd7 2274 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2275 /* set for low clocks */
cf0cfdd7 2276 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2277
2278 /* possibly force display priority to high */
2279 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2280 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2281 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2282 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2283 (rdev->disp_priority == 2)) {
cf0cfdd7 2284 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2285 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2286 }
2287 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2288 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2289 !evergreen_check_latency_hiding(&wm_low) ||
2290 (rdev->disp_priority == 2)) {
2291 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2292 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2293 }
2294
2295 a.full = dfixed_const(1000);
2296 b.full = dfixed_const(mode->clock);
2297 b.full = dfixed_div(b, a);
2298 c.full = dfixed_const(latency_watermark_a);
2299 c.full = dfixed_mul(c, b);
2300 c.full = dfixed_mul(c, radeon_crtc->hsc);
2301 c.full = dfixed_div(c, a);
2302 a.full = dfixed_const(16);
2303 c.full = dfixed_div(c, a);
2304 priority_a_mark = dfixed_trunc(c);
2305 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2306
2307 a.full = dfixed_const(1000);
2308 b.full = dfixed_const(mode->clock);
2309 b.full = dfixed_div(b, a);
2310 c.full = dfixed_const(latency_watermark_b);
2311 c.full = dfixed_mul(c, b);
2312 c.full = dfixed_mul(c, radeon_crtc->hsc);
2313 c.full = dfixed_div(c, a);
2314 a.full = dfixed_const(16);
2315 c.full = dfixed_div(c, a);
2316 priority_b_mark = dfixed_trunc(c);
2317 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2318 }
2319
2320 /* select wm A */
2321 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2322 tmp = arb_control3;
2323 tmp &= ~LATENCY_WATERMARK_MASK(3);
2324 tmp |= LATENCY_WATERMARK_MASK(1);
2325 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2326 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2327 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2328 LATENCY_HIGH_WATERMARK(line_time)));
2329 /* select wm B */
2330 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2331 tmp &= ~LATENCY_WATERMARK_MASK(3);
2332 tmp |= LATENCY_WATERMARK_MASK(2);
2333 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2334 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2335 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2336 LATENCY_HIGH_WATERMARK(line_time)));
2337 /* restore original selection */
2338 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2339
2340 /* write the priority marks */
2341 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2342 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2343
7178d2a6
AD		 2344 /* save values for DPM */
2345 radeon_crtc->line_time = line_time;
2346 radeon_crtc->wm_high = latency_watermark_a;
2347 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2348}
2349
377edc8b
AD		 2350/**
2351 * evergreen_bandwidth_update - update display watermarks callback.
2352 *
2353 * @rdev: radeon_device pointer
2354 *
2355 * Update the display watermarks based on the requested mode(s)
2356 * (evergreen+).
2357 */
0ca2ab52 2358void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2359{
f9d9c362
AD		 2360 struct drm_display_mode *mode0 = NULL;
2361 struct drm_display_mode *mode1 = NULL;
2362 u32 num_heads = 0, lb_size;
2363 int i;
2364
2365 radeon_update_display_priority(rdev);
2366
2367 for (i = 0; i < rdev->num_crtc; i++) {
2368 if (rdev->mode_info.crtcs[i]->base.enabled)
2369 num_heads++;
2370 }
2371 for (i = 0; i < rdev->num_crtc; i += 2) {
2372 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2373 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2374 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2375 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2376 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2377 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2378 }
bcc1c2a1
AD		 2379}
2380
377edc8b
AD		 2381/**
2382 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2383 *
2384 * @rdev: radeon_device pointer
2385 *
2386 * Wait for the MC (memory controller) to be idle.
2387 * (evergreen+).
2388 * Returns 0 if the MC is idle, -1 if not.
2389 */
b9952a8a 2390int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2391{
2392 unsigned i;
2393 u32 tmp;
2394
2395 for (i = 0; i < rdev->usec_timeout; i++) {
2396 /* read MC_STATUS */
2397 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2398 if (!tmp)
2399 return 0;
2400 udelay(1);
2401 }
2402 return -1;
2403}
2404
2405/*
2406 * GART
2407 */
0fcdb61e
AD		 2408void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2409{
2410 unsigned i;
2411 u32 tmp;
2412
6f2f48a9
AD		 2413 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2414
0fcdb61e
AD		 2415 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2416 for (i = 0; i < rdev->usec_timeout; i++) {
2417 /* read MC_STATUS */
2418 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2419 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2420 if (tmp == 2) {
2421 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2422 return;
2423 }
2424 if (tmp) {
2425 return;
2426 }
2427 udelay(1);
2428 }
2429}
2430
1109ca09 2431static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2432{
2433 u32 tmp;
0fcdb61e 2434 int r;
bcc1c2a1 2435
c9a1be96 2436 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2437 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2438 return -EINVAL;
2439 }
2440 r = radeon_gart_table_vram_pin(rdev);
2441 if (r)
2442 return r;
82568565 2443 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2444 /* Setup L2 cache */
2445 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2446 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2447 EFFECTIVE_L2_QUEUE_SIZE(7));
2448 WREG32(VM_L2_CNTL2, 0);
2449 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2450 /* Setup TLB control */
2451 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2452 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2453 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2454 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2455 if (rdev->flags & RADEON_IS_IGP) {
2456 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2457 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2458 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2459 } else {
2460 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2461 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2462 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2463 if ((rdev->family == CHIP_JUNIPER) ||
2464 (rdev->family == CHIP_CYPRESS) ||
2465 (rdev->family == CHIP_HEMLOCK) ||
2466 (rdev->family == CHIP_BARTS))
2467 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2468 }
bcc1c2a1
AD		 2469 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2473 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2474 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2475 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2476 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2477 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2478 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2479 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2480 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2481
0fcdb61e 2482 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2483 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2484 (unsigned)(rdev->mc.gtt_size >> 20),
2485 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2486 rdev->gart.ready = true;
2487 return 0;
2488}
2489
1109ca09 2490static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2491{
2492 u32 tmp;
bcc1c2a1
AD		 2493
2494 /* Disable all tables */
0fcdb61e
AD		 2495 WREG32(VM_CONTEXT0_CNTL, 0);
2496 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2497
2498 /* Setup L2 cache */
2499 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2500 EFFECTIVE_L2_QUEUE_SIZE(7));
2501 WREG32(VM_L2_CNTL2, 0);
2502 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2503 /* Setup TLB control */
2504 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2505 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2506 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2507 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2508 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2510 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2512 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2513}
2514
1109ca09 2515static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2516{
2517 evergreen_pcie_gart_disable(rdev);
2518 radeon_gart_table_vram_free(rdev);
2519 radeon_gart_fini(rdev);
2520}
2521
2522
1109ca09 2523static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2524{
2525 u32 tmp;
bcc1c2a1
AD		 2526
2527 /* Setup L2 cache */
2528 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2529 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2530 EFFECTIVE_L2_QUEUE_SIZE(7));
2531 WREG32(VM_L2_CNTL2, 0);
2532 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2533 /* Setup TLB control */
2534 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2535 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2536 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2537 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2538 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2539 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2540 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2541 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2542 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2543 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2544 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2545 WREG32(VM_CONTEXT0_CNTL, 0);
2546 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2547}
2548
b9952a8a 2549void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2550{
62444b74
AD		 2551 u32 crtc_enabled, tmp, frame_count, blackout;
2552 int i, j;
2553
5153550a
AD		 2554 if (!ASIC_IS_NODCE(rdev)) {
2555 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2556 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2557
5153550a
AD		 2558 /* disable VGA render */
2559 WREG32(VGA_RENDER_CONTROL, 0);
2560 }
62444b74
AD		 2561 /* blank the display controllers */
2562 for (i = 0; i < rdev->num_crtc; i++) {
2563 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2564 if (crtc_enabled) {
2565 save->crtc_enabled[i] = true;
2566 if (ASIC_IS_DCE6(rdev)) {
2567 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2568 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2569 radeon_wait_for_vblank(rdev, i);
abf1457b 2570 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2571 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2572 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2573 }
2574 } else {
2575 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2576 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2577 radeon_wait_for_vblank(rdev, i);
abf1457b 2578 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2579 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2580 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2581 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2582 }
2583 }
2584 /* wait for the next frame */
2585 frame_count = radeon_get_vblank_counter(rdev, i);
2586 for (j = 0; j < rdev->usec_timeout; j++) {
2587 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2588 break;
2589 udelay(1);
2590 }
abf1457b
AD		 2591
2592 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2593 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2594 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2595 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2596 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2597 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2598 save->crtc_enabled[i] = false;
2599 /* ***** */
804cc4a0
AD		 2600 } else {
2601 save->crtc_enabled[i] = false;
62444b74 2602 }
18007401 2603 }
bcc1c2a1 2604
62444b74
AD		 2605 radeon_mc_wait_for_idle(rdev);
2606
2607 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2608 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2609 /* Block CPU access */
2610 WREG32(BIF_FB_EN, 0);
2611 /* blackout the MC */
2612 blackout &= ~BLACKOUT_MODE_MASK;
2613 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2614 }
ed39fadd
AD		 2615 /* wait for the MC to settle */
2616 udelay(100);
968c0166
AD		 2617
2618 /* lock double buffered regs */
2619 for (i = 0; i < rdev->num_crtc; i++) {
2620 if (save->crtc_enabled[i]) {
2621 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2622 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2623 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2624 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2625 }
2626 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2627 if (!(tmp & 1)) {
2628 tmp |= 1;
2629 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2630 }
2631 }
2632 }
bcc1c2a1
AD		 2633}
2634
b9952a8a 2635void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2636{
62444b74
AD		 2637 u32 tmp, frame_count;
2638 int i, j;
18007401 2639
62444b74
AD		 2640 /* update crtc base addresses */
2641 for (i = 0; i < rdev->num_crtc; i++) {
2642 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2643 upper_32_bits(rdev->mc.vram_start));
62444b74 2644 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2645 upper_32_bits(rdev->mc.vram_start));
62444b74 2646 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2647 (u32)rdev->mc.vram_start);
62444b74 2648 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2649 (u32)rdev->mc.vram_start);
2650 }
5153550a
AD		 2651
2652 if (!ASIC_IS_NODCE(rdev)) {
2653 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2654 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2655 }
62444b74 2656
968c0166
AD		 2657 /* unlock regs and wait for update */
2658 for (i = 0; i < rdev->num_crtc; i++) {
2659 if (save->crtc_enabled[i]) {
2660 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2661 if ((tmp & 0x3) != 0) {
2662 tmp &= ~0x3;
2663 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2664 }
2665 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2666 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2667 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2668 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2669 }
2670 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2671 if (tmp & 1) {
2672 tmp &= ~1;
2673 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2674 }
2675 for (j = 0; j < rdev->usec_timeout; j++) {
2676 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2677 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2678 break;
2679 udelay(1);
2680 }
2681 }
2682 }
2683
62444b74
AD		 2684 /* unblackout the MC */
2685 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2686 tmp &= ~BLACKOUT_MODE_MASK;
2687 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2688 /* allow CPU access */
2689 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2690
2691 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2692 if (save->crtc_enabled[i]) {
62444b74
AD		 2693 if (ASIC_IS_DCE6(rdev)) {
2694 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2695 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2696 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2697 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2698 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2699 } else {
2700 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2701 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2702 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2703 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2704 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2705 }
2706 /* wait for the next frame */
2707 frame_count = radeon_get_vblank_counter(rdev, i);
2708 for (j = 0; j < rdev->usec_timeout; j++) {
2709 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2710 break;
2711 udelay(1);
2712 }
2713 }
2714 }
5153550a
AD		 2715 if (!ASIC_IS_NODCE(rdev)) {
2716 /* Unlock vga access */
2717 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2718 mdelay(1);
2719 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2720 }
bcc1c2a1
AD		 2721}
2722
755d819e 2723void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2724{
2725 struct evergreen_mc_save save;
2726 u32 tmp;
2727 int i, j;
2728
2729 /* Initialize HDP */
2730 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2731 WREG32((0x2c14 + j), 0x00000000);
2732 WREG32((0x2c18 + j), 0x00000000);
2733 WREG32((0x2c1c + j), 0x00000000);
2734 WREG32((0x2c20 + j), 0x00000000);
2735 WREG32((0x2c24 + j), 0x00000000);
2736 }
2737 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2738
2739 evergreen_mc_stop(rdev, &save);
2740 if (evergreen_mc_wait_for_idle(rdev)) {
2741 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2742 }
2743 /* Lockout access through VGA aperture*/
2744 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2745 /* Update configuration */
2746 if (rdev->flags & RADEON_IS_AGP) {
2747 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2748 /* VRAM before AGP */
2749 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2750 rdev->mc.vram_start >> 12);
2751 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2752 rdev->mc.gtt_end >> 12);
2753 } else {
2754 /* VRAM after AGP */
2755 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2756 rdev->mc.gtt_start >> 12);
2757 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2758 rdev->mc.vram_end >> 12);
2759 }
2760 } else {
2761 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2762 rdev->mc.vram_start >> 12);
2763 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2764 rdev->mc.vram_end >> 12);
2765 }
3b9832f6 2766 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2767 /* llano/ontario only */
2768 if ((rdev->family == CHIP_PALM) ||
2769 (rdev->family == CHIP_SUMO) ||
2770 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2771 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2772 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2773 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2774 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2775 }
bcc1c2a1
AD		 2776 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2777 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2778 WREG32(MC_VM_FB_LOCATION, tmp);
2779 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2780 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2781 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2782 if (rdev->flags & RADEON_IS_AGP) {
2783 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2784 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2785 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2786 } else {
2787 WREG32(MC_VM_AGP_BASE, 0);
2788 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2789 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2790 }
2791 if (evergreen_mc_wait_for_idle(rdev)) {
2792 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2793 }
2794 evergreen_mc_resume(rdev, &save);
2795 /* we need to own VRAM, so turn off the VGA renderer here
2796 * to stop it overwriting our objects */
2797 rv515_vga_render_disable(rdev);
2798}
2799
bcc1c2a1
AD		 2800/*
2801 * CP.
2802 */
12920591
AD		 2803void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2804{
876dc9f3 2805 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2806 u32 next_rptr;
7b1f2485 2807
12920591 2808 /* set to DX10/11 mode */
e32eb50d
CK		 2809 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2810 radeon_ring_write(ring, 1);
45df6803
CK		 2811
2812 if (ring->rptr_save_reg) {
89d35807 2813 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2814 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2815 radeon_ring_write(ring, ((ring->rptr_save_reg -
2816 PACKET3_SET_CONFIG_REG_START) >> 2));
2817 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2818 } else if (rdev->wb.enabled) {
2819 next_rptr = ring->wptr + 5 + 4;
2820 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2821 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2822 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2823 radeon_ring_write(ring, next_rptr);
2824 radeon_ring_write(ring, 0);
45df6803
CK		 2825 }
2826
e32eb50d
CK		 2827 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2828 radeon_ring_write(ring,
0f234f5f
AD		 2829#ifdef __BIG_ENDIAN
2830 (2 << 0) |
2831#endif
2832 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2833 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2834 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2835}
2836
bcc1c2a1
AD		 2837
2838static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2839{
fe251e2f
AD		 2840 const __be32 *fw_data;
2841 int i;
2842
2843 if (!rdev->me_fw || !rdev->pfp_fw)
2844 return -EINVAL;
bcc1c2a1 2845
fe251e2f 2846 r700_cp_stop(rdev);
0f234f5f
AD		 2847 WREG32(CP_RB_CNTL,
2848#ifdef __BIG_ENDIAN
2849 BUF_SWAP_32BIT |
2850#endif
2851 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2852
2853 fw_data = (const __be32 *)rdev->pfp_fw->data;
2854 WREG32(CP_PFP_UCODE_ADDR, 0);
2855 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2856 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2857 WREG32(CP_PFP_UCODE_ADDR, 0);
2858
2859 fw_data = (const __be32 *)rdev->me_fw->data;
2860 WREG32(CP_ME_RAM_WADDR, 0);
2861 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2862 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2863
2864 WREG32(CP_PFP_UCODE_ADDR, 0);
2865 WREG32(CP_ME_RAM_WADDR, 0);
2866 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2867 return 0;
2868}
2869
7e7b41d2
AD		 2870static int evergreen_cp_start(struct radeon_device *rdev)
2871{
e32eb50d 2872 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2873 int r, i;
7e7b41d2
AD		 2874 uint32_t cp_me;
2875
e32eb50d 2876 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2877 if (r) {
2878 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2879 return r;
2880 }
e32eb50d
CK		 2881 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2882 radeon_ring_write(ring, 0x1);
2883 radeon_ring_write(ring, 0x0);
2884 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2885 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2886 radeon_ring_write(ring, 0);
2887 radeon_ring_write(ring, 0);
2888 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2889
2890 cp_me = 0xff;
2891 WREG32(CP_ME_CNTL, cp_me);
2892
e32eb50d 2893 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2894 if (r) {
2895 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2896 return r;
2897 }
2281a378
AD		 2898
2899 /* setup clear context state */
e32eb50d
CK		 2900 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2901 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2902
2903 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2904 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2905
e32eb50d
CK		 2906 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2907 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2908
2909 /* set clear context state */
e32eb50d
CK		 2910 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2911 radeon_ring_write(ring, 0);
2281a378
AD		 2912
2913 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2914 radeon_ring_write(ring, 0xc0026f00);
2915 radeon_ring_write(ring, 0x00000000);
2916 radeon_ring_write(ring, 0x00000000);
2917 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2918
2919 /* Clear consts */
e32eb50d
CK		 2920 radeon_ring_write(ring, 0xc0036f00);
2921 radeon_ring_write(ring, 0x00000bc4);
2922 radeon_ring_write(ring, 0xffffffff);
2923 radeon_ring_write(ring, 0xffffffff);
2924 radeon_ring_write(ring, 0xffffffff);
2281a378 2925
e32eb50d
CK		 2926 radeon_ring_write(ring, 0xc0026900);
2927 radeon_ring_write(ring, 0x00000316);
2928 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2929 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2930
e32eb50d 2931 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2932
2933 return 0;
2934}
2935
1109ca09 2936static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2937{
e32eb50d 2938 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2939 u32 tmp;
2940 u32 rb_bufsz;
2941 int r;
2942
2943 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2944 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2945 SOFT_RESET_PA |
2946 SOFT_RESET_SH |
2947 SOFT_RESET_VGT |
a49a50da 2948 SOFT_RESET_SPI |
fe251e2f
AD		 2949 SOFT_RESET_SX));
2950 RREG32(GRBM_SOFT_RESET);
2951 mdelay(15);
2952 WREG32(GRBM_SOFT_RESET, 0);
2953 RREG32(GRBM_SOFT_RESET);
2954
2955 /* Set ring buffer size */
b72a8925
DV		 2956 rb_bufsz = order_base_2(ring->ring_size / 8);
2957 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2958#ifdef __BIG_ENDIAN
2959 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2960#endif
fe251e2f 2961 WREG32(CP_RB_CNTL, tmp);
15d3332f 2962 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2963 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2964
2965 /* Set the write pointer delay */
2966 WREG32(CP_RB_WPTR_DELAY, 0);
2967
2968 /* Initialize the ring buffer's read and write pointers */
2969 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2970 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2971 ring->wptr = 0;
2972 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2973
48fc7f7e 2974 /* set the wb address whether it's enabled or not */
0f234f5f 2975 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2976 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2977 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2978 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2979
2980 if (rdev->wb.enabled)
2981 WREG32(SCRATCH_UMSK, 0xff);
2982 else {
2983 tmp |= RB_NO_UPDATE;
2984 WREG32(SCRATCH_UMSK, 0);
2985 }
2986
fe251e2f
AD		 2987 mdelay(1);
2988 WREG32(CP_RB_CNTL, tmp);
2989
e32eb50d 2990 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2991 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2992
e32eb50d 2993 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2994
7e7b41d2 2995 evergreen_cp_start(rdev);
e32eb50d 2996 ring->ready = true;
f712812e 2997 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2998 if (r) {
e32eb50d 2999 ring->ready = false;
fe251e2f
AD		 3000 return r;
3001 }
3002 return 0;
3003}
bcc1c2a1
AD		 3004
3005/*
3006 * Core functions
3007 */
bcc1c2a1
AD		 3008static void evergreen_gpu_init(struct radeon_device *rdev)
3009{
416a2bd2 3010 u32 gb_addr_config;
32fcdbf4 3011 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3012 u32 sx_debug_1;
3013 u32 smx_dc_ctl0;
3014 u32 sq_config;
3015 u32 sq_lds_resource_mgmt;
3016 u32 sq_gpr_resource_mgmt_1;
3017 u32 sq_gpr_resource_mgmt_2;
3018 u32 sq_gpr_resource_mgmt_3;
3019 u32 sq_thread_resource_mgmt;
3020 u32 sq_thread_resource_mgmt_2;
3021 u32 sq_stack_resource_mgmt_1;
3022 u32 sq_stack_resource_mgmt_2;
3023 u32 sq_stack_resource_mgmt_3;
3024 u32 vgt_cache_invalidation;
f25a5c63 3025 u32 hdp_host_path_cntl, tmp;
416a2bd2 3026 u32 disabled_rb_mask;
32fcdbf4
AD		 3027 int i, j, num_shader_engines, ps_thread_count;
3028
3029 switch (rdev->family) {
3030 case CHIP_CYPRESS:
3031 case CHIP_HEMLOCK:
3032 rdev->config.evergreen.num_ses = 2;
3033 rdev->config.evergreen.max_pipes = 4;
3034 rdev->config.evergreen.max_tile_pipes = 8;
3035 rdev->config.evergreen.max_simds = 10;
3036 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3037 rdev->config.evergreen.max_gprs = 256;
3038 rdev->config.evergreen.max_threads = 248;
3039 rdev->config.evergreen.max_gs_threads = 32;
3040 rdev->config.evergreen.max_stack_entries = 512;
3041 rdev->config.evergreen.sx_num_of_sets = 4;
3042 rdev->config.evergreen.sx_max_export_size = 256;
3043 rdev->config.evergreen.sx_max_export_pos_size = 64;
3044 rdev->config.evergreen.sx_max_export_smx_size = 192;
3045 rdev->config.evergreen.max_hw_contexts = 8;
3046 rdev->config.evergreen.sq_num_cf_insts = 2;
3047
3048 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3049 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3050 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3051 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3052 break;
3053 case CHIP_JUNIPER:
3054 rdev->config.evergreen.num_ses = 1;
3055 rdev->config.evergreen.max_pipes = 4;
3056 rdev->config.evergreen.max_tile_pipes = 4;
3057 rdev->config.evergreen.max_simds = 10;
3058 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3059 rdev->config.evergreen.max_gprs = 256;
3060 rdev->config.evergreen.max_threads = 248;
3061 rdev->config.evergreen.max_gs_threads = 32;
3062 rdev->config.evergreen.max_stack_entries = 512;
3063 rdev->config.evergreen.sx_num_of_sets = 4;
3064 rdev->config.evergreen.sx_max_export_size = 256;
3065 rdev->config.evergreen.sx_max_export_pos_size = 64;
3066 rdev->config.evergreen.sx_max_export_smx_size = 192;
3067 rdev->config.evergreen.max_hw_contexts = 8;
3068 rdev->config.evergreen.sq_num_cf_insts = 2;
3069
3070 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3071 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3072 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3073 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3074 break;
3075 case CHIP_REDWOOD:
3076 rdev->config.evergreen.num_ses = 1;
3077 rdev->config.evergreen.max_pipes = 4;
3078 rdev->config.evergreen.max_tile_pipes = 4;
3079 rdev->config.evergreen.max_simds = 5;
3080 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3081 rdev->config.evergreen.max_gprs = 256;
3082 rdev->config.evergreen.max_threads = 248;
3083 rdev->config.evergreen.max_gs_threads = 32;
3084 rdev->config.evergreen.max_stack_entries = 256;
3085 rdev->config.evergreen.sx_num_of_sets = 4;
3086 rdev->config.evergreen.sx_max_export_size = 256;
3087 rdev->config.evergreen.sx_max_export_pos_size = 64;
3088 rdev->config.evergreen.sx_max_export_smx_size = 192;
3089 rdev->config.evergreen.max_hw_contexts = 8;
3090 rdev->config.evergreen.sq_num_cf_insts = 2;
3091
3092 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3093 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3094 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3095 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3096 break;
3097 case CHIP_CEDAR:
3098 default:
3099 rdev->config.evergreen.num_ses = 1;
3100 rdev->config.evergreen.max_pipes = 2;
3101 rdev->config.evergreen.max_tile_pipes = 2;
3102 rdev->config.evergreen.max_simds = 2;
3103 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3104 rdev->config.evergreen.max_gprs = 256;
3105 rdev->config.evergreen.max_threads = 192;
3106 rdev->config.evergreen.max_gs_threads = 16;
3107 rdev->config.evergreen.max_stack_entries = 256;
3108 rdev->config.evergreen.sx_num_of_sets = 4;
3109 rdev->config.evergreen.sx_max_export_size = 128;
3110 rdev->config.evergreen.sx_max_export_pos_size = 32;
3111 rdev->config.evergreen.sx_max_export_smx_size = 96;
3112 rdev->config.evergreen.max_hw_contexts = 4;
3113 rdev->config.evergreen.sq_num_cf_insts = 1;
3114
d5e455e4
AD		 3115 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3118 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3119 break;
3120 case CHIP_PALM:
3121 rdev->config.evergreen.num_ses = 1;
3122 rdev->config.evergreen.max_pipes = 2;
3123 rdev->config.evergreen.max_tile_pipes = 2;
3124 rdev->config.evergreen.max_simds = 2;
3125 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3126 rdev->config.evergreen.max_gprs = 256;
3127 rdev->config.evergreen.max_threads = 192;
3128 rdev->config.evergreen.max_gs_threads = 16;
3129 rdev->config.evergreen.max_stack_entries = 256;
3130 rdev->config.evergreen.sx_num_of_sets = 4;
3131 rdev->config.evergreen.sx_max_export_size = 128;
3132 rdev->config.evergreen.sx_max_export_pos_size = 32;
3133 rdev->config.evergreen.sx_max_export_smx_size = 96;
3134 rdev->config.evergreen.max_hw_contexts = 4;
3135 rdev->config.evergreen.sq_num_cf_insts = 1;
3136
d5c5a72f
AD		 3137 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3138 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3140 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3141 break;
3142 case CHIP_SUMO:
3143 rdev->config.evergreen.num_ses = 1;
3144 rdev->config.evergreen.max_pipes = 4;
bd25f078 3145 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3146 if (rdev->pdev->device == 0x9648)
3147 rdev->config.evergreen.max_simds = 3;
3148 else if ((rdev->pdev->device == 0x9647) ||
3149 (rdev->pdev->device == 0x964a))
3150 rdev->config.evergreen.max_simds = 4;
3151 else
3152 rdev->config.evergreen.max_simds = 5;
3153 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3154 rdev->config.evergreen.max_gprs = 256;
3155 rdev->config.evergreen.max_threads = 248;
3156 rdev->config.evergreen.max_gs_threads = 32;
3157 rdev->config.evergreen.max_stack_entries = 256;
3158 rdev->config.evergreen.sx_num_of_sets = 4;
3159 rdev->config.evergreen.sx_max_export_size = 256;
3160 rdev->config.evergreen.sx_max_export_pos_size = 64;
3161 rdev->config.evergreen.sx_max_export_smx_size = 192;
3162 rdev->config.evergreen.max_hw_contexts = 8;
3163 rdev->config.evergreen.sq_num_cf_insts = 2;
3164
3165 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3166 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3167 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3168 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3169 break;
3170 case CHIP_SUMO2:
3171 rdev->config.evergreen.num_ses = 1;
3172 rdev->config.evergreen.max_pipes = 4;
3173 rdev->config.evergreen.max_tile_pipes = 4;
3174 rdev->config.evergreen.max_simds = 2;
3175 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3176 rdev->config.evergreen.max_gprs = 256;
3177 rdev->config.evergreen.max_threads = 248;
3178 rdev->config.evergreen.max_gs_threads = 32;
3179 rdev->config.evergreen.max_stack_entries = 512;
3180 rdev->config.evergreen.sx_num_of_sets = 4;
3181 rdev->config.evergreen.sx_max_export_size = 256;
3182 rdev->config.evergreen.sx_max_export_pos_size = 64;
3183 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3184 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3185 rdev->config.evergreen.sq_num_cf_insts = 2;
3186
adb68fa2
AD		 3187 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3188 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3189 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3190 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3191 break;
3192 case CHIP_BARTS:
3193 rdev->config.evergreen.num_ses = 2;
3194 rdev->config.evergreen.max_pipes = 4;
3195 rdev->config.evergreen.max_tile_pipes = 8;
3196 rdev->config.evergreen.max_simds = 7;
3197 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3198 rdev->config.evergreen.max_gprs = 256;
3199 rdev->config.evergreen.max_threads = 248;
3200 rdev->config.evergreen.max_gs_threads = 32;
3201 rdev->config.evergreen.max_stack_entries = 512;
3202 rdev->config.evergreen.sx_num_of_sets = 4;
3203 rdev->config.evergreen.sx_max_export_size = 256;
3204 rdev->config.evergreen.sx_max_export_pos_size = 64;
3205 rdev->config.evergreen.sx_max_export_smx_size = 192;
3206 rdev->config.evergreen.max_hw_contexts = 8;
3207 rdev->config.evergreen.sq_num_cf_insts = 2;
3208
3209 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3210 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3211 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3212 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3213 break;
3214 case CHIP_TURKS:
3215 rdev->config.evergreen.num_ses = 1;
3216 rdev->config.evergreen.max_pipes = 4;
3217 rdev->config.evergreen.max_tile_pipes = 4;
3218 rdev->config.evergreen.max_simds = 6;
3219 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3220 rdev->config.evergreen.max_gprs = 256;
3221 rdev->config.evergreen.max_threads = 248;
3222 rdev->config.evergreen.max_gs_threads = 32;
3223 rdev->config.evergreen.max_stack_entries = 256;
3224 rdev->config.evergreen.sx_num_of_sets = 4;
3225 rdev->config.evergreen.sx_max_export_size = 256;
3226 rdev->config.evergreen.sx_max_export_pos_size = 64;
3227 rdev->config.evergreen.sx_max_export_smx_size = 192;
3228 rdev->config.evergreen.max_hw_contexts = 8;
3229 rdev->config.evergreen.sq_num_cf_insts = 2;
3230
3231 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3232 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3233 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3234 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3235 break;
3236 case CHIP_CAICOS:
3237 rdev->config.evergreen.num_ses = 1;
bd25f078 3238 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3239 rdev->config.evergreen.max_tile_pipes = 2;
3240 rdev->config.evergreen.max_simds = 2;
3241 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3242 rdev->config.evergreen.max_gprs = 256;
3243 rdev->config.evergreen.max_threads = 192;
3244 rdev->config.evergreen.max_gs_threads = 16;
3245 rdev->config.evergreen.max_stack_entries = 256;
3246 rdev->config.evergreen.sx_num_of_sets = 4;
3247 rdev->config.evergreen.sx_max_export_size = 128;
3248 rdev->config.evergreen.sx_max_export_pos_size = 32;
3249 rdev->config.evergreen.sx_max_export_smx_size = 96;
3250 rdev->config.evergreen.max_hw_contexts = 4;
3251 rdev->config.evergreen.sq_num_cf_insts = 1;
3252
32fcdbf4
AD		 3253 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3254 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3255 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3256 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3257 break;
3258 }
3259
3260 /* Initialize HDP */
3261 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3262 WREG32((0x2c14 + j), 0x00000000);
3263 WREG32((0x2c18 + j), 0x00000000);
3264 WREG32((0x2c1c + j), 0x00000000);
3265 WREG32((0x2c20 + j), 0x00000000);
3266 WREG32((0x2c24 + j), 0x00000000);
3267 }
3268
3269 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3270
d054ac16
AD		 3271 evergreen_fix_pci_max_read_req_size(rdev);
3272
32fcdbf4 3273 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3274 if ((rdev->family == CHIP_PALM) ||
3275 (rdev->family == CHIP_SUMO) ||
3276 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3277 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3278 else
3279 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3280
1aa52bd3
AD		 3281 /* setup tiling info dword. gb_addr_config is not adequate since it does
3282 * not have bank info, so create a custom tiling dword.
3283 * bits 3:0 num_pipes
3284 * bits 7:4 num_banks
3285 * bits 11:8 group_size
3286 * bits 15:12 row_size
3287 */
3288 rdev->config.evergreen.tile_config = 0;
3289 switch (rdev->config.evergreen.max_tile_pipes) {
3290 case 1:
3291 default:
3292 rdev->config.evergreen.tile_config |= (0 << 0);
3293 break;
3294 case 2:
3295 rdev->config.evergreen.tile_config |= (1 << 0);
3296 break;
3297 case 4:
3298 rdev->config.evergreen.tile_config |= (2 << 0);
3299 break;
3300 case 8:
3301 rdev->config.evergreen.tile_config |= (3 << 0);
3302 break;
3303 }
d698a34d 3304 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3305 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3306 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3307 else {
c8d15edc
AD		 3308 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3309 case 0: /* four banks */
29d65406 3310 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3311 break;
3312 case 1: /* eight banks */
3313 rdev->config.evergreen.tile_config |= 1 << 4;
3314 break;
3315 case 2: /* sixteen banks */
3316 default:
3317 rdev->config.evergreen.tile_config |= 2 << 4;
3318 break;
3319 }
29d65406 3320 }
416a2bd2 3321 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3322 rdev->config.evergreen.tile_config |=
3323 ((gb_addr_config & 0x30000000) >> 28) << 12;
3324
416a2bd2 3325 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3326
416a2bd2
AD		 3327 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3328 u32 efuse_straps_4;
3329 u32 efuse_straps_3;
32fcdbf4 3330
ff82bbc4
AD		 3331 efuse_straps_4 = RREG32_RCU(0x204);
3332 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3333 tmp = (((efuse_straps_4 & 0xf) << 4) |
3334 ((efuse_straps_3 & 0xf0000000) >> 28));
3335 } else {
3336 tmp = 0;
3337 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3338 u32 rb_disable_bitmap;
3339
3340 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3341 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3342 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3343 tmp <<= 4;
3344 tmp |= rb_disable_bitmap;
32fcdbf4 3345 }
416a2bd2
AD		 3346 }
3347 /* enabled rb are just the one not disabled :) */
3348 disabled_rb_mask = tmp;
cedb655a
AD		 3349 tmp = 0;
3350 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3351 tmp |= (1 << i);
3352 /* if all the backends are disabled, fix it up here */
3353 if ((disabled_rb_mask & tmp) == tmp) {
3354 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3355 disabled_rb_mask &= ~(1 << i);
3356 }
32fcdbf4 3357
416a2bd2
AD		 3358 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3359 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3360
416a2bd2
AD		 3361 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3362 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3363 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3364 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3365 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3366 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3367 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3368
f7eb9730
AD		 3369 if ((rdev->config.evergreen.max_backends == 1) &&
3370 (rdev->flags & RADEON_IS_IGP)) {
3371 if ((disabled_rb_mask & 3) == 1) {
3372 /* RB0 disabled, RB1 enabled */
3373 tmp = 0x11111111;
3374 } else {
3375 /* RB1 disabled, RB0 enabled */
3376 tmp = 0x00000000;
3377 }
3378 } else {
3379 tmp = gb_addr_config & NUM_PIPES_MASK;
3380 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3381 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3382 }
416a2bd2 3383 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3384
3385 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3386 WREG32(CGTS_TCC_DISABLE, 0);
3387 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3388 WREG32(CGTS_USER_TCC_DISABLE, 0);
3389
3390 /* set HW defaults for 3D engine */
3391 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3392 ROQ_IB2_START(0x2b)));
3393
3394 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3395
3396 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3397 SYNC_GRADIENT |
3398 SYNC_WALKER |
3399 SYNC_ALIGNER));
3400
3401 sx_debug_1 = RREG32(SX_DEBUG_1);
3402 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3403 WREG32(SX_DEBUG_1, sx_debug_1);
3404
3405
3406 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3407 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3408 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3409 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3410
b866d133
AD		 3411 if (rdev->family <= CHIP_SUMO2)
3412 WREG32(SMX_SAR_CTL0, 0x00010000);
3413
32fcdbf4
AD		 3414 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3415 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3416 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3417
3418 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3419 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3420 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3421
3422 WREG32(VGT_NUM_INSTANCES, 1);
3423 WREG32(SPI_CONFIG_CNTL, 0);
3424 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3425 WREG32(CP_PERFMON_CNTL, 0);
3426
3427 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3428 FETCH_FIFO_HIWATER(0x4) |
3429 DONE_FIFO_HIWATER(0xe0) |
3430 ALU_UPDATE_FIFO_HIWATER(0x8)));
3431
3432 sq_config = RREG32(SQ_CONFIG);
3433 sq_config &= ~(PS_PRIO(3) |
3434 VS_PRIO(3) |
3435 GS_PRIO(3) |
3436 ES_PRIO(3));
3437 sq_config |= (VC_ENABLE |
3438 EXPORT_SRC_C |
3439 PS_PRIO(0) |
3440 VS_PRIO(1) |
3441 GS_PRIO(2) |
3442 ES_PRIO(3));
3443
d5e455e4
AD		 3444 switch (rdev->family) {
3445 case CHIP_CEDAR:
3446 case CHIP_PALM:
d5c5a72f
AD		 3447 case CHIP_SUMO:
3448 case CHIP_SUMO2:
adb68fa2 3449 case CHIP_CAICOS:
32fcdbf4
AD		 3450 /* no vertex cache */
3451 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3452 break;
3453 default:
3454 break;
3455 }
32fcdbf4
AD		 3456
3457 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3458
3459 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3460 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3461 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3462 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3463 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3464 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3465 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3466
d5e455e4
AD		 3467 switch (rdev->family) {
3468 case CHIP_CEDAR:
3469 case CHIP_PALM:
d5c5a72f
AD		 3470 case CHIP_SUMO:
3471 case CHIP_SUMO2:
32fcdbf4 3472 ps_thread_count = 96;
d5e455e4
AD		 3473 break;
3474 default:
32fcdbf4 3475 ps_thread_count = 128;
d5e455e4
AD		 3476 break;
3477 }
32fcdbf4
AD		 3478
3479 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3480 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3481 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3482 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3483 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3484 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3485
3486 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3487 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3488 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3489 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3490 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3491 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3492
3493 WREG32(SQ_CONFIG, sq_config);
3494 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3495 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3496 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3497 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3498 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3499 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3500 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3501 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3502 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3503 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3504
3505 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3506 FORCE_EOV_MAX_REZ_CNT(255)));
3507
d5e455e4
AD		 3508 switch (rdev->family) {
3509 case CHIP_CEDAR:
3510 case CHIP_PALM:
d5c5a72f
AD		 3511 case CHIP_SUMO:
3512 case CHIP_SUMO2:
adb68fa2 3513 case CHIP_CAICOS:
32fcdbf4 3514 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3515 break;
3516 default:
32fcdbf4 3517 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3518 break;
3519 }
32fcdbf4
AD		 3520 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3521 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3522
3523 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3524 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3525 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3526
60a4a3e0
AD		 3527 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3528 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3529
32fcdbf4
AD		 3530 WREG32(CB_PERF_CTR0_SEL_0, 0);
3531 WREG32(CB_PERF_CTR0_SEL_1, 0);
3532 WREG32(CB_PERF_CTR1_SEL_0, 0);
3533 WREG32(CB_PERF_CTR1_SEL_1, 0);
3534 WREG32(CB_PERF_CTR2_SEL_0, 0);
3535 WREG32(CB_PERF_CTR2_SEL_1, 0);
3536 WREG32(CB_PERF_CTR3_SEL_0, 0);
3537 WREG32(CB_PERF_CTR3_SEL_1, 0);
3538
60a4a3e0
AD		 3539 /* clear render buffer base addresses */
3540 WREG32(CB_COLOR0_BASE, 0);
3541 WREG32(CB_COLOR1_BASE, 0);
3542 WREG32(CB_COLOR2_BASE, 0);
3543 WREG32(CB_COLOR3_BASE, 0);
3544 WREG32(CB_COLOR4_BASE, 0);
3545 WREG32(CB_COLOR5_BASE, 0);
3546 WREG32(CB_COLOR6_BASE, 0);
3547 WREG32(CB_COLOR7_BASE, 0);
3548 WREG32(CB_COLOR8_BASE, 0);
3549 WREG32(CB_COLOR9_BASE, 0);
3550 WREG32(CB_COLOR10_BASE, 0);
3551 WREG32(CB_COLOR11_BASE, 0);
3552
3553 /* set the shader const cache sizes to 0 */
3554 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3555 WREG32(i, 0);
3556 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3557 WREG32(i, 0);
3558
f25a5c63
AD		 3559 tmp = RREG32(HDP_MISC_CNTL);
3560 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3561 WREG32(HDP_MISC_CNTL, tmp);
3562
32fcdbf4
AD		 3563 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3564 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3565
3566 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3567
3568 udelay(50);
3569
bcc1c2a1
AD		 3570}
3571
3572int evergreen_mc_init(struct radeon_device *rdev)
3573{
bcc1c2a1
AD		 3574 u32 tmp;
3575 int chansize, numchan;
bcc1c2a1
AD		 3576
3577 /* Get VRAM informations */
3578 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3579 if ((rdev->family == CHIP_PALM) ||
3580 (rdev->family == CHIP_SUMO) ||
3581 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3582 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3583 else
3584 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3585 if (tmp & CHANSIZE_OVERRIDE) {
3586 chansize = 16;
3587 } else if (tmp & CHANSIZE_MASK) {
3588 chansize = 64;
3589 } else {
3590 chansize = 32;
3591 }
3592 tmp = RREG32(MC_SHARED_CHMAP);
3593 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3594 case 0:
3595 default:
3596 numchan = 1;
3597 break;
3598 case 1:
3599 numchan = 2;
3600 break;
3601 case 2:
3602 numchan = 4;
3603 break;
3604 case 3:
3605 numchan = 8;
3606 break;
3607 }
3608 rdev->mc.vram_width = numchan * chansize;
3609 /* Could aper size report 0 ? */
01d73a69
JC		 3610 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3611 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3612 /* Setup GPU memory space */
05b3ef69
AD		 3613 if ((rdev->family == CHIP_PALM) ||
3614 (rdev->family == CHIP_SUMO) ||
3615 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3616 /* size in bytes on fusion */
3617 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3618 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3619 } else {
05b3ef69 3620 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3621 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3622 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3623 }
51e5fcd3 3624 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3625 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3626 radeon_update_bandwidth_info(rdev);
3627
bcc1c2a1
AD		 3628 return 0;
3629}
d594e46a 3630
187e3593 3631void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3632{
64c56e8c 3633 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3634 RREG32(GRBM_STATUS));
64c56e8c 3635 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3636 RREG32(GRBM_STATUS_SE0));
64c56e8c 3637 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3638 RREG32(GRBM_STATUS_SE1));
64c56e8c 3639 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3640 RREG32(SRBM_STATUS));
a65a4369
AD		 3641 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3642 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3643 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3644 RREG32(CP_STALLED_STAT1));
3645 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3646 RREG32(CP_STALLED_STAT2));
3647 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3648 RREG32(CP_BUSY_STAT));
3649 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3650 RREG32(CP_STAT));
eaaa6983
JG		 3651 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3652 RREG32(DMA_STATUS_REG));
168757ea
AD		 3653 if (rdev->family >= CHIP_CAYMAN) {
3654 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3655 RREG32(DMA_STATUS_REG + 0x800));
3656 }
0ecebb9e
AD		 3657}
3658
168757ea 3659bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3660{
a65a4369
AD		 3661 u32 crtc_hung = 0;
3662 u32 crtc_status[6];
3663 u32 i, j, tmp;
3664
3665 for (i = 0; i < rdev->num_crtc; i++) {
3666 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3667 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3668 crtc_hung |= (1 << i);
3669 }
3670 }
3671
3672 for (j = 0; j < 10; j++) {
3673 for (i = 0; i < rdev->num_crtc; i++) {
3674 if (crtc_hung & (1 << i)) {
3675 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3676 if (tmp != crtc_status[i])
3677 crtc_hung &= ~(1 << i);
3678 }
3679 }
3680 if (crtc_hung == 0)
3681 return false;
3682 udelay(100);
3683 }
3684
3685 return true;
3686}
3687
2483b4ea 3688u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3689{
3690 u32 reset_mask = 0;
b7630473 3691 u32 tmp;
0ecebb9e 3692
a65a4369
AD		 3693 /* GRBM_STATUS */
3694 tmp = RREG32(GRBM_STATUS);
3695 if (tmp & (PA_BUSY | SC_BUSY |
3696 SH_BUSY | SX_BUSY |
3697 TA_BUSY | VGT_BUSY |
3698 DB_BUSY | CB_BUSY |
3699 SPI_BUSY | VGT_BUSY_NO_DMA))
3700 reset_mask |= RADEON_RESET_GFX;
3701
3702 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3703 CP_BUSY | CP_COHERENCY_BUSY))
3704 reset_mask |= RADEON_RESET_CP;
3705
3706 if (tmp & GRBM_EE_BUSY)
3707 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3708
a65a4369
AD		 3709 /* DMA_STATUS_REG */
3710 tmp = RREG32(DMA_STATUS_REG);
3711 if (!(tmp & DMA_IDLE))
3712 reset_mask |= RADEON_RESET_DMA;
3713
3714 /* SRBM_STATUS2 */
3715 tmp = RREG32(SRBM_STATUS2);
3716 if (tmp & DMA_BUSY)
3717 reset_mask |= RADEON_RESET_DMA;
3718
3719 /* SRBM_STATUS */
3720 tmp = RREG32(SRBM_STATUS);
3721 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3722 reset_mask |= RADEON_RESET_RLC;
3723
3724 if (tmp & IH_BUSY)
3725 reset_mask |= RADEON_RESET_IH;
3726
3727 if (tmp & SEM_BUSY)
3728 reset_mask |= RADEON_RESET_SEM;
3729
3730 if (tmp & GRBM_RQ_PENDING)
3731 reset_mask |= RADEON_RESET_GRBM;
3732
3733 if (tmp & VMC_BUSY)
3734 reset_mask |= RADEON_RESET_VMC;
3735
3736 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3737 MCC_BUSY | MCD_BUSY))
3738 reset_mask |= RADEON_RESET_MC;
3739
3740 if (evergreen_is_display_hung(rdev))
3741 reset_mask |= RADEON_RESET_DISPLAY;
3742
3743 /* VM_L2_STATUS */
3744 tmp = RREG32(VM_L2_STATUS);
3745 if (tmp & L2_BUSY)
3746 reset_mask |= RADEON_RESET_VMC;
3747
d808fc88
AD		 3748 /* Skip MC reset as it's mostly likely not hung, just busy */
3749 if (reset_mask & RADEON_RESET_MC) {
3750 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3751 reset_mask &= ~RADEON_RESET_MC;
3752 }
3753
a65a4369
AD		 3754 return reset_mask;
3755}
3756
3757static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3758{
3759 struct evergreen_mc_save save;
3760 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3761 u32 tmp;
19fc42ed 3762
0ecebb9e 3763 if (reset_mask == 0)
a65a4369 3764 return;
0ecebb9e
AD		 3765
3766 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3767
b7630473
AD		 3768 evergreen_print_gpu_status_regs(rdev);
3769
b7630473
AD		 3770 /* Disable CP parsing/prefetching */
3771 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3772
3773 if (reset_mask & RADEON_RESET_DMA) {
3774 /* Disable DMA */
3775 tmp = RREG32(DMA_RB_CNTL);
3776 tmp &= ~DMA_RB_ENABLE;
3777 WREG32(DMA_RB_CNTL, tmp);
3778 }
3779
b21b6e7a
AD		 3780 udelay(50);
3781
3782 evergreen_mc_stop(rdev, &save);
3783 if (evergreen_mc_wait_for_idle(rdev)) {
3784 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3785 }
3786
b7630473
AD		 3787 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3788 grbm_soft_reset |= SOFT_RESET_DB |
3789 SOFT_RESET_CB |
3790 SOFT_RESET_PA |
3791 SOFT_RESET_SC |
3792 SOFT_RESET_SPI |
3793 SOFT_RESET_SX |
3794 SOFT_RESET_SH |
3795 SOFT_RESET_TC |
3796 SOFT_RESET_TA |
3797 SOFT_RESET_VC |
3798 SOFT_RESET_VGT;
3799 }
3800
3801 if (reset_mask & RADEON_RESET_CP) {
3802 grbm_soft_reset |= SOFT_RESET_CP |
3803 SOFT_RESET_VGT;
3804
3805 srbm_soft_reset |= SOFT_RESET_GRBM;
3806 }
0ecebb9e
AD		 3807
3808 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3809 srbm_soft_reset |= SOFT_RESET_DMA;
3810
a65a4369
AD		 3811 if (reset_mask & RADEON_RESET_DISPLAY)
3812 srbm_soft_reset |= SOFT_RESET_DC;
3813
3814 if (reset_mask & RADEON_RESET_RLC)
3815 srbm_soft_reset |= SOFT_RESET_RLC;
3816
3817 if (reset_mask & RADEON_RESET_SEM)
3818 srbm_soft_reset |= SOFT_RESET_SEM;
3819
3820 if (reset_mask & RADEON_RESET_IH)
3821 srbm_soft_reset |= SOFT_RESET_IH;
3822
3823 if (reset_mask & RADEON_RESET_GRBM)
3824 srbm_soft_reset |= SOFT_RESET_GRBM;
3825
3826 if (reset_mask & RADEON_RESET_VMC)
3827 srbm_soft_reset |= SOFT_RESET_VMC;
3828
24178ec4
AD		 3829 if (!(rdev->flags & RADEON_IS_IGP)) {
3830 if (reset_mask & RADEON_RESET_MC)
3831 srbm_soft_reset |= SOFT_RESET_MC;
3832 }
a65a4369 3833
b7630473
AD		 3834 if (grbm_soft_reset) {
3835 tmp = RREG32(GRBM_SOFT_RESET);
3836 tmp |= grbm_soft_reset;
3837 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3838 WREG32(GRBM_SOFT_RESET, tmp);
3839 tmp = RREG32(GRBM_SOFT_RESET);
3840
3841 udelay(50);
3842
3843 tmp &= ~grbm_soft_reset;
3844 WREG32(GRBM_SOFT_RESET, tmp);
3845 tmp = RREG32(GRBM_SOFT_RESET);
3846 }
3847
3848 if (srbm_soft_reset) {
3849 tmp = RREG32(SRBM_SOFT_RESET);
3850 tmp |= srbm_soft_reset;
3851 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3852 WREG32(SRBM_SOFT_RESET, tmp);
3853 tmp = RREG32(SRBM_SOFT_RESET);
3854
3855 udelay(50);
3856
3857 tmp &= ~srbm_soft_reset;
3858 WREG32(SRBM_SOFT_RESET, tmp);
3859 tmp = RREG32(SRBM_SOFT_RESET);
3860 }
0ecebb9e
AD		 3861
3862 /* Wait a little for things to settle down */
3863 udelay(50);
3864
747943ea 3865 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3866 udelay(50);
3867
b7630473 3868 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3869}
3870
b5470b03
AD		 3871void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3872{
3873 struct evergreen_mc_save save;
3874 u32 tmp, i;
3875
3876 dev_info(rdev->dev, "GPU pci config reset\n");
3877
3878 /* disable dpm? */
3879
3880 /* Disable CP parsing/prefetching */
3881 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3882 udelay(50);
3883 /* Disable DMA */
3884 tmp = RREG32(DMA_RB_CNTL);
3885 tmp &= ~DMA_RB_ENABLE;
3886 WREG32(DMA_RB_CNTL, tmp);
3887 /* XXX other engines? */
3888
3889 /* halt the rlc */
3890 r600_rlc_stop(rdev);
3891
3892 udelay(50);
3893
3894 /* set mclk/sclk to bypass */
3895 rv770_set_clk_bypass_mode(rdev);
3896 /* disable BM */
3897 pci_clear_master(rdev->pdev);
3898 /* disable mem access */
3899 evergreen_mc_stop(rdev, &save);
3900 if (evergreen_mc_wait_for_idle(rdev)) {
3901 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3902 }
3903 /* reset */
3904 radeon_pci_config_reset(rdev);
3905 /* wait for asic to come out of reset */
3906 for (i = 0; i < rdev->usec_timeout; i++) {
3907 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3908 break;
3909 udelay(1);
3910 }
3911}
3912
a2d07b74 3913int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3914{
a65a4369
AD		 3915 u32 reset_mask;
3916
3917 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3918
3919 if (reset_mask)
3920 r600_set_bios_scratch_engine_hung(rdev, true);
3921
b5470b03 3922 /* try soft reset */
a65a4369
AD		 3923 evergreen_gpu_soft_reset(rdev, reset_mask);
3924
3925 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3926
b5470b03
AD		 3927 /* try pci config reset */
3928 if (reset_mask && radeon_hard_reset)
3929 evergreen_gpu_pci_config_reset(rdev);
3930
3931 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3932
a65a4369
AD		 3933 if (!reset_mask)
3934 r600_set_bios_scratch_engine_hung(rdev, false);
3935
3936 return 0;
747943ea
AD		 3937}
3938
123bc183
AD		 3939/**
3940 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3941 *
3942 * @rdev: radeon_device pointer
3943 * @ring: radeon_ring structure holding ring information
3944 *
3945 * Check if the GFX engine is locked up.
3946 * Returns true if the engine appears to be locked up, false if not.
3947 */
3948bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3949{
3950 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3951
3952 if (!(reset_mask & (RADEON_RESET_GFX |
3953 RADEON_RESET_COMPUTE |
3954 RADEON_RESET_CP))) {
3955 radeon_ring_lockup_update(ring);
3956 return false;
3957 }
3958 /* force CP activities */
3959 radeon_ring_force_activity(rdev, ring);
3960 return radeon_ring_test_lockup(rdev, ring);
3961}
3962
2948f5e6
AD		 3963/*
3964 * RLC
3965 */
3966#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3967#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3968
3969void sumo_rlc_fini(struct radeon_device *rdev)
3970{
3971 int r;
3972
3973 /* save restore block */
3974 if (rdev->rlc.save_restore_obj) {
3975 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3976 if (unlikely(r != 0))
3977 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3978 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3979 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3980
3981 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3982 rdev->rlc.save_restore_obj = NULL;
3983 }
3984
3985 /* clear state block */
3986 if (rdev->rlc.clear_state_obj) {
3987 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3988 if (unlikely(r != 0))
3989 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3990 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3991 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3992
3993 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3994 rdev->rlc.clear_state_obj = NULL;
3995 }
22c775ce
AD		 3996
3997 /* clear state block */
3998 if (rdev->rlc.cp_table_obj) {
3999 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4000 if (unlikely(r != 0))
4001 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4002 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4003 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4004
4005 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4006 rdev->rlc.cp_table_obj = NULL;
4007 }
2948f5e6
AD		 4008}
4009
22c775ce
AD		 4010#define CP_ME_TABLE_SIZE 96
4011
2948f5e6
AD		 4012int sumo_rlc_init(struct radeon_device *rdev)
4013{
1fd11777 4014 const u32 *src_ptr;
2948f5e6
AD		 4015 volatile u32 *dst_ptr;
4016 u32 dws, data, i, j, k, reg_num;
59a82d0e 4017 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4018 u64 reg_list_mc_addr;
1fd11777 4019 const struct cs_section_def *cs_data;
2948f5e6
AD		 4020 int r;
4021
4022 src_ptr = rdev->rlc.reg_list;
4023 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4024 if (rdev->family >= CHIP_BONAIRE) {
4025 dws += (5 * 16) + 48 + 48 + 64;
4026 }
2948f5e6
AD		 4027 cs_data = rdev->rlc.cs_data;
4028
10b7ca7e
AD		 4029 if (src_ptr) {
4030 /* save restore block */
4031 if (rdev->rlc.save_restore_obj == NULL) {
4032 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4033 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
4034 if (r) {
4035 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4036 return r;
4037 }
4038 }
4039
4040 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4041 if (unlikely(r != 0)) {
4042 sumo_rlc_fini(rdev);
4043 return r;
4044 }
4045 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4046 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4047 if (r) {
10b7ca7e
AD		 4048 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4049 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4050 sumo_rlc_fini(rdev);
2948f5e6
AD		 4051 return r;
4052 }
2948f5e6 4053
10b7ca7e
AD		 4054 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4055 if (r) {
4056 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4057 sumo_rlc_fini(rdev);
4058 return r;
4059 }
4060 /* write the sr buffer */
4061 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4062 if (rdev->family >= CHIP_TAHITI) {
4063 /* SI */
59a82d0e 4064 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4065 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4066 } else {
4067 /* ON/LN/TN */
4068 /* format:
4069 * dw0: (reg2 << 16) | reg1
4070 * dw1: reg1 save space
4071 * dw2: reg2 save space
4072 */
4073 for (i = 0; i < dws; i++) {
4074 data = src_ptr[i] >> 2;
4075 i++;
4076 if (i < dws)
4077 data |= (src_ptr[i] >> 2) << 16;
4078 j = (((i - 1) * 3) / 2);
6ba81e53 4079 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4080 }
4081 j = ((i * 3) / 2);
6ba81e53 4082 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4083 }
10b7ca7e 4084 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4085 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4086 }
2948f5e6 4087
10b7ca7e
AD		 4088 if (cs_data) {
4089 /* clear state block */
a0f38609
AD		 4090 if (rdev->family >= CHIP_BONAIRE) {
4091 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4092 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4093 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4094 dws = rdev->rlc.clear_state_size + (256 / 4);
4095 } else {
4096 reg_list_num = 0;
4097 dws = 0;
4098 for (i = 0; cs_data[i].section != NULL; i++) {
4099 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4100 reg_list_num++;
4101 dws += cs_data[i].section[j].reg_count;
4102 }
10b7ca7e 4103 }
59a82d0e
AD		 4104 reg_list_blk_index = (3 * reg_list_num + 2);
4105 dws += reg_list_blk_index;
4106 rdev->rlc.clear_state_size = dws;
2948f5e6 4107 }
2948f5e6 4108
10b7ca7e 4109 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4110 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
10b7ca7e
AD		 4111 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4112 if (r) {
4113 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4114 sumo_rlc_fini(rdev);
4115 return r;
4116 }
4117 }
4118 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4119 if (unlikely(r != 0)) {
4120 sumo_rlc_fini(rdev);
4121 return r;
4122 }
4123 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4124 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4125 if (r) {
10b7ca7e
AD		 4126 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4127 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4128 sumo_rlc_fini(rdev);
4129 return r;
4130 }
2948f5e6 4131
10b7ca7e
AD		 4132 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4133 if (r) {
4134 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4135 sumo_rlc_fini(rdev);
4136 return r;
4137 }
4138 /* set up the cs buffer */
4139 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4140 if (rdev->family >= CHIP_BONAIRE) {
4141 cik_get_csb_buffer(rdev, dst_ptr);
4142 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4143 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4144 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4145 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4146 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4147 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4148 } else {
4149 reg_list_hdr_blk_index = 0;
4150 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4151 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4152 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4153 reg_list_hdr_blk_index++;
59a82d0e
AD		 4154 for (i = 0; cs_data[i].section != NULL; i++) {
4155 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4156 reg_num = cs_data[i].section[j].reg_count;
4157 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4158 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4159 reg_list_hdr_blk_index++;
4160
4161 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4162 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4163 reg_list_hdr_blk_index++;
4164
4165 data = 0x08000000 | (reg_num * 4);
6ba81e53 4166 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4167 reg_list_hdr_blk_index++;
4168
4169 for (k = 0; k < reg_num; k++) {
4170 data = cs_data[i].section[j].extent[k];
6ba81e53 4171 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4172 }
4173 reg_list_mc_addr += reg_num * 4;
4174 reg_list_blk_index += reg_num;
10b7ca7e 4175 }
2948f5e6 4176 }
6ba81e53 4177 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4178 }
10b7ca7e
AD		 4179 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4180 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4181 }
2948f5e6 4182
22c775ce
AD		 4183 if (rdev->rlc.cp_table_size) {
4184 if (rdev->rlc.cp_table_obj == NULL) {
4185 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4186 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4187 if (r) {
4188 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4189 sumo_rlc_fini(rdev);
4190 return r;
4191 }
4192 }
4193
4194 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4195 if (unlikely(r != 0)) {
4196 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4197 sumo_rlc_fini(rdev);
4198 return r;
4199 }
4200 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4201 &rdev->rlc.cp_table_gpu_addr);
4202 if (r) {
4203 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4204 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4205 sumo_rlc_fini(rdev);
4206 return r;
4207 }
4208 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4209 if (r) {
4210 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4211 sumo_rlc_fini(rdev);
4212 return r;
4213 }
4214
4215 cik_init_cp_pg_table(rdev);
4216
4217 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4218 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4219
4220 }
2948f5e6
AD		 4221
4222 return 0;
4223}
4224
4225static void evergreen_rlc_start(struct radeon_device *rdev)
4226{
8ba10463
AD		 4227 u32 mask = RLC_ENABLE;
4228
4229 if (rdev->flags & RADEON_IS_IGP) {
4230 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4231 }
4232
4233 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4234}
4235
4236int evergreen_rlc_resume(struct radeon_device *rdev)
4237{
4238 u32 i;
4239 const __be32 *fw_data;
4240
4241 if (!rdev->rlc_fw)
4242 return -EINVAL;
4243
4244 r600_rlc_stop(rdev);
4245
4246 WREG32(RLC_HB_CNTL, 0);
4247
4248 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4249 if (rdev->family == CHIP_ARUBA) {
4250 u32 always_on_bitmap =
4251 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4252 /* find out the number of active simds */
4253 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4254 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4255 tmp = hweight32(~tmp);
4256 if (tmp == rdev->config.cayman.max_simds_per_se) {
4257 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4258 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4259 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4260 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4261 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4262 }
4263 } else {
4264 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4265 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4266 }
2948f5e6
AD		 4267 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4268 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4269 } else {
4270 WREG32(RLC_HB_BASE, 0);
4271 WREG32(RLC_HB_RPTR, 0);
4272 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4273 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4274 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4275 }
2948f5e6
AD		 4276 WREG32(RLC_MC_CNTL, 0);
4277 WREG32(RLC_UCODE_CNTL, 0);
4278
4279 fw_data = (const __be32 *)rdev->rlc_fw->data;
4280 if (rdev->family >= CHIP_ARUBA) {
4281 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4282 WREG32(RLC_UCODE_ADDR, i);
4283 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4284 }
4285 } else if (rdev->family >= CHIP_CAYMAN) {
4286 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4287 WREG32(RLC_UCODE_ADDR, i);
4288 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289 }
4290 } else {
4291 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4292 WREG32(RLC_UCODE_ADDR, i);
4293 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4294 }
4295 }
4296 WREG32(RLC_UCODE_ADDR, 0);
4297
4298 evergreen_rlc_start(rdev);
4299
4300 return 0;
4301}
4302
45f9a39b
AD		 4303/* Interrupts */
4304
4305u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4306{
46437057 4307 if (crtc >= rdev->num_crtc)
45f9a39b 4308 return 0;
46437057
AD		 4309 else
4310 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4311}
4312
4313void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4314{
4315 u32 tmp;
4316
1b37078b
AD		 4317 if (rdev->family >= CHIP_CAYMAN) {
4318 cayman_cp_int_cntl_setup(rdev, 0,
4319 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4320 cayman_cp_int_cntl_setup(rdev, 1, 0);
4321 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4322 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4323 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4324 } else
4325 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4326 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4327 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4328 WREG32(GRBM_INT_CNTL, 0);
4329 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4330 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4331 if (rdev->num_crtc >= 4) {
18007401
AD		 4332 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4333 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4334 }
4335 if (rdev->num_crtc >= 6) {
18007401
AD		 4336 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4337 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4338 }
45f9a39b
AD		 4339
4340 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4342 if (rdev->num_crtc >= 4) {
18007401
AD		 4343 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4345 }
4346 if (rdev->num_crtc >= 6) {
18007401
AD		 4347 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4348 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4349 }
45f9a39b 4350
05b3ef69
AD		 4351 /* only one DAC on DCE6 */
4352 if (!ASIC_IS_DCE6(rdev))
4353 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4354 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4355
4356 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4357 WREG32(DC_HPD1_INT_CONTROL, tmp);
4358 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4359 WREG32(DC_HPD2_INT_CONTROL, tmp);
4360 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4361 WREG32(DC_HPD3_INT_CONTROL, tmp);
4362 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4363 WREG32(DC_HPD4_INT_CONTROL, tmp);
4364 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4365 WREG32(DC_HPD5_INT_CONTROL, tmp);
4366 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4367 WREG32(DC_HPD6_INT_CONTROL, tmp);
4368
4369}
4370
4371int evergreen_irq_set(struct radeon_device *rdev)
4372{
4373 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4374 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4375 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4376 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4377 u32 grbm_int_cntl = 0;
6f34be50 4378 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4379 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4380 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4381 u32 thermal_int = 0;
45f9a39b
AD		 4382
4383 if (!rdev->irq.installed) {
fce7d61b 4384 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4385 return -EINVAL;
4386 }
4387 /* don't enable anything if the ih is disabled */
4388 if (!rdev->ih.enabled) {
4389 r600_disable_interrupts(rdev);
4390 /* force the active interrupt state to all disabled */
4391 evergreen_disable_interrupt_state(rdev);
4392 return 0;
4393 }
4394
4395 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4396 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4397 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4398 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4399 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4400 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4401 if (rdev->family == CHIP_ARUBA)
4402 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4403 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4404 else
4405 thermal_int = RREG32(CG_THERMAL_INT) &
4406 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4407
f122c610
AD		 4408 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4409 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4410 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4411 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4412 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4413 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4414
233d1ad5
AD		 4415 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4416
1b37078b
AD		 4417 if (rdev->family >= CHIP_CAYMAN) {
4418 /* enable CP interrupts on all rings */
736fc37f 4419 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4420 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4421 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4422 }
736fc37f 4423 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4424 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4425 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4426 }
736fc37f 4427 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4428 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4429 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4430 }
4431 } else {
736fc37f 4432 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4433 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4434 cp_int_cntl |= RB_INT_ENABLE;
4435 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4436 }
45f9a39b 4437 }
1b37078b 4438
233d1ad5
AD		 4439 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4440 DRM_DEBUG("r600_irq_set: sw int dma\n");
4441 dma_cntl |= TRAP_ENABLE;
4442 }
4443
f60cbd11
AD		 4444 if (rdev->family >= CHIP_CAYMAN) {
4445 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4446 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4447 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4448 dma_cntl1 |= TRAP_ENABLE;
4449 }
4450 }
4451
dc50ba7f
AD		 4452 if (rdev->irq.dpm_thermal) {
4453 DRM_DEBUG("dpm thermal\n");
4454 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4455 }
4456
6f34be50 4457 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4458 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4459 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4460 crtc1 |= VBLANK_INT_MASK;
4461 }
6f34be50 4462 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4463 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4464 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4465 crtc2 |= VBLANK_INT_MASK;
4466 }
6f34be50 4467 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4468 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4469 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4470 crtc3 |= VBLANK_INT_MASK;
4471 }
6f34be50 4472 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4473 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4474 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4475 crtc4 |= VBLANK_INT_MASK;
4476 }
6f34be50 4477 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4478 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4479 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4480 crtc5 |= VBLANK_INT_MASK;
4481 }
6f34be50 4482 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4483 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4484 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4485 crtc6 |= VBLANK_INT_MASK;
4486 }
4487 if (rdev->irq.hpd[0]) {
4488 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4489 hpd1 |= DC_HPDx_INT_EN;
4490 }
4491 if (rdev->irq.hpd[1]) {
4492 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4493 hpd2 |= DC_HPDx_INT_EN;
4494 }
4495 if (rdev->irq.hpd[2]) {
4496 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4497 hpd3 |= DC_HPDx_INT_EN;
4498 }
4499 if (rdev->irq.hpd[3]) {
4500 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4501 hpd4 |= DC_HPDx_INT_EN;
4502 }
4503 if (rdev->irq.hpd[4]) {
4504 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4505 hpd5 |= DC_HPDx_INT_EN;
4506 }
4507 if (rdev->irq.hpd[5]) {
4508 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4509 hpd6 |= DC_HPDx_INT_EN;
4510 }
f122c610
AD		 4511 if (rdev->irq.afmt[0]) {
4512 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4513 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4514 }
4515 if (rdev->irq.afmt[1]) {
4516 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4517 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4518 }
4519 if (rdev->irq.afmt[2]) {
4520 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4521 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4522 }
4523 if (rdev->irq.afmt[3]) {
4524 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4525 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4526 }
4527 if (rdev->irq.afmt[4]) {
4528 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4529 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4530 }
4531 if (rdev->irq.afmt[5]) {
4532 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4533 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4534 }
45f9a39b 4535
1b37078b
AD		 4536 if (rdev->family >= CHIP_CAYMAN) {
4537 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4538 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4539 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4540 } else
4541 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4542
4543 WREG32(DMA_CNTL, dma_cntl);
4544
f60cbd11
AD		 4545 if (rdev->family >= CHIP_CAYMAN)
4546 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4547
2031f77c 4548 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4549
4550 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4551 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4552 if (rdev->num_crtc >= 4) {
18007401
AD		 4553 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4554 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4555 }
4556 if (rdev->num_crtc >= 6) {
18007401
AD		 4557 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4558 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4559 }
45f9a39b 4560
6f34be50
AD		 4561 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4563 if (rdev->num_crtc >= 4) {
4564 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4565 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4566 }
4567 if (rdev->num_crtc >= 6) {
4568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4570 }
6f34be50 4571
45f9a39b
AD		 4572 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4573 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4574 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4575 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4576 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4577 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4578 if (rdev->family == CHIP_ARUBA)
4579 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4580 else
4581 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4582
f122c610
AD		 4583 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4584 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4589
bcc1c2a1
AD		 4590 return 0;
4591}
4592
cbdd4501 4593static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4594{
4595 u32 tmp;
4596
6f34be50
AD		 4597 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4598 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4599 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4601 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4602 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4603 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4604 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4605 if (rdev->num_crtc >= 4) {
4606 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4607 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4608 }
4609 if (rdev->num_crtc >= 6) {
4610 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4611 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4612 }
6f34be50 4613
f122c610
AD		 4614 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4615 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4618 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4619 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4620
6f34be50
AD		 4621 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4623 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4624 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4625 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4626 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4627 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4628 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4629 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4630 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4632 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4633
b7eff394
AD		 4634 if (rdev->num_crtc >= 4) {
4635 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4637 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4638 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4639 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4640 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4642 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4644 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4646 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4647 }
4648
4649 if (rdev->num_crtc >= 6) {
4650 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4651 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4652 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4653 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4655 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4656 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4657 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4659 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4660 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4661 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4662 }
45f9a39b 4663
6f34be50 4664 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4665 tmp = RREG32(DC_HPD1_INT_CONTROL);
4666 tmp |= DC_HPDx_INT_ACK;
4667 WREG32(DC_HPD1_INT_CONTROL, tmp);
4668 }
6f34be50 4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4670 tmp = RREG32(DC_HPD2_INT_CONTROL);
4671 tmp |= DC_HPDx_INT_ACK;
4672 WREG32(DC_HPD2_INT_CONTROL, tmp);
4673 }
6f34be50 4674 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4675 tmp = RREG32(DC_HPD3_INT_CONTROL);
4676 tmp |= DC_HPDx_INT_ACK;
4677 WREG32(DC_HPD3_INT_CONTROL, tmp);
4678 }
6f34be50 4679 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4680 tmp = RREG32(DC_HPD4_INT_CONTROL);
4681 tmp |= DC_HPDx_INT_ACK;
4682 WREG32(DC_HPD4_INT_CONTROL, tmp);
4683 }
6f34be50 4684 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4685 tmp = RREG32(DC_HPD5_INT_CONTROL);
4686 tmp |= DC_HPDx_INT_ACK;
4687 WREG32(DC_HPD5_INT_CONTROL, tmp);
4688 }
6f34be50 4689 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4690 tmp = RREG32(DC_HPD5_INT_CONTROL);
4691 tmp |= DC_HPDx_INT_ACK;
4692 WREG32(DC_HPD6_INT_CONTROL, tmp);
4693 }
f122c610
AD		 4694 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4695 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4696 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4697 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4698 }
4699 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4700 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4701 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4702 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4703 }
4704 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4705 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4706 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4707 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4708 }
4709 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4710 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4711 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4712 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4713 }
4714 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4715 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4716 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4717 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4718 }
4719 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4720 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4721 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4722 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4723 }
45f9a39b
AD		 4724}
4725
1109ca09 4726static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4727{
45f9a39b
AD		 4728 r600_disable_interrupts(rdev);
4729 /* Wait and acknowledge irq */
4730 mdelay(1);
6f34be50 4731 evergreen_irq_ack(rdev);
45f9a39b
AD		 4732 evergreen_disable_interrupt_state(rdev);
4733}
4734
755d819e 4735void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4736{
4737 evergreen_irq_disable(rdev);
4738 r600_rlc_stop(rdev);
4739}
4740
cbdd4501 4741static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4742{
4743 u32 wptr, tmp;
4744
724c80e1 4745 if (rdev->wb.enabled)
204ae24d 4746 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4747 else
4748 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4749
4750 if (wptr & RB_OVERFLOW) {
4751 /* When a ring buffer overflow happen start parsing interrupt
4752 * from the last not overwritten vector (wptr + 16). Hopefully
4753 * this should allow us to catchup.
4754 */
4755 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4756 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4757 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4758 tmp = RREG32(IH_RB_CNTL);
4759 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4760 WREG32(IH_RB_CNTL, tmp);
4761 }
4762 return (wptr & rdev->ih.ptr_mask);
4763}
4764
4765int evergreen_irq_process(struct radeon_device *rdev)
4766{
682f1a54
DA		 4767 u32 wptr;
4768 u32 rptr;
45f9a39b
AD		 4769 u32 src_id, src_data;
4770 u32 ring_index;
45f9a39b 4771 bool queue_hotplug = false;
f122c610 4772 bool queue_hdmi = false;
dc50ba7f 4773 bool queue_thermal = false;
54e2e49c 4774 u32 status, addr;
45f9a39b 4775
682f1a54 4776 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4777 return IRQ_NONE;
4778
682f1a54 4779 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4780
4781restart_ih:
4782 /* is somebody else already processing irqs? */
4783 if (atomic_xchg(&rdev->ih.lock, 1))
4784 return IRQ_NONE;
4785
682f1a54
DA		 4786 rptr = rdev->ih.rptr;
4787 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4788
964f6645
BH		 4789 /* Order reading of wptr vs. reading of IH ring data */
4790 rmb();
4791
45f9a39b 4792 /* display interrupts */
6f34be50 4793 evergreen_irq_ack(rdev);
45f9a39b 4794
45f9a39b
AD		 4795 while (rptr != wptr) {
4796 /* wptr/rptr are in bytes! */
4797 ring_index = rptr / 4;
0f234f5f
AD		 4798 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4799 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4800
4801 switch (src_id) {
4802 case 1: /* D1 vblank/vline */
4803 switch (src_data) {
4804 case 0: /* D1 vblank */
6f34be50 4805 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4806 if (rdev->irq.crtc_vblank_int[0]) {
4807 drm_handle_vblank(rdev->ddev, 0);
4808 rdev->pm.vblank_sync = true;
4809 wake_up(&rdev->irq.vblank_queue);
4810 }
736fc37f 4811 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4812 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4813 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4814 DRM_DEBUG("IH: D1 vblank\n");
4815 }
4816 break;
4817 case 1: /* D1 vline */
6f34be50
AD		 4818 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4819 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4820 DRM_DEBUG("IH: D1 vline\n");
4821 }
4822 break;
4823 default:
4824 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4825 break;
4826 }
4827 break;
4828 case 2: /* D2 vblank/vline */
4829 switch (src_data) {
4830 case 0: /* D2 vblank */
6f34be50 4831 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4832 if (rdev->irq.crtc_vblank_int[1]) {
4833 drm_handle_vblank(rdev->ddev, 1);
4834 rdev->pm.vblank_sync = true;
4835 wake_up(&rdev->irq.vblank_queue);
4836 }
736fc37f 4837 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4838 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4839 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4840 DRM_DEBUG("IH: D2 vblank\n");
4841 }
4842 break;
4843 case 1: /* D2 vline */
6f34be50
AD		 4844 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4845 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4846 DRM_DEBUG("IH: D2 vline\n");
4847 }
4848 break;
4849 default:
4850 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4851 break;
4852 }
4853 break;
4854 case 3: /* D3 vblank/vline */
4855 switch (src_data) {
4856 case 0: /* D3 vblank */
6f34be50
AD		 4857 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4858 if (rdev->irq.crtc_vblank_int[2]) {
4859 drm_handle_vblank(rdev->ddev, 2);
4860 rdev->pm.vblank_sync = true;
4861 wake_up(&rdev->irq.vblank_queue);
4862 }
736fc37f 4863 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4864 radeon_crtc_handle_flip(rdev, 2);
4865 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4866 DRM_DEBUG("IH: D3 vblank\n");
4867 }
4868 break;
4869 case 1: /* D3 vline */
6f34be50
AD		 4870 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4871 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4872 DRM_DEBUG("IH: D3 vline\n");
4873 }
4874 break;
4875 default:
4876 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4877 break;
4878 }
4879 break;
4880 case 4: /* D4 vblank/vline */
4881 switch (src_data) {
4882 case 0: /* D4 vblank */
6f34be50
AD		 4883 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4884 if (rdev->irq.crtc_vblank_int[3]) {
4885 drm_handle_vblank(rdev->ddev, 3);
4886 rdev->pm.vblank_sync = true;
4887 wake_up(&rdev->irq.vblank_queue);
4888 }
736fc37f 4889 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4890 radeon_crtc_handle_flip(rdev, 3);
4891 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4892 DRM_DEBUG("IH: D4 vblank\n");
4893 }
4894 break;
4895 case 1: /* D4 vline */
6f34be50
AD		 4896 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4897 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4898 DRM_DEBUG("IH: D4 vline\n");
4899 }
4900 break;
4901 default:
4902 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4903 break;
4904 }
4905 break;
4906 case 5: /* D5 vblank/vline */
4907 switch (src_data) {
4908 case 0: /* D5 vblank */
6f34be50
AD		 4909 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4910 if (rdev->irq.crtc_vblank_int[4]) {
4911 drm_handle_vblank(rdev->ddev, 4);
4912 rdev->pm.vblank_sync = true;
4913 wake_up(&rdev->irq.vblank_queue);
4914 }
736fc37f 4915 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4916 radeon_crtc_handle_flip(rdev, 4);
4917 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4918 DRM_DEBUG("IH: D5 vblank\n");
4919 }
4920 break;
4921 case 1: /* D5 vline */
6f34be50
AD		 4922 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4923 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4924 DRM_DEBUG("IH: D5 vline\n");
4925 }
4926 break;
4927 default:
4928 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4929 break;
4930 }
4931 break;
4932 case 6: /* D6 vblank/vline */
4933 switch (src_data) {
4934 case 0: /* D6 vblank */
6f34be50
AD		 4935 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4936 if (rdev->irq.crtc_vblank_int[5]) {
4937 drm_handle_vblank(rdev->ddev, 5);
4938 rdev->pm.vblank_sync = true;
4939 wake_up(&rdev->irq.vblank_queue);
4940 }
736fc37f 4941 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4942 radeon_crtc_handle_flip(rdev, 5);
4943 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4944 DRM_DEBUG("IH: D6 vblank\n");
4945 }
4946 break;
4947 case 1: /* D6 vline */
6f34be50
AD		 4948 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4949 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4950 DRM_DEBUG("IH: D6 vline\n");
4951 }
4952 break;
4953 default:
4954 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4955 break;
4956 }
4957 break;
4958 case 42: /* HPD hotplug */
4959 switch (src_data) {
4960 case 0:
6f34be50
AD		 4961 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4962 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4963 queue_hotplug = true;
4964 DRM_DEBUG("IH: HPD1\n");
4965 }
4966 break;
4967 case 1:
6f34be50
AD		 4968 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4969 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4970 queue_hotplug = true;
4971 DRM_DEBUG("IH: HPD2\n");
4972 }
4973 break;
4974 case 2:
6f34be50
AD		 4975 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4976 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4977 queue_hotplug = true;
4978 DRM_DEBUG("IH: HPD3\n");
4979 }
4980 break;
4981 case 3:
6f34be50
AD		 4982 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4983 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4984 queue_hotplug = true;
4985 DRM_DEBUG("IH: HPD4\n");
4986 }
4987 break;
4988 case 4:
6f34be50
AD		 4989 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4990 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4991 queue_hotplug = true;
4992 DRM_DEBUG("IH: HPD5\n");
4993 }
4994 break;
4995 case 5:
6f34be50
AD		 4996 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4997 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4998 queue_hotplug = true;
4999 DRM_DEBUG("IH: HPD6\n");
5000 }
5001 break;
5002 default:
5003 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5004 break;
5005 }
5006 break;
f122c610
AD		 5007 case 44: /* hdmi */
5008 switch (src_data) {
5009 case 0:
5010 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5011 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5012 queue_hdmi = true;
5013 DRM_DEBUG("IH: HDMI0\n");
5014 }
5015 break;
5016 case 1:
5017 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5018 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5019 queue_hdmi = true;
5020 DRM_DEBUG("IH: HDMI1\n");
5021 }
5022 break;
5023 case 2:
5024 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5025 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5026 queue_hdmi = true;
5027 DRM_DEBUG("IH: HDMI2\n");
5028 }
5029 break;
5030 case 3:
5031 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5032 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5033 queue_hdmi = true;
5034 DRM_DEBUG("IH: HDMI3\n");
5035 }
5036 break;
5037 case 4:
5038 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5039 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5040 queue_hdmi = true;
5041 DRM_DEBUG("IH: HDMI4\n");
5042 }
5043 break;
5044 case 5:
5045 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5046 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5047 queue_hdmi = true;
5048 DRM_DEBUG("IH: HDMI5\n");
5049 }
5050 break;
5051 default:
5052 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5053 break;
5054 }
f2ba57b5
CK		 5055 case 124: /* UVD */
5056 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5057 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5058 break;
ae133a11
CK		 5059 case 146:
5060 case 147:
54e2e49c
AD		 5061 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5062 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 5063 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5064 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5065 addr);
ae133a11 5066 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5067 status);
5068 cayman_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 5069 /* reset addr and status */
5070 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5071 break;
45f9a39b
AD		 5072 case 176: /* CP_INT in ring buffer */
5073 case 177: /* CP_INT in IB1 */
5074 case 178: /* CP_INT in IB2 */
5075 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5076 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5077 break;
5078 case 181: /* CP EOP event */
5079 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5080 if (rdev->family >= CHIP_CAYMAN) {
5081 switch (src_data) {
5082 case 0:
5083 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5084 break;
5085 case 1:
5086 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5087 break;
5088 case 2:
5089 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5090 break;
5091 }
5092 } else
5093 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5094 break;
233d1ad5
AD		 5095 case 224: /* DMA trap event */
5096 DRM_DEBUG("IH: DMA trap\n");
5097 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5098 break;
dc50ba7f
AD		 5099 case 230: /* thermal low to high */
5100 DRM_DEBUG("IH: thermal low to high\n");
5101 rdev->pm.dpm.thermal.high_to_low = false;
5102 queue_thermal = true;
5103 break;
5104 case 231: /* thermal high to low */
5105 DRM_DEBUG("IH: thermal high to low\n");
5106 rdev->pm.dpm.thermal.high_to_low = true;
5107 queue_thermal = true;
5108 break;
2031f77c 5109 case 233: /* GUI IDLE */
303c805c 5110 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5111 break;
f60cbd11
AD		 5112 case 244: /* DMA trap event */
5113 if (rdev->family >= CHIP_CAYMAN) {
5114 DRM_DEBUG("IH: DMA1 trap\n");
5115 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5116 }
5117 break;
45f9a39b
AD		 5118 default:
5119 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5120 break;
5121 }
5122
5123 /* wptr/rptr are in bytes! */
5124 rptr += 16;
5125 rptr &= rdev->ih.ptr_mask;
5126 }
45f9a39b 5127 if (queue_hotplug)
32c87fca 5128 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5129 if (queue_hdmi)
5130 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5131 if (queue_thermal && rdev->pm.dpm_enabled)
5132 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 5133 rdev->ih.rptr = rptr;
5134 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5135 atomic_set(&rdev->ih.lock, 0);
5136
5137 /* make sure wptr hasn't changed while processing */
5138 wptr = evergreen_get_ih_wptr(rdev);
5139 if (wptr != rptr)
5140 goto restart_ih;
5141
45f9a39b
AD		 5142 return IRQ_HANDLED;
5143}
5144
bcc1c2a1
AD		 5145static int evergreen_startup(struct radeon_device *rdev)
5146{
f2ba57b5 5147 struct radeon_ring *ring;
bcc1c2a1
AD		 5148 int r;
5149
9e46a48d 5150 /* enable pcie gen2 link */
cd54033a 5151 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5152 /* enable aspm */
5153 evergreen_program_aspm(rdev);
9e46a48d 5154
e5903d39
AD		 5155 /* scratch needs to be initialized before MC */
5156 r = r600_vram_scratch_init(rdev);
5157 if (r)
5158 return r;
5159
6fab3feb
AD		 5160 evergreen_mc_program(rdev);
5161
6c7bccea 5162 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5163 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5164 if (r) {
0af62b01 5165 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5166 return r;
5167 }
5168 }
fe251e2f 5169
bcc1c2a1 5170 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5171 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5172 } else {
5173 r = evergreen_pcie_gart_enable(rdev);
5174 if (r)
5175 return r;
5176 }
bcc1c2a1 5177 evergreen_gpu_init(rdev);
bcc1c2a1 5178
2948f5e6
AD		 5179 /* allocate rlc buffers */
5180 if (rdev->flags & RADEON_IS_IGP) {
5181 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5182 rdev->rlc.reg_list_size =
5183 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5184 rdev->rlc.cs_data = evergreen_cs_data;
5185 r = sumo_rlc_init(rdev);
5186 if (r) {
5187 DRM_ERROR("Failed to init rlc BOs!\n");
5188 return r;
5189 }
5190 }
5191
724c80e1
AD		 5192 /* allocate wb buffer */
5193 r = radeon_wb_init(rdev);
5194 if (r)
5195 return r;
5196
30eb77f4
JG		 5197 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5198 if (r) {
5199 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5200 return r;
5201 }
5202
233d1ad5
AD		 5203 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5204 if (r) {
5205 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5206 return r;
5207 }
5208
e409b128 5209 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5210 if (!r) {
5211 r = radeon_fence_driver_start_ring(rdev,
5212 R600_RING_TYPE_UVD_INDEX);
5213 if (r)
5214 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5215 }
5216
5217 if (r)
5218 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5219
bcc1c2a1 5220 /* Enable IRQ */
e49f3959
AH		 5221 if (!rdev->irq.installed) {
5222 r = radeon_irq_kms_init(rdev);
5223 if (r)
5224 return r;
5225 }
5226
bcc1c2a1
AD		 5227 r = r600_irq_init(rdev);
5228 if (r) {
5229 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5230 radeon_irq_kms_fini(rdev);
5231 return r;
5232 }
45f9a39b 5233 evergreen_irq_set(rdev);
bcc1c2a1 5234
f2ba57b5 5235 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5236 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5237 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5238 if (r)
5239 return r;
233d1ad5
AD		 5240
5241 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5242 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5243 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5244 if (r)
5245 return r;
5246
bcc1c2a1
AD		 5247 r = evergreen_cp_load_microcode(rdev);
5248 if (r)
5249 return r;
fe251e2f 5250 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5251 if (r)
5252 return r;
5253 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5254 if (r)
5255 return r;
fe251e2f 5256
f2ba57b5
CK		 5257 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5258 if (ring->ring_size) {
02c9f7fa 5259 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5260 RADEON_CP_PACKET2);
f2ba57b5 5261 if (!r)
e409b128 5262 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5263
5264 if (r)
5265 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5266 }
5267
2898c348
CK		 5268 r = radeon_ib_pool_init(rdev);
5269 if (r) {
5270 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5271 return r;
2898c348 5272 }
b15ba512 5273
69d2ae57
RM		 5274 r = r600_audio_init(rdev);
5275 if (r) {
5276 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5277 return r;
5278 }
5279
bcc1c2a1
AD		 5280 return 0;
5281}
5282
5283int evergreen_resume(struct radeon_device *rdev)
5284{
5285 int r;
5286
86f5c9ed
AD		 5287 /* reset the asic, the gfx blocks are often in a bad state
5288 * after the driver is unloaded or after a resume
5289 */
5290 if (radeon_asic_reset(rdev))
5291 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5292 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5293 * posting will perform necessary task to bring back GPU into good
5294 * shape.
5295 */
5296 /* post card */
5297 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5298
d4788db3
AD		 5299 /* init golden registers */
5300 evergreen_init_golden_registers(rdev);
5301
6c7bccea
AD		 5302 radeon_pm_resume(rdev);
5303
b15ba512 5304 rdev->accel_working = true;
bcc1c2a1
AD		 5305 r = evergreen_startup(rdev);
5306 if (r) {
755d819e 5307 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5308 rdev->accel_working = false;
bcc1c2a1
AD		 5309 return r;
5310 }
fe251e2f 5311
bcc1c2a1
AD		 5312 return r;
5313
5314}
5315
5316int evergreen_suspend(struct radeon_device *rdev)
5317{
6c7bccea 5318 radeon_pm_suspend(rdev);
69d2ae57 5319 r600_audio_fini(rdev);
e409b128 5320 uvd_v1_0_fini(rdev);
f2ba57b5 5321 radeon_uvd_suspend(rdev);
bcc1c2a1 5322 r700_cp_stop(rdev);
233d1ad5 5323 r600_dma_stop(rdev);
45f9a39b 5324 evergreen_irq_suspend(rdev);
724c80e1 5325 radeon_wb_disable(rdev);
bcc1c2a1 5326 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5327
5328 return 0;
5329}
5330
bcc1c2a1
AD		 5331/* Plan is to move initialization in that function and use
5332 * helper function so that radeon_device_init pretty much
5333 * do nothing more than calling asic specific function. This
5334 * should also allow to remove a bunch of callback function
5335 * like vram_info.
5336 */
5337int evergreen_init(struct radeon_device *rdev)
5338{
5339 int r;
5340
bcc1c2a1
AD		 5341 /* Read BIOS */
5342 if (!radeon_get_bios(rdev)) {
5343 if (ASIC_IS_AVIVO(rdev))
5344 return -EINVAL;
5345 }
5346 /* Must be an ATOMBIOS */
5347 if (!rdev->is_atom_bios) {
755d819e 5348 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5349 return -EINVAL;
5350 }
5351 r = radeon_atombios_init(rdev);
5352 if (r)
5353 return r;
86f5c9ed
AD		 5354 /* reset the asic, the gfx blocks are often in a bad state
5355 * after the driver is unloaded or after a resume
5356 */
5357 if (radeon_asic_reset(rdev))
5358 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5359 /* Post card if necessary */
fd909c37 5360 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5361 if (!rdev->bios) {
5362 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5363 return -EINVAL;
5364 }
5365 DRM_INFO("GPU not posted. posting now...\n");
5366 atom_asic_init(rdev->mode_info.atom_context);
5367 }
d4788db3
AD		 5368 /* init golden registers */
5369 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5370 /* Initialize scratch registers */
5371 r600_scratch_init(rdev);
5372 /* Initialize surface registers */
5373 radeon_surface_init(rdev);
5374 /* Initialize clocks */
5375 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5376 /* Fence driver */
5377 r = radeon_fence_driver_init(rdev);
5378 if (r)
5379 return r;
d594e46a
JG		 5380 /* initialize AGP */
5381 if (rdev->flags & RADEON_IS_AGP) {
5382 r = radeon_agp_init(rdev);
5383 if (r)
5384 radeon_agp_disable(rdev);
5385 }
5386 /* initialize memory controller */
bcc1c2a1
AD		 5387 r = evergreen_mc_init(rdev);
5388 if (r)
5389 return r;
5390 /* Memory manager */
5391 r = radeon_bo_init(rdev);
5392 if (r)
5393 return r;
45f9a39b 5394
01ac8794
AD		 5395 if (ASIC_IS_DCE5(rdev)) {
5396 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5397 r = ni_init_microcode(rdev);
5398 if (r) {
5399 DRM_ERROR("Failed to load firmware!\n");
5400 return r;
5401 }
5402 }
5403 } else {
5404 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5405 r = r600_init_microcode(rdev);
5406 if (r) {
5407 DRM_ERROR("Failed to load firmware!\n");
5408 return r;
5409 }
5410 }
5411 }
5412
6c7bccea
AD		 5413 /* Initialize power management */
5414 radeon_pm_init(rdev);
5415
e32eb50d
CK		 5416 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5417 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5418
233d1ad5
AD		 5419 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5420 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5421
f2ba57b5
CK		 5422 r = radeon_uvd_init(rdev);
5423 if (!r) {
5424 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5425 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5426 4096);
5427 }
5428
bcc1c2a1
AD		 5429 rdev->ih.ring_obj = NULL;
5430 r600_ih_ring_init(rdev, 64 * 1024);
5431
5432 r = r600_pcie_gart_init(rdev);
5433 if (r)
5434 return r;
0fcdb61e 5435
148a03bc 5436 rdev->accel_working = true;
bcc1c2a1
AD		 5437 r = evergreen_startup(rdev);
5438 if (r) {
fe251e2f
AD		 5439 dev_err(rdev->dev, "disabling GPU acceleration\n");
5440 r700_cp_fini(rdev);
233d1ad5 5441 r600_dma_fini(rdev);
fe251e2f 5442 r600_irq_fini(rdev);
2948f5e6
AD		 5443 if (rdev->flags & RADEON_IS_IGP)
5444 sumo_rlc_fini(rdev);
724c80e1 5445 radeon_wb_fini(rdev);
2898c348 5446 radeon_ib_pool_fini(rdev);
fe251e2f 5447 radeon_irq_kms_fini(rdev);
0fcdb61e 5448 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5449 rdev->accel_working = false;
5450 }
77e00f2e
AD		 5451
5452 /* Don't start up if the MC ucode is missing on BTC parts.
5453 * The default clocks and voltages before the MC ucode
5454 * is loaded are not suffient for advanced operations.
5455 */
5456 if (ASIC_IS_DCE5(rdev)) {
5457 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5458 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5459 return -EINVAL;
5460 }
5461 }
5462
bcc1c2a1
AD		 5463 return 0;
5464}
5465
5466void evergreen_fini(struct radeon_device *rdev)
5467{
6c7bccea 5468 radeon_pm_fini(rdev);
69d2ae57 5469 r600_audio_fini(rdev);
45f9a39b 5470 r700_cp_fini(rdev);
233d1ad5 5471 r600_dma_fini(rdev);
bcc1c2a1 5472 r600_irq_fini(rdev);
2948f5e6
AD		 5473 if (rdev->flags & RADEON_IS_IGP)
5474 sumo_rlc_fini(rdev);
724c80e1 5475 radeon_wb_fini(rdev);
2898c348 5476 radeon_ib_pool_fini(rdev);
bcc1c2a1 5477 radeon_irq_kms_fini(rdev);
bcc1c2a1 5478 evergreen_pcie_gart_fini(rdev);
e409b128 5479 uvd_v1_0_fini(rdev);
f2ba57b5 5480 radeon_uvd_fini(rdev);
16cdf04d 5481 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5482 radeon_gem_fini(rdev);
5483 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5484 radeon_agp_fini(rdev);
5485 radeon_bo_fini(rdev);
5486 radeon_atombios_fini(rdev);
5487 kfree(rdev->bios);
5488 rdev->bios = NULL;
bcc1c2a1 5489}
9e46a48d 5490
b07759bf 5491void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5492{
7e0e4196 5493 u32 link_width_cntl, speed_cntl;
9e46a48d 5494
d42dd579
AD		 5495 if (radeon_pcie_gen2 == 0)
5496 return;
5497
9e46a48d
AD		 5498 if (rdev->flags & RADEON_IS_IGP)
5499 return;
5500
5501 if (!(rdev->flags & RADEON_IS_PCIE))
5502 return;
5503
5504 /* x2 cards have a special sequence */
5505 if (ASIC_IS_X2(rdev))
5506 return;
5507
7e0e4196
KSS		 5508 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5509 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5510 return;
5511
492d2b61 5512 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5513 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5514 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5515 return;
5516 }
5517
197bbb3d
DA		 5518 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5519
9e46a48d
AD		 5520 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5521 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5522
492d2b61 5523 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5524 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5525 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5526
492d2b61 5527 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5528 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5529 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5530
492d2b61 5531 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5532 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5533 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5534
492d2b61 5535 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5536 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5537 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5538
492d2b61 5539 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5540 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5541 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5542
5543 } else {
492d2b61 5544 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5545 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5546 if (1)
5547 link_width_cntl |= LC_UPCONFIGURE_DIS;
5548 else
5549 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5550 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5551 }
5552}
f52382d7
AD		 5553
5554void evergreen_program_aspm(struct radeon_device *rdev)
5555{
5556 u32 data, orig;
5557 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5558 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5559 /* fusion_platform = true
5560 * if the system is a fusion system
5561 * (APU or DGPU in a fusion system).
5562 * todo: check if the system is a fusion platform.
5563 */
5564 bool fusion_platform = false;
5565
1294d4a3
AD		 5566 if (radeon_aspm == 0)
5567 return;
5568
f52382d7
AD		 5569 if (!(rdev->flags & RADEON_IS_PCIE))
5570 return;
5571
5572 switch (rdev->family) {
5573 case CHIP_CYPRESS:
5574 case CHIP_HEMLOCK:
5575 case CHIP_JUNIPER:
5576 case CHIP_REDWOOD:
5577 case CHIP_CEDAR:
5578 case CHIP_SUMO:
5579 case CHIP_SUMO2:
5580 case CHIP_PALM:
5581 case CHIP_ARUBA:
5582 disable_l0s = true;
5583 break;
5584 default:
5585 disable_l0s = false;
5586 break;
5587 }
5588
5589 if (rdev->flags & RADEON_IS_IGP)
5590 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5591
5592 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5593 if (fusion_platform)
5594 data &= ~MULTI_PIF;
5595 else
5596 data |= MULTI_PIF;
5597 if (data != orig)
5598 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5599
5600 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5601 if (fusion_platform)
5602 data &= ~MULTI_PIF;
5603 else
5604 data |= MULTI_PIF;
5605 if (data != orig)
5606 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5607
5608 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5609 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5610 if (!disable_l0s) {
5611 if (rdev->family >= CHIP_BARTS)
5612 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5613 else
5614 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5615 }
5616
5617 if (!disable_l1) {
5618 if (rdev->family >= CHIP_BARTS)
5619 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5620 else
5621 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5622
5623 if (!disable_plloff_in_l1) {
5624 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5625 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5626 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5627 if (data != orig)
5628 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5629
5630 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5631 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5632 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5633 if (data != orig)
5634 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5635
5636 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5637 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5638 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5639 if (data != orig)
5640 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5641
5642 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5643 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5644 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5645 if (data != orig)
5646 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5647
5648 if (rdev->family >= CHIP_BARTS) {
5649 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5650 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5651 data |= PLL_RAMP_UP_TIME_0(4);
5652 if (data != orig)
5653 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5654
5655 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5656 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5657 data |= PLL_RAMP_UP_TIME_1(4);
5658 if (data != orig)
5659 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5660
5661 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5662 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5663 data |= PLL_RAMP_UP_TIME_0(4);
5664 if (data != orig)
5665 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5666
5667 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5668 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5669 data |= PLL_RAMP_UP_TIME_1(4);
5670 if (data != orig)
5671 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5672 }
5673
5674 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5675 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5676 data |= LC_DYN_LANES_PWR_STATE(3);
5677 if (data != orig)
5678 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5679
5680 if (rdev->family >= CHIP_BARTS) {
5681 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5682 data &= ~LS2_EXIT_TIME_MASK;
5683 data |= LS2_EXIT_TIME(1);
5684 if (data != orig)
5685 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5686
5687 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5688 data &= ~LS2_EXIT_TIME_MASK;
5689 data |= LS2_EXIT_TIME(1);
5690 if (data != orig)
5691 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5692 }
5693 }
5694 }
5695
5696 /* evergreen parts only */
5697 if (rdev->family < CHIP_BARTS)
5698 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5699
5700 if (pcie_lc_cntl != pcie_lc_cntl_old)
5701 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5702}
Linux kernel - Deliverables
This page took 0.602876 seconds and 5 git commands to generate.