projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/radeon/kms: add dpm support for trinity asics
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 static const u32 crtc_offsets[6] =
39 {
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46 };
47
48 #include "clearstate_evergreen.h"
49
50 static u32 sumo_rlc_save_restore_register_list[] =
51 {
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133 };
134 static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
136 static void evergreen_gpu_init(struct radeon_device *rdev);
137 void evergreen_fini(struct radeon_device *rdev);
138 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141
142 static const u32 evergreen_golden_registers[] =
143 {
144 0x3f90, 0xffff0000, 0xff000000,
145 0x9148, 0xffff0000, 0xff000000,
146 0x3f94, 0xffff0000, 0xff000000,
147 0x914c, 0xffff0000, 0xff000000,
148 0x9b7c, 0xffffffff, 0x00000000,
149 0x8a14, 0xffffffff, 0x00000007,
150 0x8b10, 0xffffffff, 0x00000000,
151 0x960c, 0xffffffff, 0x54763210,
152 0x88c4, 0xffffffff, 0x000000c2,
153 0x88d4, 0xffffffff, 0x00000010,
154 0x8974, 0xffffffff, 0x00000000,
155 0xc78, 0x00000080, 0x00000080,
156 0x5eb4, 0xffffffff, 0x00000002,
157 0x5e78, 0xffffffff, 0x001000f0,
158 0x6104, 0x01000300, 0x00000000,
159 0x5bc0, 0x00300000, 0x00000000,
160 0x7030, 0xffffffff, 0x00000011,
161 0x7c30, 0xffffffff, 0x00000011,
162 0x10830, 0xffffffff, 0x00000011,
163 0x11430, 0xffffffff, 0x00000011,
164 0x12030, 0xffffffff, 0x00000011,
165 0x12c30, 0xffffffff, 0x00000011,
166 0xd02c, 0xffffffff, 0x08421000,
167 0x240c, 0xffffffff, 0x00000380,
168 0x8b24, 0xffffffff, 0x00ff0fff,
169 0x28a4c, 0x06000000, 0x06000000,
170 0x10c, 0x00000001, 0x00000001,
171 0x8d00, 0xffffffff, 0x100e4848,
172 0x8d04, 0xffffffff, 0x00164745,
173 0x8c00, 0xffffffff, 0xe4000003,
174 0x8c04, 0xffffffff, 0x40600060,
175 0x8c08, 0xffffffff, 0x001c001c,
176 0x8cf0, 0xffffffff, 0x08e00620,
177 0x8c20, 0xffffffff, 0x00800080,
178 0x8c24, 0xffffffff, 0x00800080,
179 0x8c18, 0xffffffff, 0x20202078,
180 0x8c1c, 0xffffffff, 0x00001010,
181 0x28350, 0xffffffff, 0x00000000,
182 0xa008, 0xffffffff, 0x00010000,
183 0x5cc, 0xffffffff, 0x00000001,
184 0x9508, 0xffffffff, 0x00000002,
185 0x913c, 0x0000000f, 0x0000000a
186 };
187
188 static const u32 evergreen_golden_registers2[] =
189 {
190 0x2f4c, 0xffffffff, 0x00000000,
191 0x54f4, 0xffffffff, 0x00000000,
192 0x54f0, 0xffffffff, 0x00000000,
193 0x5498, 0xffffffff, 0x00000000,
194 0x549c, 0xffffffff, 0x00000000,
195 0x5494, 0xffffffff, 0x00000000,
196 0x53cc, 0xffffffff, 0x00000000,
197 0x53c8, 0xffffffff, 0x00000000,
198 0x53c4, 0xffffffff, 0x00000000,
199 0x53c0, 0xffffffff, 0x00000000,
200 0x53bc, 0xffffffff, 0x00000000,
201 0x53b8, 0xffffffff, 0x00000000,
202 0x53b4, 0xffffffff, 0x00000000,
203 0x53b0, 0xffffffff, 0x00000000
204 };
205
206 static const u32 cypress_mgcg_init[] =
207 {
208 0x802c, 0xffffffff, 0xc0000000,
209 0x5448, 0xffffffff, 0x00000100,
210 0x55e4, 0xffffffff, 0x00000100,
211 0x160c, 0xffffffff, 0x00000100,
212 0x5644, 0xffffffff, 0x00000100,
213 0xc164, 0xffffffff, 0x00000100,
214 0x8a18, 0xffffffff, 0x00000100,
215 0x897c, 0xffffffff, 0x06000100,
216 0x8b28, 0xffffffff, 0x00000100,
217 0x9144, 0xffffffff, 0x00000100,
218 0x9a60, 0xffffffff, 0x00000100,
219 0x9868, 0xffffffff, 0x00000100,
220 0x8d58, 0xffffffff, 0x00000100,
221 0x9510, 0xffffffff, 0x00000100,
222 0x949c, 0xffffffff, 0x00000100,
223 0x9654, 0xffffffff, 0x00000100,
224 0x9030, 0xffffffff, 0x00000100,
225 0x9034, 0xffffffff, 0x00000100,
226 0x9038, 0xffffffff, 0x00000100,
227 0x903c, 0xffffffff, 0x00000100,
228 0x9040, 0xffffffff, 0x00000100,
229 0xa200, 0xffffffff, 0x00000100,
230 0xa204, 0xffffffff, 0x00000100,
231 0xa208, 0xffffffff, 0x00000100,
232 0xa20c, 0xffffffff, 0x00000100,
233 0x971c, 0xffffffff, 0x00000100,
234 0x977c, 0xffffffff, 0x00000100,
235 0x3f80, 0xffffffff, 0x00000100,
236 0xa210, 0xffffffff, 0x00000100,
237 0xa214, 0xffffffff, 0x00000100,
238 0x4d8, 0xffffffff, 0x00000100,
239 0x9784, 0xffffffff, 0x00000100,
240 0x9698, 0xffffffff, 0x00000100,
241 0x4d4, 0xffffffff, 0x00000200,
242 0x30cc, 0xffffffff, 0x00000100,
243 0xd0c0, 0xffffffff, 0xff000100,
244 0x802c, 0xffffffff, 0x40000000,
245 0x915c, 0xffffffff, 0x00010000,
246 0x9160, 0xffffffff, 0x00030002,
247 0x9178, 0xffffffff, 0x00070000,
248 0x917c, 0xffffffff, 0x00030002,
249 0x9180, 0xffffffff, 0x00050004,
250 0x918c, 0xffffffff, 0x00010006,
251 0x9190, 0xffffffff, 0x00090008,
252 0x9194, 0xffffffff, 0x00070000,
253 0x9198, 0xffffffff, 0x00030002,
254 0x919c, 0xffffffff, 0x00050004,
255 0x91a8, 0xffffffff, 0x00010006,
256 0x91ac, 0xffffffff, 0x00090008,
257 0x91b0, 0xffffffff, 0x00070000,
258 0x91b4, 0xffffffff, 0x00030002,
259 0x91b8, 0xffffffff, 0x00050004,
260 0x91c4, 0xffffffff, 0x00010006,
261 0x91c8, 0xffffffff, 0x00090008,
262 0x91cc, 0xffffffff, 0x00070000,
263 0x91d0, 0xffffffff, 0x00030002,
264 0x91d4, 0xffffffff, 0x00050004,
265 0x91e0, 0xffffffff, 0x00010006,
266 0x91e4, 0xffffffff, 0x00090008,
267 0x91e8, 0xffffffff, 0x00000000,
268 0x91ec, 0xffffffff, 0x00070000,
269 0x91f0, 0xffffffff, 0x00030002,
270 0x91f4, 0xffffffff, 0x00050004,
271 0x9200, 0xffffffff, 0x00010006,
272 0x9204, 0xffffffff, 0x00090008,
273 0x9208, 0xffffffff, 0x00070000,
274 0x920c, 0xffffffff, 0x00030002,
275 0x9210, 0xffffffff, 0x00050004,
276 0x921c, 0xffffffff, 0x00010006,
277 0x9220, 0xffffffff, 0x00090008,
278 0x9224, 0xffffffff, 0x00070000,
279 0x9228, 0xffffffff, 0x00030002,
280 0x922c, 0xffffffff, 0x00050004,
281 0x9238, 0xffffffff, 0x00010006,
282 0x923c, 0xffffffff, 0x00090008,
283 0x9240, 0xffffffff, 0x00070000,
284 0x9244, 0xffffffff, 0x00030002,
285 0x9248, 0xffffffff, 0x00050004,
286 0x9254, 0xffffffff, 0x00010006,
287 0x9258, 0xffffffff, 0x00090008,
288 0x925c, 0xffffffff, 0x00070000,
289 0x9260, 0xffffffff, 0x00030002,
290 0x9264, 0xffffffff, 0x00050004,
291 0x9270, 0xffffffff, 0x00010006,
292 0x9274, 0xffffffff, 0x00090008,
293 0x9278, 0xffffffff, 0x00070000,
294 0x927c, 0xffffffff, 0x00030002,
295 0x9280, 0xffffffff, 0x00050004,
296 0x928c, 0xffffffff, 0x00010006,
297 0x9290, 0xffffffff, 0x00090008,
298 0x9294, 0xffffffff, 0x00000000,
299 0x929c, 0xffffffff, 0x00000001,
300 0x802c, 0xffffffff, 0x40010000,
301 0x915c, 0xffffffff, 0x00010000,
302 0x9160, 0xffffffff, 0x00030002,
303 0x9178, 0xffffffff, 0x00070000,
304 0x917c, 0xffffffff, 0x00030002,
305 0x9180, 0xffffffff, 0x00050004,
306 0x918c, 0xffffffff, 0x00010006,
307 0x9190, 0xffffffff, 0x00090008,
308 0x9194, 0xffffffff, 0x00070000,
309 0x9198, 0xffffffff, 0x00030002,
310 0x919c, 0xffffffff, 0x00050004,
311 0x91a8, 0xffffffff, 0x00010006,
312 0x91ac, 0xffffffff, 0x00090008,
313 0x91b0, 0xffffffff, 0x00070000,
314 0x91b4, 0xffffffff, 0x00030002,
315 0x91b8, 0xffffffff, 0x00050004,
316 0x91c4, 0xffffffff, 0x00010006,
317 0x91c8, 0xffffffff, 0x00090008,
318 0x91cc, 0xffffffff, 0x00070000,
319 0x91d0, 0xffffffff, 0x00030002,
320 0x91d4, 0xffffffff, 0x00050004,
321 0x91e0, 0xffffffff, 0x00010006,
322 0x91e4, 0xffffffff, 0x00090008,
323 0x91e8, 0xffffffff, 0x00000000,
324 0x91ec, 0xffffffff, 0x00070000,
325 0x91f0, 0xffffffff, 0x00030002,
326 0x91f4, 0xffffffff, 0x00050004,
327 0x9200, 0xffffffff, 0x00010006,
328 0x9204, 0xffffffff, 0x00090008,
329 0x9208, 0xffffffff, 0x00070000,
330 0x920c, 0xffffffff, 0x00030002,
331 0x9210, 0xffffffff, 0x00050004,
332 0x921c, 0xffffffff, 0x00010006,
333 0x9220, 0xffffffff, 0x00090008,
334 0x9224, 0xffffffff, 0x00070000,
335 0x9228, 0xffffffff, 0x00030002,
336 0x922c, 0xffffffff, 0x00050004,
337 0x9238, 0xffffffff, 0x00010006,
338 0x923c, 0xffffffff, 0x00090008,
339 0x9240, 0xffffffff, 0x00070000,
340 0x9244, 0xffffffff, 0x00030002,
341 0x9248, 0xffffffff, 0x00050004,
342 0x9254, 0xffffffff, 0x00010006,
343 0x9258, 0xffffffff, 0x00090008,
344 0x925c, 0xffffffff, 0x00070000,
345 0x9260, 0xffffffff, 0x00030002,
346 0x9264, 0xffffffff, 0x00050004,
347 0x9270, 0xffffffff, 0x00010006,
348 0x9274, 0xffffffff, 0x00090008,
349 0x9278, 0xffffffff, 0x00070000,
350 0x927c, 0xffffffff, 0x00030002,
351 0x9280, 0xffffffff, 0x00050004,
352 0x928c, 0xffffffff, 0x00010006,
353 0x9290, 0xffffffff, 0x00090008,
354 0x9294, 0xffffffff, 0x00000000,
355 0x929c, 0xffffffff, 0x00000001,
356 0x802c, 0xffffffff, 0xc0000000
357 };
358
359 static const u32 redwood_mgcg_init[] =
360 {
361 0x802c, 0xffffffff, 0xc0000000,
362 0x5448, 0xffffffff, 0x00000100,
363 0x55e4, 0xffffffff, 0x00000100,
364 0x160c, 0xffffffff, 0x00000100,
365 0x5644, 0xffffffff, 0x00000100,
366 0xc164, 0xffffffff, 0x00000100,
367 0x8a18, 0xffffffff, 0x00000100,
368 0x897c, 0xffffffff, 0x06000100,
369 0x8b28, 0xffffffff, 0x00000100,
370 0x9144, 0xffffffff, 0x00000100,
371 0x9a60, 0xffffffff, 0x00000100,
372 0x9868, 0xffffffff, 0x00000100,
373 0x8d58, 0xffffffff, 0x00000100,
374 0x9510, 0xffffffff, 0x00000100,
375 0x949c, 0xffffffff, 0x00000100,
376 0x9654, 0xffffffff, 0x00000100,
377 0x9030, 0xffffffff, 0x00000100,
378 0x9034, 0xffffffff, 0x00000100,
379 0x9038, 0xffffffff, 0x00000100,
380 0x903c, 0xffffffff, 0x00000100,
381 0x9040, 0xffffffff, 0x00000100,
382 0xa200, 0xffffffff, 0x00000100,
383 0xa204, 0xffffffff, 0x00000100,
384 0xa208, 0xffffffff, 0x00000100,
385 0xa20c, 0xffffffff, 0x00000100,
386 0x971c, 0xffffffff, 0x00000100,
387 0x977c, 0xffffffff, 0x00000100,
388 0x3f80, 0xffffffff, 0x00000100,
389 0xa210, 0xffffffff, 0x00000100,
390 0xa214, 0xffffffff, 0x00000100,
391 0x4d8, 0xffffffff, 0x00000100,
392 0x9784, 0xffffffff, 0x00000100,
393 0x9698, 0xffffffff, 0x00000100,
394 0x4d4, 0xffffffff, 0x00000200,
395 0x30cc, 0xffffffff, 0x00000100,
396 0xd0c0, 0xffffffff, 0xff000100,
397 0x802c, 0xffffffff, 0x40000000,
398 0x915c, 0xffffffff, 0x00010000,
399 0x9160, 0xffffffff, 0x00030002,
400 0x9178, 0xffffffff, 0x00070000,
401 0x917c, 0xffffffff, 0x00030002,
402 0x9180, 0xffffffff, 0x00050004,
403 0x918c, 0xffffffff, 0x00010006,
404 0x9190, 0xffffffff, 0x00090008,
405 0x9194, 0xffffffff, 0x00070000,
406 0x9198, 0xffffffff, 0x00030002,
407 0x919c, 0xffffffff, 0x00050004,
408 0x91a8, 0xffffffff, 0x00010006,
409 0x91ac, 0xffffffff, 0x00090008,
410 0x91b0, 0xffffffff, 0x00070000,
411 0x91b4, 0xffffffff, 0x00030002,
412 0x91b8, 0xffffffff, 0x00050004,
413 0x91c4, 0xffffffff, 0x00010006,
414 0x91c8, 0xffffffff, 0x00090008,
415 0x91cc, 0xffffffff, 0x00070000,
416 0x91d0, 0xffffffff, 0x00030002,
417 0x91d4, 0xffffffff, 0x00050004,
418 0x91e0, 0xffffffff, 0x00010006,
419 0x91e4, 0xffffffff, 0x00090008,
420 0x91e8, 0xffffffff, 0x00000000,
421 0x91ec, 0xffffffff, 0x00070000,
422 0x91f0, 0xffffffff, 0x00030002,
423 0x91f4, 0xffffffff, 0x00050004,
424 0x9200, 0xffffffff, 0x00010006,
425 0x9204, 0xffffffff, 0x00090008,
426 0x9294, 0xffffffff, 0x00000000,
427 0x929c, 0xffffffff, 0x00000001,
428 0x802c, 0xffffffff, 0xc0000000
429 };
430
431 static const u32 cedar_golden_registers[] =
432 {
433 0x3f90, 0xffff0000, 0xff000000,
434 0x9148, 0xffff0000, 0xff000000,
435 0x3f94, 0xffff0000, 0xff000000,
436 0x914c, 0xffff0000, 0xff000000,
437 0x9b7c, 0xffffffff, 0x00000000,
438 0x8a14, 0xffffffff, 0x00000007,
439 0x8b10, 0xffffffff, 0x00000000,
440 0x960c, 0xffffffff, 0x54763210,
441 0x88c4, 0xffffffff, 0x000000c2,
442 0x88d4, 0xffffffff, 0x00000000,
443 0x8974, 0xffffffff, 0x00000000,
444 0xc78, 0x00000080, 0x00000080,
445 0x5eb4, 0xffffffff, 0x00000002,
446 0x5e78, 0xffffffff, 0x001000f0,
447 0x6104, 0x01000300, 0x00000000,
448 0x5bc0, 0x00300000, 0x00000000,
449 0x7030, 0xffffffff, 0x00000011,
450 0x7c30, 0xffffffff, 0x00000011,
451 0x10830, 0xffffffff, 0x00000011,
452 0x11430, 0xffffffff, 0x00000011,
453 0xd02c, 0xffffffff, 0x08421000,
454 0x240c, 0xffffffff, 0x00000380,
455 0x8b24, 0xffffffff, 0x00ff0fff,
456 0x28a4c, 0x06000000, 0x06000000,
457 0x10c, 0x00000001, 0x00000001,
458 0x8d00, 0xffffffff, 0x100e4848,
459 0x8d04, 0xffffffff, 0x00164745,
460 0x8c00, 0xffffffff, 0xe4000003,
461 0x8c04, 0xffffffff, 0x40600060,
462 0x8c08, 0xffffffff, 0x001c001c,
463 0x8cf0, 0xffffffff, 0x08e00410,
464 0x8c20, 0xffffffff, 0x00800080,
465 0x8c24, 0xffffffff, 0x00800080,
466 0x8c18, 0xffffffff, 0x20202078,
467 0x8c1c, 0xffffffff, 0x00001010,
468 0x28350, 0xffffffff, 0x00000000,
469 0xa008, 0xffffffff, 0x00010000,
470 0x5cc, 0xffffffff, 0x00000001,
471 0x9508, 0xffffffff, 0x00000002
472 };
473
474 static const u32 cedar_mgcg_init[] =
475 {
476 0x802c, 0xffffffff, 0xc0000000,
477 0x5448, 0xffffffff, 0x00000100,
478 0x55e4, 0xffffffff, 0x00000100,
479 0x160c, 0xffffffff, 0x00000100,
480 0x5644, 0xffffffff, 0x00000100,
481 0xc164, 0xffffffff, 0x00000100,
482 0x8a18, 0xffffffff, 0x00000100,
483 0x897c, 0xffffffff, 0x06000100,
484 0x8b28, 0xffffffff, 0x00000100,
485 0x9144, 0xffffffff, 0x00000100,
486 0x9a60, 0xffffffff, 0x00000100,
487 0x9868, 0xffffffff, 0x00000100,
488 0x8d58, 0xffffffff, 0x00000100,
489 0x9510, 0xffffffff, 0x00000100,
490 0x949c, 0xffffffff, 0x00000100,
491 0x9654, 0xffffffff, 0x00000100,
492 0x9030, 0xffffffff, 0x00000100,
493 0x9034, 0xffffffff, 0x00000100,
494 0x9038, 0xffffffff, 0x00000100,
495 0x903c, 0xffffffff, 0x00000100,
496 0x9040, 0xffffffff, 0x00000100,
497 0xa200, 0xffffffff, 0x00000100,
498 0xa204, 0xffffffff, 0x00000100,
499 0xa208, 0xffffffff, 0x00000100,
500 0xa20c, 0xffffffff, 0x00000100,
501 0x971c, 0xffffffff, 0x00000100,
502 0x977c, 0xffffffff, 0x00000100,
503 0x3f80, 0xffffffff, 0x00000100,
504 0xa210, 0xffffffff, 0x00000100,
505 0xa214, 0xffffffff, 0x00000100,
506 0x4d8, 0xffffffff, 0x00000100,
507 0x9784, 0xffffffff, 0x00000100,
508 0x9698, 0xffffffff, 0x00000100,
509 0x4d4, 0xffffffff, 0x00000200,
510 0x30cc, 0xffffffff, 0x00000100,
511 0xd0c0, 0xffffffff, 0xff000100,
512 0x802c, 0xffffffff, 0x40000000,
513 0x915c, 0xffffffff, 0x00010000,
514 0x9178, 0xffffffff, 0x00050000,
515 0x917c, 0xffffffff, 0x00030002,
516 0x918c, 0xffffffff, 0x00010004,
517 0x9190, 0xffffffff, 0x00070006,
518 0x9194, 0xffffffff, 0x00050000,
519 0x9198, 0xffffffff, 0x00030002,
520 0x91a8, 0xffffffff, 0x00010004,
521 0x91ac, 0xffffffff, 0x00070006,
522 0x91e8, 0xffffffff, 0x00000000,
523 0x9294, 0xffffffff, 0x00000000,
524 0x929c, 0xffffffff, 0x00000001,
525 0x802c, 0xffffffff, 0xc0000000
526 };
527
528 static const u32 juniper_mgcg_init[] =
529 {
530 0x802c, 0xffffffff, 0xc0000000,
531 0x5448, 0xffffffff, 0x00000100,
532 0x55e4, 0xffffffff, 0x00000100,
533 0x160c, 0xffffffff, 0x00000100,
534 0x5644, 0xffffffff, 0x00000100,
535 0xc164, 0xffffffff, 0x00000100,
536 0x8a18, 0xffffffff, 0x00000100,
537 0x897c, 0xffffffff, 0x06000100,
538 0x8b28, 0xffffffff, 0x00000100,
539 0x9144, 0xffffffff, 0x00000100,
540 0x9a60, 0xffffffff, 0x00000100,
541 0x9868, 0xffffffff, 0x00000100,
542 0x8d58, 0xffffffff, 0x00000100,
543 0x9510, 0xffffffff, 0x00000100,
544 0x949c, 0xffffffff, 0x00000100,
545 0x9654, 0xffffffff, 0x00000100,
546 0x9030, 0xffffffff, 0x00000100,
547 0x9034, 0xffffffff, 0x00000100,
548 0x9038, 0xffffffff, 0x00000100,
549 0x903c, 0xffffffff, 0x00000100,
550 0x9040, 0xffffffff, 0x00000100,
551 0xa200, 0xffffffff, 0x00000100,
552 0xa204, 0xffffffff, 0x00000100,
553 0xa208, 0xffffffff, 0x00000100,
554 0xa20c, 0xffffffff, 0x00000100,
555 0x971c, 0xffffffff, 0x00000100,
556 0xd0c0, 0xffffffff, 0xff000100,
557 0x802c, 0xffffffff, 0x40000000,
558 0x915c, 0xffffffff, 0x00010000,
559 0x9160, 0xffffffff, 0x00030002,
560 0x9178, 0xffffffff, 0x00070000,
561 0x917c, 0xffffffff, 0x00030002,
562 0x9180, 0xffffffff, 0x00050004,
563 0x918c, 0xffffffff, 0x00010006,
564 0x9190, 0xffffffff, 0x00090008,
565 0x9194, 0xffffffff, 0x00070000,
566 0x9198, 0xffffffff, 0x00030002,
567 0x919c, 0xffffffff, 0x00050004,
568 0x91a8, 0xffffffff, 0x00010006,
569 0x91ac, 0xffffffff, 0x00090008,
570 0x91b0, 0xffffffff, 0x00070000,
571 0x91b4, 0xffffffff, 0x00030002,
572 0x91b8, 0xffffffff, 0x00050004,
573 0x91c4, 0xffffffff, 0x00010006,
574 0x91c8, 0xffffffff, 0x00090008,
575 0x91cc, 0xffffffff, 0x00070000,
576 0x91d0, 0xffffffff, 0x00030002,
577 0x91d4, 0xffffffff, 0x00050004,
578 0x91e0, 0xffffffff, 0x00010006,
579 0x91e4, 0xffffffff, 0x00090008,
580 0x91e8, 0xffffffff, 0x00000000,
581 0x91ec, 0xffffffff, 0x00070000,
582 0x91f0, 0xffffffff, 0x00030002,
583 0x91f4, 0xffffffff, 0x00050004,
584 0x9200, 0xffffffff, 0x00010006,
585 0x9204, 0xffffffff, 0x00090008,
586 0x9208, 0xffffffff, 0x00070000,
587 0x920c, 0xffffffff, 0x00030002,
588 0x9210, 0xffffffff, 0x00050004,
589 0x921c, 0xffffffff, 0x00010006,
590 0x9220, 0xffffffff, 0x00090008,
591 0x9224, 0xffffffff, 0x00070000,
592 0x9228, 0xffffffff, 0x00030002,
593 0x922c, 0xffffffff, 0x00050004,
594 0x9238, 0xffffffff, 0x00010006,
595 0x923c, 0xffffffff, 0x00090008,
596 0x9240, 0xffffffff, 0x00070000,
597 0x9244, 0xffffffff, 0x00030002,
598 0x9248, 0xffffffff, 0x00050004,
599 0x9254, 0xffffffff, 0x00010006,
600 0x9258, 0xffffffff, 0x00090008,
601 0x925c, 0xffffffff, 0x00070000,
602 0x9260, 0xffffffff, 0x00030002,
603 0x9264, 0xffffffff, 0x00050004,
604 0x9270, 0xffffffff, 0x00010006,
605 0x9274, 0xffffffff, 0x00090008,
606 0x9278, 0xffffffff, 0x00070000,
607 0x927c, 0xffffffff, 0x00030002,
608 0x9280, 0xffffffff, 0x00050004,
609 0x928c, 0xffffffff, 0x00010006,
610 0x9290, 0xffffffff, 0x00090008,
611 0x9294, 0xffffffff, 0x00000000,
612 0x929c, 0xffffffff, 0x00000001,
613 0x802c, 0xffffffff, 0xc0000000,
614 0x977c, 0xffffffff, 0x00000100,
615 0x3f80, 0xffffffff, 0x00000100,
616 0xa210, 0xffffffff, 0x00000100,
617 0xa214, 0xffffffff, 0x00000100,
618 0x4d8, 0xffffffff, 0x00000100,
619 0x9784, 0xffffffff, 0x00000100,
620 0x9698, 0xffffffff, 0x00000100,
621 0x4d4, 0xffffffff, 0x00000200,
622 0x30cc, 0xffffffff, 0x00000100,
623 0x802c, 0xffffffff, 0xc0000000
624 };
625
626 static const u32 supersumo_golden_registers[] =
627 {
628 0x5eb4, 0xffffffff, 0x00000002,
629 0x5cc, 0xffffffff, 0x00000001,
630 0x7030, 0xffffffff, 0x00000011,
631 0x7c30, 0xffffffff, 0x00000011,
632 0x6104, 0x01000300, 0x00000000,
633 0x5bc0, 0x00300000, 0x00000000,
634 0x8c04, 0xffffffff, 0x40600060,
635 0x8c08, 0xffffffff, 0x001c001c,
636 0x8c20, 0xffffffff, 0x00800080,
637 0x8c24, 0xffffffff, 0x00800080,
638 0x8c18, 0xffffffff, 0x20202078,
639 0x8c1c, 0xffffffff, 0x00001010,
640 0x918c, 0xffffffff, 0x00010006,
641 0x91a8, 0xffffffff, 0x00010006,
642 0x91c4, 0xffffffff, 0x00010006,
643 0x91e0, 0xffffffff, 0x00010006,
644 0x9200, 0xffffffff, 0x00010006,
645 0x9150, 0xffffffff, 0x6e944040,
646 0x917c, 0xffffffff, 0x00030002,
647 0x9180, 0xffffffff, 0x00050004,
648 0x9198, 0xffffffff, 0x00030002,
649 0x919c, 0xffffffff, 0x00050004,
650 0x91b4, 0xffffffff, 0x00030002,
651 0x91b8, 0xffffffff, 0x00050004,
652 0x91d0, 0xffffffff, 0x00030002,
653 0x91d4, 0xffffffff, 0x00050004,
654 0x91f0, 0xffffffff, 0x00030002,
655 0x91f4, 0xffffffff, 0x00050004,
656 0x915c, 0xffffffff, 0x00010000,
657 0x9160, 0xffffffff, 0x00030002,
658 0x3f90, 0xffff0000, 0xff000000,
659 0x9178, 0xffffffff, 0x00070000,
660 0x9194, 0xffffffff, 0x00070000,
661 0x91b0, 0xffffffff, 0x00070000,
662 0x91cc, 0xffffffff, 0x00070000,
663 0x91ec, 0xffffffff, 0x00070000,
664 0x9148, 0xffff0000, 0xff000000,
665 0x9190, 0xffffffff, 0x00090008,
666 0x91ac, 0xffffffff, 0x00090008,
667 0x91c8, 0xffffffff, 0x00090008,
668 0x91e4, 0xffffffff, 0x00090008,
669 0x9204, 0xffffffff, 0x00090008,
670 0x3f94, 0xffff0000, 0xff000000,
671 0x914c, 0xffff0000, 0xff000000,
672 0x929c, 0xffffffff, 0x00000001,
673 0x8a18, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x5644, 0xffffffff, 0x00000100,
677 0x9b7c, 0xffffffff, 0x00000000,
678 0x8030, 0xffffffff, 0x0000100a,
679 0x8a14, 0xffffffff, 0x00000007,
680 0x8b24, 0xffffffff, 0x00ff0fff,
681 0x8b10, 0xffffffff, 0x00000000,
682 0x28a4c, 0x06000000, 0x06000000,
683 0x4d8, 0xffffffff, 0x00000100,
684 0x913c, 0xffff000f, 0x0100000a,
685 0x960c, 0xffffffff, 0x54763210,
686 0x88c4, 0xffffffff, 0x000000c2,
687 0x88d4, 0xffffffff, 0x00000010,
688 0x8974, 0xffffffff, 0x00000000,
689 0xc78, 0x00000080, 0x00000080,
690 0x5e78, 0xffffffff, 0x001000f0,
691 0xd02c, 0xffffffff, 0x08421000,
692 0xa008, 0xffffffff, 0x00010000,
693 0x8d00, 0xffffffff, 0x100e4848,
694 0x8d04, 0xffffffff, 0x00164745,
695 0x8c00, 0xffffffff, 0xe4000003,
696 0x8cf0, 0x1fffffff, 0x08e00620,
697 0x28350, 0xffffffff, 0x00000000,
698 0x9508, 0xffffffff, 0x00000002
699 };
700
701 static const u32 sumo_golden_registers[] =
702 {
703 0x900c, 0x00ffffff, 0x0017071f,
704 0x8c18, 0xffffffff, 0x10101060,
705 0x8c1c, 0xffffffff, 0x00001010,
706 0x8c30, 0x0000000f, 0x00000005,
707 0x9688, 0x0000000f, 0x00000007
708 };
709
710 static const u32 wrestler_golden_registers[] =
711 {
712 0x5eb4, 0xffffffff, 0x00000002,
713 0x5cc, 0xffffffff, 0x00000001,
714 0x7030, 0xffffffff, 0x00000011,
715 0x7c30, 0xffffffff, 0x00000011,
716 0x6104, 0x01000300, 0x00000000,
717 0x5bc0, 0x00300000, 0x00000000,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x9150, 0xffffffff, 0x6e944040,
721 0x917c, 0xffffffff, 0x00030002,
722 0x9198, 0xffffffff, 0x00030002,
723 0x915c, 0xffffffff, 0x00010000,
724 0x3f90, 0xffff0000, 0xff000000,
725 0x9178, 0xffffffff, 0x00070000,
726 0x9194, 0xffffffff, 0x00070000,
727 0x9148, 0xffff0000, 0xff000000,
728 0x9190, 0xffffffff, 0x00090008,
729 0x91ac, 0xffffffff, 0x00090008,
730 0x3f94, 0xffff0000, 0xff000000,
731 0x914c, 0xffff0000, 0xff000000,
732 0x929c, 0xffffffff, 0x00000001,
733 0x8a18, 0xffffffff, 0x00000100,
734 0x8b28, 0xffffffff, 0x00000100,
735 0x9144, 0xffffffff, 0x00000100,
736 0x9b7c, 0xffffffff, 0x00000000,
737 0x8030, 0xffffffff, 0x0000100a,
738 0x8a14, 0xffffffff, 0x00000001,
739 0x8b24, 0xffffffff, 0x00ff0fff,
740 0x8b10, 0xffffffff, 0x00000000,
741 0x28a4c, 0x06000000, 0x06000000,
742 0x4d8, 0xffffffff, 0x00000100,
743 0x913c, 0xffff000f, 0x0100000a,
744 0x960c, 0xffffffff, 0x54763210,
745 0x88c4, 0xffffffff, 0x000000c2,
746 0x88d4, 0xffffffff, 0x00000010,
747 0x8974, 0xffffffff, 0x00000000,
748 0xc78, 0x00000080, 0x00000080,
749 0x5e78, 0xffffffff, 0x001000f0,
750 0xd02c, 0xffffffff, 0x08421000,
751 0xa008, 0xffffffff, 0x00010000,
752 0x8d00, 0xffffffff, 0x100e4848,
753 0x8d04, 0xffffffff, 0x00164745,
754 0x8c00, 0xffffffff, 0xe4000003,
755 0x8cf0, 0x1fffffff, 0x08e00410,
756 0x28350, 0xffffffff, 0x00000000,
757 0x9508, 0xffffffff, 0x00000002,
758 0x900c, 0xffffffff, 0x0017071f,
759 0x8c18, 0xffffffff, 0x10101060,
760 0x8c1c, 0xffffffff, 0x00001010
761 };
762
763 static const u32 barts_golden_registers[] =
764 {
765 0x5eb4, 0xffffffff, 0x00000002,
766 0x5e78, 0x8f311ff1, 0x001000f0,
767 0x3f90, 0xffff0000, 0xff000000,
768 0x9148, 0xffff0000, 0xff000000,
769 0x3f94, 0xffff0000, 0xff000000,
770 0x914c, 0xffff0000, 0xff000000,
771 0xc78, 0x00000080, 0x00000080,
772 0xbd4, 0x70073777, 0x00010001,
773 0xd02c, 0xbfffff1f, 0x08421000,
774 0xd0b8, 0x03773777, 0x02011003,
775 0x5bc0, 0x00200000, 0x50100000,
776 0x98f8, 0x33773777, 0x02011003,
777 0x98fc, 0xffffffff, 0x76543210,
778 0x7030, 0x31000311, 0x00000011,
779 0x2f48, 0x00000007, 0x02011003,
780 0x6b28, 0x00000010, 0x00000012,
781 0x7728, 0x00000010, 0x00000012,
782 0x10328, 0x00000010, 0x00000012,
783 0x10f28, 0x00000010, 0x00000012,
784 0x11b28, 0x00000010, 0x00000012,
785 0x12728, 0x00000010, 0x00000012,
786 0x240c, 0x000007ff, 0x00000380,
787 0x8a14, 0xf000001f, 0x00000007,
788 0x8b24, 0x3fff3fff, 0x00ff0fff,
789 0x8b10, 0x0000ff0f, 0x00000000,
790 0x28a4c, 0x07ffffff, 0x06000000,
791 0x10c, 0x00000001, 0x00010003,
792 0xa02c, 0xffffffff, 0x0000009b,
793 0x913c, 0x0000000f, 0x0100000a,
794 0x8d00, 0xffff7f7f, 0x100e4848,
795 0x8d04, 0x00ffffff, 0x00164745,
796 0x8c00, 0xfffc0003, 0xe4000003,
797 0x8c04, 0xf8ff00ff, 0x40600060,
798 0x8c08, 0x00ff00ff, 0x001c001c,
799 0x8cf0, 0x1fff1fff, 0x08e00620,
800 0x8c20, 0x0fff0fff, 0x00800080,
801 0x8c24, 0x0fff0fff, 0x00800080,
802 0x8c18, 0xffffffff, 0x20202078,
803 0x8c1c, 0x0000ffff, 0x00001010,
804 0x28350, 0x00000f01, 0x00000000,
805 0x9508, 0x3700001f, 0x00000002,
806 0x960c, 0xffffffff, 0x54763210,
807 0x88c4, 0x001f3ae3, 0x000000c2,
808 0x88d4, 0x0000001f, 0x00000010,
809 0x8974, 0xffffffff, 0x00000000
810 };
811
812 static const u32 turks_golden_registers[] =
813 {
814 0x5eb4, 0xffffffff, 0x00000002,
815 0x5e78, 0x8f311ff1, 0x001000f0,
816 0x8c8, 0x00003000, 0x00001070,
817 0x8cc, 0x000fffff, 0x00040035,
818 0x3f90, 0xffff0000, 0xfff00000,
819 0x9148, 0xffff0000, 0xfff00000,
820 0x3f94, 0xffff0000, 0xfff00000,
821 0x914c, 0xffff0000, 0xfff00000,
822 0xc78, 0x00000080, 0x00000080,
823 0xbd4, 0x00073007, 0x00010002,
824 0xd02c, 0xbfffff1f, 0x08421000,
825 0xd0b8, 0x03773777, 0x02010002,
826 0x5bc0, 0x00200000, 0x50100000,
827 0x98f8, 0x33773777, 0x00010002,
828 0x98fc, 0xffffffff, 0x33221100,
829 0x7030, 0x31000311, 0x00000011,
830 0x2f48, 0x33773777, 0x00010002,
831 0x6b28, 0x00000010, 0x00000012,
832 0x7728, 0x00000010, 0x00000012,
833 0x10328, 0x00000010, 0x00000012,
834 0x10f28, 0x00000010, 0x00000012,
835 0x11b28, 0x00000010, 0x00000012,
836 0x12728, 0x00000010, 0x00000012,
837 0x240c, 0x000007ff, 0x00000380,
838 0x8a14, 0xf000001f, 0x00000007,
839 0x8b24, 0x3fff3fff, 0x00ff0fff,
840 0x8b10, 0x0000ff0f, 0x00000000,
841 0x28a4c, 0x07ffffff, 0x06000000,
842 0x10c, 0x00000001, 0x00010003,
843 0xa02c, 0xffffffff, 0x0000009b,
844 0x913c, 0x0000000f, 0x0100000a,
845 0x8d00, 0xffff7f7f, 0x100e4848,
846 0x8d04, 0x00ffffff, 0x00164745,
847 0x8c00, 0xfffc0003, 0xe4000003,
848 0x8c04, 0xf8ff00ff, 0x40600060,
849 0x8c08, 0x00ff00ff, 0x001c001c,
850 0x8cf0, 0x1fff1fff, 0x08e00410,
851 0x8c20, 0x0fff0fff, 0x00800080,
852 0x8c24, 0x0fff0fff, 0x00800080,
853 0x8c18, 0xffffffff, 0x20202078,
854 0x8c1c, 0x0000ffff, 0x00001010,
855 0x28350, 0x00000f01, 0x00000000,
856 0x9508, 0x3700001f, 0x00000002,
857 0x960c, 0xffffffff, 0x54763210,
858 0x88c4, 0x001f3ae3, 0x000000c2,
859 0x88d4, 0x0000001f, 0x00000010,
860 0x8974, 0xffffffff, 0x00000000
861 };
862
863 static const u32 caicos_golden_registers[] =
864 {
865 0x5eb4, 0xffffffff, 0x00000002,
866 0x5e78, 0x8f311ff1, 0x001000f0,
867 0x8c8, 0x00003420, 0x00001450,
868 0x8cc, 0x000fffff, 0x00040035,
869 0x3f90, 0xffff0000, 0xfffc0000,
870 0x9148, 0xffff0000, 0xfffc0000,
871 0x3f94, 0xffff0000, 0xfffc0000,
872 0x914c, 0xffff0000, 0xfffc0000,
873 0xc78, 0x00000080, 0x00000080,
874 0xbd4, 0x00073007, 0x00010001,
875 0xd02c, 0xbfffff1f, 0x08421000,
876 0xd0b8, 0x03773777, 0x02010001,
877 0x5bc0, 0x00200000, 0x50100000,
878 0x98f8, 0x33773777, 0x02010001,
879 0x98fc, 0xffffffff, 0x33221100,
880 0x7030, 0x31000311, 0x00000011,
881 0x2f48, 0x33773777, 0x02010001,
882 0x6b28, 0x00000010, 0x00000012,
883 0x7728, 0x00000010, 0x00000012,
884 0x10328, 0x00000010, 0x00000012,
885 0x10f28, 0x00000010, 0x00000012,
886 0x11b28, 0x00000010, 0x00000012,
887 0x12728, 0x00000010, 0x00000012,
888 0x240c, 0x000007ff, 0x00000380,
889 0x8a14, 0xf000001f, 0x00000001,
890 0x8b24, 0x3fff3fff, 0x00ff0fff,
891 0x8b10, 0x0000ff0f, 0x00000000,
892 0x28a4c, 0x07ffffff, 0x06000000,
893 0x10c, 0x00000001, 0x00010003,
894 0xa02c, 0xffffffff, 0x0000009b,
895 0x913c, 0x0000000f, 0x0100000a,
896 0x8d00, 0xffff7f7f, 0x100e4848,
897 0x8d04, 0x00ffffff, 0x00164745,
898 0x8c00, 0xfffc0003, 0xe4000003,
899 0x8c04, 0xf8ff00ff, 0x40600060,
900 0x8c08, 0x00ff00ff, 0x001c001c,
901 0x8cf0, 0x1fff1fff, 0x08e00410,
902 0x8c20, 0x0fff0fff, 0x00800080,
903 0x8c24, 0x0fff0fff, 0x00800080,
904 0x8c18, 0xffffffff, 0x20202078,
905 0x8c1c, 0x0000ffff, 0x00001010,
906 0x28350, 0x00000f01, 0x00000000,
907 0x9508, 0x3700001f, 0x00000002,
908 0x960c, 0xffffffff, 0x54763210,
909 0x88c4, 0x001f3ae3, 0x000000c2,
910 0x88d4, 0x0000001f, 0x00000010,
911 0x8974, 0xffffffff, 0x00000000
912 };
913
914 static void evergreen_init_golden_registers(struct radeon_device *rdev)
915 {
916 switch (rdev->family) {
917 case CHIP_CYPRESS:
918 case CHIP_HEMLOCK:
919 radeon_program_register_sequence(rdev,
920 evergreen_golden_registers,
921 (const u32)ARRAY_SIZE(evergreen_golden_registers));
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers2,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
925 radeon_program_register_sequence(rdev,
926 cypress_mgcg_init,
927 (const u32)ARRAY_SIZE(cypress_mgcg_init));
928 break;
929 case CHIP_JUNIPER:
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers));
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers2,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
936 radeon_program_register_sequence(rdev,
937 juniper_mgcg_init,
938 (const u32)ARRAY_SIZE(juniper_mgcg_init));
939 break;
940 case CHIP_REDWOOD:
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers));
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers2,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
947 radeon_program_register_sequence(rdev,
948 redwood_mgcg_init,
949 (const u32)ARRAY_SIZE(redwood_mgcg_init));
950 break;
951 case CHIP_CEDAR:
952 radeon_program_register_sequence(rdev,
953 cedar_golden_registers,
954 (const u32)ARRAY_SIZE(cedar_golden_registers));
955 radeon_program_register_sequence(rdev,
956 evergreen_golden_registers2,
957 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
958 radeon_program_register_sequence(rdev,
959 cedar_mgcg_init,
960 (const u32)ARRAY_SIZE(cedar_mgcg_init));
961 break;
962 case CHIP_PALM:
963 radeon_program_register_sequence(rdev,
964 wrestler_golden_registers,
965 (const u32)ARRAY_SIZE(wrestler_golden_registers));
966 break;
967 case CHIP_SUMO:
968 radeon_program_register_sequence(rdev,
969 supersumo_golden_registers,
970 (const u32)ARRAY_SIZE(supersumo_golden_registers));
971 break;
972 case CHIP_SUMO2:
973 radeon_program_register_sequence(rdev,
974 supersumo_golden_registers,
975 (const u32)ARRAY_SIZE(supersumo_golden_registers));
976 radeon_program_register_sequence(rdev,
977 sumo_golden_registers,
978 (const u32)ARRAY_SIZE(sumo_golden_registers));
979 break;
980 case CHIP_BARTS:
981 radeon_program_register_sequence(rdev,
982 barts_golden_registers,
983 (const u32)ARRAY_SIZE(barts_golden_registers));
984 break;
985 case CHIP_TURKS:
986 radeon_program_register_sequence(rdev,
987 turks_golden_registers,
988 (const u32)ARRAY_SIZE(turks_golden_registers));
989 break;
990 case CHIP_CAICOS:
991 radeon_program_register_sequence(rdev,
992 caicos_golden_registers,
993 (const u32)ARRAY_SIZE(caicos_golden_registers));
994 break;
995 default:
996 break;
997 }
998 }
999
1000 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1001 unsigned *bankh, unsigned *mtaspect,
1002 unsigned *tile_split)
1003 {
1004 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1005 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1006 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1007 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1008 switch (*bankw) {
1009 default:
1010 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1011 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1012 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1013 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1014 }
1015 switch (*bankh) {
1016 default:
1017 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1018 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1019 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1020 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1021 }
1022 switch (*mtaspect) {
1023 default:
1024 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1025 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1026 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1027 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1028 }
1029 }
1030
1031 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1032 u32 cntl_reg, u32 status_reg)
1033 {
1034 int r, i;
1035 struct atom_clock_dividers dividers;
1036
1037 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1038 clock, false, &dividers);
1039 if (r)
1040 return r;
1041
1042 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1043
1044 for (i = 0; i < 100; i++) {
1045 if (RREG32(status_reg) & DCLK_STATUS)
1046 break;
1047 mdelay(10);
1048 }
1049 if (i == 100)
1050 return -ETIMEDOUT;
1051
1052 return 0;
1053 }
1054
1055 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1056 {
1057 int r = 0;
1058 u32 cg_scratch = RREG32(CG_SCRATCH1);
1059
1060 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1061 if (r)
1062 goto done;
1063 cg_scratch &= 0xffff0000;
1064 cg_scratch |= vclk / 100; /* Mhz */
1065
1066 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1067 if (r)
1068 goto done;
1069 cg_scratch &= 0x0000ffff;
1070 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1071
1072 done:
1073 WREG32(CG_SCRATCH1, cg_scratch);
1074
1075 return r;
1076 }
1077
1078 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1079 {
1080 /* start off with something large */
1081 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1082 int r;
1083
1084 /* bypass vclk and dclk with bclk */
1085 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1086 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1087 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1088
1089 /* put PLL in bypass mode */
1090 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1091
1092 if (!vclk || !dclk) {
1093 /* keep the Bypass mode, put PLL to sleep */
1094 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1095 return 0;
1096 }
1097
1098 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1099 16384, 0x03FFFFFF, 0, 128, 5,
1100 &fb_div, &vclk_div, &dclk_div);
1101 if (r)
1102 return r;
1103
1104 /* set VCO_MODE to 1 */
1105 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1106
1107 /* toggle UPLL_SLEEP to 1 then back to 0 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1109 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1110
1111 /* deassert UPLL_RESET */
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1113
1114 mdelay(1);
1115
1116 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1117 if (r)
1118 return r;
1119
1120 /* assert UPLL_RESET again */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1122
1123 /* disable spread spectrum. */
1124 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1125
1126 /* set feedback divider */
1127 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1128
1129 /* set ref divider to 0 */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1131
1132 if (fb_div < 307200)
1133 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1134 else
1135 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1136
1137 /* set PDIV_A and PDIV_B */
1138 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1139 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1140 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1141
1142 /* give the PLL some time to settle */
1143 mdelay(15);
1144
1145 /* deassert PLL_RESET */
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1147
1148 mdelay(15);
1149
1150 /* switch from bypass mode to normal mode */
1151 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1152
1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1154 if (r)
1155 return r;
1156
1157 /* switch VCLK and DCLK selection */
1158 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1159 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1160 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1161
1162 mdelay(100);
1163
1164 return 0;
1165 }
1166
1167 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1168 {
1169 u16 ctl, v;
1170 int err;
1171
1172 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1173 if (err)
1174 return;
1175
1176 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1177
1178 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1179 * to avoid hangs or perfomance issues
1180 */
1181 if ((v == 0) || (v == 6) || (v == 7)) {
1182 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1183 ctl |= (2 << 12);
1184 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1185 }
1186 }
1187
1188 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1189 {
1190 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1191 return true;
1192 else
1193 return false;
1194 }
1195
1196 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1197 {
1198 u32 pos1, pos2;
1199
1200 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1201 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202
1203 if (pos1 != pos2)
1204 return true;
1205 else
1206 return false;
1207 }
1208
1209 /**
1210 * dce4_wait_for_vblank - vblank wait asic callback.
1211 *
1212 * @rdev: radeon_device pointer
1213 * @crtc: crtc to wait for vblank on
1214 *
1215 * Wait for vblank on the requested crtc (evergreen+).
1216 */
1217 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1218 {
1219 unsigned i = 0;
1220
1221 if (crtc >= rdev->num_crtc)
1222 return;
1223
1224 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1225 return;
1226
1227 /* depending on when we hit vblank, we may be close to active; if so,
1228 * wait for another frame.
1229 */
1230 while (dce4_is_in_vblank(rdev, crtc)) {
1231 if (i++ % 100 == 0) {
1232 if (!dce4_is_counter_moving(rdev, crtc))
1233 break;
1234 }
1235 }
1236
1237 while (!dce4_is_in_vblank(rdev, crtc)) {
1238 if (i++ % 100 == 0) {
1239 if (!dce4_is_counter_moving(rdev, crtc))
1240 break;
1241 }
1242 }
1243 }
1244
1245 /**
1246 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1247 *
1248 * @rdev: radeon_device pointer
1249 * @crtc: crtc to prepare for pageflip on
1250 *
1251 * Pre-pageflip callback (evergreen+).
1252 * Enables the pageflip irq (vblank irq).
1253 */
1254 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1255 {
1256 /* enable the pflip int */
1257 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1258 }
1259
1260 /**
1261 * evergreen_post_page_flip - pos-pageflip callback.
1262 *
1263 * @rdev: radeon_device pointer
1264 * @crtc: crtc to cleanup pageflip on
1265 *
1266 * Post-pageflip callback (evergreen+).
1267 * Disables the pageflip irq (vblank irq).
1268 */
1269 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1270 {
1271 /* disable the pflip int */
1272 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1273 }
1274
1275 /**
1276 * evergreen_page_flip - pageflip callback.
1277 *
1278 * @rdev: radeon_device pointer
1279 * @crtc_id: crtc to cleanup pageflip on
1280 * @crtc_base: new address of the crtc (GPU MC address)
1281 *
1282 * Does the actual pageflip (evergreen+).
1283 * During vblank we take the crtc lock and wait for the update_pending
1284 * bit to go high, when it does, we release the lock, and allow the
1285 * double buffered update to take place.
1286 * Returns the current update pending status.
1287 */
1288 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1289 {
1290 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1291 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1292 int i;
1293
1294 /* Lock the graphics update lock */
1295 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1296 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1297
1298 /* update the scanout addresses */
1299 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1300 upper_32_bits(crtc_base));
1301 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1302 (u32)crtc_base);
1303
1304 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1305 upper_32_bits(crtc_base));
1306 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1307 (u32)crtc_base);
1308
1309 /* Wait for update_pending to go high. */
1310 for (i = 0; i < rdev->usec_timeout; i++) {
1311 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1312 break;
1313 udelay(1);
1314 }
1315 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1316
1317 /* Unlock the lock, so double-buffering can take place inside vblank */
1318 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1319 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1320
1321 /* Return current update_pending status: */
1322 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1323 }
1324
1325 /* get temperature in millidegrees */
1326 int evergreen_get_temp(struct radeon_device *rdev)
1327 {
1328 u32 temp, toffset;
1329 int actual_temp = 0;
1330
1331 if (rdev->family == CHIP_JUNIPER) {
1332 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1333 TOFFSET_SHIFT;
1334 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1335 TS0_ADC_DOUT_SHIFT;
1336
1337 if (toffset & 0x100)
1338 actual_temp = temp / 2 - (0x200 - toffset);
1339 else
1340 actual_temp = temp / 2 + toffset;
1341
1342 actual_temp = actual_temp * 1000;
1343
1344 } else {
1345 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1346 ASIC_T_SHIFT;
1347
1348 if (temp & 0x400)
1349 actual_temp = -256;
1350 else if (temp & 0x200)
1351 actual_temp = 255;
1352 else if (temp & 0x100) {
1353 actual_temp = temp & 0x1ff;
1354 actual_temp |= ~0x1ff;
1355 } else
1356 actual_temp = temp & 0xff;
1357
1358 actual_temp = (actual_temp * 1000) / 2;
1359 }
1360
1361 return actual_temp;
1362 }
1363
1364 int sumo_get_temp(struct radeon_device *rdev)
1365 {
1366 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1367 int actual_temp = temp - 49;
1368
1369 return actual_temp * 1000;
1370 }
1371
1372 /**
1373 * sumo_pm_init_profile - Initialize power profiles callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 *
1377 * Initialize the power states used in profile mode
1378 * (sumo, trinity, SI).
1379 * Used for profile mode only.
1380 */
1381 void sumo_pm_init_profile(struct radeon_device *rdev)
1382 {
1383 int idx;
1384
1385 /* default */
1386 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1390
1391 /* low,mid sh/mh */
1392 if (rdev->flags & RADEON_IS_MOBILITY)
1393 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1394 else
1395 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1396
1397 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1401
1402 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1406
1407 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1411
1412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1416
1417 /* high sh/mh */
1418 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1419 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1423 rdev->pm.power_state[idx].num_clock_modes - 1;
1424
1425 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1429 rdev->pm.power_state[idx].num_clock_modes - 1;
1430 }
1431
1432 /**
1433 * btc_pm_init_profile - Initialize power profiles callback.
1434 *
1435 * @rdev: radeon_device pointer
1436 *
1437 * Initialize the power states used in profile mode
1438 * (BTC, cayman).
1439 * Used for profile mode only.
1440 */
1441 void btc_pm_init_profile(struct radeon_device *rdev)
1442 {
1443 int idx;
1444
1445 /* default */
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1450 /* starting with BTC, there is one state that is used for both
1451 * MH and SH. Difference is that we always use the high clock index for
1452 * mclk.
1453 */
1454 if (rdev->flags & RADEON_IS_MOBILITY)
1455 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1456 else
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1458 /* low sh */
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1463 /* mid sh */
1464 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1468 /* high sh */
1469 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1473 /* low mh */
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1478 /* mid mh */
1479 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1483 /* high mh */
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1488 }
1489
1490 /**
1491 * evergreen_pm_misc - set additional pm hw parameters callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Set non-clock parameters associated with a power state
1496 * (voltage, etc.) (evergreen+).
1497 */
1498 void evergreen_pm_misc(struct radeon_device *rdev)
1499 {
1500 int req_ps_idx = rdev->pm.requested_power_state_index;
1501 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1502 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1503 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1504
1505 if (voltage->type == VOLTAGE_SW) {
1506 /* 0xff01 is a flag rather then an actual voltage */
1507 if (voltage->voltage == 0xff01)
1508 return;
1509 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1510 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1511 rdev->pm.current_vddc = voltage->voltage;
1512 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1513 }
1514
1515 /* starting with BTC, there is one state that is used for both
1516 * MH and SH. Difference is that we always use the high clock index for
1517 * mclk and vddci.
1518 */
1519 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1520 (rdev->family >= CHIP_BARTS) &&
1521 rdev->pm.active_crtc_count &&
1522 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1523 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1524 voltage = &rdev->pm.power_state[req_ps_idx].
1525 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1526
1527 /* 0xff01 is a flag rather then an actual voltage */
1528 if (voltage->vddci == 0xff01)
1529 return;
1530 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1531 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1532 rdev->pm.current_vddci = voltage->vddci;
1533 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1534 }
1535 }
1536 }
1537
1538 /**
1539 * evergreen_pm_prepare - pre-power state change callback.
1540 *
1541 * @rdev: radeon_device pointer
1542 *
1543 * Prepare for a power state change (evergreen+).
1544 */
1545 void evergreen_pm_prepare(struct radeon_device *rdev)
1546 {
1547 struct drm_device *ddev = rdev->ddev;
1548 struct drm_crtc *crtc;
1549 struct radeon_crtc *radeon_crtc;
1550 u32 tmp;
1551
1552 /* disable any active CRTCs */
1553 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1554 radeon_crtc = to_radeon_crtc(crtc);
1555 if (radeon_crtc->enabled) {
1556 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1557 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1558 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1559 }
1560 }
1561 }
1562
1563 /**
1564 * evergreen_pm_finish - post-power state change callback.
1565 *
1566 * @rdev: radeon_device pointer
1567 *
1568 * Clean up after a power state change (evergreen+).
1569 */
1570 void evergreen_pm_finish(struct radeon_device *rdev)
1571 {
1572 struct drm_device *ddev = rdev->ddev;
1573 struct drm_crtc *crtc;
1574 struct radeon_crtc *radeon_crtc;
1575 u32 tmp;
1576
1577 /* enable any active CRTCs */
1578 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1579 radeon_crtc = to_radeon_crtc(crtc);
1580 if (radeon_crtc->enabled) {
1581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1582 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1583 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1584 }
1585 }
1586 }
1587
1588 /**
1589 * evergreen_hpd_sense - hpd sense callback.
1590 *
1591 * @rdev: radeon_device pointer
1592 * @hpd: hpd (hotplug detect) pin
1593 *
1594 * Checks if a digital monitor is connected (evergreen+).
1595 * Returns true if connected, false if not connected.
1596 */
1597 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1598 {
1599 bool connected = false;
1600
1601 switch (hpd) {
1602 case RADEON_HPD_1:
1603 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1604 connected = true;
1605 break;
1606 case RADEON_HPD_2:
1607 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1608 connected = true;
1609 break;
1610 case RADEON_HPD_3:
1611 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1612 connected = true;
1613 break;
1614 case RADEON_HPD_4:
1615 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1616 connected = true;
1617 break;
1618 case RADEON_HPD_5:
1619 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1620 connected = true;
1621 break;
1622 case RADEON_HPD_6:
1623 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1624 connected = true;
1625 break;
1626 default:
1627 break;
1628 }
1629
1630 return connected;
1631 }
1632
1633 /**
1634 * evergreen_hpd_set_polarity - hpd set polarity callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 * @hpd: hpd (hotplug detect) pin
1638 *
1639 * Set the polarity of the hpd pin (evergreen+).
1640 */
1641 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1642 enum radeon_hpd_id hpd)
1643 {
1644 u32 tmp;
1645 bool connected = evergreen_hpd_sense(rdev, hpd);
1646
1647 switch (hpd) {
1648 case RADEON_HPD_1:
1649 tmp = RREG32(DC_HPD1_INT_CONTROL);
1650 if (connected)
1651 tmp &= ~DC_HPDx_INT_POLARITY;
1652 else
1653 tmp |= DC_HPDx_INT_POLARITY;
1654 WREG32(DC_HPD1_INT_CONTROL, tmp);
1655 break;
1656 case RADEON_HPD_2:
1657 tmp = RREG32(DC_HPD2_INT_CONTROL);
1658 if (connected)
1659 tmp &= ~DC_HPDx_INT_POLARITY;
1660 else
1661 tmp |= DC_HPDx_INT_POLARITY;
1662 WREG32(DC_HPD2_INT_CONTROL, tmp);
1663 break;
1664 case RADEON_HPD_3:
1665 tmp = RREG32(DC_HPD3_INT_CONTROL);
1666 if (connected)
1667 tmp &= ~DC_HPDx_INT_POLARITY;
1668 else
1669 tmp |= DC_HPDx_INT_POLARITY;
1670 WREG32(DC_HPD3_INT_CONTROL, tmp);
1671 break;
1672 case RADEON_HPD_4:
1673 tmp = RREG32(DC_HPD4_INT_CONTROL);
1674 if (connected)
1675 tmp &= ~DC_HPDx_INT_POLARITY;
1676 else
1677 tmp |= DC_HPDx_INT_POLARITY;
1678 WREG32(DC_HPD4_INT_CONTROL, tmp);
1679 break;
1680 case RADEON_HPD_5:
1681 tmp = RREG32(DC_HPD5_INT_CONTROL);
1682 if (connected)
1683 tmp &= ~DC_HPDx_INT_POLARITY;
1684 else
1685 tmp |= DC_HPDx_INT_POLARITY;
1686 WREG32(DC_HPD5_INT_CONTROL, tmp);
1687 break;
1688 case RADEON_HPD_6:
1689 tmp = RREG32(DC_HPD6_INT_CONTROL);
1690 if (connected)
1691 tmp &= ~DC_HPDx_INT_POLARITY;
1692 else
1693 tmp |= DC_HPDx_INT_POLARITY;
1694 WREG32(DC_HPD6_INT_CONTROL, tmp);
1695 break;
1696 default:
1697 break;
1698 }
1699 }
1700
1701 /**
1702 * evergreen_hpd_init - hpd setup callback.
1703 *
1704 * @rdev: radeon_device pointer
1705 *
1706 * Setup the hpd pins used by the card (evergreen+).
1707 * Enable the pin, set the polarity, and enable the hpd interrupts.
1708 */
1709 void evergreen_hpd_init(struct radeon_device *rdev)
1710 {
1711 struct drm_device *dev = rdev->ddev;
1712 struct drm_connector *connector;
1713 unsigned enabled = 0;
1714 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1715 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1716
1717 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1718 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1719
1720 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1721 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1722 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1723 * aux dp channel on imac and help (but not completely fix)
1724 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1725 * also avoid interrupt storms during dpms.
1726 */
1727 continue;
1728 }
1729 switch (radeon_connector->hpd.hpd) {
1730 case RADEON_HPD_1:
1731 WREG32(DC_HPD1_CONTROL, tmp);
1732 break;
1733 case RADEON_HPD_2:
1734 WREG32(DC_HPD2_CONTROL, tmp);
1735 break;
1736 case RADEON_HPD_3:
1737 WREG32(DC_HPD3_CONTROL, tmp);
1738 break;
1739 case RADEON_HPD_4:
1740 WREG32(DC_HPD4_CONTROL, tmp);
1741 break;
1742 case RADEON_HPD_5:
1743 WREG32(DC_HPD5_CONTROL, tmp);
1744 break;
1745 case RADEON_HPD_6:
1746 WREG32(DC_HPD6_CONTROL, tmp);
1747 break;
1748 default:
1749 break;
1750 }
1751 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1752 enabled |= 1 << radeon_connector->hpd.hpd;
1753 }
1754 radeon_irq_kms_enable_hpd(rdev, enabled);
1755 }
1756
1757 /**
1758 * evergreen_hpd_fini - hpd tear down callback.
1759 *
1760 * @rdev: radeon_device pointer
1761 *
1762 * Tear down the hpd pins used by the card (evergreen+).
1763 * Disable the hpd interrupts.
1764 */
1765 void evergreen_hpd_fini(struct radeon_device *rdev)
1766 {
1767 struct drm_device *dev = rdev->ddev;
1768 struct drm_connector *connector;
1769 unsigned disabled = 0;
1770
1771 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1772 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1773 switch (radeon_connector->hpd.hpd) {
1774 case RADEON_HPD_1:
1775 WREG32(DC_HPD1_CONTROL, 0);
1776 break;
1777 case RADEON_HPD_2:
1778 WREG32(DC_HPD2_CONTROL, 0);
1779 break;
1780 case RADEON_HPD_3:
1781 WREG32(DC_HPD3_CONTROL, 0);
1782 break;
1783 case RADEON_HPD_4:
1784 WREG32(DC_HPD4_CONTROL, 0);
1785 break;
1786 case RADEON_HPD_5:
1787 WREG32(DC_HPD5_CONTROL, 0);
1788 break;
1789 case RADEON_HPD_6:
1790 WREG32(DC_HPD6_CONTROL, 0);
1791 break;
1792 default:
1793 break;
1794 }
1795 disabled |= 1 << radeon_connector->hpd.hpd;
1796 }
1797 radeon_irq_kms_disable_hpd(rdev, disabled);
1798 }
1799
1800 /* watermark setup */
1801
1802 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1803 struct radeon_crtc *radeon_crtc,
1804 struct drm_display_mode *mode,
1805 struct drm_display_mode *other_mode)
1806 {
1807 u32 tmp;
1808 /*
1809 * Line Buffer Setup
1810 * There are 3 line buffers, each one shared by 2 display controllers.
1811 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1812 * the display controllers. The paritioning is done via one of four
1813 * preset allocations specified in bits 2:0:
1814 * first display controller
1815 * 0 - first half of lb (3840 * 2)
1816 * 1 - first 3/4 of lb (5760 * 2)
1817 * 2 - whole lb (7680 * 2), other crtc must be disabled
1818 * 3 - first 1/4 of lb (1920 * 2)
1819 * second display controller
1820 * 4 - second half of lb (3840 * 2)
1821 * 5 - second 3/4 of lb (5760 * 2)
1822 * 6 - whole lb (7680 * 2), other crtc must be disabled
1823 * 7 - last 1/4 of lb (1920 * 2)
1824 */
1825 /* this can get tricky if we have two large displays on a paired group
1826 * of crtcs. Ideally for multiple large displays we'd assign them to
1827 * non-linked crtcs for maximum line buffer allocation.
1828 */
1829 if (radeon_crtc->base.enabled && mode) {
1830 if (other_mode)
1831 tmp = 0; /* 1/2 */
1832 else
1833 tmp = 2; /* whole */
1834 } else
1835 tmp = 0;
1836
1837 /* second controller of the pair uses second half of the lb */
1838 if (radeon_crtc->crtc_id % 2)
1839 tmp += 4;
1840 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1841
1842 if (radeon_crtc->base.enabled && mode) {
1843 switch (tmp) {
1844 case 0:
1845 case 4:
1846 default:
1847 if (ASIC_IS_DCE5(rdev))
1848 return 4096 * 2;
1849 else
1850 return 3840 * 2;
1851 case 1:
1852 case 5:
1853 if (ASIC_IS_DCE5(rdev))
1854 return 6144 * 2;
1855 else
1856 return 5760 * 2;
1857 case 2:
1858 case 6:
1859 if (ASIC_IS_DCE5(rdev))
1860 return 8192 * 2;
1861 else
1862 return 7680 * 2;
1863 case 3:
1864 case 7:
1865 if (ASIC_IS_DCE5(rdev))
1866 return 2048 * 2;
1867 else
1868 return 1920 * 2;
1869 }
1870 }
1871
1872 /* controller not enabled, so no lb used */
1873 return 0;
1874 }
1875
1876 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1877 {
1878 u32 tmp = RREG32(MC_SHARED_CHMAP);
1879
1880 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1881 case 0:
1882 default:
1883 return 1;
1884 case 1:
1885 return 2;
1886 case 2:
1887 return 4;
1888 case 3:
1889 return 8;
1890 }
1891 }
1892
1893 struct evergreen_wm_params {
1894 u32 dram_channels; /* number of dram channels */
1895 u32 yclk; /* bandwidth per dram data pin in kHz */
1896 u32 sclk; /* engine clock in kHz */
1897 u32 disp_clk; /* display clock in kHz */
1898 u32 src_width; /* viewport width */
1899 u32 active_time; /* active display time in ns */
1900 u32 blank_time; /* blank time in ns */
1901 bool interlaced; /* mode is interlaced */
1902 fixed20_12 vsc; /* vertical scale ratio */
1903 u32 num_heads; /* number of active crtcs */
1904 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1905 u32 lb_size; /* line buffer allocated to pipe */
1906 u32 vtaps; /* vertical scaler taps */
1907 };
1908
1909 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1910 {
1911 /* Calculate DRAM Bandwidth and the part allocated to display. */
1912 fixed20_12 dram_efficiency; /* 0.7 */
1913 fixed20_12 yclk, dram_channels, bandwidth;
1914 fixed20_12 a;
1915
1916 a.full = dfixed_const(1000);
1917 yclk.full = dfixed_const(wm->yclk);
1918 yclk.full = dfixed_div(yclk, a);
1919 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1920 a.full = dfixed_const(10);
1921 dram_efficiency.full = dfixed_const(7);
1922 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1923 bandwidth.full = dfixed_mul(dram_channels, yclk);
1924 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1925
1926 return dfixed_trunc(bandwidth);
1927 }
1928
1929 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1930 {
1931 /* Calculate DRAM Bandwidth and the part allocated to display. */
1932 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1933 fixed20_12 yclk, dram_channels, bandwidth;
1934 fixed20_12 a;
1935
1936 a.full = dfixed_const(1000);
1937 yclk.full = dfixed_const(wm->yclk);
1938 yclk.full = dfixed_div(yclk, a);
1939 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940 a.full = dfixed_const(10);
1941 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1942 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1943 bandwidth.full = dfixed_mul(dram_channels, yclk);
1944 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1945
1946 return dfixed_trunc(bandwidth);
1947 }
1948
1949 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1950 {
1951 /* Calculate the display Data return Bandwidth */
1952 fixed20_12 return_efficiency; /* 0.8 */
1953 fixed20_12 sclk, bandwidth;
1954 fixed20_12 a;
1955
1956 a.full = dfixed_const(1000);
1957 sclk.full = dfixed_const(wm->sclk);
1958 sclk.full = dfixed_div(sclk, a);
1959 a.full = dfixed_const(10);
1960 return_efficiency.full = dfixed_const(8);
1961 return_efficiency.full = dfixed_div(return_efficiency, a);
1962 a.full = dfixed_const(32);
1963 bandwidth.full = dfixed_mul(a, sclk);
1964 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1965
1966 return dfixed_trunc(bandwidth);
1967 }
1968
1969 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1970 {
1971 /* Calculate the DMIF Request Bandwidth */
1972 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1973 fixed20_12 disp_clk, bandwidth;
1974 fixed20_12 a;
1975
1976 a.full = dfixed_const(1000);
1977 disp_clk.full = dfixed_const(wm->disp_clk);
1978 disp_clk.full = dfixed_div(disp_clk, a);
1979 a.full = dfixed_const(10);
1980 disp_clk_request_efficiency.full = dfixed_const(8);
1981 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1982 a.full = dfixed_const(32);
1983 bandwidth.full = dfixed_mul(a, disp_clk);
1984 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1985
1986 return dfixed_trunc(bandwidth);
1987 }
1988
1989 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1990 {
1991 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1992 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1993 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1994 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1995
1996 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1997 }
1998
1999 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2000 {
2001 /* Calculate the display mode Average Bandwidth
2002 * DisplayMode should contain the source and destination dimensions,
2003 * timing, etc.
2004 */
2005 fixed20_12 bpp;
2006 fixed20_12 line_time;
2007 fixed20_12 src_width;
2008 fixed20_12 bandwidth;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1000);
2012 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2013 line_time.full = dfixed_div(line_time, a);
2014 bpp.full = dfixed_const(wm->bytes_per_pixel);
2015 src_width.full = dfixed_const(wm->src_width);
2016 bandwidth.full = dfixed_mul(src_width, bpp);
2017 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2018 bandwidth.full = dfixed_div(bandwidth, line_time);
2019
2020 return dfixed_trunc(bandwidth);
2021 }
2022
2023 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2024 {
2025 /* First calcualte the latency in ns */
2026 u32 mc_latency = 2000; /* 2000 ns. */
2027 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2028 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2029 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2030 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2031 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2032 (wm->num_heads * cursor_line_pair_return_time);
2033 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2034 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2035 fixed20_12 a, b, c;
2036
2037 if (wm->num_heads == 0)
2038 return 0;
2039
2040 a.full = dfixed_const(2);
2041 b.full = dfixed_const(1);
2042 if ((wm->vsc.full > a.full) ||
2043 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2044 (wm->vtaps >= 5) ||
2045 ((wm->vsc.full >= a.full) && wm->interlaced))
2046 max_src_lines_per_dst_line = 4;
2047 else
2048 max_src_lines_per_dst_line = 2;
2049
2050 a.full = dfixed_const(available_bandwidth);
2051 b.full = dfixed_const(wm->num_heads);
2052 a.full = dfixed_div(a, b);
2053
2054 b.full = dfixed_const(1000);
2055 c.full = dfixed_const(wm->disp_clk);
2056 b.full = dfixed_div(c, b);
2057 c.full = dfixed_const(wm->bytes_per_pixel);
2058 b.full = dfixed_mul(b, c);
2059
2060 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2061
2062 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2063 b.full = dfixed_const(1000);
2064 c.full = dfixed_const(lb_fill_bw);
2065 b.full = dfixed_div(c, b);
2066 a.full = dfixed_div(a, b);
2067 line_fill_time = dfixed_trunc(a);
2068
2069 if (line_fill_time < wm->active_time)
2070 return latency;
2071 else
2072 return latency + (line_fill_time - wm->active_time);
2073
2074 }
2075
2076 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2077 {
2078 if (evergreen_average_bandwidth(wm) <=
2079 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2080 return true;
2081 else
2082 return false;
2083 };
2084
2085 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2086 {
2087 if (evergreen_average_bandwidth(wm) <=
2088 (evergreen_available_bandwidth(wm) / wm->num_heads))
2089 return true;
2090 else
2091 return false;
2092 };
2093
2094 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2095 {
2096 u32 lb_partitions = wm->lb_size / wm->src_width;
2097 u32 line_time = wm->active_time + wm->blank_time;
2098 u32 latency_tolerant_lines;
2099 u32 latency_hiding;
2100 fixed20_12 a;
2101
2102 a.full = dfixed_const(1);
2103 if (wm->vsc.full > a.full)
2104 latency_tolerant_lines = 1;
2105 else {
2106 if (lb_partitions <= (wm->vtaps + 1))
2107 latency_tolerant_lines = 1;
2108 else
2109 latency_tolerant_lines = 2;
2110 }
2111
2112 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2113
2114 if (evergreen_latency_watermark(wm) <= latency_hiding)
2115 return true;
2116 else
2117 return false;
2118 }
2119
2120 static void evergreen_program_watermarks(struct radeon_device *rdev,
2121 struct radeon_crtc *radeon_crtc,
2122 u32 lb_size, u32 num_heads)
2123 {
2124 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2125 struct evergreen_wm_params wm_low, wm_high;
2126 u32 dram_channels;
2127 u32 pixel_period;
2128 u32 line_time = 0;
2129 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2130 u32 priority_a_mark = 0, priority_b_mark = 0;
2131 u32 priority_a_cnt = PRIORITY_OFF;
2132 u32 priority_b_cnt = PRIORITY_OFF;
2133 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2134 u32 tmp, arb_control3;
2135 fixed20_12 a, b, c;
2136
2137 if (radeon_crtc->base.enabled && num_heads && mode) {
2138 pixel_period = 1000000 / (u32)mode->clock;
2139 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2140 priority_a_cnt = 0;
2141 priority_b_cnt = 0;
2142 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2143
2144 /* watermark for high clocks */
2145 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2146 wm_high.yclk =
2147 radeon_dpm_get_mclk(rdev, false) * 10;
2148 wm_high.sclk =
2149 radeon_dpm_get_sclk(rdev, false) * 10;
2150 } else {
2151 wm_high.yclk = rdev->pm.current_mclk * 10;
2152 wm_high.sclk = rdev->pm.current_sclk * 10;
2153 }
2154
2155 wm_high.disp_clk = mode->clock;
2156 wm_high.src_width = mode->crtc_hdisplay;
2157 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2158 wm_high.blank_time = line_time - wm_high.active_time;
2159 wm_high.interlaced = false;
2160 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2161 wm_high.interlaced = true;
2162 wm_high.vsc = radeon_crtc->vsc;
2163 wm_high.vtaps = 1;
2164 if (radeon_crtc->rmx_type != RMX_OFF)
2165 wm_high.vtaps = 2;
2166 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2167 wm_high.lb_size = lb_size;
2168 wm_high.dram_channels = dram_channels;
2169 wm_high.num_heads = num_heads;
2170
2171 /* watermark for low clocks */
2172 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2173 wm_low.yclk =
2174 radeon_dpm_get_mclk(rdev, true) * 10;
2175 wm_low.sclk =
2176 radeon_dpm_get_sclk(rdev, true) * 10;
2177 } else {
2178 wm_low.yclk = rdev->pm.current_mclk * 10;
2179 wm_low.sclk = rdev->pm.current_sclk * 10;
2180 }
2181
2182 wm_low.disp_clk = mode->clock;
2183 wm_low.src_width = mode->crtc_hdisplay;
2184 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2185 wm_low.blank_time = line_time - wm_low.active_time;
2186 wm_low.interlaced = false;
2187 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2188 wm_low.interlaced = true;
2189 wm_low.vsc = radeon_crtc->vsc;
2190 wm_low.vtaps = 1;
2191 if (radeon_crtc->rmx_type != RMX_OFF)
2192 wm_low.vtaps = 2;
2193 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2194 wm_low.lb_size = lb_size;
2195 wm_low.dram_channels = dram_channels;
2196 wm_low.num_heads = num_heads;
2197
2198 /* set for high clocks */
2199 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2200 /* set for low clocks */
2201 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2202
2203 /* possibly force display priority to high */
2204 /* should really do this at mode validation time... */
2205 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2206 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2207 !evergreen_check_latency_hiding(&wm_high) ||
2208 (rdev->disp_priority == 2)) {
2209 DRM_DEBUG_KMS("force priority a to high\n");
2210 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2211 }
2212 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2213 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2214 !evergreen_check_latency_hiding(&wm_low) ||
2215 (rdev->disp_priority == 2)) {
2216 DRM_DEBUG_KMS("force priority b to high\n");
2217 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2218 }
2219
2220 a.full = dfixed_const(1000);
2221 b.full = dfixed_const(mode->clock);
2222 b.full = dfixed_div(b, a);
2223 c.full = dfixed_const(latency_watermark_a);
2224 c.full = dfixed_mul(c, b);
2225 c.full = dfixed_mul(c, radeon_crtc->hsc);
2226 c.full = dfixed_div(c, a);
2227 a.full = dfixed_const(16);
2228 c.full = dfixed_div(c, a);
2229 priority_a_mark = dfixed_trunc(c);
2230 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2231
2232 a.full = dfixed_const(1000);
2233 b.full = dfixed_const(mode->clock);
2234 b.full = dfixed_div(b, a);
2235 c.full = dfixed_const(latency_watermark_b);
2236 c.full = dfixed_mul(c, b);
2237 c.full = dfixed_mul(c, radeon_crtc->hsc);
2238 c.full = dfixed_div(c, a);
2239 a.full = dfixed_const(16);
2240 c.full = dfixed_div(c, a);
2241 priority_b_mark = dfixed_trunc(c);
2242 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2243 }
2244
2245 /* select wm A */
2246 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2247 tmp = arb_control3;
2248 tmp &= ~LATENCY_WATERMARK_MASK(3);
2249 tmp |= LATENCY_WATERMARK_MASK(1);
2250 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2251 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2252 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2253 LATENCY_HIGH_WATERMARK(line_time)));
2254 /* select wm B */
2255 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2256 tmp &= ~LATENCY_WATERMARK_MASK(3);
2257 tmp |= LATENCY_WATERMARK_MASK(2);
2258 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2259 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2260 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2261 LATENCY_HIGH_WATERMARK(line_time)));
2262 /* restore original selection */
2263 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2264
2265 /* write the priority marks */
2266 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2267 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2268
2269 }
2270
2271 /**
2272 * evergreen_bandwidth_update - update display watermarks callback.
2273 *
2274 * @rdev: radeon_device pointer
2275 *
2276 * Update the display watermarks based on the requested mode(s)
2277 * (evergreen+).
2278 */
2279 void evergreen_bandwidth_update(struct radeon_device *rdev)
2280 {
2281 struct drm_display_mode *mode0 = NULL;
2282 struct drm_display_mode *mode1 = NULL;
2283 u32 num_heads = 0, lb_size;
2284 int i;
2285
2286 radeon_update_display_priority(rdev);
2287
2288 for (i = 0; i < rdev->num_crtc; i++) {
2289 if (rdev->mode_info.crtcs[i]->base.enabled)
2290 num_heads++;
2291 }
2292 for (i = 0; i < rdev->num_crtc; i += 2) {
2293 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2294 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2295 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2296 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2297 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2298 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2299 }
2300 }
2301
2302 /**
2303 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2304 *
2305 * @rdev: radeon_device pointer
2306 *
2307 * Wait for the MC (memory controller) to be idle.
2308 * (evergreen+).
2309 * Returns 0 if the MC is idle, -1 if not.
2310 */
2311 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2312 {
2313 unsigned i;
2314 u32 tmp;
2315
2316 for (i = 0; i < rdev->usec_timeout; i++) {
2317 /* read MC_STATUS */
2318 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2319 if (!tmp)
2320 return 0;
2321 udelay(1);
2322 }
2323 return -1;
2324 }
2325
2326 /*
2327 * GART
2328 */
2329 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2330 {
2331 unsigned i;
2332 u32 tmp;
2333
2334 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2335
2336 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2337 for (i = 0; i < rdev->usec_timeout; i++) {
2338 /* read MC_STATUS */
2339 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2340 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2341 if (tmp == 2) {
2342 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2343 return;
2344 }
2345 if (tmp) {
2346 return;
2347 }
2348 udelay(1);
2349 }
2350 }
2351
2352 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2353 {
2354 u32 tmp;
2355 int r;
2356
2357 if (rdev->gart.robj == NULL) {
2358 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2359 return -EINVAL;
2360 }
2361 r = radeon_gart_table_vram_pin(rdev);
2362 if (r)
2363 return r;
2364 radeon_gart_restore(rdev);
2365 /* Setup L2 cache */
2366 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2367 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2368 EFFECTIVE_L2_QUEUE_SIZE(7));
2369 WREG32(VM_L2_CNTL2, 0);
2370 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2371 /* Setup TLB control */
2372 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2373 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2374 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2375 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2376 if (rdev->flags & RADEON_IS_IGP) {
2377 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2378 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2379 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2380 } else {
2381 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2382 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2383 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2384 if ((rdev->family == CHIP_JUNIPER) ||
2385 (rdev->family == CHIP_CYPRESS) ||
2386 (rdev->family == CHIP_HEMLOCK) ||
2387 (rdev->family == CHIP_BARTS))
2388 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2389 }
2390 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2391 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2392 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2393 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2394 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2395 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2396 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2397 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2398 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2399 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2400 (u32)(rdev->dummy_page.addr >> 12));
2401 WREG32(VM_CONTEXT1_CNTL, 0);
2402
2403 evergreen_pcie_gart_tlb_flush(rdev);
2404 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2405 (unsigned)(rdev->mc.gtt_size >> 20),
2406 (unsigned long long)rdev->gart.table_addr);
2407 rdev->gart.ready = true;
2408 return 0;
2409 }
2410
2411 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2412 {
2413 u32 tmp;
2414
2415 /* Disable all tables */
2416 WREG32(VM_CONTEXT0_CNTL, 0);
2417 WREG32(VM_CONTEXT1_CNTL, 0);
2418
2419 /* Setup L2 cache */
2420 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2421 EFFECTIVE_L2_QUEUE_SIZE(7));
2422 WREG32(VM_L2_CNTL2, 0);
2423 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2424 /* Setup TLB control */
2425 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2426 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2430 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2431 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2432 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2433 radeon_gart_table_vram_unpin(rdev);
2434 }
2435
2436 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2437 {
2438 evergreen_pcie_gart_disable(rdev);
2439 radeon_gart_table_vram_free(rdev);
2440 radeon_gart_fini(rdev);
2441 }
2442
2443
2444 static void evergreen_agp_enable(struct radeon_device *rdev)
2445 {
2446 u32 tmp;
2447
2448 /* Setup L2 cache */
2449 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2450 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2451 EFFECTIVE_L2_QUEUE_SIZE(7));
2452 WREG32(VM_L2_CNTL2, 0);
2453 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2454 /* Setup TLB control */
2455 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2456 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2457 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2458 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2459 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2462 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2463 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2464 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2465 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2466 WREG32(VM_CONTEXT0_CNTL, 0);
2467 WREG32(VM_CONTEXT1_CNTL, 0);
2468 }
2469
2470 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2471 {
2472 u32 crtc_enabled, tmp, frame_count, blackout;
2473 int i, j;
2474
2475 if (!ASIC_IS_NODCE(rdev)) {
2476 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2477 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2478
2479 /* disable VGA render */
2480 WREG32(VGA_RENDER_CONTROL, 0);
2481 }
2482 /* blank the display controllers */
2483 for (i = 0; i < rdev->num_crtc; i++) {
2484 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2485 if (crtc_enabled) {
2486 save->crtc_enabled[i] = true;
2487 if (ASIC_IS_DCE6(rdev)) {
2488 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2489 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2490 radeon_wait_for_vblank(rdev, i);
2491 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2492 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2493 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2494 }
2495 } else {
2496 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2497 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2498 radeon_wait_for_vblank(rdev, i);
2499 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2500 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2501 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2502 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2503 }
2504 }
2505 /* wait for the next frame */
2506 frame_count = radeon_get_vblank_counter(rdev, i);
2507 for (j = 0; j < rdev->usec_timeout; j++) {
2508 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2509 break;
2510 udelay(1);
2511 }
2512
2513 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2514 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2515 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2516 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2517 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2518 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2519 save->crtc_enabled[i] = false;
2520 /* ***** */
2521 } else {
2522 save->crtc_enabled[i] = false;
2523 }
2524 }
2525
2526 radeon_mc_wait_for_idle(rdev);
2527
2528 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2529 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2530 /* Block CPU access */
2531 WREG32(BIF_FB_EN, 0);
2532 /* blackout the MC */
2533 blackout &= ~BLACKOUT_MODE_MASK;
2534 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2535 }
2536 /* wait for the MC to settle */
2537 udelay(100);
2538
2539 /* lock double buffered regs */
2540 for (i = 0; i < rdev->num_crtc; i++) {
2541 if (save->crtc_enabled[i]) {
2542 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2543 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2544 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2545 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2546 }
2547 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2548 if (!(tmp & 1)) {
2549 tmp |= 1;
2550 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2551 }
2552 }
2553 }
2554 }
2555
2556 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2557 {
2558 u32 tmp, frame_count;
2559 int i, j;
2560
2561 /* update crtc base addresses */
2562 for (i = 0; i < rdev->num_crtc; i++) {
2563 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2564 upper_32_bits(rdev->mc.vram_start));
2565 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2566 upper_32_bits(rdev->mc.vram_start));
2567 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2568 (u32)rdev->mc.vram_start);
2569 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2570 (u32)rdev->mc.vram_start);
2571 }
2572
2573 if (!ASIC_IS_NODCE(rdev)) {
2574 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2575 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2576 }
2577
2578 /* unlock regs and wait for update */
2579 for (i = 0; i < rdev->num_crtc; i++) {
2580 if (save->crtc_enabled[i]) {
2581 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2582 if ((tmp & 0x3) != 0) {
2583 tmp &= ~0x3;
2584 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2585 }
2586 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2587 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2588 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2589 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2590 }
2591 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2592 if (tmp & 1) {
2593 tmp &= ~1;
2594 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2595 }
2596 for (j = 0; j < rdev->usec_timeout; j++) {
2597 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2598 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2599 break;
2600 udelay(1);
2601 }
2602 }
2603 }
2604
2605 /* unblackout the MC */
2606 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2607 tmp &= ~BLACKOUT_MODE_MASK;
2608 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2609 /* allow CPU access */
2610 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2611
2612 for (i = 0; i < rdev->num_crtc; i++) {
2613 if (save->crtc_enabled[i]) {
2614 if (ASIC_IS_DCE6(rdev)) {
2615 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2616 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2617 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2618 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2619 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2620 } else {
2621 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2622 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2623 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2624 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2625 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2626 }
2627 /* wait for the next frame */
2628 frame_count = radeon_get_vblank_counter(rdev, i);
2629 for (j = 0; j < rdev->usec_timeout; j++) {
2630 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2631 break;
2632 udelay(1);
2633 }
2634 }
2635 }
2636 if (!ASIC_IS_NODCE(rdev)) {
2637 /* Unlock vga access */
2638 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2639 mdelay(1);
2640 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2641 }
2642 }
2643
2644 void evergreen_mc_program(struct radeon_device *rdev)
2645 {
2646 struct evergreen_mc_save save;
2647 u32 tmp;
2648 int i, j;
2649
2650 /* Initialize HDP */
2651 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2652 WREG32((0x2c14 + j), 0x00000000);
2653 WREG32((0x2c18 + j), 0x00000000);
2654 WREG32((0x2c1c + j), 0x00000000);
2655 WREG32((0x2c20 + j), 0x00000000);
2656 WREG32((0x2c24 + j), 0x00000000);
2657 }
2658 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2659
2660 evergreen_mc_stop(rdev, &save);
2661 if (evergreen_mc_wait_for_idle(rdev)) {
2662 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2663 }
2664 /* Lockout access through VGA aperture*/
2665 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2666 /* Update configuration */
2667 if (rdev->flags & RADEON_IS_AGP) {
2668 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2669 /* VRAM before AGP */
2670 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2671 rdev->mc.vram_start >> 12);
2672 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2673 rdev->mc.gtt_end >> 12);
2674 } else {
2675 /* VRAM after AGP */
2676 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2677 rdev->mc.gtt_start >> 12);
2678 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2679 rdev->mc.vram_end >> 12);
2680 }
2681 } else {
2682 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2683 rdev->mc.vram_start >> 12);
2684 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2685 rdev->mc.vram_end >> 12);
2686 }
2687 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2688 /* llano/ontario only */
2689 if ((rdev->family == CHIP_PALM) ||
2690 (rdev->family == CHIP_SUMO) ||
2691 (rdev->family == CHIP_SUMO2)) {
2692 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2693 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2694 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2695 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2696 }
2697 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2698 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2699 WREG32(MC_VM_FB_LOCATION, tmp);
2700 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2701 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2702 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2703 if (rdev->flags & RADEON_IS_AGP) {
2704 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2705 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2706 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2707 } else {
2708 WREG32(MC_VM_AGP_BASE, 0);
2709 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2710 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2711 }
2712 if (evergreen_mc_wait_for_idle(rdev)) {
2713 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2714 }
2715 evergreen_mc_resume(rdev, &save);
2716 /* we need to own VRAM, so turn off the VGA renderer here
2717 * to stop it overwriting our objects */
2718 rv515_vga_render_disable(rdev);
2719 }
2720
2721 /*
2722 * CP.
2723 */
2724 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2725 {
2726 struct radeon_ring *ring = &rdev->ring[ib->ring];
2727 u32 next_rptr;
2728
2729 /* set to DX10/11 mode */
2730 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2731 radeon_ring_write(ring, 1);
2732
2733 if (ring->rptr_save_reg) {
2734 next_rptr = ring->wptr + 3 + 4;
2735 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2736 radeon_ring_write(ring, ((ring->rptr_save_reg -
2737 PACKET3_SET_CONFIG_REG_START) >> 2));
2738 radeon_ring_write(ring, next_rptr);
2739 } else if (rdev->wb.enabled) {
2740 next_rptr = ring->wptr + 5 + 4;
2741 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2742 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2743 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2744 radeon_ring_write(ring, next_rptr);
2745 radeon_ring_write(ring, 0);
2746 }
2747
2748 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2749 radeon_ring_write(ring,
2750 #ifdef __BIG_ENDIAN
2751 (2 << 0) |
2752 #endif
2753 (ib->gpu_addr & 0xFFFFFFFC));
2754 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2755 radeon_ring_write(ring, ib->length_dw);
2756 }
2757
2758
2759 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2760 {
2761 const __be32 *fw_data;
2762 int i;
2763
2764 if (!rdev->me_fw || !rdev->pfp_fw)
2765 return -EINVAL;
2766
2767 r700_cp_stop(rdev);
2768 WREG32(CP_RB_CNTL,
2769 #ifdef __BIG_ENDIAN
2770 BUF_SWAP_32BIT |
2771 #endif
2772 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2773
2774 fw_data = (const __be32 *)rdev->pfp_fw->data;
2775 WREG32(CP_PFP_UCODE_ADDR, 0);
2776 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2777 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2778 WREG32(CP_PFP_UCODE_ADDR, 0);
2779
2780 fw_data = (const __be32 *)rdev->me_fw->data;
2781 WREG32(CP_ME_RAM_WADDR, 0);
2782 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2783 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2784
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2786 WREG32(CP_ME_RAM_WADDR, 0);
2787 WREG32(CP_ME_RAM_RADDR, 0);
2788 return 0;
2789 }
2790
2791 static int evergreen_cp_start(struct radeon_device *rdev)
2792 {
2793 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2794 int r, i;
2795 uint32_t cp_me;
2796
2797 r = radeon_ring_lock(rdev, ring, 7);
2798 if (r) {
2799 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2800 return r;
2801 }
2802 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2803 radeon_ring_write(ring, 0x1);
2804 radeon_ring_write(ring, 0x0);
2805 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2806 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2807 radeon_ring_write(ring, 0);
2808 radeon_ring_write(ring, 0);
2809 radeon_ring_unlock_commit(rdev, ring);
2810
2811 cp_me = 0xff;
2812 WREG32(CP_ME_CNTL, cp_me);
2813
2814 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2815 if (r) {
2816 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2817 return r;
2818 }
2819
2820 /* setup clear context state */
2821 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2822 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2823
2824 for (i = 0; i < evergreen_default_size; i++)
2825 radeon_ring_write(ring, evergreen_default_state[i]);
2826
2827 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2829
2830 /* set clear context state */
2831 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2832 radeon_ring_write(ring, 0);
2833
2834 /* SQ_VTX_BASE_VTX_LOC */
2835 radeon_ring_write(ring, 0xc0026f00);
2836 radeon_ring_write(ring, 0x00000000);
2837 radeon_ring_write(ring, 0x00000000);
2838 radeon_ring_write(ring, 0x00000000);
2839
2840 /* Clear consts */
2841 radeon_ring_write(ring, 0xc0036f00);
2842 radeon_ring_write(ring, 0x00000bc4);
2843 radeon_ring_write(ring, 0xffffffff);
2844 radeon_ring_write(ring, 0xffffffff);
2845 radeon_ring_write(ring, 0xffffffff);
2846
2847 radeon_ring_write(ring, 0xc0026900);
2848 radeon_ring_write(ring, 0x00000316);
2849 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2850 radeon_ring_write(ring, 0x00000010); /* */
2851
2852 radeon_ring_unlock_commit(rdev, ring);
2853
2854 return 0;
2855 }
2856
2857 static int evergreen_cp_resume(struct radeon_device *rdev)
2858 {
2859 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2860 u32 tmp;
2861 u32 rb_bufsz;
2862 int r;
2863
2864 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2865 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2866 SOFT_RESET_PA |
2867 SOFT_RESET_SH |
2868 SOFT_RESET_VGT |
2869 SOFT_RESET_SPI |
2870 SOFT_RESET_SX));
2871 RREG32(GRBM_SOFT_RESET);
2872 mdelay(15);
2873 WREG32(GRBM_SOFT_RESET, 0);
2874 RREG32(GRBM_SOFT_RESET);
2875
2876 /* Set ring buffer size */
2877 rb_bufsz = drm_order(ring->ring_size / 8);
2878 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2879 #ifdef __BIG_ENDIAN
2880 tmp |= BUF_SWAP_32BIT;
2881 #endif
2882 WREG32(CP_RB_CNTL, tmp);
2883 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2884 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2885
2886 /* Set the write pointer delay */
2887 WREG32(CP_RB_WPTR_DELAY, 0);
2888
2889 /* Initialize the ring buffer's read and write pointers */
2890 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2891 WREG32(CP_RB_RPTR_WR, 0);
2892 ring->wptr = 0;
2893 WREG32(CP_RB_WPTR, ring->wptr);
2894
2895 /* set the wb address whether it's enabled or not */
2896 WREG32(CP_RB_RPTR_ADDR,
2897 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2898 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2899 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2900
2901 if (rdev->wb.enabled)
2902 WREG32(SCRATCH_UMSK, 0xff);
2903 else {
2904 tmp |= RB_NO_UPDATE;
2905 WREG32(SCRATCH_UMSK, 0);
2906 }
2907
2908 mdelay(1);
2909 WREG32(CP_RB_CNTL, tmp);
2910
2911 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2912 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2913
2914 ring->rptr = RREG32(CP_RB_RPTR);
2915
2916 evergreen_cp_start(rdev);
2917 ring->ready = true;
2918 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2919 if (r) {
2920 ring->ready = false;
2921 return r;
2922 }
2923 return 0;
2924 }
2925
2926 /*
2927 * Core functions
2928 */
2929 static void evergreen_gpu_init(struct radeon_device *rdev)
2930 {
2931 u32 gb_addr_config;
2932 u32 mc_shared_chmap, mc_arb_ramcfg;
2933 u32 sx_debug_1;
2934 u32 smx_dc_ctl0;
2935 u32 sq_config;
2936 u32 sq_lds_resource_mgmt;
2937 u32 sq_gpr_resource_mgmt_1;
2938 u32 sq_gpr_resource_mgmt_2;
2939 u32 sq_gpr_resource_mgmt_3;
2940 u32 sq_thread_resource_mgmt;
2941 u32 sq_thread_resource_mgmt_2;
2942 u32 sq_stack_resource_mgmt_1;
2943 u32 sq_stack_resource_mgmt_2;
2944 u32 sq_stack_resource_mgmt_3;
2945 u32 vgt_cache_invalidation;
2946 u32 hdp_host_path_cntl, tmp;
2947 u32 disabled_rb_mask;
2948 int i, j, num_shader_engines, ps_thread_count;
2949
2950 switch (rdev->family) {
2951 case CHIP_CYPRESS:
2952 case CHIP_HEMLOCK:
2953 rdev->config.evergreen.num_ses = 2;
2954 rdev->config.evergreen.max_pipes = 4;
2955 rdev->config.evergreen.max_tile_pipes = 8;
2956 rdev->config.evergreen.max_simds = 10;
2957 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2958 rdev->config.evergreen.max_gprs = 256;
2959 rdev->config.evergreen.max_threads = 248;
2960 rdev->config.evergreen.max_gs_threads = 32;
2961 rdev->config.evergreen.max_stack_entries = 512;
2962 rdev->config.evergreen.sx_num_of_sets = 4;
2963 rdev->config.evergreen.sx_max_export_size = 256;
2964 rdev->config.evergreen.sx_max_export_pos_size = 64;
2965 rdev->config.evergreen.sx_max_export_smx_size = 192;
2966 rdev->config.evergreen.max_hw_contexts = 8;
2967 rdev->config.evergreen.sq_num_cf_insts = 2;
2968
2969 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2970 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2971 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2972 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2973 break;
2974 case CHIP_JUNIPER:
2975 rdev->config.evergreen.num_ses = 1;
2976 rdev->config.evergreen.max_pipes = 4;
2977 rdev->config.evergreen.max_tile_pipes = 4;
2978 rdev->config.evergreen.max_simds = 10;
2979 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2980 rdev->config.evergreen.max_gprs = 256;
2981 rdev->config.evergreen.max_threads = 248;
2982 rdev->config.evergreen.max_gs_threads = 32;
2983 rdev->config.evergreen.max_stack_entries = 512;
2984 rdev->config.evergreen.sx_num_of_sets = 4;
2985 rdev->config.evergreen.sx_max_export_size = 256;
2986 rdev->config.evergreen.sx_max_export_pos_size = 64;
2987 rdev->config.evergreen.sx_max_export_smx_size = 192;
2988 rdev->config.evergreen.max_hw_contexts = 8;
2989 rdev->config.evergreen.sq_num_cf_insts = 2;
2990
2991 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2992 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2993 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2994 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
2995 break;
2996 case CHIP_REDWOOD:
2997 rdev->config.evergreen.num_ses = 1;
2998 rdev->config.evergreen.max_pipes = 4;
2999 rdev->config.evergreen.max_tile_pipes = 4;
3000 rdev->config.evergreen.max_simds = 5;
3001 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3002 rdev->config.evergreen.max_gprs = 256;
3003 rdev->config.evergreen.max_threads = 248;
3004 rdev->config.evergreen.max_gs_threads = 32;
3005 rdev->config.evergreen.max_stack_entries = 256;
3006 rdev->config.evergreen.sx_num_of_sets = 4;
3007 rdev->config.evergreen.sx_max_export_size = 256;
3008 rdev->config.evergreen.sx_max_export_pos_size = 64;
3009 rdev->config.evergreen.sx_max_export_smx_size = 192;
3010 rdev->config.evergreen.max_hw_contexts = 8;
3011 rdev->config.evergreen.sq_num_cf_insts = 2;
3012
3013 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3014 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3015 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3016 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3017 break;
3018 case CHIP_CEDAR:
3019 default:
3020 rdev->config.evergreen.num_ses = 1;
3021 rdev->config.evergreen.max_pipes = 2;
3022 rdev->config.evergreen.max_tile_pipes = 2;
3023 rdev->config.evergreen.max_simds = 2;
3024 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3025 rdev->config.evergreen.max_gprs = 256;
3026 rdev->config.evergreen.max_threads = 192;
3027 rdev->config.evergreen.max_gs_threads = 16;
3028 rdev->config.evergreen.max_stack_entries = 256;
3029 rdev->config.evergreen.sx_num_of_sets = 4;
3030 rdev->config.evergreen.sx_max_export_size = 128;
3031 rdev->config.evergreen.sx_max_export_pos_size = 32;
3032 rdev->config.evergreen.sx_max_export_smx_size = 96;
3033 rdev->config.evergreen.max_hw_contexts = 4;
3034 rdev->config.evergreen.sq_num_cf_insts = 1;
3035
3036 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3037 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3038 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3039 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3040 break;
3041 case CHIP_PALM:
3042 rdev->config.evergreen.num_ses = 1;
3043 rdev->config.evergreen.max_pipes = 2;
3044 rdev->config.evergreen.max_tile_pipes = 2;
3045 rdev->config.evergreen.max_simds = 2;
3046 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3047 rdev->config.evergreen.max_gprs = 256;
3048 rdev->config.evergreen.max_threads = 192;
3049 rdev->config.evergreen.max_gs_threads = 16;
3050 rdev->config.evergreen.max_stack_entries = 256;
3051 rdev->config.evergreen.sx_num_of_sets = 4;
3052 rdev->config.evergreen.sx_max_export_size = 128;
3053 rdev->config.evergreen.sx_max_export_pos_size = 32;
3054 rdev->config.evergreen.sx_max_export_smx_size = 96;
3055 rdev->config.evergreen.max_hw_contexts = 4;
3056 rdev->config.evergreen.sq_num_cf_insts = 1;
3057
3058 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3059 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3060 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3061 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3062 break;
3063 case CHIP_SUMO:
3064 rdev->config.evergreen.num_ses = 1;
3065 rdev->config.evergreen.max_pipes = 4;
3066 rdev->config.evergreen.max_tile_pipes = 4;
3067 if (rdev->pdev->device == 0x9648)
3068 rdev->config.evergreen.max_simds = 3;
3069 else if ((rdev->pdev->device == 0x9647) ||
3070 (rdev->pdev->device == 0x964a))
3071 rdev->config.evergreen.max_simds = 4;
3072 else
3073 rdev->config.evergreen.max_simds = 5;
3074 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3075 rdev->config.evergreen.max_gprs = 256;
3076 rdev->config.evergreen.max_threads = 248;
3077 rdev->config.evergreen.max_gs_threads = 32;
3078 rdev->config.evergreen.max_stack_entries = 256;
3079 rdev->config.evergreen.sx_num_of_sets = 4;
3080 rdev->config.evergreen.sx_max_export_size = 256;
3081 rdev->config.evergreen.sx_max_export_pos_size = 64;
3082 rdev->config.evergreen.sx_max_export_smx_size = 192;
3083 rdev->config.evergreen.max_hw_contexts = 8;
3084 rdev->config.evergreen.sq_num_cf_insts = 2;
3085
3086 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3087 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3088 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3089 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3090 break;
3091 case CHIP_SUMO2:
3092 rdev->config.evergreen.num_ses = 1;
3093 rdev->config.evergreen.max_pipes = 4;
3094 rdev->config.evergreen.max_tile_pipes = 4;
3095 rdev->config.evergreen.max_simds = 2;
3096 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3097 rdev->config.evergreen.max_gprs = 256;
3098 rdev->config.evergreen.max_threads = 248;
3099 rdev->config.evergreen.max_gs_threads = 32;
3100 rdev->config.evergreen.max_stack_entries = 512;
3101 rdev->config.evergreen.sx_num_of_sets = 4;
3102 rdev->config.evergreen.sx_max_export_size = 256;
3103 rdev->config.evergreen.sx_max_export_pos_size = 64;
3104 rdev->config.evergreen.sx_max_export_smx_size = 192;
3105 rdev->config.evergreen.max_hw_contexts = 8;
3106 rdev->config.evergreen.sq_num_cf_insts = 2;
3107
3108 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3109 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3111 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3112 break;
3113 case CHIP_BARTS:
3114 rdev->config.evergreen.num_ses = 2;
3115 rdev->config.evergreen.max_pipes = 4;
3116 rdev->config.evergreen.max_tile_pipes = 8;
3117 rdev->config.evergreen.max_simds = 7;
3118 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3119 rdev->config.evergreen.max_gprs = 256;
3120 rdev->config.evergreen.max_threads = 248;
3121 rdev->config.evergreen.max_gs_threads = 32;
3122 rdev->config.evergreen.max_stack_entries = 512;
3123 rdev->config.evergreen.sx_num_of_sets = 4;
3124 rdev->config.evergreen.sx_max_export_size = 256;
3125 rdev->config.evergreen.sx_max_export_pos_size = 64;
3126 rdev->config.evergreen.sx_max_export_smx_size = 192;
3127 rdev->config.evergreen.max_hw_contexts = 8;
3128 rdev->config.evergreen.sq_num_cf_insts = 2;
3129
3130 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3131 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3132 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3133 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3134 break;
3135 case CHIP_TURKS:
3136 rdev->config.evergreen.num_ses = 1;
3137 rdev->config.evergreen.max_pipes = 4;
3138 rdev->config.evergreen.max_tile_pipes = 4;
3139 rdev->config.evergreen.max_simds = 6;
3140 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3141 rdev->config.evergreen.max_gprs = 256;
3142 rdev->config.evergreen.max_threads = 248;
3143 rdev->config.evergreen.max_gs_threads = 32;
3144 rdev->config.evergreen.max_stack_entries = 256;
3145 rdev->config.evergreen.sx_num_of_sets = 4;
3146 rdev->config.evergreen.sx_max_export_size = 256;
3147 rdev->config.evergreen.sx_max_export_pos_size = 64;
3148 rdev->config.evergreen.sx_max_export_smx_size = 192;
3149 rdev->config.evergreen.max_hw_contexts = 8;
3150 rdev->config.evergreen.sq_num_cf_insts = 2;
3151
3152 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3153 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3154 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3155 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3156 break;
3157 case CHIP_CAICOS:
3158 rdev->config.evergreen.num_ses = 1;
3159 rdev->config.evergreen.max_pipes = 2;
3160 rdev->config.evergreen.max_tile_pipes = 2;
3161 rdev->config.evergreen.max_simds = 2;
3162 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 192;
3165 rdev->config.evergreen.max_gs_threads = 16;
3166 rdev->config.evergreen.max_stack_entries = 256;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 128;
3169 rdev->config.evergreen.sx_max_export_pos_size = 32;
3170 rdev->config.evergreen.sx_max_export_smx_size = 96;
3171 rdev->config.evergreen.max_hw_contexts = 4;
3172 rdev->config.evergreen.sq_num_cf_insts = 1;
3173
3174 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3178 break;
3179 }
3180
3181 /* Initialize HDP */
3182 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3183 WREG32((0x2c14 + j), 0x00000000);
3184 WREG32((0x2c18 + j), 0x00000000);
3185 WREG32((0x2c1c + j), 0x00000000);
3186 WREG32((0x2c20 + j), 0x00000000);
3187 WREG32((0x2c24 + j), 0x00000000);
3188 }
3189
3190 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3191
3192 evergreen_fix_pci_max_read_req_size(rdev);
3193
3194 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3195 if ((rdev->family == CHIP_PALM) ||
3196 (rdev->family == CHIP_SUMO) ||
3197 (rdev->family == CHIP_SUMO2))
3198 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3199 else
3200 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3201
3202 /* setup tiling info dword. gb_addr_config is not adequate since it does
3203 * not have bank info, so create a custom tiling dword.
3204 * bits 3:0 num_pipes
3205 * bits 7:4 num_banks
3206 * bits 11:8 group_size
3207 * bits 15:12 row_size
3208 */
3209 rdev->config.evergreen.tile_config = 0;
3210 switch (rdev->config.evergreen.max_tile_pipes) {
3211 case 1:
3212 default:
3213 rdev->config.evergreen.tile_config |= (0 << 0);
3214 break;
3215 case 2:
3216 rdev->config.evergreen.tile_config |= (1 << 0);
3217 break;
3218 case 4:
3219 rdev->config.evergreen.tile_config |= (2 << 0);
3220 break;
3221 case 8:
3222 rdev->config.evergreen.tile_config |= (3 << 0);
3223 break;
3224 }
3225 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3226 if (rdev->flags & RADEON_IS_IGP)
3227 rdev->config.evergreen.tile_config |= 1 << 4;
3228 else {
3229 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3230 case 0: /* four banks */
3231 rdev->config.evergreen.tile_config |= 0 << 4;
3232 break;
3233 case 1: /* eight banks */
3234 rdev->config.evergreen.tile_config |= 1 << 4;
3235 break;
3236 case 2: /* sixteen banks */
3237 default:
3238 rdev->config.evergreen.tile_config |= 2 << 4;
3239 break;
3240 }
3241 }
3242 rdev->config.evergreen.tile_config |= 0 << 8;
3243 rdev->config.evergreen.tile_config |=
3244 ((gb_addr_config & 0x30000000) >> 28) << 12;
3245
3246 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3247
3248 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3249 u32 efuse_straps_4;
3250 u32 efuse_straps_3;
3251
3252 efuse_straps_4 = RREG32_RCU(0x204);
3253 efuse_straps_3 = RREG32_RCU(0x203);
3254 tmp = (((efuse_straps_4 & 0xf) << 4) |
3255 ((efuse_straps_3 & 0xf0000000) >> 28));
3256 } else {
3257 tmp = 0;
3258 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3259 u32 rb_disable_bitmap;
3260
3261 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3262 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3263 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3264 tmp <<= 4;
3265 tmp |= rb_disable_bitmap;
3266 }
3267 }
3268 /* enabled rb are just the one not disabled :) */
3269 disabled_rb_mask = tmp;
3270 tmp = 0;
3271 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3272 tmp |= (1 << i);
3273 /* if all the backends are disabled, fix it up here */
3274 if ((disabled_rb_mask & tmp) == tmp) {
3275 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3276 disabled_rb_mask &= ~(1 << i);
3277 }
3278
3279 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3280 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3281
3282 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3283 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3284 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3285 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3286 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3287 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3288 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3289
3290 if ((rdev->config.evergreen.max_backends == 1) &&
3291 (rdev->flags & RADEON_IS_IGP)) {
3292 if ((disabled_rb_mask & 3) == 1) {
3293 /* RB0 disabled, RB1 enabled */
3294 tmp = 0x11111111;
3295 } else {
3296 /* RB1 disabled, RB0 enabled */
3297 tmp = 0x00000000;
3298 }
3299 } else {
3300 tmp = gb_addr_config & NUM_PIPES_MASK;
3301 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3302 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3303 }
3304 WREG32(GB_BACKEND_MAP, tmp);
3305
3306 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3307 WREG32(CGTS_TCC_DISABLE, 0);
3308 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3309 WREG32(CGTS_USER_TCC_DISABLE, 0);
3310
3311 /* set HW defaults for 3D engine */
3312 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3313 ROQ_IB2_START(0x2b)));
3314
3315 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3316
3317 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3318 SYNC_GRADIENT |
3319 SYNC_WALKER |
3320 SYNC_ALIGNER));
3321
3322 sx_debug_1 = RREG32(SX_DEBUG_1);
3323 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3324 WREG32(SX_DEBUG_1, sx_debug_1);
3325
3326
3327 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3328 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3329 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3330 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3331
3332 if (rdev->family <= CHIP_SUMO2)
3333 WREG32(SMX_SAR_CTL0, 0x00010000);
3334
3335 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3336 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3337 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3338
3339 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3340 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3341 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3342
3343 WREG32(VGT_NUM_INSTANCES, 1);
3344 WREG32(SPI_CONFIG_CNTL, 0);
3345 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3346 WREG32(CP_PERFMON_CNTL, 0);
3347
3348 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3349 FETCH_FIFO_HIWATER(0x4) |
3350 DONE_FIFO_HIWATER(0xe0) |
3351 ALU_UPDATE_FIFO_HIWATER(0x8)));
3352
3353 sq_config = RREG32(SQ_CONFIG);
3354 sq_config &= ~(PS_PRIO(3) |
3355 VS_PRIO(3) |
3356 GS_PRIO(3) |
3357 ES_PRIO(3));
3358 sq_config |= (VC_ENABLE |
3359 EXPORT_SRC_C |
3360 PS_PRIO(0) |
3361 VS_PRIO(1) |
3362 GS_PRIO(2) |
3363 ES_PRIO(3));
3364
3365 switch (rdev->family) {
3366 case CHIP_CEDAR:
3367 case CHIP_PALM:
3368 case CHIP_SUMO:
3369 case CHIP_SUMO2:
3370 case CHIP_CAICOS:
3371 /* no vertex cache */
3372 sq_config &= ~VC_ENABLE;
3373 break;
3374 default:
3375 break;
3376 }
3377
3378 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3379
3380 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3381 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3382 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3383 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3384 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3385 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3386 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3387
3388 switch (rdev->family) {
3389 case CHIP_CEDAR:
3390 case CHIP_PALM:
3391 case CHIP_SUMO:
3392 case CHIP_SUMO2:
3393 ps_thread_count = 96;
3394 break;
3395 default:
3396 ps_thread_count = 128;
3397 break;
3398 }
3399
3400 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3401 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3402 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3403 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3404 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3405 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3406
3407 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3408 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3409 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3410 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3411 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3412 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3413
3414 WREG32(SQ_CONFIG, sq_config);
3415 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3416 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3417 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3418 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3419 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3420 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3421 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3422 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3423 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3424 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3425
3426 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3427 FORCE_EOV_MAX_REZ_CNT(255)));
3428
3429 switch (rdev->family) {
3430 case CHIP_CEDAR:
3431 case CHIP_PALM:
3432 case CHIP_SUMO:
3433 case CHIP_SUMO2:
3434 case CHIP_CAICOS:
3435 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3436 break;
3437 default:
3438 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3439 break;
3440 }
3441 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3442 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3443
3444 WREG32(VGT_GS_VERTEX_REUSE, 16);
3445 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3446 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3447
3448 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3449 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3450
3451 WREG32(CB_PERF_CTR0_SEL_0, 0);
3452 WREG32(CB_PERF_CTR0_SEL_1, 0);
3453 WREG32(CB_PERF_CTR1_SEL_0, 0);
3454 WREG32(CB_PERF_CTR1_SEL_1, 0);
3455 WREG32(CB_PERF_CTR2_SEL_0, 0);
3456 WREG32(CB_PERF_CTR2_SEL_1, 0);
3457 WREG32(CB_PERF_CTR3_SEL_0, 0);
3458 WREG32(CB_PERF_CTR3_SEL_1, 0);
3459
3460 /* clear render buffer base addresses */
3461 WREG32(CB_COLOR0_BASE, 0);
3462 WREG32(CB_COLOR1_BASE, 0);
3463 WREG32(CB_COLOR2_BASE, 0);
3464 WREG32(CB_COLOR3_BASE, 0);
3465 WREG32(CB_COLOR4_BASE, 0);
3466 WREG32(CB_COLOR5_BASE, 0);
3467 WREG32(CB_COLOR6_BASE, 0);
3468 WREG32(CB_COLOR7_BASE, 0);
3469 WREG32(CB_COLOR8_BASE, 0);
3470 WREG32(CB_COLOR9_BASE, 0);
3471 WREG32(CB_COLOR10_BASE, 0);
3472 WREG32(CB_COLOR11_BASE, 0);
3473
3474 /* set the shader const cache sizes to 0 */
3475 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3476 WREG32(i, 0);
3477 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3478 WREG32(i, 0);
3479
3480 tmp = RREG32(HDP_MISC_CNTL);
3481 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3482 WREG32(HDP_MISC_CNTL, tmp);
3483
3484 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3485 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3486
3487 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3488
3489 udelay(50);
3490
3491 }
3492
3493 int evergreen_mc_init(struct radeon_device *rdev)
3494 {
3495 u32 tmp;
3496 int chansize, numchan;
3497
3498 /* Get VRAM informations */
3499 rdev->mc.vram_is_ddr = true;
3500 if ((rdev->family == CHIP_PALM) ||
3501 (rdev->family == CHIP_SUMO) ||
3502 (rdev->family == CHIP_SUMO2))
3503 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3504 else
3505 tmp = RREG32(MC_ARB_RAMCFG);
3506 if (tmp & CHANSIZE_OVERRIDE) {
3507 chansize = 16;
3508 } else if (tmp & CHANSIZE_MASK) {
3509 chansize = 64;
3510 } else {
3511 chansize = 32;
3512 }
3513 tmp = RREG32(MC_SHARED_CHMAP);
3514 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3515 case 0:
3516 default:
3517 numchan = 1;
3518 break;
3519 case 1:
3520 numchan = 2;
3521 break;
3522 case 2:
3523 numchan = 4;
3524 break;
3525 case 3:
3526 numchan = 8;
3527 break;
3528 }
3529 rdev->mc.vram_width = numchan * chansize;
3530 /* Could aper size report 0 ? */
3531 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3532 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3533 /* Setup GPU memory space */
3534 if ((rdev->family == CHIP_PALM) ||
3535 (rdev->family == CHIP_SUMO) ||
3536 (rdev->family == CHIP_SUMO2)) {
3537 /* size in bytes on fusion */
3538 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3539 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3540 } else {
3541 /* size in MB on evergreen/cayman/tn */
3542 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3543 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3544 }
3545 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3546 r700_vram_gtt_location(rdev, &rdev->mc);
3547 radeon_update_bandwidth_info(rdev);
3548
3549 return 0;
3550 }
3551
3552 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3553 {
3554 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3555 RREG32(GRBM_STATUS));
3556 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3557 RREG32(GRBM_STATUS_SE0));
3558 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3559 RREG32(GRBM_STATUS_SE1));
3560 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3561 RREG32(SRBM_STATUS));
3562 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3563 RREG32(SRBM_STATUS2));
3564 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3565 RREG32(CP_STALLED_STAT1));
3566 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3567 RREG32(CP_STALLED_STAT2));
3568 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3569 RREG32(CP_BUSY_STAT));
3570 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3571 RREG32(CP_STAT));
3572 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3573 RREG32(DMA_STATUS_REG));
3574 if (rdev->family >= CHIP_CAYMAN) {
3575 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3576 RREG32(DMA_STATUS_REG + 0x800));
3577 }
3578 }
3579
3580 bool evergreen_is_display_hung(struct radeon_device *rdev)
3581 {
3582 u32 crtc_hung = 0;
3583 u32 crtc_status[6];
3584 u32 i, j, tmp;
3585
3586 for (i = 0; i < rdev->num_crtc; i++) {
3587 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3588 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3589 crtc_hung |= (1 << i);
3590 }
3591 }
3592
3593 for (j = 0; j < 10; j++) {
3594 for (i = 0; i < rdev->num_crtc; i++) {
3595 if (crtc_hung & (1 << i)) {
3596 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3597 if (tmp != crtc_status[i])
3598 crtc_hung &= ~(1 << i);
3599 }
3600 }
3601 if (crtc_hung == 0)
3602 return false;
3603 udelay(100);
3604 }
3605
3606 return true;
3607 }
3608
3609 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3610 {
3611 u32 reset_mask = 0;
3612 u32 tmp;
3613
3614 /* GRBM_STATUS */
3615 tmp = RREG32(GRBM_STATUS);
3616 if (tmp & (PA_BUSY | SC_BUSY |
3617 SH_BUSY | SX_BUSY |
3618 TA_BUSY | VGT_BUSY |
3619 DB_BUSY | CB_BUSY |
3620 SPI_BUSY | VGT_BUSY_NO_DMA))
3621 reset_mask |= RADEON_RESET_GFX;
3622
3623 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3624 CP_BUSY | CP_COHERENCY_BUSY))
3625 reset_mask |= RADEON_RESET_CP;
3626
3627 if (tmp & GRBM_EE_BUSY)
3628 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3629
3630 /* DMA_STATUS_REG */
3631 tmp = RREG32(DMA_STATUS_REG);
3632 if (!(tmp & DMA_IDLE))
3633 reset_mask |= RADEON_RESET_DMA;
3634
3635 /* SRBM_STATUS2 */
3636 tmp = RREG32(SRBM_STATUS2);
3637 if (tmp & DMA_BUSY)
3638 reset_mask |= RADEON_RESET_DMA;
3639
3640 /* SRBM_STATUS */
3641 tmp = RREG32(SRBM_STATUS);
3642 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3643 reset_mask |= RADEON_RESET_RLC;
3644
3645 if (tmp & IH_BUSY)
3646 reset_mask |= RADEON_RESET_IH;
3647
3648 if (tmp & SEM_BUSY)
3649 reset_mask |= RADEON_RESET_SEM;
3650
3651 if (tmp & GRBM_RQ_PENDING)
3652 reset_mask |= RADEON_RESET_GRBM;
3653
3654 if (tmp & VMC_BUSY)
3655 reset_mask |= RADEON_RESET_VMC;
3656
3657 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3658 MCC_BUSY | MCD_BUSY))
3659 reset_mask |= RADEON_RESET_MC;
3660
3661 if (evergreen_is_display_hung(rdev))
3662 reset_mask |= RADEON_RESET_DISPLAY;
3663
3664 /* VM_L2_STATUS */
3665 tmp = RREG32(VM_L2_STATUS);
3666 if (tmp & L2_BUSY)
3667 reset_mask |= RADEON_RESET_VMC;
3668
3669 /* Skip MC reset as it's mostly likely not hung, just busy */
3670 if (reset_mask & RADEON_RESET_MC) {
3671 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3672 reset_mask &= ~RADEON_RESET_MC;
3673 }
3674
3675 return reset_mask;
3676 }
3677
3678 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3679 {
3680 struct evergreen_mc_save save;
3681 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3682 u32 tmp;
3683
3684 if (reset_mask == 0)
3685 return;
3686
3687 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3688
3689 evergreen_print_gpu_status_regs(rdev);
3690
3691 /* Disable CP parsing/prefetching */
3692 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3693
3694 if (reset_mask & RADEON_RESET_DMA) {
3695 /* Disable DMA */
3696 tmp = RREG32(DMA_RB_CNTL);
3697 tmp &= ~DMA_RB_ENABLE;
3698 WREG32(DMA_RB_CNTL, tmp);
3699 }
3700
3701 udelay(50);
3702
3703 evergreen_mc_stop(rdev, &save);
3704 if (evergreen_mc_wait_for_idle(rdev)) {
3705 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3706 }
3707
3708 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3709 grbm_soft_reset |= SOFT_RESET_DB |
3710 SOFT_RESET_CB |
3711 SOFT_RESET_PA |
3712 SOFT_RESET_SC |
3713 SOFT_RESET_SPI |
3714 SOFT_RESET_SX |
3715 SOFT_RESET_SH |
3716 SOFT_RESET_TC |
3717 SOFT_RESET_TA |
3718 SOFT_RESET_VC |
3719 SOFT_RESET_VGT;
3720 }
3721
3722 if (reset_mask & RADEON_RESET_CP) {
3723 grbm_soft_reset |= SOFT_RESET_CP |
3724 SOFT_RESET_VGT;
3725
3726 srbm_soft_reset |= SOFT_RESET_GRBM;
3727 }
3728
3729 if (reset_mask & RADEON_RESET_DMA)
3730 srbm_soft_reset |= SOFT_RESET_DMA;
3731
3732 if (reset_mask & RADEON_RESET_DISPLAY)
3733 srbm_soft_reset |= SOFT_RESET_DC;
3734
3735 if (reset_mask & RADEON_RESET_RLC)
3736 srbm_soft_reset |= SOFT_RESET_RLC;
3737
3738 if (reset_mask & RADEON_RESET_SEM)
3739 srbm_soft_reset |= SOFT_RESET_SEM;
3740
3741 if (reset_mask & RADEON_RESET_IH)
3742 srbm_soft_reset |= SOFT_RESET_IH;
3743
3744 if (reset_mask & RADEON_RESET_GRBM)
3745 srbm_soft_reset |= SOFT_RESET_GRBM;
3746
3747 if (reset_mask & RADEON_RESET_VMC)
3748 srbm_soft_reset |= SOFT_RESET_VMC;
3749
3750 if (!(rdev->flags & RADEON_IS_IGP)) {
3751 if (reset_mask & RADEON_RESET_MC)
3752 srbm_soft_reset |= SOFT_RESET_MC;
3753 }
3754
3755 if (grbm_soft_reset) {
3756 tmp = RREG32(GRBM_SOFT_RESET);
3757 tmp |= grbm_soft_reset;
3758 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3759 WREG32(GRBM_SOFT_RESET, tmp);
3760 tmp = RREG32(GRBM_SOFT_RESET);
3761
3762 udelay(50);
3763
3764 tmp &= ~grbm_soft_reset;
3765 WREG32(GRBM_SOFT_RESET, tmp);
3766 tmp = RREG32(GRBM_SOFT_RESET);
3767 }
3768
3769 if (srbm_soft_reset) {
3770 tmp = RREG32(SRBM_SOFT_RESET);
3771 tmp |= srbm_soft_reset;
3772 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3773 WREG32(SRBM_SOFT_RESET, tmp);
3774 tmp = RREG32(SRBM_SOFT_RESET);
3775
3776 udelay(50);
3777
3778 tmp &= ~srbm_soft_reset;
3779 WREG32(SRBM_SOFT_RESET, tmp);
3780 tmp = RREG32(SRBM_SOFT_RESET);
3781 }
3782
3783 /* Wait a little for things to settle down */
3784 udelay(50);
3785
3786 evergreen_mc_resume(rdev, &save);
3787 udelay(50);
3788
3789 evergreen_print_gpu_status_regs(rdev);
3790 }
3791
3792 int evergreen_asic_reset(struct radeon_device *rdev)
3793 {
3794 u32 reset_mask;
3795
3796 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3797
3798 if (reset_mask)
3799 r600_set_bios_scratch_engine_hung(rdev, true);
3800
3801 evergreen_gpu_soft_reset(rdev, reset_mask);
3802
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (!reset_mask)
3806 r600_set_bios_scratch_engine_hung(rdev, false);
3807
3808 return 0;
3809 }
3810
3811 /**
3812 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3813 *
3814 * @rdev: radeon_device pointer
3815 * @ring: radeon_ring structure holding ring information
3816 *
3817 * Check if the GFX engine is locked up.
3818 * Returns true if the engine appears to be locked up, false if not.
3819 */
3820 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3821 {
3822 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3823
3824 if (!(reset_mask & (RADEON_RESET_GFX |
3825 RADEON_RESET_COMPUTE |
3826 RADEON_RESET_CP))) {
3827 radeon_ring_lockup_update(ring);
3828 return false;
3829 }
3830 /* force CP activities */
3831 radeon_ring_force_activity(rdev, ring);
3832 return radeon_ring_test_lockup(rdev, ring);
3833 }
3834
3835 /**
3836 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3837 *
3838 * @rdev: radeon_device pointer
3839 * @ring: radeon_ring structure holding ring information
3840 *
3841 * Check if the async DMA engine is locked up.
3842 * Returns true if the engine appears to be locked up, false if not.
3843 */
3844 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3845 {
3846 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3847
3848 if (!(reset_mask & RADEON_RESET_DMA)) {
3849 radeon_ring_lockup_update(ring);
3850 return false;
3851 }
3852 /* force ring activities */
3853 radeon_ring_force_activity(rdev, ring);
3854 return radeon_ring_test_lockup(rdev, ring);
3855 }
3856
3857 /*
3858 * RLC
3859 */
3860 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3861 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3862
3863 void sumo_rlc_fini(struct radeon_device *rdev)
3864 {
3865 int r;
3866
3867 /* save restore block */
3868 if (rdev->rlc.save_restore_obj) {
3869 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3870 if (unlikely(r != 0))
3871 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3872 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3873 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3874
3875 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3876 rdev->rlc.save_restore_obj = NULL;
3877 }
3878
3879 /* clear state block */
3880 if (rdev->rlc.clear_state_obj) {
3881 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3882 if (unlikely(r != 0))
3883 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3884 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3885 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3886
3887 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3888 rdev->rlc.clear_state_obj = NULL;
3889 }
3890 }
3891
3892 int sumo_rlc_init(struct radeon_device *rdev)
3893 {
3894 u32 *src_ptr;
3895 volatile u32 *dst_ptr;
3896 u32 dws, data, i, j, k, reg_num;
3897 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3898 u64 reg_list_mc_addr;
3899 struct cs_section_def *cs_data;
3900 int r;
3901
3902 src_ptr = rdev->rlc.reg_list;
3903 dws = rdev->rlc.reg_list_size;
3904 cs_data = rdev->rlc.cs_data;
3905
3906 /* save restore block */
3907 if (rdev->rlc.save_restore_obj == NULL) {
3908 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3909 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3910 if (r) {
3911 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3912 return r;
3913 }
3914 }
3915
3916 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3917 if (unlikely(r != 0)) {
3918 sumo_rlc_fini(rdev);
3919 return r;
3920 }
3921 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3922 &rdev->rlc.save_restore_gpu_addr);
3923 if (r) {
3924 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3925 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3926 sumo_rlc_fini(rdev);
3927 return r;
3928 }
3929 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3930 if (r) {
3931 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3932 sumo_rlc_fini(rdev);
3933 return r;
3934 }
3935 /* write the sr buffer */
3936 dst_ptr = rdev->rlc.sr_ptr;
3937 /* format:
3938 * dw0: (reg2 << 16) | reg1
3939 * dw1: reg1 save space
3940 * dw2: reg2 save space
3941 */
3942 for (i = 0; i < dws; i++) {
3943 data = src_ptr[i] >> 2;
3944 i++;
3945 if (i < dws)
3946 data |= (src_ptr[i] >> 2) << 16;
3947 j = (((i - 1) * 3) / 2);
3948 dst_ptr[j] = data;
3949 }
3950 j = ((i * 3) / 2);
3951 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3952
3953 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3954 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3955
3956 /* clear state block */
3957 reg_list_num = 0;
3958 dws = 0;
3959 for (i = 0; cs_data[i].section != NULL; i++) {
3960 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3961 reg_list_num++;
3962 dws += cs_data[i].section[j].reg_count;
3963 }
3964 }
3965 reg_list_blk_index = (3 * reg_list_num + 2);
3966 dws += reg_list_blk_index;
3967
3968 if (rdev->rlc.clear_state_obj == NULL) {
3969 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3970 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3971 if (r) {
3972 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3973 sumo_rlc_fini(rdev);
3974 return r;
3975 }
3976 }
3977 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3978 if (unlikely(r != 0)) {
3979 sumo_rlc_fini(rdev);
3980 return r;
3981 }
3982 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3983 &rdev->rlc.clear_state_gpu_addr);
3984 if (r) {
3985
3986 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3987 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3988 sumo_rlc_fini(rdev);
3989 return r;
3990 }
3991 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3992 if (r) {
3993 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3994 sumo_rlc_fini(rdev);
3995 return r;
3996 }
3997 /* set up the cs buffer */
3998 dst_ptr = rdev->rlc.cs_ptr;
3999 reg_list_hdr_blk_index = 0;
4000 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4001 data = upper_32_bits(reg_list_mc_addr);
4002 dst_ptr[reg_list_hdr_blk_index] = data;
4003 reg_list_hdr_blk_index++;
4004 for (i = 0; cs_data[i].section != NULL; i++) {
4005 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4006 reg_num = cs_data[i].section[j].reg_count;
4007 data = reg_list_mc_addr & 0xffffffff;
4008 dst_ptr[reg_list_hdr_blk_index] = data;
4009 reg_list_hdr_blk_index++;
4010
4011 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4012 dst_ptr[reg_list_hdr_blk_index] = data;
4013 reg_list_hdr_blk_index++;
4014
4015 data = 0x08000000 | (reg_num * 4);
4016 dst_ptr[reg_list_hdr_blk_index] = data;
4017 reg_list_hdr_blk_index++;
4018
4019 for (k = 0; k < reg_num; k++) {
4020 data = cs_data[i].section[j].extent[k];
4021 dst_ptr[reg_list_blk_index + k] = data;
4022 }
4023 reg_list_mc_addr += reg_num * 4;
4024 reg_list_blk_index += reg_num;
4025 }
4026 }
4027 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4028
4029 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4030 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4031
4032 return 0;
4033 }
4034
4035 static void evergreen_rlc_start(struct radeon_device *rdev)
4036 {
4037 if (rdev->flags & RADEON_IS_IGP)
4038 WREG32(RLC_CNTL, RLC_ENABLE | GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC);
4039 else
4040 WREG32(RLC_CNTL, RLC_ENABLE);
4041 }
4042
4043 int evergreen_rlc_resume(struct radeon_device *rdev)
4044 {
4045 u32 i;
4046 const __be32 *fw_data;
4047
4048 if (!rdev->rlc_fw)
4049 return -EINVAL;
4050
4051 r600_rlc_stop(rdev);
4052
4053 WREG32(RLC_HB_CNTL, 0);
4054
4055 if (rdev->flags & RADEON_IS_IGP) {
4056 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4057 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4058 } else {
4059 WREG32(RLC_HB_BASE, 0);
4060 WREG32(RLC_HB_RPTR, 0);
4061 WREG32(RLC_HB_WPTR, 0);
4062 }
4063 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4064 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4065 WREG32(RLC_MC_CNTL, 0);
4066 WREG32(RLC_UCODE_CNTL, 0);
4067
4068 fw_data = (const __be32 *)rdev->rlc_fw->data;
4069 if (rdev->family >= CHIP_ARUBA) {
4070 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4071 WREG32(RLC_UCODE_ADDR, i);
4072 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4073 }
4074 } else if (rdev->family >= CHIP_CAYMAN) {
4075 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4076 WREG32(RLC_UCODE_ADDR, i);
4077 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4078 }
4079 } else {
4080 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4081 WREG32(RLC_UCODE_ADDR, i);
4082 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4083 }
4084 }
4085 WREG32(RLC_UCODE_ADDR, 0);
4086
4087 evergreen_rlc_start(rdev);
4088
4089 return 0;
4090 }
4091
4092 /* Interrupts */
4093
4094 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4095 {
4096 if (crtc >= rdev->num_crtc)
4097 return 0;
4098 else
4099 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4100 }
4101
4102 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4103 {
4104 u32 tmp;
4105
4106 if (rdev->family >= CHIP_CAYMAN) {
4107 cayman_cp_int_cntl_setup(rdev, 0,
4108 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4109 cayman_cp_int_cntl_setup(rdev, 1, 0);
4110 cayman_cp_int_cntl_setup(rdev, 2, 0);
4111 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4112 WREG32(CAYMAN_DMA1_CNTL, tmp);
4113 } else
4114 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4115 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4116 WREG32(DMA_CNTL, tmp);
4117 WREG32(GRBM_INT_CNTL, 0);
4118 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4119 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4120 if (rdev->num_crtc >= 4) {
4121 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4122 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4123 }
4124 if (rdev->num_crtc >= 6) {
4125 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4126 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4127 }
4128
4129 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4130 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4131 if (rdev->num_crtc >= 4) {
4132 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4133 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4134 }
4135 if (rdev->num_crtc >= 6) {
4136 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4137 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4138 }
4139
4140 /* only one DAC on DCE6 */
4141 if (!ASIC_IS_DCE6(rdev))
4142 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4143 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4144
4145 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4146 WREG32(DC_HPD1_INT_CONTROL, tmp);
4147 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4148 WREG32(DC_HPD2_INT_CONTROL, tmp);
4149 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4150 WREG32(DC_HPD3_INT_CONTROL, tmp);
4151 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4152 WREG32(DC_HPD4_INT_CONTROL, tmp);
4153 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4154 WREG32(DC_HPD5_INT_CONTROL, tmp);
4155 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4156 WREG32(DC_HPD6_INT_CONTROL, tmp);
4157
4158 }
4159
4160 int evergreen_irq_set(struct radeon_device *rdev)
4161 {
4162 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4163 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4164 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4165 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4166 u32 grbm_int_cntl = 0;
4167 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4168 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4169 u32 dma_cntl, dma_cntl1 = 0;
4170 u32 thermal_int = 0;
4171
4172 if (!rdev->irq.installed) {
4173 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4174 return -EINVAL;
4175 }
4176 /* don't enable anything if the ih is disabled */
4177 if (!rdev->ih.enabled) {
4178 r600_disable_interrupts(rdev);
4179 /* force the active interrupt state to all disabled */
4180 evergreen_disable_interrupt_state(rdev);
4181 return 0;
4182 }
4183
4184 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4185 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4186 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4187 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4188 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4189 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4190 if (rdev->family == CHIP_ARUBA)
4191 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4192 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4193 else
4194 thermal_int = RREG32(CG_THERMAL_INT) &
4195 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4196
4197 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4198 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4199 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4200 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4201 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4202 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4203
4204 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4205
4206 if (rdev->family >= CHIP_CAYMAN) {
4207 /* enable CP interrupts on all rings */
4208 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4209 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4210 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4211 }
4212 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4213 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4214 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4215 }
4216 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4217 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4218 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4219 }
4220 } else {
4221 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4222 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4223 cp_int_cntl |= RB_INT_ENABLE;
4224 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4225 }
4226 }
4227
4228 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4229 DRM_DEBUG("r600_irq_set: sw int dma\n");
4230 dma_cntl |= TRAP_ENABLE;
4231 }
4232
4233 if (rdev->family >= CHIP_CAYMAN) {
4234 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4235 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4236 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4237 dma_cntl1 |= TRAP_ENABLE;
4238 }
4239 }
4240
4241 if (rdev->irq.dpm_thermal) {
4242 DRM_DEBUG("dpm thermal\n");
4243 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4244 }
4245
4246 if (rdev->irq.crtc_vblank_int[0] ||
4247 atomic_read(&rdev->irq.pflip[0])) {
4248 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4249 crtc1 |= VBLANK_INT_MASK;
4250 }
4251 if (rdev->irq.crtc_vblank_int[1] ||
4252 atomic_read(&rdev->irq.pflip[1])) {
4253 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4254 crtc2 |= VBLANK_INT_MASK;
4255 }
4256 if (rdev->irq.crtc_vblank_int[2] ||
4257 atomic_read(&rdev->irq.pflip[2])) {
4258 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4259 crtc3 |= VBLANK_INT_MASK;
4260 }
4261 if (rdev->irq.crtc_vblank_int[3] ||
4262 atomic_read(&rdev->irq.pflip[3])) {
4263 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4264 crtc4 |= VBLANK_INT_MASK;
4265 }
4266 if (rdev->irq.crtc_vblank_int[4] ||
4267 atomic_read(&rdev->irq.pflip[4])) {
4268 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4269 crtc5 |= VBLANK_INT_MASK;
4270 }
4271 if (rdev->irq.crtc_vblank_int[5] ||
4272 atomic_read(&rdev->irq.pflip[5])) {
4273 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4274 crtc6 |= VBLANK_INT_MASK;
4275 }
4276 if (rdev->irq.hpd[0]) {
4277 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4278 hpd1 |= DC_HPDx_INT_EN;
4279 }
4280 if (rdev->irq.hpd[1]) {
4281 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4282 hpd2 |= DC_HPDx_INT_EN;
4283 }
4284 if (rdev->irq.hpd[2]) {
4285 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4286 hpd3 |= DC_HPDx_INT_EN;
4287 }
4288 if (rdev->irq.hpd[3]) {
4289 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4290 hpd4 |= DC_HPDx_INT_EN;
4291 }
4292 if (rdev->irq.hpd[4]) {
4293 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4294 hpd5 |= DC_HPDx_INT_EN;
4295 }
4296 if (rdev->irq.hpd[5]) {
4297 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4298 hpd6 |= DC_HPDx_INT_EN;
4299 }
4300 if (rdev->irq.afmt[0]) {
4301 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4302 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4303 }
4304 if (rdev->irq.afmt[1]) {
4305 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4306 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4307 }
4308 if (rdev->irq.afmt[2]) {
4309 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4310 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4311 }
4312 if (rdev->irq.afmt[3]) {
4313 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4314 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4315 }
4316 if (rdev->irq.afmt[4]) {
4317 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4318 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4319 }
4320 if (rdev->irq.afmt[5]) {
4321 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4322 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4323 }
4324
4325 if (rdev->family >= CHIP_CAYMAN) {
4326 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4327 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4328 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4329 } else
4330 WREG32(CP_INT_CNTL, cp_int_cntl);
4331
4332 WREG32(DMA_CNTL, dma_cntl);
4333
4334 if (rdev->family >= CHIP_CAYMAN)
4335 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4336
4337 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4338
4339 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4340 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4341 if (rdev->num_crtc >= 4) {
4342 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4343 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4344 }
4345 if (rdev->num_crtc >= 6) {
4346 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4347 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4348 }
4349
4350 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4351 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4352 if (rdev->num_crtc >= 4) {
4353 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4354 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4355 }
4356 if (rdev->num_crtc >= 6) {
4357 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4358 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4359 }
4360
4361 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4362 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4363 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4364 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4365 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4366 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4367 if (rdev->family == CHIP_ARUBA)
4368 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4369 else
4370 WREG32(CG_THERMAL_INT, thermal_int);
4371
4372 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4373 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4374 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4375 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4376 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4377 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4378
4379 return 0;
4380 }
4381
4382 static void evergreen_irq_ack(struct radeon_device *rdev)
4383 {
4384 u32 tmp;
4385
4386 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4387 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4388 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4389 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4390 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4391 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4392 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4393 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4394 if (rdev->num_crtc >= 4) {
4395 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4396 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4397 }
4398 if (rdev->num_crtc >= 6) {
4399 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4400 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4401 }
4402
4403 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4404 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4405 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4406 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4407 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4408 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4409
4410 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4411 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4412 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4413 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4414 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4415 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4416 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4417 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4418 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4419 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4420 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4421 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4422
4423 if (rdev->num_crtc >= 4) {
4424 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4425 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4426 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4427 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4428 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4429 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4430 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4431 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4432 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4433 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4434 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4435 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4436 }
4437
4438 if (rdev->num_crtc >= 6) {
4439 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4440 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4441 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4442 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4443 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4444 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4445 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4446 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4447 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4448 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4449 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4450 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4451 }
4452
4453 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4454 tmp = RREG32(DC_HPD1_INT_CONTROL);
4455 tmp |= DC_HPDx_INT_ACK;
4456 WREG32(DC_HPD1_INT_CONTROL, tmp);
4457 }
4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4459 tmp = RREG32(DC_HPD2_INT_CONTROL);
4460 tmp |= DC_HPDx_INT_ACK;
4461 WREG32(DC_HPD2_INT_CONTROL, tmp);
4462 }
4463 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4464 tmp = RREG32(DC_HPD3_INT_CONTROL);
4465 tmp |= DC_HPDx_INT_ACK;
4466 WREG32(DC_HPD3_INT_CONTROL, tmp);
4467 }
4468 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4469 tmp = RREG32(DC_HPD4_INT_CONTROL);
4470 tmp |= DC_HPDx_INT_ACK;
4471 WREG32(DC_HPD4_INT_CONTROL, tmp);
4472 }
4473 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4474 tmp = RREG32(DC_HPD5_INT_CONTROL);
4475 tmp |= DC_HPDx_INT_ACK;
4476 WREG32(DC_HPD5_INT_CONTROL, tmp);
4477 }
4478 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4479 tmp = RREG32(DC_HPD5_INT_CONTROL);
4480 tmp |= DC_HPDx_INT_ACK;
4481 WREG32(DC_HPD6_INT_CONTROL, tmp);
4482 }
4483 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4484 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4485 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4486 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4487 }
4488 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4489 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4490 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4491 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4492 }
4493 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4494 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4495 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4496 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4497 }
4498 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4499 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4500 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4501 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4502 }
4503 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4504 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4505 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4506 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4507 }
4508 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4509 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4510 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4511 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4512 }
4513 }
4514
4515 static void evergreen_irq_disable(struct radeon_device *rdev)
4516 {
4517 r600_disable_interrupts(rdev);
4518 /* Wait and acknowledge irq */
4519 mdelay(1);
4520 evergreen_irq_ack(rdev);
4521 evergreen_disable_interrupt_state(rdev);
4522 }
4523
4524 void evergreen_irq_suspend(struct radeon_device *rdev)
4525 {
4526 evergreen_irq_disable(rdev);
4527 r600_rlc_stop(rdev);
4528 }
4529
4530 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4531 {
4532 u32 wptr, tmp;
4533
4534 if (rdev->wb.enabled)
4535 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4536 else
4537 wptr = RREG32(IH_RB_WPTR);
4538
4539 if (wptr & RB_OVERFLOW) {
4540 /* When a ring buffer overflow happen start parsing interrupt
4541 * from the last not overwritten vector (wptr + 16). Hopefully
4542 * this should allow us to catchup.
4543 */
4544 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4545 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4546 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4547 tmp = RREG32(IH_RB_CNTL);
4548 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4549 WREG32(IH_RB_CNTL, tmp);
4550 }
4551 return (wptr & rdev->ih.ptr_mask);
4552 }
4553
4554 int evergreen_irq_process(struct radeon_device *rdev)
4555 {
4556 u32 wptr;
4557 u32 rptr;
4558 u32 src_id, src_data;
4559 u32 ring_index;
4560 bool queue_hotplug = false;
4561 bool queue_hdmi = false;
4562 bool queue_thermal = false;
4563
4564 if (!rdev->ih.enabled || rdev->shutdown)
4565 return IRQ_NONE;
4566
4567 wptr = evergreen_get_ih_wptr(rdev);
4568
4569 restart_ih:
4570 /* is somebody else already processing irqs? */
4571 if (atomic_xchg(&rdev->ih.lock, 1))
4572 return IRQ_NONE;
4573
4574 rptr = rdev->ih.rptr;
4575 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4576
4577 /* Order reading of wptr vs. reading of IH ring data */
4578 rmb();
4579
4580 /* display interrupts */
4581 evergreen_irq_ack(rdev);
4582
4583 while (rptr != wptr) {
4584 /* wptr/rptr are in bytes! */
4585 ring_index = rptr / 4;
4586 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4587 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4588
4589 switch (src_id) {
4590 case 1: /* D1 vblank/vline */
4591 switch (src_data) {
4592 case 0: /* D1 vblank */
4593 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4594 if (rdev->irq.crtc_vblank_int[0]) {
4595 drm_handle_vblank(rdev->ddev, 0);
4596 rdev->pm.vblank_sync = true;
4597 wake_up(&rdev->irq.vblank_queue);
4598 }
4599 if (atomic_read(&rdev->irq.pflip[0]))
4600 radeon_crtc_handle_flip(rdev, 0);
4601 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4602 DRM_DEBUG("IH: D1 vblank\n");
4603 }
4604 break;
4605 case 1: /* D1 vline */
4606 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4607 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4608 DRM_DEBUG("IH: D1 vline\n");
4609 }
4610 break;
4611 default:
4612 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4613 break;
4614 }
4615 break;
4616 case 2: /* D2 vblank/vline */
4617 switch (src_data) {
4618 case 0: /* D2 vblank */
4619 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4620 if (rdev->irq.crtc_vblank_int[1]) {
4621 drm_handle_vblank(rdev->ddev, 1);
4622 rdev->pm.vblank_sync = true;
4623 wake_up(&rdev->irq.vblank_queue);
4624 }
4625 if (atomic_read(&rdev->irq.pflip[1]))
4626 radeon_crtc_handle_flip(rdev, 1);
4627 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4628 DRM_DEBUG("IH: D2 vblank\n");
4629 }
4630 break;
4631 case 1: /* D2 vline */
4632 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4633 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4634 DRM_DEBUG("IH: D2 vline\n");
4635 }
4636 break;
4637 default:
4638 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4639 break;
4640 }
4641 break;
4642 case 3: /* D3 vblank/vline */
4643 switch (src_data) {
4644 case 0: /* D3 vblank */
4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4646 if (rdev->irq.crtc_vblank_int[2]) {
4647 drm_handle_vblank(rdev->ddev, 2);
4648 rdev->pm.vblank_sync = true;
4649 wake_up(&rdev->irq.vblank_queue);
4650 }
4651 if (atomic_read(&rdev->irq.pflip[2]))
4652 radeon_crtc_handle_flip(rdev, 2);
4653 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4654 DRM_DEBUG("IH: D3 vblank\n");
4655 }
4656 break;
4657 case 1: /* D3 vline */
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4659 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4660 DRM_DEBUG("IH: D3 vline\n");
4661 }
4662 break;
4663 default:
4664 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4665 break;
4666 }
4667 break;
4668 case 4: /* D4 vblank/vline */
4669 switch (src_data) {
4670 case 0: /* D4 vblank */
4671 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4672 if (rdev->irq.crtc_vblank_int[3]) {
4673 drm_handle_vblank(rdev->ddev, 3);
4674 rdev->pm.vblank_sync = true;
4675 wake_up(&rdev->irq.vblank_queue);
4676 }
4677 if (atomic_read(&rdev->irq.pflip[3]))
4678 radeon_crtc_handle_flip(rdev, 3);
4679 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4680 DRM_DEBUG("IH: D4 vblank\n");
4681 }
4682 break;
4683 case 1: /* D4 vline */
4684 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4685 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4686 DRM_DEBUG("IH: D4 vline\n");
4687 }
4688 break;
4689 default:
4690 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4691 break;
4692 }
4693 break;
4694 case 5: /* D5 vblank/vline */
4695 switch (src_data) {
4696 case 0: /* D5 vblank */
4697 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4698 if (rdev->irq.crtc_vblank_int[4]) {
4699 drm_handle_vblank(rdev->ddev, 4);
4700 rdev->pm.vblank_sync = true;
4701 wake_up(&rdev->irq.vblank_queue);
4702 }
4703 if (atomic_read(&rdev->irq.pflip[4]))
4704 radeon_crtc_handle_flip(rdev, 4);
4705 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4706 DRM_DEBUG("IH: D5 vblank\n");
4707 }
4708 break;
4709 case 1: /* D5 vline */
4710 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4711 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4712 DRM_DEBUG("IH: D5 vline\n");
4713 }
4714 break;
4715 default:
4716 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4717 break;
4718 }
4719 break;
4720 case 6: /* D6 vblank/vline */
4721 switch (src_data) {
4722 case 0: /* D6 vblank */
4723 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4724 if (rdev->irq.crtc_vblank_int[5]) {
4725 drm_handle_vblank(rdev->ddev, 5);
4726 rdev->pm.vblank_sync = true;
4727 wake_up(&rdev->irq.vblank_queue);
4728 }
4729 if (atomic_read(&rdev->irq.pflip[5]))
4730 radeon_crtc_handle_flip(rdev, 5);
4731 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4732 DRM_DEBUG("IH: D6 vblank\n");
4733 }
4734 break;
4735 case 1: /* D6 vline */
4736 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4737 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4738 DRM_DEBUG("IH: D6 vline\n");
4739 }
4740 break;
4741 default:
4742 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4743 break;
4744 }
4745 break;
4746 case 42: /* HPD hotplug */
4747 switch (src_data) {
4748 case 0:
4749 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4750 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4751 queue_hotplug = true;
4752 DRM_DEBUG("IH: HPD1\n");
4753 }
4754 break;
4755 case 1:
4756 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4757 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4758 queue_hotplug = true;
4759 DRM_DEBUG("IH: HPD2\n");
4760 }
4761 break;
4762 case 2:
4763 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4764 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4765 queue_hotplug = true;
4766 DRM_DEBUG("IH: HPD3\n");
4767 }
4768 break;
4769 case 3:
4770 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4771 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4772 queue_hotplug = true;
4773 DRM_DEBUG("IH: HPD4\n");
4774 }
4775 break;
4776 case 4:
4777 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4778 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4779 queue_hotplug = true;
4780 DRM_DEBUG("IH: HPD5\n");
4781 }
4782 break;
4783 case 5:
4784 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4785 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4786 queue_hotplug = true;
4787 DRM_DEBUG("IH: HPD6\n");
4788 }
4789 break;
4790 default:
4791 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4792 break;
4793 }
4794 break;
4795 case 44: /* hdmi */
4796 switch (src_data) {
4797 case 0:
4798 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4799 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4800 queue_hdmi = true;
4801 DRM_DEBUG("IH: HDMI0\n");
4802 }
4803 break;
4804 case 1:
4805 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4806 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4807 queue_hdmi = true;
4808 DRM_DEBUG("IH: HDMI1\n");
4809 }
4810 break;
4811 case 2:
4812 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4813 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4814 queue_hdmi = true;
4815 DRM_DEBUG("IH: HDMI2\n");
4816 }
4817 break;
4818 case 3:
4819 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4820 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4821 queue_hdmi = true;
4822 DRM_DEBUG("IH: HDMI3\n");
4823 }
4824 break;
4825 case 4:
4826 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4827 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4828 queue_hdmi = true;
4829 DRM_DEBUG("IH: HDMI4\n");
4830 }
4831 break;
4832 case 5:
4833 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4834 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4835 queue_hdmi = true;
4836 DRM_DEBUG("IH: HDMI5\n");
4837 }
4838 break;
4839 default:
4840 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4841 break;
4842 }
4843 case 124: /* UVD */
4844 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4845 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4846 break;
4847 case 146:
4848 case 147:
4849 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4850 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4851 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4852 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4853 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4854 /* reset addr and status */
4855 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4856 break;
4857 case 176: /* CP_INT in ring buffer */
4858 case 177: /* CP_INT in IB1 */
4859 case 178: /* CP_INT in IB2 */
4860 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862 break;
4863 case 181: /* CP EOP event */
4864 DRM_DEBUG("IH: CP EOP\n");
4865 if (rdev->family >= CHIP_CAYMAN) {
4866 switch (src_data) {
4867 case 0:
4868 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869 break;
4870 case 1:
4871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872 break;
4873 case 2:
4874 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875 break;
4876 }
4877 } else
4878 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879 break;
4880 case 224: /* DMA trap event */
4881 DRM_DEBUG("IH: DMA trap\n");
4882 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883 break;
4884 case 230: /* thermal low to high */
4885 DRM_DEBUG("IH: thermal low to high\n");
4886 rdev->pm.dpm.thermal.high_to_low = false;
4887 queue_thermal = true;
4888 break;
4889 case 231: /* thermal high to low */
4890 DRM_DEBUG("IH: thermal high to low\n");
4891 rdev->pm.dpm.thermal.high_to_low = true;
4892 queue_thermal = true;
4893 break;
4894 case 233: /* GUI IDLE */
4895 DRM_DEBUG("IH: GUI idle\n");
4896 break;
4897 case 244: /* DMA trap event */
4898 if (rdev->family >= CHIP_CAYMAN) {
4899 DRM_DEBUG("IH: DMA1 trap\n");
4900 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901 }
4902 break;
4903 default:
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905 break;
4906 }
4907
4908 /* wptr/rptr are in bytes! */
4909 rptr += 16;
4910 rptr &= rdev->ih.ptr_mask;
4911 }
4912 if (queue_hotplug)
4913 schedule_work(&rdev->hotplug_work);
4914 if (queue_hdmi)
4915 schedule_work(&rdev->audio_work);
4916 if (queue_thermal && rdev->pm.dpm_enabled)
4917 schedule_work(&rdev->pm.dpm.thermal.work);
4918 rdev->ih.rptr = rptr;
4919 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4920 atomic_set(&rdev->ih.lock, 0);
4921
4922 /* make sure wptr hasn't changed while processing */
4923 wptr = evergreen_get_ih_wptr(rdev);
4924 if (wptr != rptr)
4925 goto restart_ih;
4926
4927 return IRQ_HANDLED;
4928 }
4929
4930 /**
4931 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4932 *
4933 * @rdev: radeon_device pointer
4934 * @fence: radeon fence object
4935 *
4936 * Add a DMA fence packet to the ring to write
4937 * the fence seq number and DMA trap packet to generate
4938 * an interrupt if needed (evergreen-SI).
4939 */
4940 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4941 struct radeon_fence *fence)
4942 {
4943 struct radeon_ring *ring = &rdev->ring[fence->ring];
4944 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4945 /* write the fence */
4946 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
4947 radeon_ring_write(ring, addr & 0xfffffffc);
4948 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4949 radeon_ring_write(ring, fence->seq);
4950 /* generate an interrupt */
4951 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
4952 /* flush HDP */
4953 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4954 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4955 radeon_ring_write(ring, 1);
4956 }
4957
4958 /**
4959 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4960 *
4961 * @rdev: radeon_device pointer
4962 * @ib: IB object to schedule
4963 *
4964 * Schedule an IB in the DMA ring (evergreen).
4965 */
4966 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4967 struct radeon_ib *ib)
4968 {
4969 struct radeon_ring *ring = &rdev->ring[ib->ring];
4970
4971 if (rdev->wb.enabled) {
4972 u32 next_rptr = ring->wptr + 4;
4973 while ((next_rptr & 7) != 5)
4974 next_rptr++;
4975 next_rptr += 3;
4976 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
4977 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4978 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4979 radeon_ring_write(ring, next_rptr);
4980 }
4981
4982 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4983 * Pad as necessary with NOPs.
4984 */
4985 while ((ring->wptr & 7) != 5)
4986 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4987 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
4988 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4989 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4990
4991 }
4992
4993 /**
4994 * evergreen_copy_dma - copy pages using the DMA engine
4995 *
4996 * @rdev: radeon_device pointer
4997 * @src_offset: src GPU address
4998 * @dst_offset: dst GPU address
4999 * @num_gpu_pages: number of GPU pages to xfer
5000 * @fence: radeon fence object
5001 *
5002 * Copy GPU paging using the DMA engine (evergreen-cayman).
5003 * Used by the radeon ttm implementation to move pages if
5004 * registered as the asic copy callback.
5005 */
5006 int evergreen_copy_dma(struct radeon_device *rdev,
5007 uint64_t src_offset, uint64_t dst_offset,
5008 unsigned num_gpu_pages,
5009 struct radeon_fence **fence)
5010 {
5011 struct radeon_semaphore *sem = NULL;
5012 int ring_index = rdev->asic->copy.dma_ring_index;
5013 struct radeon_ring *ring = &rdev->ring[ring_index];
5014 u32 size_in_dw, cur_size_in_dw;
5015 int i, num_loops;
5016 int r = 0;
5017
5018 r = radeon_semaphore_create(rdev, &sem);
5019 if (r) {
5020 DRM_ERROR("radeon: moving bo (%d).\n", r);
5021 return r;
5022 }
5023
5024 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5025 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5026 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5027 if (r) {
5028 DRM_ERROR("radeon: moving bo (%d).\n", r);
5029 radeon_semaphore_free(rdev, &sem, NULL);
5030 return r;
5031 }
5032
5033 if (radeon_fence_need_sync(*fence, ring->idx)) {
5034 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5035 ring->idx);
5036 radeon_fence_note_sync(*fence, ring->idx);
5037 } else {
5038 radeon_semaphore_free(rdev, &sem, NULL);
5039 }
5040
5041 for (i = 0; i < num_loops; i++) {
5042 cur_size_in_dw = size_in_dw;
5043 if (cur_size_in_dw > 0xFFFFF)
5044 cur_size_in_dw = 0xFFFFF;
5045 size_in_dw -= cur_size_in_dw;
5046 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
5047 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5048 radeon_ring_write(ring, src_offset & 0xfffffffc);
5049 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5050 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5051 src_offset += cur_size_in_dw * 4;
5052 dst_offset += cur_size_in_dw * 4;
5053 }
5054
5055 r = radeon_fence_emit(rdev, fence, ring->idx);
5056 if (r) {
5057 radeon_ring_unlock_undo(rdev, ring);
5058 return r;
5059 }
5060
5061 radeon_ring_unlock_commit(rdev, ring);
5062 radeon_semaphore_free(rdev, &sem, *fence);
5063
5064 return r;
5065 }
5066
5067 static int evergreen_startup(struct radeon_device *rdev)
5068 {
5069 struct radeon_ring *ring;
5070 int r;
5071
5072 /* enable pcie gen2 link */
5073 evergreen_pcie_gen2_enable(rdev);
5074
5075 if (ASIC_IS_DCE5(rdev)) {
5076 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5077 r = ni_init_microcode(rdev);
5078 if (r) {
5079 DRM_ERROR("Failed to load firmware!\n");
5080 return r;
5081 }
5082 }
5083 r = ni_mc_load_microcode(rdev);
5084 if (r) {
5085 DRM_ERROR("Failed to load MC firmware!\n");
5086 return r;
5087 }
5088 } else {
5089 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5090 r = r600_init_microcode(rdev);
5091 if (r) {
5092 DRM_ERROR("Failed to load firmware!\n");
5093 return r;
5094 }
5095 }
5096 }
5097
5098 r = r600_vram_scratch_init(rdev);
5099 if (r)
5100 return r;
5101
5102 evergreen_mc_program(rdev);
5103 if (rdev->flags & RADEON_IS_AGP) {
5104 evergreen_agp_enable(rdev);
5105 } else {
5106 r = evergreen_pcie_gart_enable(rdev);
5107 if (r)
5108 return r;
5109 }
5110 evergreen_gpu_init(rdev);
5111
5112 r = evergreen_blit_init(rdev);
5113 if (r) {
5114 r600_blit_fini(rdev);
5115 rdev->asic->copy.copy = NULL;
5116 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
5117 }
5118
5119 /* allocate rlc buffers */
5120 if (rdev->flags & RADEON_IS_IGP) {
5121 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5122 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5123 rdev->rlc.cs_data = evergreen_cs_data;
5124 r = sumo_rlc_init(rdev);
5125 if (r) {
5126 DRM_ERROR("Failed to init rlc BOs!\n");
5127 return r;
5128 }
5129 }
5130
5131 /* allocate wb buffer */
5132 r = radeon_wb_init(rdev);
5133 if (r)
5134 return r;
5135
5136 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5137 if (r) {
5138 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5139 return r;
5140 }
5141
5142 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5143 if (r) {
5144 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5145 return r;
5146 }
5147
5148 r = rv770_uvd_resume(rdev);
5149 if (!r) {
5150 r = radeon_fence_driver_start_ring(rdev,
5151 R600_RING_TYPE_UVD_INDEX);
5152 if (r)
5153 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5154 }
5155
5156 if (r)
5157 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5158
5159 /* Enable IRQ */
5160 if (!rdev->irq.installed) {
5161 r = radeon_irq_kms_init(rdev);
5162 if (r)
5163 return r;
5164 }
5165
5166 r = r600_irq_init(rdev);
5167 if (r) {
5168 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5169 radeon_irq_kms_fini(rdev);
5170 return r;
5171 }
5172 evergreen_irq_set(rdev);
5173
5174 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5175 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5176 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5177 0, 0xfffff, RADEON_CP_PACKET2);
5178 if (r)
5179 return r;
5180
5181 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5182 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5183 DMA_RB_RPTR, DMA_RB_WPTR,
5184 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5185 if (r)
5186 return r;
5187
5188 r = evergreen_cp_load_microcode(rdev);
5189 if (r)
5190 return r;
5191 r = evergreen_cp_resume(rdev);
5192 if (r)
5193 return r;
5194 r = r600_dma_resume(rdev);
5195 if (r)
5196 return r;
5197
5198 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5199 if (ring->ring_size) {
5200 r = radeon_ring_init(rdev, ring, ring->ring_size,
5201 R600_WB_UVD_RPTR_OFFSET,
5202 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5203 0, 0xfffff, RADEON_CP_PACKET2);
5204 if (!r)
5205 r = r600_uvd_init(rdev);
5206
5207 if (r)
5208 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5209 }
5210
5211 r = radeon_ib_pool_init(rdev);
5212 if (r) {
5213 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5214 return r;
5215 }
5216
5217 r = r600_audio_init(rdev);
5218 if (r) {
5219 DRM_ERROR("radeon: audio init failed\n");
5220 return r;
5221 }
5222
5223 return 0;
5224 }
5225
5226 int evergreen_resume(struct radeon_device *rdev)
5227 {
5228 int r;
5229
5230 /* reset the asic, the gfx blocks are often in a bad state
5231 * after the driver is unloaded or after a resume
5232 */
5233 if (radeon_asic_reset(rdev))
5234 dev_warn(rdev->dev, "GPU reset failed !\n");
5235 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5236 * posting will perform necessary task to bring back GPU into good
5237 * shape.
5238 */
5239 /* post card */
5240 atom_asic_init(rdev->mode_info.atom_context);
5241
5242 /* init golden registers */
5243 evergreen_init_golden_registers(rdev);
5244
5245 rdev->accel_working = true;
5246 r = evergreen_startup(rdev);
5247 if (r) {
5248 DRM_ERROR("evergreen startup failed on resume\n");
5249 rdev->accel_working = false;
5250 return r;
5251 }
5252
5253 return r;
5254
5255 }
5256
5257 int evergreen_suspend(struct radeon_device *rdev)
5258 {
5259 r600_audio_fini(rdev);
5260 radeon_uvd_suspend(rdev);
5261 r700_cp_stop(rdev);
5262 r600_dma_stop(rdev);
5263 r600_uvd_rbc_stop(rdev);
5264 evergreen_irq_suspend(rdev);
5265 radeon_wb_disable(rdev);
5266 evergreen_pcie_gart_disable(rdev);
5267
5268 return 0;
5269 }
5270
5271 /* Plan is to move initialization in that function and use
5272 * helper function so that radeon_device_init pretty much
5273 * do nothing more than calling asic specific function. This
5274 * should also allow to remove a bunch of callback function
5275 * like vram_info.
5276 */
5277 int evergreen_init(struct radeon_device *rdev)
5278 {
5279 int r;
5280
5281 /* Read BIOS */
5282 if (!radeon_get_bios(rdev)) {
5283 if (ASIC_IS_AVIVO(rdev))
5284 return -EINVAL;
5285 }
5286 /* Must be an ATOMBIOS */
5287 if (!rdev->is_atom_bios) {
5288 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5289 return -EINVAL;
5290 }
5291 r = radeon_atombios_init(rdev);
5292 if (r)
5293 return r;
5294 /* reset the asic, the gfx blocks are often in a bad state
5295 * after the driver is unloaded or after a resume
5296 */
5297 if (radeon_asic_reset(rdev))
5298 dev_warn(rdev->dev, "GPU reset failed !\n");
5299 /* Post card if necessary */
5300 if (!radeon_card_posted(rdev)) {
5301 if (!rdev->bios) {
5302 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5303 return -EINVAL;
5304 }
5305 DRM_INFO("GPU not posted. posting now...\n");
5306 atom_asic_init(rdev->mode_info.atom_context);
5307 }
5308 /* init golden registers */
5309 evergreen_init_golden_registers(rdev);
5310 /* Initialize scratch registers */
5311 r600_scratch_init(rdev);
5312 /* Initialize surface registers */
5313 radeon_surface_init(rdev);
5314 /* Initialize clocks */
5315 radeon_get_clock_info(rdev->ddev);
5316 /* Fence driver */
5317 r = radeon_fence_driver_init(rdev);
5318 if (r)
5319 return r;
5320 /* initialize AGP */
5321 if (rdev->flags & RADEON_IS_AGP) {
5322 r = radeon_agp_init(rdev);
5323 if (r)
5324 radeon_agp_disable(rdev);
5325 }
5326 /* initialize memory controller */
5327 r = evergreen_mc_init(rdev);
5328 if (r)
5329 return r;
5330 /* Memory manager */
5331 r = radeon_bo_init(rdev);
5332 if (r)
5333 return r;
5334
5335 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5336 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5337
5338 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5339 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5340
5341 r = radeon_uvd_init(rdev);
5342 if (!r) {
5343 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5344 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5345 4096);
5346 }
5347
5348 rdev->ih.ring_obj = NULL;
5349 r600_ih_ring_init(rdev, 64 * 1024);
5350
5351 r = r600_pcie_gart_init(rdev);
5352 if (r)
5353 return r;
5354
5355 rdev->accel_working = true;
5356 r = evergreen_startup(rdev);
5357 if (r) {
5358 dev_err(rdev->dev, "disabling GPU acceleration\n");
5359 r700_cp_fini(rdev);
5360 r600_dma_fini(rdev);
5361 r600_irq_fini(rdev);
5362 if (rdev->flags & RADEON_IS_IGP)
5363 sumo_rlc_fini(rdev);
5364 radeon_wb_fini(rdev);
5365 radeon_ib_pool_fini(rdev);
5366 radeon_irq_kms_fini(rdev);
5367 evergreen_pcie_gart_fini(rdev);
5368 rdev->accel_working = false;
5369 }
5370
5371 /* Don't start up if the MC ucode is missing on BTC parts.
5372 * The default clocks and voltages before the MC ucode
5373 * is loaded are not suffient for advanced operations.
5374 */
5375 if (ASIC_IS_DCE5(rdev)) {
5376 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5377 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5378 return -EINVAL;
5379 }
5380 }
5381
5382 return 0;
5383 }
5384
5385 void evergreen_fini(struct radeon_device *rdev)
5386 {
5387 r600_audio_fini(rdev);
5388 r600_blit_fini(rdev);
5389 r700_cp_fini(rdev);
5390 r600_dma_fini(rdev);
5391 r600_irq_fini(rdev);
5392 if (rdev->flags & RADEON_IS_IGP)
5393 sumo_rlc_fini(rdev);
5394 radeon_wb_fini(rdev);
5395 radeon_ib_pool_fini(rdev);
5396 radeon_irq_kms_fini(rdev);
5397 evergreen_pcie_gart_fini(rdev);
5398 radeon_uvd_fini(rdev);
5399 r600_vram_scratch_fini(rdev);
5400 radeon_gem_fini(rdev);
5401 radeon_fence_driver_fini(rdev);
5402 radeon_agp_fini(rdev);
5403 radeon_bo_fini(rdev);
5404 radeon_atombios_fini(rdev);
5405 kfree(rdev->bios);
5406 rdev->bios = NULL;
5407 }
5408
5409 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5410 {
5411 u32 link_width_cntl, speed_cntl;
5412
5413 if (radeon_pcie_gen2 == 0)
5414 return;
5415
5416 if (rdev->flags & RADEON_IS_IGP)
5417 return;
5418
5419 if (!(rdev->flags & RADEON_IS_PCIE))
5420 return;
5421
5422 /* x2 cards have a special sequence */
5423 if (ASIC_IS_X2(rdev))
5424 return;
5425
5426 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5427 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5428 return;
5429
5430 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5431 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5432 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5433 return;
5434 }
5435
5436 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5437
5438 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5439 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5440
5441 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5442 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5443 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5444
5445 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5446 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5447 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5448
5449 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5450 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5451 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5452
5453 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5454 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5455 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5456
5457 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5458 speed_cntl |= LC_GEN2_EN_STRAP;
5459 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5460
5461 } else {
5462 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5463 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5464 if (1)
5465 link_width_cntl |= LC_UPCONFIGURE_DIS;
5466 else
5467 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5468 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5469 }
5470 }
Linux kernel - Deliverables
This page took 0.582951 seconds and 6 git commands to generate.