projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'drm-nouveau-fixes-3.10' of git://anongit.freedesktop.org/git/nouveau...
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36
37 #define EVERGREEN_PFP_UCODE_SIZE 1120
38 #define EVERGREEN_PM4_UCODE_SIZE 1376
39
40 static const u32 crtc_offsets[6] =
41 {
42 EVERGREEN_CRTC0_REGISTER_OFFSET,
43 EVERGREEN_CRTC1_REGISTER_OFFSET,
44 EVERGREEN_CRTC2_REGISTER_OFFSET,
45 EVERGREEN_CRTC3_REGISTER_OFFSET,
46 EVERGREEN_CRTC4_REGISTER_OFFSET,
47 EVERGREEN_CRTC5_REGISTER_OFFSET
48 };
49
50 static void evergreen_gpu_init(struct radeon_device *rdev);
51 void evergreen_fini(struct radeon_device *rdev);
52 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
53 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
54 int ring, u32 cp_int_cntl);
55
56 static const u32 evergreen_golden_registers[] =
57 {
58 0x3f90, 0xffff0000, 0xff000000,
59 0x9148, 0xffff0000, 0xff000000,
60 0x3f94, 0xffff0000, 0xff000000,
61 0x914c, 0xffff0000, 0xff000000,
62 0x9b7c, 0xffffffff, 0x00000000,
63 0x8a14, 0xffffffff, 0x00000007,
64 0x8b10, 0xffffffff, 0x00000000,
65 0x960c, 0xffffffff, 0x54763210,
66 0x88c4, 0xffffffff, 0x000000c2,
67 0x88d4, 0xffffffff, 0x00000010,
68 0x8974, 0xffffffff, 0x00000000,
69 0xc78, 0x00000080, 0x00000080,
70 0x5eb4, 0xffffffff, 0x00000002,
71 0x5e78, 0xffffffff, 0x001000f0,
72 0x6104, 0x01000300, 0x00000000,
73 0x5bc0, 0x00300000, 0x00000000,
74 0x7030, 0xffffffff, 0x00000011,
75 0x7c30, 0xffffffff, 0x00000011,
76 0x10830, 0xffffffff, 0x00000011,
77 0x11430, 0xffffffff, 0x00000011,
78 0x12030, 0xffffffff, 0x00000011,
79 0x12c30, 0xffffffff, 0x00000011,
80 0xd02c, 0xffffffff, 0x08421000,
81 0x240c, 0xffffffff, 0x00000380,
82 0x8b24, 0xffffffff, 0x00ff0fff,
83 0x28a4c, 0x06000000, 0x06000000,
84 0x10c, 0x00000001, 0x00000001,
85 0x8d00, 0xffffffff, 0x100e4848,
86 0x8d04, 0xffffffff, 0x00164745,
87 0x8c00, 0xffffffff, 0xe4000003,
88 0x8c04, 0xffffffff, 0x40600060,
89 0x8c08, 0xffffffff, 0x001c001c,
90 0x8cf0, 0xffffffff, 0x08e00620,
91 0x8c20, 0xffffffff, 0x00800080,
92 0x8c24, 0xffffffff, 0x00800080,
93 0x8c18, 0xffffffff, 0x20202078,
94 0x8c1c, 0xffffffff, 0x00001010,
95 0x28350, 0xffffffff, 0x00000000,
96 0xa008, 0xffffffff, 0x00010000,
97 0x5cc, 0xffffffff, 0x00000001,
98 0x9508, 0xffffffff, 0x00000002,
99 0x913c, 0x0000000f, 0x0000000a
100 };
101
102 static const u32 evergreen_golden_registers2[] =
103 {
104 0x2f4c, 0xffffffff, 0x00000000,
105 0x54f4, 0xffffffff, 0x00000000,
106 0x54f0, 0xffffffff, 0x00000000,
107 0x5498, 0xffffffff, 0x00000000,
108 0x549c, 0xffffffff, 0x00000000,
109 0x5494, 0xffffffff, 0x00000000,
110 0x53cc, 0xffffffff, 0x00000000,
111 0x53c8, 0xffffffff, 0x00000000,
112 0x53c4, 0xffffffff, 0x00000000,
113 0x53c0, 0xffffffff, 0x00000000,
114 0x53bc, 0xffffffff, 0x00000000,
115 0x53b8, 0xffffffff, 0x00000000,
116 0x53b4, 0xffffffff, 0x00000000,
117 0x53b0, 0xffffffff, 0x00000000
118 };
119
120 static const u32 cypress_mgcg_init[] =
121 {
122 0x802c, 0xffffffff, 0xc0000000,
123 0x5448, 0xffffffff, 0x00000100,
124 0x55e4, 0xffffffff, 0x00000100,
125 0x160c, 0xffffffff, 0x00000100,
126 0x5644, 0xffffffff, 0x00000100,
127 0xc164, 0xffffffff, 0x00000100,
128 0x8a18, 0xffffffff, 0x00000100,
129 0x897c, 0xffffffff, 0x06000100,
130 0x8b28, 0xffffffff, 0x00000100,
131 0x9144, 0xffffffff, 0x00000100,
132 0x9a60, 0xffffffff, 0x00000100,
133 0x9868, 0xffffffff, 0x00000100,
134 0x8d58, 0xffffffff, 0x00000100,
135 0x9510, 0xffffffff, 0x00000100,
136 0x949c, 0xffffffff, 0x00000100,
137 0x9654, 0xffffffff, 0x00000100,
138 0x9030, 0xffffffff, 0x00000100,
139 0x9034, 0xffffffff, 0x00000100,
140 0x9038, 0xffffffff, 0x00000100,
141 0x903c, 0xffffffff, 0x00000100,
142 0x9040, 0xffffffff, 0x00000100,
143 0xa200, 0xffffffff, 0x00000100,
144 0xa204, 0xffffffff, 0x00000100,
145 0xa208, 0xffffffff, 0x00000100,
146 0xa20c, 0xffffffff, 0x00000100,
147 0x971c, 0xffffffff, 0x00000100,
148 0x977c, 0xffffffff, 0x00000100,
149 0x3f80, 0xffffffff, 0x00000100,
150 0xa210, 0xffffffff, 0x00000100,
151 0xa214, 0xffffffff, 0x00000100,
152 0x4d8, 0xffffffff, 0x00000100,
153 0x9784, 0xffffffff, 0x00000100,
154 0x9698, 0xffffffff, 0x00000100,
155 0x4d4, 0xffffffff, 0x00000200,
156 0x30cc, 0xffffffff, 0x00000100,
157 0xd0c0, 0xffffffff, 0xff000100,
158 0x802c, 0xffffffff, 0x40000000,
159 0x915c, 0xffffffff, 0x00010000,
160 0x9160, 0xffffffff, 0x00030002,
161 0x9178, 0xffffffff, 0x00070000,
162 0x917c, 0xffffffff, 0x00030002,
163 0x9180, 0xffffffff, 0x00050004,
164 0x918c, 0xffffffff, 0x00010006,
165 0x9190, 0xffffffff, 0x00090008,
166 0x9194, 0xffffffff, 0x00070000,
167 0x9198, 0xffffffff, 0x00030002,
168 0x919c, 0xffffffff, 0x00050004,
169 0x91a8, 0xffffffff, 0x00010006,
170 0x91ac, 0xffffffff, 0x00090008,
171 0x91b0, 0xffffffff, 0x00070000,
172 0x91b4, 0xffffffff, 0x00030002,
173 0x91b8, 0xffffffff, 0x00050004,
174 0x91c4, 0xffffffff, 0x00010006,
175 0x91c8, 0xffffffff, 0x00090008,
176 0x91cc, 0xffffffff, 0x00070000,
177 0x91d0, 0xffffffff, 0x00030002,
178 0x91d4, 0xffffffff, 0x00050004,
179 0x91e0, 0xffffffff, 0x00010006,
180 0x91e4, 0xffffffff, 0x00090008,
181 0x91e8, 0xffffffff, 0x00000000,
182 0x91ec, 0xffffffff, 0x00070000,
183 0x91f0, 0xffffffff, 0x00030002,
184 0x91f4, 0xffffffff, 0x00050004,
185 0x9200, 0xffffffff, 0x00010006,
186 0x9204, 0xffffffff, 0x00090008,
187 0x9208, 0xffffffff, 0x00070000,
188 0x920c, 0xffffffff, 0x00030002,
189 0x9210, 0xffffffff, 0x00050004,
190 0x921c, 0xffffffff, 0x00010006,
191 0x9220, 0xffffffff, 0x00090008,
192 0x9224, 0xffffffff, 0x00070000,
193 0x9228, 0xffffffff, 0x00030002,
194 0x922c, 0xffffffff, 0x00050004,
195 0x9238, 0xffffffff, 0x00010006,
196 0x923c, 0xffffffff, 0x00090008,
197 0x9240, 0xffffffff, 0x00070000,
198 0x9244, 0xffffffff, 0x00030002,
199 0x9248, 0xffffffff, 0x00050004,
200 0x9254, 0xffffffff, 0x00010006,
201 0x9258, 0xffffffff, 0x00090008,
202 0x925c, 0xffffffff, 0x00070000,
203 0x9260, 0xffffffff, 0x00030002,
204 0x9264, 0xffffffff, 0x00050004,
205 0x9270, 0xffffffff, 0x00010006,
206 0x9274, 0xffffffff, 0x00090008,
207 0x9278, 0xffffffff, 0x00070000,
208 0x927c, 0xffffffff, 0x00030002,
209 0x9280, 0xffffffff, 0x00050004,
210 0x928c, 0xffffffff, 0x00010006,
211 0x9290, 0xffffffff, 0x00090008,
212 0x9294, 0xffffffff, 0x00000000,
213 0x929c, 0xffffffff, 0x00000001,
214 0x802c, 0xffffffff, 0x40010000,
215 0x915c, 0xffffffff, 0x00010000,
216 0x9160, 0xffffffff, 0x00030002,
217 0x9178, 0xffffffff, 0x00070000,
218 0x917c, 0xffffffff, 0x00030002,
219 0x9180, 0xffffffff, 0x00050004,
220 0x918c, 0xffffffff, 0x00010006,
221 0x9190, 0xffffffff, 0x00090008,
222 0x9194, 0xffffffff, 0x00070000,
223 0x9198, 0xffffffff, 0x00030002,
224 0x919c, 0xffffffff, 0x00050004,
225 0x91a8, 0xffffffff, 0x00010006,
226 0x91ac, 0xffffffff, 0x00090008,
227 0x91b0, 0xffffffff, 0x00070000,
228 0x91b4, 0xffffffff, 0x00030002,
229 0x91b8, 0xffffffff, 0x00050004,
230 0x91c4, 0xffffffff, 0x00010006,
231 0x91c8, 0xffffffff, 0x00090008,
232 0x91cc, 0xffffffff, 0x00070000,
233 0x91d0, 0xffffffff, 0x00030002,
234 0x91d4, 0xffffffff, 0x00050004,
235 0x91e0, 0xffffffff, 0x00010006,
236 0x91e4, 0xffffffff, 0x00090008,
237 0x91e8, 0xffffffff, 0x00000000,
238 0x91ec, 0xffffffff, 0x00070000,
239 0x91f0, 0xffffffff, 0x00030002,
240 0x91f4, 0xffffffff, 0x00050004,
241 0x9200, 0xffffffff, 0x00010006,
242 0x9204, 0xffffffff, 0x00090008,
243 0x9208, 0xffffffff, 0x00070000,
244 0x920c, 0xffffffff, 0x00030002,
245 0x9210, 0xffffffff, 0x00050004,
246 0x921c, 0xffffffff, 0x00010006,
247 0x9220, 0xffffffff, 0x00090008,
248 0x9224, 0xffffffff, 0x00070000,
249 0x9228, 0xffffffff, 0x00030002,
250 0x922c, 0xffffffff, 0x00050004,
251 0x9238, 0xffffffff, 0x00010006,
252 0x923c, 0xffffffff, 0x00090008,
253 0x9240, 0xffffffff, 0x00070000,
254 0x9244, 0xffffffff, 0x00030002,
255 0x9248, 0xffffffff, 0x00050004,
256 0x9254, 0xffffffff, 0x00010006,
257 0x9258, 0xffffffff, 0x00090008,
258 0x925c, 0xffffffff, 0x00070000,
259 0x9260, 0xffffffff, 0x00030002,
260 0x9264, 0xffffffff, 0x00050004,
261 0x9270, 0xffffffff, 0x00010006,
262 0x9274, 0xffffffff, 0x00090008,
263 0x9278, 0xffffffff, 0x00070000,
264 0x927c, 0xffffffff, 0x00030002,
265 0x9280, 0xffffffff, 0x00050004,
266 0x928c, 0xffffffff, 0x00010006,
267 0x9290, 0xffffffff, 0x00090008,
268 0x9294, 0xffffffff, 0x00000000,
269 0x929c, 0xffffffff, 0x00000001,
270 0x802c, 0xffffffff, 0xc0000000
271 };
272
273 static const u32 redwood_mgcg_init[] =
274 {
275 0x802c, 0xffffffff, 0xc0000000,
276 0x5448, 0xffffffff, 0x00000100,
277 0x55e4, 0xffffffff, 0x00000100,
278 0x160c, 0xffffffff, 0x00000100,
279 0x5644, 0xffffffff, 0x00000100,
280 0xc164, 0xffffffff, 0x00000100,
281 0x8a18, 0xffffffff, 0x00000100,
282 0x897c, 0xffffffff, 0x06000100,
283 0x8b28, 0xffffffff, 0x00000100,
284 0x9144, 0xffffffff, 0x00000100,
285 0x9a60, 0xffffffff, 0x00000100,
286 0x9868, 0xffffffff, 0x00000100,
287 0x8d58, 0xffffffff, 0x00000100,
288 0x9510, 0xffffffff, 0x00000100,
289 0x949c, 0xffffffff, 0x00000100,
290 0x9654, 0xffffffff, 0x00000100,
291 0x9030, 0xffffffff, 0x00000100,
292 0x9034, 0xffffffff, 0x00000100,
293 0x9038, 0xffffffff, 0x00000100,
294 0x903c, 0xffffffff, 0x00000100,
295 0x9040, 0xffffffff, 0x00000100,
296 0xa200, 0xffffffff, 0x00000100,
297 0xa204, 0xffffffff, 0x00000100,
298 0xa208, 0xffffffff, 0x00000100,
299 0xa20c, 0xffffffff, 0x00000100,
300 0x971c, 0xffffffff, 0x00000100,
301 0x977c, 0xffffffff, 0x00000100,
302 0x3f80, 0xffffffff, 0x00000100,
303 0xa210, 0xffffffff, 0x00000100,
304 0xa214, 0xffffffff, 0x00000100,
305 0x4d8, 0xffffffff, 0x00000100,
306 0x9784, 0xffffffff, 0x00000100,
307 0x9698, 0xffffffff, 0x00000100,
308 0x4d4, 0xffffffff, 0x00000200,
309 0x30cc, 0xffffffff, 0x00000100,
310 0xd0c0, 0xffffffff, 0xff000100,
311 0x802c, 0xffffffff, 0x40000000,
312 0x915c, 0xffffffff, 0x00010000,
313 0x9160, 0xffffffff, 0x00030002,
314 0x9178, 0xffffffff, 0x00070000,
315 0x917c, 0xffffffff, 0x00030002,
316 0x9180, 0xffffffff, 0x00050004,
317 0x918c, 0xffffffff, 0x00010006,
318 0x9190, 0xffffffff, 0x00090008,
319 0x9194, 0xffffffff, 0x00070000,
320 0x9198, 0xffffffff, 0x00030002,
321 0x919c, 0xffffffff, 0x00050004,
322 0x91a8, 0xffffffff, 0x00010006,
323 0x91ac, 0xffffffff, 0x00090008,
324 0x91b0, 0xffffffff, 0x00070000,
325 0x91b4, 0xffffffff, 0x00030002,
326 0x91b8, 0xffffffff, 0x00050004,
327 0x91c4, 0xffffffff, 0x00010006,
328 0x91c8, 0xffffffff, 0x00090008,
329 0x91cc, 0xffffffff, 0x00070000,
330 0x91d0, 0xffffffff, 0x00030002,
331 0x91d4, 0xffffffff, 0x00050004,
332 0x91e0, 0xffffffff, 0x00010006,
333 0x91e4, 0xffffffff, 0x00090008,
334 0x91e8, 0xffffffff, 0x00000000,
335 0x91ec, 0xffffffff, 0x00070000,
336 0x91f0, 0xffffffff, 0x00030002,
337 0x91f4, 0xffffffff, 0x00050004,
338 0x9200, 0xffffffff, 0x00010006,
339 0x9204, 0xffffffff, 0x00090008,
340 0x9294, 0xffffffff, 0x00000000,
341 0x929c, 0xffffffff, 0x00000001,
342 0x802c, 0xffffffff, 0xc0000000
343 };
344
345 static const u32 cedar_golden_registers[] =
346 {
347 0x3f90, 0xffff0000, 0xff000000,
348 0x9148, 0xffff0000, 0xff000000,
349 0x3f94, 0xffff0000, 0xff000000,
350 0x914c, 0xffff0000, 0xff000000,
351 0x9b7c, 0xffffffff, 0x00000000,
352 0x8a14, 0xffffffff, 0x00000007,
353 0x8b10, 0xffffffff, 0x00000000,
354 0x960c, 0xffffffff, 0x54763210,
355 0x88c4, 0xffffffff, 0x000000c2,
356 0x88d4, 0xffffffff, 0x00000000,
357 0x8974, 0xffffffff, 0x00000000,
358 0xc78, 0x00000080, 0x00000080,
359 0x5eb4, 0xffffffff, 0x00000002,
360 0x5e78, 0xffffffff, 0x001000f0,
361 0x6104, 0x01000300, 0x00000000,
362 0x5bc0, 0x00300000, 0x00000000,
363 0x7030, 0xffffffff, 0x00000011,
364 0x7c30, 0xffffffff, 0x00000011,
365 0x10830, 0xffffffff, 0x00000011,
366 0x11430, 0xffffffff, 0x00000011,
367 0xd02c, 0xffffffff, 0x08421000,
368 0x240c, 0xffffffff, 0x00000380,
369 0x8b24, 0xffffffff, 0x00ff0fff,
370 0x28a4c, 0x06000000, 0x06000000,
371 0x10c, 0x00000001, 0x00000001,
372 0x8d00, 0xffffffff, 0x100e4848,
373 0x8d04, 0xffffffff, 0x00164745,
374 0x8c00, 0xffffffff, 0xe4000003,
375 0x8c04, 0xffffffff, 0x40600060,
376 0x8c08, 0xffffffff, 0x001c001c,
377 0x8cf0, 0xffffffff, 0x08e00410,
378 0x8c20, 0xffffffff, 0x00800080,
379 0x8c24, 0xffffffff, 0x00800080,
380 0x8c18, 0xffffffff, 0x20202078,
381 0x8c1c, 0xffffffff, 0x00001010,
382 0x28350, 0xffffffff, 0x00000000,
383 0xa008, 0xffffffff, 0x00010000,
384 0x5cc, 0xffffffff, 0x00000001,
385 0x9508, 0xffffffff, 0x00000002
386 };
387
388 static const u32 cedar_mgcg_init[] =
389 {
390 0x802c, 0xffffffff, 0xc0000000,
391 0x5448, 0xffffffff, 0x00000100,
392 0x55e4, 0xffffffff, 0x00000100,
393 0x160c, 0xffffffff, 0x00000100,
394 0x5644, 0xffffffff, 0x00000100,
395 0xc164, 0xffffffff, 0x00000100,
396 0x8a18, 0xffffffff, 0x00000100,
397 0x897c, 0xffffffff, 0x06000100,
398 0x8b28, 0xffffffff, 0x00000100,
399 0x9144, 0xffffffff, 0x00000100,
400 0x9a60, 0xffffffff, 0x00000100,
401 0x9868, 0xffffffff, 0x00000100,
402 0x8d58, 0xffffffff, 0x00000100,
403 0x9510, 0xffffffff, 0x00000100,
404 0x949c, 0xffffffff, 0x00000100,
405 0x9654, 0xffffffff, 0x00000100,
406 0x9030, 0xffffffff, 0x00000100,
407 0x9034, 0xffffffff, 0x00000100,
408 0x9038, 0xffffffff, 0x00000100,
409 0x903c, 0xffffffff, 0x00000100,
410 0x9040, 0xffffffff, 0x00000100,
411 0xa200, 0xffffffff, 0x00000100,
412 0xa204, 0xffffffff, 0x00000100,
413 0xa208, 0xffffffff, 0x00000100,
414 0xa20c, 0xffffffff, 0x00000100,
415 0x971c, 0xffffffff, 0x00000100,
416 0x977c, 0xffffffff, 0x00000100,
417 0x3f80, 0xffffffff, 0x00000100,
418 0xa210, 0xffffffff, 0x00000100,
419 0xa214, 0xffffffff, 0x00000100,
420 0x4d8, 0xffffffff, 0x00000100,
421 0x9784, 0xffffffff, 0x00000100,
422 0x9698, 0xffffffff, 0x00000100,
423 0x4d4, 0xffffffff, 0x00000200,
424 0x30cc, 0xffffffff, 0x00000100,
425 0xd0c0, 0xffffffff, 0xff000100,
426 0x802c, 0xffffffff, 0x40000000,
427 0x915c, 0xffffffff, 0x00010000,
428 0x9178, 0xffffffff, 0x00050000,
429 0x917c, 0xffffffff, 0x00030002,
430 0x918c, 0xffffffff, 0x00010004,
431 0x9190, 0xffffffff, 0x00070006,
432 0x9194, 0xffffffff, 0x00050000,
433 0x9198, 0xffffffff, 0x00030002,
434 0x91a8, 0xffffffff, 0x00010004,
435 0x91ac, 0xffffffff, 0x00070006,
436 0x91e8, 0xffffffff, 0x00000000,
437 0x9294, 0xffffffff, 0x00000000,
438 0x929c, 0xffffffff, 0x00000001,
439 0x802c, 0xffffffff, 0xc0000000
440 };
441
442 static const u32 juniper_mgcg_init[] =
443 {
444 0x802c, 0xffffffff, 0xc0000000,
445 0x5448, 0xffffffff, 0x00000100,
446 0x55e4, 0xffffffff, 0x00000100,
447 0x160c, 0xffffffff, 0x00000100,
448 0x5644, 0xffffffff, 0x00000100,
449 0xc164, 0xffffffff, 0x00000100,
450 0x8a18, 0xffffffff, 0x00000100,
451 0x897c, 0xffffffff, 0x06000100,
452 0x8b28, 0xffffffff, 0x00000100,
453 0x9144, 0xffffffff, 0x00000100,
454 0x9a60, 0xffffffff, 0x00000100,
455 0x9868, 0xffffffff, 0x00000100,
456 0x8d58, 0xffffffff, 0x00000100,
457 0x9510, 0xffffffff, 0x00000100,
458 0x949c, 0xffffffff, 0x00000100,
459 0x9654, 0xffffffff, 0x00000100,
460 0x9030, 0xffffffff, 0x00000100,
461 0x9034, 0xffffffff, 0x00000100,
462 0x9038, 0xffffffff, 0x00000100,
463 0x903c, 0xffffffff, 0x00000100,
464 0x9040, 0xffffffff, 0x00000100,
465 0xa200, 0xffffffff, 0x00000100,
466 0xa204, 0xffffffff, 0x00000100,
467 0xa208, 0xffffffff, 0x00000100,
468 0xa20c, 0xffffffff, 0x00000100,
469 0x971c, 0xffffffff, 0x00000100,
470 0xd0c0, 0xffffffff, 0xff000100,
471 0x802c, 0xffffffff, 0x40000000,
472 0x915c, 0xffffffff, 0x00010000,
473 0x9160, 0xffffffff, 0x00030002,
474 0x9178, 0xffffffff, 0x00070000,
475 0x917c, 0xffffffff, 0x00030002,
476 0x9180, 0xffffffff, 0x00050004,
477 0x918c, 0xffffffff, 0x00010006,
478 0x9190, 0xffffffff, 0x00090008,
479 0x9194, 0xffffffff, 0x00070000,
480 0x9198, 0xffffffff, 0x00030002,
481 0x919c, 0xffffffff, 0x00050004,
482 0x91a8, 0xffffffff, 0x00010006,
483 0x91ac, 0xffffffff, 0x00090008,
484 0x91b0, 0xffffffff, 0x00070000,
485 0x91b4, 0xffffffff, 0x00030002,
486 0x91b8, 0xffffffff, 0x00050004,
487 0x91c4, 0xffffffff, 0x00010006,
488 0x91c8, 0xffffffff, 0x00090008,
489 0x91cc, 0xffffffff, 0x00070000,
490 0x91d0, 0xffffffff, 0x00030002,
491 0x91d4, 0xffffffff, 0x00050004,
492 0x91e0, 0xffffffff, 0x00010006,
493 0x91e4, 0xffffffff, 0x00090008,
494 0x91e8, 0xffffffff, 0x00000000,
495 0x91ec, 0xffffffff, 0x00070000,
496 0x91f0, 0xffffffff, 0x00030002,
497 0x91f4, 0xffffffff, 0x00050004,
498 0x9200, 0xffffffff, 0x00010006,
499 0x9204, 0xffffffff, 0x00090008,
500 0x9208, 0xffffffff, 0x00070000,
501 0x920c, 0xffffffff, 0x00030002,
502 0x9210, 0xffffffff, 0x00050004,
503 0x921c, 0xffffffff, 0x00010006,
504 0x9220, 0xffffffff, 0x00090008,
505 0x9224, 0xffffffff, 0x00070000,
506 0x9228, 0xffffffff, 0x00030002,
507 0x922c, 0xffffffff, 0x00050004,
508 0x9238, 0xffffffff, 0x00010006,
509 0x923c, 0xffffffff, 0x00090008,
510 0x9240, 0xffffffff, 0x00070000,
511 0x9244, 0xffffffff, 0x00030002,
512 0x9248, 0xffffffff, 0x00050004,
513 0x9254, 0xffffffff, 0x00010006,
514 0x9258, 0xffffffff, 0x00090008,
515 0x925c, 0xffffffff, 0x00070000,
516 0x9260, 0xffffffff, 0x00030002,
517 0x9264, 0xffffffff, 0x00050004,
518 0x9270, 0xffffffff, 0x00010006,
519 0x9274, 0xffffffff, 0x00090008,
520 0x9278, 0xffffffff, 0x00070000,
521 0x927c, 0xffffffff, 0x00030002,
522 0x9280, 0xffffffff, 0x00050004,
523 0x928c, 0xffffffff, 0x00010006,
524 0x9290, 0xffffffff, 0x00090008,
525 0x9294, 0xffffffff, 0x00000000,
526 0x929c, 0xffffffff, 0x00000001,
527 0x802c, 0xffffffff, 0xc0000000,
528 0x977c, 0xffffffff, 0x00000100,
529 0x3f80, 0xffffffff, 0x00000100,
530 0xa210, 0xffffffff, 0x00000100,
531 0xa214, 0xffffffff, 0x00000100,
532 0x4d8, 0xffffffff, 0x00000100,
533 0x9784, 0xffffffff, 0x00000100,
534 0x9698, 0xffffffff, 0x00000100,
535 0x4d4, 0xffffffff, 0x00000200,
536 0x30cc, 0xffffffff, 0x00000100,
537 0x802c, 0xffffffff, 0xc0000000
538 };
539
540 static const u32 supersumo_golden_registers[] =
541 {
542 0x5eb4, 0xffffffff, 0x00000002,
543 0x5cc, 0xffffffff, 0x00000001,
544 0x7030, 0xffffffff, 0x00000011,
545 0x7c30, 0xffffffff, 0x00000011,
546 0x6104, 0x01000300, 0x00000000,
547 0x5bc0, 0x00300000, 0x00000000,
548 0x8c04, 0xffffffff, 0x40600060,
549 0x8c08, 0xffffffff, 0x001c001c,
550 0x8c20, 0xffffffff, 0x00800080,
551 0x8c24, 0xffffffff, 0x00800080,
552 0x8c18, 0xffffffff, 0x20202078,
553 0x8c1c, 0xffffffff, 0x00001010,
554 0x918c, 0xffffffff, 0x00010006,
555 0x91a8, 0xffffffff, 0x00010006,
556 0x91c4, 0xffffffff, 0x00010006,
557 0x91e0, 0xffffffff, 0x00010006,
558 0x9200, 0xffffffff, 0x00010006,
559 0x9150, 0xffffffff, 0x6e944040,
560 0x917c, 0xffffffff, 0x00030002,
561 0x9180, 0xffffffff, 0x00050004,
562 0x9198, 0xffffffff, 0x00030002,
563 0x919c, 0xffffffff, 0x00050004,
564 0x91b4, 0xffffffff, 0x00030002,
565 0x91b8, 0xffffffff, 0x00050004,
566 0x91d0, 0xffffffff, 0x00030002,
567 0x91d4, 0xffffffff, 0x00050004,
568 0x91f0, 0xffffffff, 0x00030002,
569 0x91f4, 0xffffffff, 0x00050004,
570 0x915c, 0xffffffff, 0x00010000,
571 0x9160, 0xffffffff, 0x00030002,
572 0x3f90, 0xffff0000, 0xff000000,
573 0x9178, 0xffffffff, 0x00070000,
574 0x9194, 0xffffffff, 0x00070000,
575 0x91b0, 0xffffffff, 0x00070000,
576 0x91cc, 0xffffffff, 0x00070000,
577 0x91ec, 0xffffffff, 0x00070000,
578 0x9148, 0xffff0000, 0xff000000,
579 0x9190, 0xffffffff, 0x00090008,
580 0x91ac, 0xffffffff, 0x00090008,
581 0x91c8, 0xffffffff, 0x00090008,
582 0x91e4, 0xffffffff, 0x00090008,
583 0x9204, 0xffffffff, 0x00090008,
584 0x3f94, 0xffff0000, 0xff000000,
585 0x914c, 0xffff0000, 0xff000000,
586 0x929c, 0xffffffff, 0x00000001,
587 0x8a18, 0xffffffff, 0x00000100,
588 0x8b28, 0xffffffff, 0x00000100,
589 0x9144, 0xffffffff, 0x00000100,
590 0x5644, 0xffffffff, 0x00000100,
591 0x9b7c, 0xffffffff, 0x00000000,
592 0x8030, 0xffffffff, 0x0000100a,
593 0x8a14, 0xffffffff, 0x00000007,
594 0x8b24, 0xffffffff, 0x00ff0fff,
595 0x8b10, 0xffffffff, 0x00000000,
596 0x28a4c, 0x06000000, 0x06000000,
597 0x4d8, 0xffffffff, 0x00000100,
598 0x913c, 0xffff000f, 0x0100000a,
599 0x960c, 0xffffffff, 0x54763210,
600 0x88c4, 0xffffffff, 0x000000c2,
601 0x88d4, 0xffffffff, 0x00000010,
602 0x8974, 0xffffffff, 0x00000000,
603 0xc78, 0x00000080, 0x00000080,
604 0x5e78, 0xffffffff, 0x001000f0,
605 0xd02c, 0xffffffff, 0x08421000,
606 0xa008, 0xffffffff, 0x00010000,
607 0x8d00, 0xffffffff, 0x100e4848,
608 0x8d04, 0xffffffff, 0x00164745,
609 0x8c00, 0xffffffff, 0xe4000003,
610 0x8cf0, 0x1fffffff, 0x08e00620,
611 0x28350, 0xffffffff, 0x00000000,
612 0x9508, 0xffffffff, 0x00000002
613 };
614
615 static const u32 sumo_golden_registers[] =
616 {
617 0x900c, 0x00ffffff, 0x0017071f,
618 0x8c18, 0xffffffff, 0x10101060,
619 0x8c1c, 0xffffffff, 0x00001010,
620 0x8c30, 0x0000000f, 0x00000005,
621 0x9688, 0x0000000f, 0x00000007
622 };
623
624 static const u32 wrestler_golden_registers[] =
625 {
626 0x5eb4, 0xffffffff, 0x00000002,
627 0x5cc, 0xffffffff, 0x00000001,
628 0x7030, 0xffffffff, 0x00000011,
629 0x7c30, 0xffffffff, 0x00000011,
630 0x6104, 0x01000300, 0x00000000,
631 0x5bc0, 0x00300000, 0x00000000,
632 0x918c, 0xffffffff, 0x00010006,
633 0x91a8, 0xffffffff, 0x00010006,
634 0x9150, 0xffffffff, 0x6e944040,
635 0x917c, 0xffffffff, 0x00030002,
636 0x9198, 0xffffffff, 0x00030002,
637 0x915c, 0xffffffff, 0x00010000,
638 0x3f90, 0xffff0000, 0xff000000,
639 0x9178, 0xffffffff, 0x00070000,
640 0x9194, 0xffffffff, 0x00070000,
641 0x9148, 0xffff0000, 0xff000000,
642 0x9190, 0xffffffff, 0x00090008,
643 0x91ac, 0xffffffff, 0x00090008,
644 0x3f94, 0xffff0000, 0xff000000,
645 0x914c, 0xffff0000, 0xff000000,
646 0x929c, 0xffffffff, 0x00000001,
647 0x8a18, 0xffffffff, 0x00000100,
648 0x8b28, 0xffffffff, 0x00000100,
649 0x9144, 0xffffffff, 0x00000100,
650 0x9b7c, 0xffffffff, 0x00000000,
651 0x8030, 0xffffffff, 0x0000100a,
652 0x8a14, 0xffffffff, 0x00000001,
653 0x8b24, 0xffffffff, 0x00ff0fff,
654 0x8b10, 0xffffffff, 0x00000000,
655 0x28a4c, 0x06000000, 0x06000000,
656 0x4d8, 0xffffffff, 0x00000100,
657 0x913c, 0xffff000f, 0x0100000a,
658 0x960c, 0xffffffff, 0x54763210,
659 0x88c4, 0xffffffff, 0x000000c2,
660 0x88d4, 0xffffffff, 0x00000010,
661 0x8974, 0xffffffff, 0x00000000,
662 0xc78, 0x00000080, 0x00000080,
663 0x5e78, 0xffffffff, 0x001000f0,
664 0xd02c, 0xffffffff, 0x08421000,
665 0xa008, 0xffffffff, 0x00010000,
666 0x8d00, 0xffffffff, 0x100e4848,
667 0x8d04, 0xffffffff, 0x00164745,
668 0x8c00, 0xffffffff, 0xe4000003,
669 0x8cf0, 0x1fffffff, 0x08e00410,
670 0x28350, 0xffffffff, 0x00000000,
671 0x9508, 0xffffffff, 0x00000002,
672 0x900c, 0xffffffff, 0x0017071f,
673 0x8c18, 0xffffffff, 0x10101060,
674 0x8c1c, 0xffffffff, 0x00001010
675 };
676
677 static const u32 barts_golden_registers[] =
678 {
679 0x5eb4, 0xffffffff, 0x00000002,
680 0x5e78, 0x8f311ff1, 0x001000f0,
681 0x3f90, 0xffff0000, 0xff000000,
682 0x9148, 0xffff0000, 0xff000000,
683 0x3f94, 0xffff0000, 0xff000000,
684 0x914c, 0xffff0000, 0xff000000,
685 0xc78, 0x00000080, 0x00000080,
686 0xbd4, 0x70073777, 0x00010001,
687 0xd02c, 0xbfffff1f, 0x08421000,
688 0xd0b8, 0x03773777, 0x02011003,
689 0x5bc0, 0x00200000, 0x50100000,
690 0x98f8, 0x33773777, 0x02011003,
691 0x98fc, 0xffffffff, 0x76543210,
692 0x7030, 0x31000311, 0x00000011,
693 0x2f48, 0x00000007, 0x02011003,
694 0x6b28, 0x00000010, 0x00000012,
695 0x7728, 0x00000010, 0x00000012,
696 0x10328, 0x00000010, 0x00000012,
697 0x10f28, 0x00000010, 0x00000012,
698 0x11b28, 0x00000010, 0x00000012,
699 0x12728, 0x00000010, 0x00000012,
700 0x240c, 0x000007ff, 0x00000380,
701 0x8a14, 0xf000001f, 0x00000007,
702 0x8b24, 0x3fff3fff, 0x00ff0fff,
703 0x8b10, 0x0000ff0f, 0x00000000,
704 0x28a4c, 0x07ffffff, 0x06000000,
705 0x10c, 0x00000001, 0x00010003,
706 0xa02c, 0xffffffff, 0x0000009b,
707 0x913c, 0x0000000f, 0x0100000a,
708 0x8d00, 0xffff7f7f, 0x100e4848,
709 0x8d04, 0x00ffffff, 0x00164745,
710 0x8c00, 0xfffc0003, 0xe4000003,
711 0x8c04, 0xf8ff00ff, 0x40600060,
712 0x8c08, 0x00ff00ff, 0x001c001c,
713 0x8cf0, 0x1fff1fff, 0x08e00620,
714 0x8c20, 0x0fff0fff, 0x00800080,
715 0x8c24, 0x0fff0fff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0x0000ffff, 0x00001010,
718 0x28350, 0x00000f01, 0x00000000,
719 0x9508, 0x3700001f, 0x00000002,
720 0x960c, 0xffffffff, 0x54763210,
721 0x88c4, 0x001f3ae3, 0x000000c2,
722 0x88d4, 0x0000001f, 0x00000010,
723 0x8974, 0xffffffff, 0x00000000
724 };
725
726 static const u32 turks_golden_registers[] =
727 {
728 0x5eb4, 0xffffffff, 0x00000002,
729 0x5e78, 0x8f311ff1, 0x001000f0,
730 0x8c8, 0x00003000, 0x00001070,
731 0x8cc, 0x000fffff, 0x00040035,
732 0x3f90, 0xffff0000, 0xfff00000,
733 0x9148, 0xffff0000, 0xfff00000,
734 0x3f94, 0xffff0000, 0xfff00000,
735 0x914c, 0xffff0000, 0xfff00000,
736 0xc78, 0x00000080, 0x00000080,
737 0xbd4, 0x00073007, 0x00010002,
738 0xd02c, 0xbfffff1f, 0x08421000,
739 0xd0b8, 0x03773777, 0x02010002,
740 0x5bc0, 0x00200000, 0x50100000,
741 0x98f8, 0x33773777, 0x00010002,
742 0x98fc, 0xffffffff, 0x33221100,
743 0x7030, 0x31000311, 0x00000011,
744 0x2f48, 0x33773777, 0x00010002,
745 0x6b28, 0x00000010, 0x00000012,
746 0x7728, 0x00000010, 0x00000012,
747 0x10328, 0x00000010, 0x00000012,
748 0x10f28, 0x00000010, 0x00000012,
749 0x11b28, 0x00000010, 0x00000012,
750 0x12728, 0x00000010, 0x00000012,
751 0x240c, 0x000007ff, 0x00000380,
752 0x8a14, 0xf000001f, 0x00000007,
753 0x8b24, 0x3fff3fff, 0x00ff0fff,
754 0x8b10, 0x0000ff0f, 0x00000000,
755 0x28a4c, 0x07ffffff, 0x06000000,
756 0x10c, 0x00000001, 0x00010003,
757 0xa02c, 0xffffffff, 0x0000009b,
758 0x913c, 0x0000000f, 0x0100000a,
759 0x8d00, 0xffff7f7f, 0x100e4848,
760 0x8d04, 0x00ffffff, 0x00164745,
761 0x8c00, 0xfffc0003, 0xe4000003,
762 0x8c04, 0xf8ff00ff, 0x40600060,
763 0x8c08, 0x00ff00ff, 0x001c001c,
764 0x8cf0, 0x1fff1fff, 0x08e00410,
765 0x8c20, 0x0fff0fff, 0x00800080,
766 0x8c24, 0x0fff0fff, 0x00800080,
767 0x8c18, 0xffffffff, 0x20202078,
768 0x8c1c, 0x0000ffff, 0x00001010,
769 0x28350, 0x00000f01, 0x00000000,
770 0x9508, 0x3700001f, 0x00000002,
771 0x960c, 0xffffffff, 0x54763210,
772 0x88c4, 0x001f3ae3, 0x000000c2,
773 0x88d4, 0x0000001f, 0x00000010,
774 0x8974, 0xffffffff, 0x00000000
775 };
776
777 static const u32 caicos_golden_registers[] =
778 {
779 0x5eb4, 0xffffffff, 0x00000002,
780 0x5e78, 0x8f311ff1, 0x001000f0,
781 0x8c8, 0x00003420, 0x00001450,
782 0x8cc, 0x000fffff, 0x00040035,
783 0x3f90, 0xffff0000, 0xfffc0000,
784 0x9148, 0xffff0000, 0xfffc0000,
785 0x3f94, 0xffff0000, 0xfffc0000,
786 0x914c, 0xffff0000, 0xfffc0000,
787 0xc78, 0x00000080, 0x00000080,
788 0xbd4, 0x00073007, 0x00010001,
789 0xd02c, 0xbfffff1f, 0x08421000,
790 0xd0b8, 0x03773777, 0x02010001,
791 0x5bc0, 0x00200000, 0x50100000,
792 0x98f8, 0x33773777, 0x02010001,
793 0x98fc, 0xffffffff, 0x33221100,
794 0x7030, 0x31000311, 0x00000011,
795 0x2f48, 0x33773777, 0x02010001,
796 0x6b28, 0x00000010, 0x00000012,
797 0x7728, 0x00000010, 0x00000012,
798 0x10328, 0x00000010, 0x00000012,
799 0x10f28, 0x00000010, 0x00000012,
800 0x11b28, 0x00000010, 0x00000012,
801 0x12728, 0x00000010, 0x00000012,
802 0x240c, 0x000007ff, 0x00000380,
803 0x8a14, 0xf000001f, 0x00000001,
804 0x8b24, 0x3fff3fff, 0x00ff0fff,
805 0x8b10, 0x0000ff0f, 0x00000000,
806 0x28a4c, 0x07ffffff, 0x06000000,
807 0x10c, 0x00000001, 0x00010003,
808 0xa02c, 0xffffffff, 0x0000009b,
809 0x913c, 0x0000000f, 0x0100000a,
810 0x8d00, 0xffff7f7f, 0x100e4848,
811 0x8d04, 0x00ffffff, 0x00164745,
812 0x8c00, 0xfffc0003, 0xe4000003,
813 0x8c04, 0xf8ff00ff, 0x40600060,
814 0x8c08, 0x00ff00ff, 0x001c001c,
815 0x8cf0, 0x1fff1fff, 0x08e00410,
816 0x8c20, 0x0fff0fff, 0x00800080,
817 0x8c24, 0x0fff0fff, 0x00800080,
818 0x8c18, 0xffffffff, 0x20202078,
819 0x8c1c, 0x0000ffff, 0x00001010,
820 0x28350, 0x00000f01, 0x00000000,
821 0x9508, 0x3700001f, 0x00000002,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0x001f3ae3, 0x000000c2,
824 0x88d4, 0x0000001f, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000
826 };
827
828 static void evergreen_init_golden_registers(struct radeon_device *rdev)
829 {
830 switch (rdev->family) {
831 case CHIP_CYPRESS:
832 case CHIP_HEMLOCK:
833 radeon_program_register_sequence(rdev,
834 evergreen_golden_registers,
835 (const u32)ARRAY_SIZE(evergreen_golden_registers));
836 radeon_program_register_sequence(rdev,
837 evergreen_golden_registers2,
838 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
839 radeon_program_register_sequence(rdev,
840 cypress_mgcg_init,
841 (const u32)ARRAY_SIZE(cypress_mgcg_init));
842 break;
843 case CHIP_JUNIPER:
844 radeon_program_register_sequence(rdev,
845 evergreen_golden_registers,
846 (const u32)ARRAY_SIZE(evergreen_golden_registers));
847 radeon_program_register_sequence(rdev,
848 evergreen_golden_registers2,
849 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
850 radeon_program_register_sequence(rdev,
851 juniper_mgcg_init,
852 (const u32)ARRAY_SIZE(juniper_mgcg_init));
853 break;
854 case CHIP_REDWOOD:
855 radeon_program_register_sequence(rdev,
856 evergreen_golden_registers,
857 (const u32)ARRAY_SIZE(evergreen_golden_registers));
858 radeon_program_register_sequence(rdev,
859 evergreen_golden_registers2,
860 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
861 radeon_program_register_sequence(rdev,
862 redwood_mgcg_init,
863 (const u32)ARRAY_SIZE(redwood_mgcg_init));
864 break;
865 case CHIP_CEDAR:
866 radeon_program_register_sequence(rdev,
867 cedar_golden_registers,
868 (const u32)ARRAY_SIZE(cedar_golden_registers));
869 radeon_program_register_sequence(rdev,
870 evergreen_golden_registers2,
871 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
872 radeon_program_register_sequence(rdev,
873 cedar_mgcg_init,
874 (const u32)ARRAY_SIZE(cedar_mgcg_init));
875 break;
876 case CHIP_PALM:
877 radeon_program_register_sequence(rdev,
878 wrestler_golden_registers,
879 (const u32)ARRAY_SIZE(wrestler_golden_registers));
880 break;
881 case CHIP_SUMO:
882 radeon_program_register_sequence(rdev,
883 supersumo_golden_registers,
884 (const u32)ARRAY_SIZE(supersumo_golden_registers));
885 break;
886 case CHIP_SUMO2:
887 radeon_program_register_sequence(rdev,
888 supersumo_golden_registers,
889 (const u32)ARRAY_SIZE(supersumo_golden_registers));
890 radeon_program_register_sequence(rdev,
891 sumo_golden_registers,
892 (const u32)ARRAY_SIZE(sumo_golden_registers));
893 break;
894 case CHIP_BARTS:
895 radeon_program_register_sequence(rdev,
896 barts_golden_registers,
897 (const u32)ARRAY_SIZE(barts_golden_registers));
898 break;
899 case CHIP_TURKS:
900 radeon_program_register_sequence(rdev,
901 turks_golden_registers,
902 (const u32)ARRAY_SIZE(turks_golden_registers));
903 break;
904 case CHIP_CAICOS:
905 radeon_program_register_sequence(rdev,
906 caicos_golden_registers,
907 (const u32)ARRAY_SIZE(caicos_golden_registers));
908 break;
909 default:
910 break;
911 }
912 }
913
914 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
915 unsigned *bankh, unsigned *mtaspect,
916 unsigned *tile_split)
917 {
918 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
919 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
920 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
921 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
922 switch (*bankw) {
923 default:
924 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
925 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
926 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
927 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
928 }
929 switch (*bankh) {
930 default:
931 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
932 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
933 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
934 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
935 }
936 switch (*mtaspect) {
937 default:
938 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
939 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
940 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
941 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
942 }
943 }
944
945 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
946 u32 cntl_reg, u32 status_reg)
947 {
948 int r, i;
949 struct atom_clock_dividers dividers;
950
951 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
952 clock, false, &dividers);
953 if (r)
954 return r;
955
956 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
957
958 for (i = 0; i < 100; i++) {
959 if (RREG32(status_reg) & DCLK_STATUS)
960 break;
961 mdelay(10);
962 }
963 if (i == 100)
964 return -ETIMEDOUT;
965
966 return 0;
967 }
968
969 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
970 {
971 int r = 0;
972 u32 cg_scratch = RREG32(CG_SCRATCH1);
973
974 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
975 if (r)
976 goto done;
977 cg_scratch &= 0xffff0000;
978 cg_scratch |= vclk / 100; /* Mhz */
979
980 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
981 if (r)
982 goto done;
983 cg_scratch &= 0x0000ffff;
984 cg_scratch |= (dclk / 100) << 16; /* Mhz */
985
986 done:
987 WREG32(CG_SCRATCH1, cg_scratch);
988
989 return r;
990 }
991
992 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
993 {
994 /* start off with something large */
995 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
996 int r;
997
998 /* bypass vclk and dclk with bclk */
999 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1000 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1001 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1002
1003 /* put PLL in bypass mode */
1004 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1005
1006 if (!vclk || !dclk) {
1007 /* keep the Bypass mode, put PLL to sleep */
1008 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1009 return 0;
1010 }
1011
1012 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1013 16384, 0x03FFFFFF, 0, 128, 5,
1014 &fb_div, &vclk_div, &dclk_div);
1015 if (r)
1016 return r;
1017
1018 /* set VCO_MODE to 1 */
1019 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1020
1021 /* toggle UPLL_SLEEP to 1 then back to 0 */
1022 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1023 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1024
1025 /* deassert UPLL_RESET */
1026 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1027
1028 mdelay(1);
1029
1030 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1031 if (r)
1032 return r;
1033
1034 /* assert UPLL_RESET again */
1035 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1036
1037 /* disable spread spectrum. */
1038 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1039
1040 /* set feedback divider */
1041 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1042
1043 /* set ref divider to 0 */
1044 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1045
1046 if (fb_div < 307200)
1047 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1048 else
1049 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1050
1051 /* set PDIV_A and PDIV_B */
1052 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1053 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1054 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1055
1056 /* give the PLL some time to settle */
1057 mdelay(15);
1058
1059 /* deassert PLL_RESET */
1060 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1061
1062 mdelay(15);
1063
1064 /* switch from bypass mode to normal mode */
1065 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1066
1067 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1068 if (r)
1069 return r;
1070
1071 /* switch VCLK and DCLK selection */
1072 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1073 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1074 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1075
1076 mdelay(100);
1077
1078 return 0;
1079 }
1080
1081 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1082 {
1083 u16 ctl, v;
1084 int err;
1085
1086 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1087 if (err)
1088 return;
1089
1090 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1091
1092 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1093 * to avoid hangs or perfomance issues
1094 */
1095 if ((v == 0) || (v == 6) || (v == 7)) {
1096 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1097 ctl |= (2 << 12);
1098 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1099 }
1100 }
1101
1102 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1103 {
1104 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1105 return true;
1106 else
1107 return false;
1108 }
1109
1110 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1111 {
1112 u32 pos1, pos2;
1113
1114 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1115 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1116
1117 if (pos1 != pos2)
1118 return true;
1119 else
1120 return false;
1121 }
1122
1123 /**
1124 * dce4_wait_for_vblank - vblank wait asic callback.
1125 *
1126 * @rdev: radeon_device pointer
1127 * @crtc: crtc to wait for vblank on
1128 *
1129 * Wait for vblank on the requested crtc (evergreen+).
1130 */
1131 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1132 {
1133 unsigned i = 0;
1134
1135 if (crtc >= rdev->num_crtc)
1136 return;
1137
1138 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1139 return;
1140
1141 /* depending on when we hit vblank, we may be close to active; if so,
1142 * wait for another frame.
1143 */
1144 while (dce4_is_in_vblank(rdev, crtc)) {
1145 if (i++ % 100 == 0) {
1146 if (!dce4_is_counter_moving(rdev, crtc))
1147 break;
1148 }
1149 }
1150
1151 while (!dce4_is_in_vblank(rdev, crtc)) {
1152 if (i++ % 100 == 0) {
1153 if (!dce4_is_counter_moving(rdev, crtc))
1154 break;
1155 }
1156 }
1157 }
1158
1159 /**
1160 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1161 *
1162 * @rdev: radeon_device pointer
1163 * @crtc: crtc to prepare for pageflip on
1164 *
1165 * Pre-pageflip callback (evergreen+).
1166 * Enables the pageflip irq (vblank irq).
1167 */
1168 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1169 {
1170 /* enable the pflip int */
1171 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1172 }
1173
1174 /**
1175 * evergreen_post_page_flip - pos-pageflip callback.
1176 *
1177 * @rdev: radeon_device pointer
1178 * @crtc: crtc to cleanup pageflip on
1179 *
1180 * Post-pageflip callback (evergreen+).
1181 * Disables the pageflip irq (vblank irq).
1182 */
1183 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1184 {
1185 /* disable the pflip int */
1186 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1187 }
1188
1189 /**
1190 * evergreen_page_flip - pageflip callback.
1191 *
1192 * @rdev: radeon_device pointer
1193 * @crtc_id: crtc to cleanup pageflip on
1194 * @crtc_base: new address of the crtc (GPU MC address)
1195 *
1196 * Does the actual pageflip (evergreen+).
1197 * During vblank we take the crtc lock and wait for the update_pending
1198 * bit to go high, when it does, we release the lock, and allow the
1199 * double buffered update to take place.
1200 * Returns the current update pending status.
1201 */
1202 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1203 {
1204 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1205 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1206 int i;
1207
1208 /* Lock the graphics update lock */
1209 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1210 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1211
1212 /* update the scanout addresses */
1213 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1214 upper_32_bits(crtc_base));
1215 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1216 (u32)crtc_base);
1217
1218 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1219 upper_32_bits(crtc_base));
1220 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1221 (u32)crtc_base);
1222
1223 /* Wait for update_pending to go high. */
1224 for (i = 0; i < rdev->usec_timeout; i++) {
1225 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1226 break;
1227 udelay(1);
1228 }
1229 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1230
1231 /* Unlock the lock, so double-buffering can take place inside vblank */
1232 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1233 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1234
1235 /* Return current update_pending status: */
1236 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1237 }
1238
1239 /* get temperature in millidegrees */
1240 int evergreen_get_temp(struct radeon_device *rdev)
1241 {
1242 u32 temp, toffset;
1243 int actual_temp = 0;
1244
1245 if (rdev->family == CHIP_JUNIPER) {
1246 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1247 TOFFSET_SHIFT;
1248 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1249 TS0_ADC_DOUT_SHIFT;
1250
1251 if (toffset & 0x100)
1252 actual_temp = temp / 2 - (0x200 - toffset);
1253 else
1254 actual_temp = temp / 2 + toffset;
1255
1256 actual_temp = actual_temp * 1000;
1257
1258 } else {
1259 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1260 ASIC_T_SHIFT;
1261
1262 if (temp & 0x400)
1263 actual_temp = -256;
1264 else if (temp & 0x200)
1265 actual_temp = 255;
1266 else if (temp & 0x100) {
1267 actual_temp = temp & 0x1ff;
1268 actual_temp |= ~0x1ff;
1269 } else
1270 actual_temp = temp & 0xff;
1271
1272 actual_temp = (actual_temp * 1000) / 2;
1273 }
1274
1275 return actual_temp;
1276 }
1277
1278 int sumo_get_temp(struct radeon_device *rdev)
1279 {
1280 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1281 int actual_temp = temp - 49;
1282
1283 return actual_temp * 1000;
1284 }
1285
1286 /**
1287 * sumo_pm_init_profile - Initialize power profiles callback.
1288 *
1289 * @rdev: radeon_device pointer
1290 *
1291 * Initialize the power states used in profile mode
1292 * (sumo, trinity, SI).
1293 * Used for profile mode only.
1294 */
1295 void sumo_pm_init_profile(struct radeon_device *rdev)
1296 {
1297 int idx;
1298
1299 /* default */
1300 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1301 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1302 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1303 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1304
1305 /* low,mid sh/mh */
1306 if (rdev->flags & RADEON_IS_MOBILITY)
1307 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1308 else
1309 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1310
1311 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1312 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1313 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1314 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1315
1316 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1317 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1318 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1319 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1320
1321 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1322 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1323 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1324 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1325
1326 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1327 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1328 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1329 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1330
1331 /* high sh/mh */
1332 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1333 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1334 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1335 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1336 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1337 rdev->pm.power_state[idx].num_clock_modes - 1;
1338
1339 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1340 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1341 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1342 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1343 rdev->pm.power_state[idx].num_clock_modes - 1;
1344 }
1345
1346 /**
1347 * btc_pm_init_profile - Initialize power profiles callback.
1348 *
1349 * @rdev: radeon_device pointer
1350 *
1351 * Initialize the power states used in profile mode
1352 * (BTC, cayman).
1353 * Used for profile mode only.
1354 */
1355 void btc_pm_init_profile(struct radeon_device *rdev)
1356 {
1357 int idx;
1358
1359 /* default */
1360 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1361 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1362 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1363 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1364 /* starting with BTC, there is one state that is used for both
1365 * MH and SH. Difference is that we always use the high clock index for
1366 * mclk.
1367 */
1368 if (rdev->flags & RADEON_IS_MOBILITY)
1369 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1370 else
1371 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1372 /* low sh */
1373 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1374 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1375 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1376 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1377 /* mid sh */
1378 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1379 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1380 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1381 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1382 /* high sh */
1383 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1384 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1385 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1386 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1387 /* low mh */
1388 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1389 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1390 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1391 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1392 /* mid mh */
1393 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1394 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1395 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1396 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1397 /* high mh */
1398 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1400 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1401 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1402 }
1403
1404 /**
1405 * evergreen_pm_misc - set additional pm hw parameters callback.
1406 *
1407 * @rdev: radeon_device pointer
1408 *
1409 * Set non-clock parameters associated with a power state
1410 * (voltage, etc.) (evergreen+).
1411 */
1412 void evergreen_pm_misc(struct radeon_device *rdev)
1413 {
1414 int req_ps_idx = rdev->pm.requested_power_state_index;
1415 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1416 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1417 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1418
1419 if (voltage->type == VOLTAGE_SW) {
1420 /* 0xff01 is a flag rather then an actual voltage */
1421 if (voltage->voltage == 0xff01)
1422 return;
1423 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1424 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1425 rdev->pm.current_vddc = voltage->voltage;
1426 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1427 }
1428
1429 /* starting with BTC, there is one state that is used for both
1430 * MH and SH. Difference is that we always use the high clock index for
1431 * mclk and vddci.
1432 */
1433 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1434 (rdev->family >= CHIP_BARTS) &&
1435 rdev->pm.active_crtc_count &&
1436 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1437 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1438 voltage = &rdev->pm.power_state[req_ps_idx].
1439 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1440
1441 /* 0xff01 is a flag rather then an actual voltage */
1442 if (voltage->vddci == 0xff01)
1443 return;
1444 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1445 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1446 rdev->pm.current_vddci = voltage->vddci;
1447 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1448 }
1449 }
1450 }
1451
1452 /**
1453 * evergreen_pm_prepare - pre-power state change callback.
1454 *
1455 * @rdev: radeon_device pointer
1456 *
1457 * Prepare for a power state change (evergreen+).
1458 */
1459 void evergreen_pm_prepare(struct radeon_device *rdev)
1460 {
1461 struct drm_device *ddev = rdev->ddev;
1462 struct drm_crtc *crtc;
1463 struct radeon_crtc *radeon_crtc;
1464 u32 tmp;
1465
1466 /* disable any active CRTCs */
1467 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1468 radeon_crtc = to_radeon_crtc(crtc);
1469 if (radeon_crtc->enabled) {
1470 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1471 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1472 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1473 }
1474 }
1475 }
1476
1477 /**
1478 * evergreen_pm_finish - post-power state change callback.
1479 *
1480 * @rdev: radeon_device pointer
1481 *
1482 * Clean up after a power state change (evergreen+).
1483 */
1484 void evergreen_pm_finish(struct radeon_device *rdev)
1485 {
1486 struct drm_device *ddev = rdev->ddev;
1487 struct drm_crtc *crtc;
1488 struct radeon_crtc *radeon_crtc;
1489 u32 tmp;
1490
1491 /* enable any active CRTCs */
1492 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1493 radeon_crtc = to_radeon_crtc(crtc);
1494 if (radeon_crtc->enabled) {
1495 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1496 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1497 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1498 }
1499 }
1500 }
1501
1502 /**
1503 * evergreen_hpd_sense - hpd sense callback.
1504 *
1505 * @rdev: radeon_device pointer
1506 * @hpd: hpd (hotplug detect) pin
1507 *
1508 * Checks if a digital monitor is connected (evergreen+).
1509 * Returns true if connected, false if not connected.
1510 */
1511 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1512 {
1513 bool connected = false;
1514
1515 switch (hpd) {
1516 case RADEON_HPD_1:
1517 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1518 connected = true;
1519 break;
1520 case RADEON_HPD_2:
1521 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1522 connected = true;
1523 break;
1524 case RADEON_HPD_3:
1525 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1526 connected = true;
1527 break;
1528 case RADEON_HPD_4:
1529 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1530 connected = true;
1531 break;
1532 case RADEON_HPD_5:
1533 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1534 connected = true;
1535 break;
1536 case RADEON_HPD_6:
1537 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1538 connected = true;
1539 break;
1540 default:
1541 break;
1542 }
1543
1544 return connected;
1545 }
1546
1547 /**
1548 * evergreen_hpd_set_polarity - hpd set polarity callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 * @hpd: hpd (hotplug detect) pin
1552 *
1553 * Set the polarity of the hpd pin (evergreen+).
1554 */
1555 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1556 enum radeon_hpd_id hpd)
1557 {
1558 u32 tmp;
1559 bool connected = evergreen_hpd_sense(rdev, hpd);
1560
1561 switch (hpd) {
1562 case RADEON_HPD_1:
1563 tmp = RREG32(DC_HPD1_INT_CONTROL);
1564 if (connected)
1565 tmp &= ~DC_HPDx_INT_POLARITY;
1566 else
1567 tmp |= DC_HPDx_INT_POLARITY;
1568 WREG32(DC_HPD1_INT_CONTROL, tmp);
1569 break;
1570 case RADEON_HPD_2:
1571 tmp = RREG32(DC_HPD2_INT_CONTROL);
1572 if (connected)
1573 tmp &= ~DC_HPDx_INT_POLARITY;
1574 else
1575 tmp |= DC_HPDx_INT_POLARITY;
1576 WREG32(DC_HPD2_INT_CONTROL, tmp);
1577 break;
1578 case RADEON_HPD_3:
1579 tmp = RREG32(DC_HPD3_INT_CONTROL);
1580 if (connected)
1581 tmp &= ~DC_HPDx_INT_POLARITY;
1582 else
1583 tmp |= DC_HPDx_INT_POLARITY;
1584 WREG32(DC_HPD3_INT_CONTROL, tmp);
1585 break;
1586 case RADEON_HPD_4:
1587 tmp = RREG32(DC_HPD4_INT_CONTROL);
1588 if (connected)
1589 tmp &= ~DC_HPDx_INT_POLARITY;
1590 else
1591 tmp |= DC_HPDx_INT_POLARITY;
1592 WREG32(DC_HPD4_INT_CONTROL, tmp);
1593 break;
1594 case RADEON_HPD_5:
1595 tmp = RREG32(DC_HPD5_INT_CONTROL);
1596 if (connected)
1597 tmp &= ~DC_HPDx_INT_POLARITY;
1598 else
1599 tmp |= DC_HPDx_INT_POLARITY;
1600 WREG32(DC_HPD5_INT_CONTROL, tmp);
1601 break;
1602 case RADEON_HPD_6:
1603 tmp = RREG32(DC_HPD6_INT_CONTROL);
1604 if (connected)
1605 tmp &= ~DC_HPDx_INT_POLARITY;
1606 else
1607 tmp |= DC_HPDx_INT_POLARITY;
1608 WREG32(DC_HPD6_INT_CONTROL, tmp);
1609 break;
1610 default:
1611 break;
1612 }
1613 }
1614
1615 /**
1616 * evergreen_hpd_init - hpd setup callback.
1617 *
1618 * @rdev: radeon_device pointer
1619 *
1620 * Setup the hpd pins used by the card (evergreen+).
1621 * Enable the pin, set the polarity, and enable the hpd interrupts.
1622 */
1623 void evergreen_hpd_init(struct radeon_device *rdev)
1624 {
1625 struct drm_device *dev = rdev->ddev;
1626 struct drm_connector *connector;
1627 unsigned enabled = 0;
1628 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1629 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1630
1631 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1632 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1633
1634 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1635 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1636 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1637 * aux dp channel on imac and help (but not completely fix)
1638 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1639 * also avoid interrupt storms during dpms.
1640 */
1641 continue;
1642 }
1643 switch (radeon_connector->hpd.hpd) {
1644 case RADEON_HPD_1:
1645 WREG32(DC_HPD1_CONTROL, tmp);
1646 break;
1647 case RADEON_HPD_2:
1648 WREG32(DC_HPD2_CONTROL, tmp);
1649 break;
1650 case RADEON_HPD_3:
1651 WREG32(DC_HPD3_CONTROL, tmp);
1652 break;
1653 case RADEON_HPD_4:
1654 WREG32(DC_HPD4_CONTROL, tmp);
1655 break;
1656 case RADEON_HPD_5:
1657 WREG32(DC_HPD5_CONTROL, tmp);
1658 break;
1659 case RADEON_HPD_6:
1660 WREG32(DC_HPD6_CONTROL, tmp);
1661 break;
1662 default:
1663 break;
1664 }
1665 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1666 enabled |= 1 << radeon_connector->hpd.hpd;
1667 }
1668 radeon_irq_kms_enable_hpd(rdev, enabled);
1669 }
1670
1671 /**
1672 * evergreen_hpd_fini - hpd tear down callback.
1673 *
1674 * @rdev: radeon_device pointer
1675 *
1676 * Tear down the hpd pins used by the card (evergreen+).
1677 * Disable the hpd interrupts.
1678 */
1679 void evergreen_hpd_fini(struct radeon_device *rdev)
1680 {
1681 struct drm_device *dev = rdev->ddev;
1682 struct drm_connector *connector;
1683 unsigned disabled = 0;
1684
1685 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1686 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1687 switch (radeon_connector->hpd.hpd) {
1688 case RADEON_HPD_1:
1689 WREG32(DC_HPD1_CONTROL, 0);
1690 break;
1691 case RADEON_HPD_2:
1692 WREG32(DC_HPD2_CONTROL, 0);
1693 break;
1694 case RADEON_HPD_3:
1695 WREG32(DC_HPD3_CONTROL, 0);
1696 break;
1697 case RADEON_HPD_4:
1698 WREG32(DC_HPD4_CONTROL, 0);
1699 break;
1700 case RADEON_HPD_5:
1701 WREG32(DC_HPD5_CONTROL, 0);
1702 break;
1703 case RADEON_HPD_6:
1704 WREG32(DC_HPD6_CONTROL, 0);
1705 break;
1706 default:
1707 break;
1708 }
1709 disabled |= 1 << radeon_connector->hpd.hpd;
1710 }
1711 radeon_irq_kms_disable_hpd(rdev, disabled);
1712 }
1713
1714 /* watermark setup */
1715
1716 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1717 struct radeon_crtc *radeon_crtc,
1718 struct drm_display_mode *mode,
1719 struct drm_display_mode *other_mode)
1720 {
1721 u32 tmp;
1722 /*
1723 * Line Buffer Setup
1724 * There are 3 line buffers, each one shared by 2 display controllers.
1725 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1726 * the display controllers. The paritioning is done via one of four
1727 * preset allocations specified in bits 2:0:
1728 * first display controller
1729 * 0 - first half of lb (3840 * 2)
1730 * 1 - first 3/4 of lb (5760 * 2)
1731 * 2 - whole lb (7680 * 2), other crtc must be disabled
1732 * 3 - first 1/4 of lb (1920 * 2)
1733 * second display controller
1734 * 4 - second half of lb (3840 * 2)
1735 * 5 - second 3/4 of lb (5760 * 2)
1736 * 6 - whole lb (7680 * 2), other crtc must be disabled
1737 * 7 - last 1/4 of lb (1920 * 2)
1738 */
1739 /* this can get tricky if we have two large displays on a paired group
1740 * of crtcs. Ideally for multiple large displays we'd assign them to
1741 * non-linked crtcs for maximum line buffer allocation.
1742 */
1743 if (radeon_crtc->base.enabled && mode) {
1744 if (other_mode)
1745 tmp = 0; /* 1/2 */
1746 else
1747 tmp = 2; /* whole */
1748 } else
1749 tmp = 0;
1750
1751 /* second controller of the pair uses second half of the lb */
1752 if (radeon_crtc->crtc_id % 2)
1753 tmp += 4;
1754 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1755
1756 if (radeon_crtc->base.enabled && mode) {
1757 switch (tmp) {
1758 case 0:
1759 case 4:
1760 default:
1761 if (ASIC_IS_DCE5(rdev))
1762 return 4096 * 2;
1763 else
1764 return 3840 * 2;
1765 case 1:
1766 case 5:
1767 if (ASIC_IS_DCE5(rdev))
1768 return 6144 * 2;
1769 else
1770 return 5760 * 2;
1771 case 2:
1772 case 6:
1773 if (ASIC_IS_DCE5(rdev))
1774 return 8192 * 2;
1775 else
1776 return 7680 * 2;
1777 case 3:
1778 case 7:
1779 if (ASIC_IS_DCE5(rdev))
1780 return 2048 * 2;
1781 else
1782 return 1920 * 2;
1783 }
1784 }
1785
1786 /* controller not enabled, so no lb used */
1787 return 0;
1788 }
1789
1790 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1791 {
1792 u32 tmp = RREG32(MC_SHARED_CHMAP);
1793
1794 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1795 case 0:
1796 default:
1797 return 1;
1798 case 1:
1799 return 2;
1800 case 2:
1801 return 4;
1802 case 3:
1803 return 8;
1804 }
1805 }
1806
1807 struct evergreen_wm_params {
1808 u32 dram_channels; /* number of dram channels */
1809 u32 yclk; /* bandwidth per dram data pin in kHz */
1810 u32 sclk; /* engine clock in kHz */
1811 u32 disp_clk; /* display clock in kHz */
1812 u32 src_width; /* viewport width */
1813 u32 active_time; /* active display time in ns */
1814 u32 blank_time; /* blank time in ns */
1815 bool interlaced; /* mode is interlaced */
1816 fixed20_12 vsc; /* vertical scale ratio */
1817 u32 num_heads; /* number of active crtcs */
1818 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1819 u32 lb_size; /* line buffer allocated to pipe */
1820 u32 vtaps; /* vertical scaler taps */
1821 };
1822
1823 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1824 {
1825 /* Calculate DRAM Bandwidth and the part allocated to display. */
1826 fixed20_12 dram_efficiency; /* 0.7 */
1827 fixed20_12 yclk, dram_channels, bandwidth;
1828 fixed20_12 a;
1829
1830 a.full = dfixed_const(1000);
1831 yclk.full = dfixed_const(wm->yclk);
1832 yclk.full = dfixed_div(yclk, a);
1833 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1834 a.full = dfixed_const(10);
1835 dram_efficiency.full = dfixed_const(7);
1836 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1837 bandwidth.full = dfixed_mul(dram_channels, yclk);
1838 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1839
1840 return dfixed_trunc(bandwidth);
1841 }
1842
1843 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1844 {
1845 /* Calculate DRAM Bandwidth and the part allocated to display. */
1846 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1847 fixed20_12 yclk, dram_channels, bandwidth;
1848 fixed20_12 a;
1849
1850 a.full = dfixed_const(1000);
1851 yclk.full = dfixed_const(wm->yclk);
1852 yclk.full = dfixed_div(yclk, a);
1853 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1854 a.full = dfixed_const(10);
1855 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1856 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1857 bandwidth.full = dfixed_mul(dram_channels, yclk);
1858 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1859
1860 return dfixed_trunc(bandwidth);
1861 }
1862
1863 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1864 {
1865 /* Calculate the display Data return Bandwidth */
1866 fixed20_12 return_efficiency; /* 0.8 */
1867 fixed20_12 sclk, bandwidth;
1868 fixed20_12 a;
1869
1870 a.full = dfixed_const(1000);
1871 sclk.full = dfixed_const(wm->sclk);
1872 sclk.full = dfixed_div(sclk, a);
1873 a.full = dfixed_const(10);
1874 return_efficiency.full = dfixed_const(8);
1875 return_efficiency.full = dfixed_div(return_efficiency, a);
1876 a.full = dfixed_const(32);
1877 bandwidth.full = dfixed_mul(a, sclk);
1878 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1879
1880 return dfixed_trunc(bandwidth);
1881 }
1882
1883 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1884 {
1885 /* Calculate the DMIF Request Bandwidth */
1886 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1887 fixed20_12 disp_clk, bandwidth;
1888 fixed20_12 a;
1889
1890 a.full = dfixed_const(1000);
1891 disp_clk.full = dfixed_const(wm->disp_clk);
1892 disp_clk.full = dfixed_div(disp_clk, a);
1893 a.full = dfixed_const(10);
1894 disp_clk_request_efficiency.full = dfixed_const(8);
1895 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1896 a.full = dfixed_const(32);
1897 bandwidth.full = dfixed_mul(a, disp_clk);
1898 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1899
1900 return dfixed_trunc(bandwidth);
1901 }
1902
1903 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1904 {
1905 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1906 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1907 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1908 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1909
1910 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1911 }
1912
1913 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
1914 {
1915 /* Calculate the display mode Average Bandwidth
1916 * DisplayMode should contain the source and destination dimensions,
1917 * timing, etc.
1918 */
1919 fixed20_12 bpp;
1920 fixed20_12 line_time;
1921 fixed20_12 src_width;
1922 fixed20_12 bandwidth;
1923 fixed20_12 a;
1924
1925 a.full = dfixed_const(1000);
1926 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1927 line_time.full = dfixed_div(line_time, a);
1928 bpp.full = dfixed_const(wm->bytes_per_pixel);
1929 src_width.full = dfixed_const(wm->src_width);
1930 bandwidth.full = dfixed_mul(src_width, bpp);
1931 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1932 bandwidth.full = dfixed_div(bandwidth, line_time);
1933
1934 return dfixed_trunc(bandwidth);
1935 }
1936
1937 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
1938 {
1939 /* First calcualte the latency in ns */
1940 u32 mc_latency = 2000; /* 2000 ns. */
1941 u32 available_bandwidth = evergreen_available_bandwidth(wm);
1942 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1943 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1944 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1945 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1946 (wm->num_heads * cursor_line_pair_return_time);
1947 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1948 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1949 fixed20_12 a, b, c;
1950
1951 if (wm->num_heads == 0)
1952 return 0;
1953
1954 a.full = dfixed_const(2);
1955 b.full = dfixed_const(1);
1956 if ((wm->vsc.full > a.full) ||
1957 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1958 (wm->vtaps >= 5) ||
1959 ((wm->vsc.full >= a.full) && wm->interlaced))
1960 max_src_lines_per_dst_line = 4;
1961 else
1962 max_src_lines_per_dst_line = 2;
1963
1964 a.full = dfixed_const(available_bandwidth);
1965 b.full = dfixed_const(wm->num_heads);
1966 a.full = dfixed_div(a, b);
1967
1968 b.full = dfixed_const(1000);
1969 c.full = dfixed_const(wm->disp_clk);
1970 b.full = dfixed_div(c, b);
1971 c.full = dfixed_const(wm->bytes_per_pixel);
1972 b.full = dfixed_mul(b, c);
1973
1974 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
1975
1976 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1977 b.full = dfixed_const(1000);
1978 c.full = dfixed_const(lb_fill_bw);
1979 b.full = dfixed_div(c, b);
1980 a.full = dfixed_div(a, b);
1981 line_fill_time = dfixed_trunc(a);
1982
1983 if (line_fill_time < wm->active_time)
1984 return latency;
1985 else
1986 return latency + (line_fill_time - wm->active_time);
1987
1988 }
1989
1990 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1991 {
1992 if (evergreen_average_bandwidth(wm) <=
1993 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
1994 return true;
1995 else
1996 return false;
1997 };
1998
1999 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2000 {
2001 if (evergreen_average_bandwidth(wm) <=
2002 (evergreen_available_bandwidth(wm) / wm->num_heads))
2003 return true;
2004 else
2005 return false;
2006 };
2007
2008 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2009 {
2010 u32 lb_partitions = wm->lb_size / wm->src_width;
2011 u32 line_time = wm->active_time + wm->blank_time;
2012 u32 latency_tolerant_lines;
2013 u32 latency_hiding;
2014 fixed20_12 a;
2015
2016 a.full = dfixed_const(1);
2017 if (wm->vsc.full > a.full)
2018 latency_tolerant_lines = 1;
2019 else {
2020 if (lb_partitions <= (wm->vtaps + 1))
2021 latency_tolerant_lines = 1;
2022 else
2023 latency_tolerant_lines = 2;
2024 }
2025
2026 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2027
2028 if (evergreen_latency_watermark(wm) <= latency_hiding)
2029 return true;
2030 else
2031 return false;
2032 }
2033
2034 static void evergreen_program_watermarks(struct radeon_device *rdev,
2035 struct radeon_crtc *radeon_crtc,
2036 u32 lb_size, u32 num_heads)
2037 {
2038 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2039 struct evergreen_wm_params wm;
2040 u32 pixel_period;
2041 u32 line_time = 0;
2042 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2043 u32 priority_a_mark = 0, priority_b_mark = 0;
2044 u32 priority_a_cnt = PRIORITY_OFF;
2045 u32 priority_b_cnt = PRIORITY_OFF;
2046 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2047 u32 tmp, arb_control3;
2048 fixed20_12 a, b, c;
2049
2050 if (radeon_crtc->base.enabled && num_heads && mode) {
2051 pixel_period = 1000000 / (u32)mode->clock;
2052 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2053 priority_a_cnt = 0;
2054 priority_b_cnt = 0;
2055
2056 wm.yclk = rdev->pm.current_mclk * 10;
2057 wm.sclk = rdev->pm.current_sclk * 10;
2058 wm.disp_clk = mode->clock;
2059 wm.src_width = mode->crtc_hdisplay;
2060 wm.active_time = mode->crtc_hdisplay * pixel_period;
2061 wm.blank_time = line_time - wm.active_time;
2062 wm.interlaced = false;
2063 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2064 wm.interlaced = true;
2065 wm.vsc = radeon_crtc->vsc;
2066 wm.vtaps = 1;
2067 if (radeon_crtc->rmx_type != RMX_OFF)
2068 wm.vtaps = 2;
2069 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
2070 wm.lb_size = lb_size;
2071 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
2072 wm.num_heads = num_heads;
2073
2074 /* set for high clocks */
2075 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
2076 /* set for low clocks */
2077 /* wm.yclk = low clk; wm.sclk = low clk */
2078 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
2079
2080 /* possibly force display priority to high */
2081 /* should really do this at mode validation time... */
2082 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
2083 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
2084 !evergreen_check_latency_hiding(&wm) ||
2085 (rdev->disp_priority == 2)) {
2086 DRM_DEBUG_KMS("force priority to high\n");
2087 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2088 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2089 }
2090
2091 a.full = dfixed_const(1000);
2092 b.full = dfixed_const(mode->clock);
2093 b.full = dfixed_div(b, a);
2094 c.full = dfixed_const(latency_watermark_a);
2095 c.full = dfixed_mul(c, b);
2096 c.full = dfixed_mul(c, radeon_crtc->hsc);
2097 c.full = dfixed_div(c, a);
2098 a.full = dfixed_const(16);
2099 c.full = dfixed_div(c, a);
2100 priority_a_mark = dfixed_trunc(c);
2101 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2102
2103 a.full = dfixed_const(1000);
2104 b.full = dfixed_const(mode->clock);
2105 b.full = dfixed_div(b, a);
2106 c.full = dfixed_const(latency_watermark_b);
2107 c.full = dfixed_mul(c, b);
2108 c.full = dfixed_mul(c, radeon_crtc->hsc);
2109 c.full = dfixed_div(c, a);
2110 a.full = dfixed_const(16);
2111 c.full = dfixed_div(c, a);
2112 priority_b_mark = dfixed_trunc(c);
2113 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2114 }
2115
2116 /* select wm A */
2117 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2118 tmp = arb_control3;
2119 tmp &= ~LATENCY_WATERMARK_MASK(3);
2120 tmp |= LATENCY_WATERMARK_MASK(1);
2121 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2122 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2123 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2124 LATENCY_HIGH_WATERMARK(line_time)));
2125 /* select wm B */
2126 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2127 tmp &= ~LATENCY_WATERMARK_MASK(3);
2128 tmp |= LATENCY_WATERMARK_MASK(2);
2129 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2130 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2131 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2132 LATENCY_HIGH_WATERMARK(line_time)));
2133 /* restore original selection */
2134 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2135
2136 /* write the priority marks */
2137 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2138 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2139
2140 }
2141
2142 /**
2143 * evergreen_bandwidth_update - update display watermarks callback.
2144 *
2145 * @rdev: radeon_device pointer
2146 *
2147 * Update the display watermarks based on the requested mode(s)
2148 * (evergreen+).
2149 */
2150 void evergreen_bandwidth_update(struct radeon_device *rdev)
2151 {
2152 struct drm_display_mode *mode0 = NULL;
2153 struct drm_display_mode *mode1 = NULL;
2154 u32 num_heads = 0, lb_size;
2155 int i;
2156
2157 radeon_update_display_priority(rdev);
2158
2159 for (i = 0; i < rdev->num_crtc; i++) {
2160 if (rdev->mode_info.crtcs[i]->base.enabled)
2161 num_heads++;
2162 }
2163 for (i = 0; i < rdev->num_crtc; i += 2) {
2164 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2165 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2166 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2167 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2168 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2169 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2170 }
2171 }
2172
2173 /**
2174 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2175 *
2176 * @rdev: radeon_device pointer
2177 *
2178 * Wait for the MC (memory controller) to be idle.
2179 * (evergreen+).
2180 * Returns 0 if the MC is idle, -1 if not.
2181 */
2182 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2183 {
2184 unsigned i;
2185 u32 tmp;
2186
2187 for (i = 0; i < rdev->usec_timeout; i++) {
2188 /* read MC_STATUS */
2189 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2190 if (!tmp)
2191 return 0;
2192 udelay(1);
2193 }
2194 return -1;
2195 }
2196
2197 /*
2198 * GART
2199 */
2200 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2201 {
2202 unsigned i;
2203 u32 tmp;
2204
2205 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2206
2207 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2208 for (i = 0; i < rdev->usec_timeout; i++) {
2209 /* read MC_STATUS */
2210 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2211 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2212 if (tmp == 2) {
2213 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2214 return;
2215 }
2216 if (tmp) {
2217 return;
2218 }
2219 udelay(1);
2220 }
2221 }
2222
2223 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2224 {
2225 u32 tmp;
2226 int r;
2227
2228 if (rdev->gart.robj == NULL) {
2229 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2230 return -EINVAL;
2231 }
2232 r = radeon_gart_table_vram_pin(rdev);
2233 if (r)
2234 return r;
2235 radeon_gart_restore(rdev);
2236 /* Setup L2 cache */
2237 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2238 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2239 EFFECTIVE_L2_QUEUE_SIZE(7));
2240 WREG32(VM_L2_CNTL2, 0);
2241 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2242 /* Setup TLB control */
2243 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2244 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2245 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2246 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2247 if (rdev->flags & RADEON_IS_IGP) {
2248 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2249 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2250 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2251 } else {
2252 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2253 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2254 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2255 if ((rdev->family == CHIP_JUNIPER) ||
2256 (rdev->family == CHIP_CYPRESS) ||
2257 (rdev->family == CHIP_HEMLOCK) ||
2258 (rdev->family == CHIP_BARTS))
2259 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2260 }
2261 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2262 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2263 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2264 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2265 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2266 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2267 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2268 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2269 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2270 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2271 (u32)(rdev->dummy_page.addr >> 12));
2272 WREG32(VM_CONTEXT1_CNTL, 0);
2273
2274 evergreen_pcie_gart_tlb_flush(rdev);
2275 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2276 (unsigned)(rdev->mc.gtt_size >> 20),
2277 (unsigned long long)rdev->gart.table_addr);
2278 rdev->gart.ready = true;
2279 return 0;
2280 }
2281
2282 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2283 {
2284 u32 tmp;
2285
2286 /* Disable all tables */
2287 WREG32(VM_CONTEXT0_CNTL, 0);
2288 WREG32(VM_CONTEXT1_CNTL, 0);
2289
2290 /* Setup L2 cache */
2291 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2292 EFFECTIVE_L2_QUEUE_SIZE(7));
2293 WREG32(VM_L2_CNTL2, 0);
2294 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2295 /* Setup TLB control */
2296 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2297 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2298 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2299 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2300 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2301 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2302 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2303 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2304 radeon_gart_table_vram_unpin(rdev);
2305 }
2306
2307 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2308 {
2309 evergreen_pcie_gart_disable(rdev);
2310 radeon_gart_table_vram_free(rdev);
2311 radeon_gart_fini(rdev);
2312 }
2313
2314
2315 static void evergreen_agp_enable(struct radeon_device *rdev)
2316 {
2317 u32 tmp;
2318
2319 /* Setup L2 cache */
2320 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2321 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2322 EFFECTIVE_L2_QUEUE_SIZE(7));
2323 WREG32(VM_L2_CNTL2, 0);
2324 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2325 /* Setup TLB control */
2326 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2327 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2328 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2329 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2330 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2331 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2332 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2333 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2334 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2335 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2336 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2337 WREG32(VM_CONTEXT0_CNTL, 0);
2338 WREG32(VM_CONTEXT1_CNTL, 0);
2339 }
2340
2341 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2342 {
2343 u32 crtc_enabled, tmp, frame_count, blackout;
2344 int i, j;
2345
2346 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2347 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2348
2349 /* disable VGA render */
2350 WREG32(VGA_RENDER_CONTROL, 0);
2351 /* blank the display controllers */
2352 for (i = 0; i < rdev->num_crtc; i++) {
2353 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2354 if (crtc_enabled) {
2355 save->crtc_enabled[i] = true;
2356 if (ASIC_IS_DCE6(rdev)) {
2357 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2358 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2359 radeon_wait_for_vblank(rdev, i);
2360 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2361 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2362 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2363 }
2364 } else {
2365 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2366 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2367 radeon_wait_for_vblank(rdev, i);
2368 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2369 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2370 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2371 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2372 }
2373 }
2374 /* wait for the next frame */
2375 frame_count = radeon_get_vblank_counter(rdev, i);
2376 for (j = 0; j < rdev->usec_timeout; j++) {
2377 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2378 break;
2379 udelay(1);
2380 }
2381
2382 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2383 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2384 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2385 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2386 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2387 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2388 save->crtc_enabled[i] = false;
2389 /* ***** */
2390 } else {
2391 save->crtc_enabled[i] = false;
2392 }
2393 }
2394
2395 radeon_mc_wait_for_idle(rdev);
2396
2397 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2398 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2399 /* Block CPU access */
2400 WREG32(BIF_FB_EN, 0);
2401 /* blackout the MC */
2402 blackout &= ~BLACKOUT_MODE_MASK;
2403 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2404 }
2405 /* wait for the MC to settle */
2406 udelay(100);
2407
2408 /* lock double buffered regs */
2409 for (i = 0; i < rdev->num_crtc; i++) {
2410 if (save->crtc_enabled[i]) {
2411 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2412 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2413 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2414 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2415 }
2416 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2417 if (!(tmp & 1)) {
2418 tmp |= 1;
2419 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2420 }
2421 }
2422 }
2423 }
2424
2425 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2426 {
2427 u32 tmp, frame_count;
2428 int i, j;
2429
2430 /* update crtc base addresses */
2431 for (i = 0; i < rdev->num_crtc; i++) {
2432 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2433 upper_32_bits(rdev->mc.vram_start));
2434 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2435 upper_32_bits(rdev->mc.vram_start));
2436 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2437 (u32)rdev->mc.vram_start);
2438 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2439 (u32)rdev->mc.vram_start);
2440 }
2441 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2442 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2443
2444 /* unlock regs and wait for update */
2445 for (i = 0; i < rdev->num_crtc; i++) {
2446 if (save->crtc_enabled[i]) {
2447 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2448 if ((tmp & 0x3) != 0) {
2449 tmp &= ~0x3;
2450 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2451 }
2452 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2453 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2454 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2455 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2456 }
2457 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2458 if (tmp & 1) {
2459 tmp &= ~1;
2460 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2461 }
2462 for (j = 0; j < rdev->usec_timeout; j++) {
2463 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2464 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2465 break;
2466 udelay(1);
2467 }
2468 }
2469 }
2470
2471 /* unblackout the MC */
2472 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2473 tmp &= ~BLACKOUT_MODE_MASK;
2474 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2475 /* allow CPU access */
2476 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2477
2478 for (i = 0; i < rdev->num_crtc; i++) {
2479 if (save->crtc_enabled[i]) {
2480 if (ASIC_IS_DCE6(rdev)) {
2481 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2482 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2483 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2484 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2485 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2486 } else {
2487 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2488 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2489 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2490 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2491 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2492 }
2493 /* wait for the next frame */
2494 frame_count = radeon_get_vblank_counter(rdev, i);
2495 for (j = 0; j < rdev->usec_timeout; j++) {
2496 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2497 break;
2498 udelay(1);
2499 }
2500 }
2501 }
2502 /* Unlock vga access */
2503 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2504 mdelay(1);
2505 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2506 }
2507
2508 void evergreen_mc_program(struct radeon_device *rdev)
2509 {
2510 struct evergreen_mc_save save;
2511 u32 tmp;
2512 int i, j;
2513
2514 /* Initialize HDP */
2515 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2516 WREG32((0x2c14 + j), 0x00000000);
2517 WREG32((0x2c18 + j), 0x00000000);
2518 WREG32((0x2c1c + j), 0x00000000);
2519 WREG32((0x2c20 + j), 0x00000000);
2520 WREG32((0x2c24 + j), 0x00000000);
2521 }
2522 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2523
2524 evergreen_mc_stop(rdev, &save);
2525 if (evergreen_mc_wait_for_idle(rdev)) {
2526 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2527 }
2528 /* Lockout access through VGA aperture*/
2529 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2530 /* Update configuration */
2531 if (rdev->flags & RADEON_IS_AGP) {
2532 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2533 /* VRAM before AGP */
2534 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2535 rdev->mc.vram_start >> 12);
2536 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2537 rdev->mc.gtt_end >> 12);
2538 } else {
2539 /* VRAM after AGP */
2540 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2541 rdev->mc.gtt_start >> 12);
2542 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2543 rdev->mc.vram_end >> 12);
2544 }
2545 } else {
2546 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2547 rdev->mc.vram_start >> 12);
2548 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2549 rdev->mc.vram_end >> 12);
2550 }
2551 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2552 /* llano/ontario only */
2553 if ((rdev->family == CHIP_PALM) ||
2554 (rdev->family == CHIP_SUMO) ||
2555 (rdev->family == CHIP_SUMO2)) {
2556 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2557 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2558 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2559 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2560 }
2561 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2562 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2563 WREG32(MC_VM_FB_LOCATION, tmp);
2564 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2565 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2566 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2567 if (rdev->flags & RADEON_IS_AGP) {
2568 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2569 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2570 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2571 } else {
2572 WREG32(MC_VM_AGP_BASE, 0);
2573 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2574 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2575 }
2576 if (evergreen_mc_wait_for_idle(rdev)) {
2577 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2578 }
2579 evergreen_mc_resume(rdev, &save);
2580 /* we need to own VRAM, so turn off the VGA renderer here
2581 * to stop it overwriting our objects */
2582 rv515_vga_render_disable(rdev);
2583 }
2584
2585 /*
2586 * CP.
2587 */
2588 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2589 {
2590 struct radeon_ring *ring = &rdev->ring[ib->ring];
2591 u32 next_rptr;
2592
2593 /* set to DX10/11 mode */
2594 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2595 radeon_ring_write(ring, 1);
2596
2597 if (ring->rptr_save_reg) {
2598 next_rptr = ring->wptr + 3 + 4;
2599 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2600 radeon_ring_write(ring, ((ring->rptr_save_reg -
2601 PACKET3_SET_CONFIG_REG_START) >> 2));
2602 radeon_ring_write(ring, next_rptr);
2603 } else if (rdev->wb.enabled) {
2604 next_rptr = ring->wptr + 5 + 4;
2605 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2606 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2607 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2608 radeon_ring_write(ring, next_rptr);
2609 radeon_ring_write(ring, 0);
2610 }
2611
2612 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2613 radeon_ring_write(ring,
2614 #ifdef __BIG_ENDIAN
2615 (2 << 0) |
2616 #endif
2617 (ib->gpu_addr & 0xFFFFFFFC));
2618 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2619 radeon_ring_write(ring, ib->length_dw);
2620 }
2621
2622
2623 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2624 {
2625 const __be32 *fw_data;
2626 int i;
2627
2628 if (!rdev->me_fw || !rdev->pfp_fw)
2629 return -EINVAL;
2630
2631 r700_cp_stop(rdev);
2632 WREG32(CP_RB_CNTL,
2633 #ifdef __BIG_ENDIAN
2634 BUF_SWAP_32BIT |
2635 #endif
2636 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2637
2638 fw_data = (const __be32 *)rdev->pfp_fw->data;
2639 WREG32(CP_PFP_UCODE_ADDR, 0);
2640 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2641 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2642 WREG32(CP_PFP_UCODE_ADDR, 0);
2643
2644 fw_data = (const __be32 *)rdev->me_fw->data;
2645 WREG32(CP_ME_RAM_WADDR, 0);
2646 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2647 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2648
2649 WREG32(CP_PFP_UCODE_ADDR, 0);
2650 WREG32(CP_ME_RAM_WADDR, 0);
2651 WREG32(CP_ME_RAM_RADDR, 0);
2652 return 0;
2653 }
2654
2655 static int evergreen_cp_start(struct radeon_device *rdev)
2656 {
2657 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2658 int r, i;
2659 uint32_t cp_me;
2660
2661 r = radeon_ring_lock(rdev, ring, 7);
2662 if (r) {
2663 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2664 return r;
2665 }
2666 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2667 radeon_ring_write(ring, 0x1);
2668 radeon_ring_write(ring, 0x0);
2669 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2670 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2671 radeon_ring_write(ring, 0);
2672 radeon_ring_write(ring, 0);
2673 radeon_ring_unlock_commit(rdev, ring);
2674
2675 cp_me = 0xff;
2676 WREG32(CP_ME_CNTL, cp_me);
2677
2678 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2679 if (r) {
2680 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2681 return r;
2682 }
2683
2684 /* setup clear context state */
2685 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2686 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2687
2688 for (i = 0; i < evergreen_default_size; i++)
2689 radeon_ring_write(ring, evergreen_default_state[i]);
2690
2691 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2692 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2693
2694 /* set clear context state */
2695 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2696 radeon_ring_write(ring, 0);
2697
2698 /* SQ_VTX_BASE_VTX_LOC */
2699 radeon_ring_write(ring, 0xc0026f00);
2700 radeon_ring_write(ring, 0x00000000);
2701 radeon_ring_write(ring, 0x00000000);
2702 radeon_ring_write(ring, 0x00000000);
2703
2704 /* Clear consts */
2705 radeon_ring_write(ring, 0xc0036f00);
2706 radeon_ring_write(ring, 0x00000bc4);
2707 radeon_ring_write(ring, 0xffffffff);
2708 radeon_ring_write(ring, 0xffffffff);
2709 radeon_ring_write(ring, 0xffffffff);
2710
2711 radeon_ring_write(ring, 0xc0026900);
2712 radeon_ring_write(ring, 0x00000316);
2713 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2714 radeon_ring_write(ring, 0x00000010); /* */
2715
2716 radeon_ring_unlock_commit(rdev, ring);
2717
2718 return 0;
2719 }
2720
2721 static int evergreen_cp_resume(struct radeon_device *rdev)
2722 {
2723 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2724 u32 tmp;
2725 u32 rb_bufsz;
2726 int r;
2727
2728 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2729 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2730 SOFT_RESET_PA |
2731 SOFT_RESET_SH |
2732 SOFT_RESET_VGT |
2733 SOFT_RESET_SPI |
2734 SOFT_RESET_SX));
2735 RREG32(GRBM_SOFT_RESET);
2736 mdelay(15);
2737 WREG32(GRBM_SOFT_RESET, 0);
2738 RREG32(GRBM_SOFT_RESET);
2739
2740 /* Set ring buffer size */
2741 rb_bufsz = drm_order(ring->ring_size / 8);
2742 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2743 #ifdef __BIG_ENDIAN
2744 tmp |= BUF_SWAP_32BIT;
2745 #endif
2746 WREG32(CP_RB_CNTL, tmp);
2747 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2748 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2749
2750 /* Set the write pointer delay */
2751 WREG32(CP_RB_WPTR_DELAY, 0);
2752
2753 /* Initialize the ring buffer's read and write pointers */
2754 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2755 WREG32(CP_RB_RPTR_WR, 0);
2756 ring->wptr = 0;
2757 WREG32(CP_RB_WPTR, ring->wptr);
2758
2759 /* set the wb address whether it's enabled or not */
2760 WREG32(CP_RB_RPTR_ADDR,
2761 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2762 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2763 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2764
2765 if (rdev->wb.enabled)
2766 WREG32(SCRATCH_UMSK, 0xff);
2767 else {
2768 tmp |= RB_NO_UPDATE;
2769 WREG32(SCRATCH_UMSK, 0);
2770 }
2771
2772 mdelay(1);
2773 WREG32(CP_RB_CNTL, tmp);
2774
2775 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2776 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2777
2778 ring->rptr = RREG32(CP_RB_RPTR);
2779
2780 evergreen_cp_start(rdev);
2781 ring->ready = true;
2782 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2783 if (r) {
2784 ring->ready = false;
2785 return r;
2786 }
2787 return 0;
2788 }
2789
2790 /*
2791 * Core functions
2792 */
2793 static void evergreen_gpu_init(struct radeon_device *rdev)
2794 {
2795 u32 gb_addr_config;
2796 u32 mc_shared_chmap, mc_arb_ramcfg;
2797 u32 sx_debug_1;
2798 u32 smx_dc_ctl0;
2799 u32 sq_config;
2800 u32 sq_lds_resource_mgmt;
2801 u32 sq_gpr_resource_mgmt_1;
2802 u32 sq_gpr_resource_mgmt_2;
2803 u32 sq_gpr_resource_mgmt_3;
2804 u32 sq_thread_resource_mgmt;
2805 u32 sq_thread_resource_mgmt_2;
2806 u32 sq_stack_resource_mgmt_1;
2807 u32 sq_stack_resource_mgmt_2;
2808 u32 sq_stack_resource_mgmt_3;
2809 u32 vgt_cache_invalidation;
2810 u32 hdp_host_path_cntl, tmp;
2811 u32 disabled_rb_mask;
2812 int i, j, num_shader_engines, ps_thread_count;
2813
2814 switch (rdev->family) {
2815 case CHIP_CYPRESS:
2816 case CHIP_HEMLOCK:
2817 rdev->config.evergreen.num_ses = 2;
2818 rdev->config.evergreen.max_pipes = 4;
2819 rdev->config.evergreen.max_tile_pipes = 8;
2820 rdev->config.evergreen.max_simds = 10;
2821 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2822 rdev->config.evergreen.max_gprs = 256;
2823 rdev->config.evergreen.max_threads = 248;
2824 rdev->config.evergreen.max_gs_threads = 32;
2825 rdev->config.evergreen.max_stack_entries = 512;
2826 rdev->config.evergreen.sx_num_of_sets = 4;
2827 rdev->config.evergreen.sx_max_export_size = 256;
2828 rdev->config.evergreen.sx_max_export_pos_size = 64;
2829 rdev->config.evergreen.sx_max_export_smx_size = 192;
2830 rdev->config.evergreen.max_hw_contexts = 8;
2831 rdev->config.evergreen.sq_num_cf_insts = 2;
2832
2833 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2834 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2835 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2836 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2837 break;
2838 case CHIP_JUNIPER:
2839 rdev->config.evergreen.num_ses = 1;
2840 rdev->config.evergreen.max_pipes = 4;
2841 rdev->config.evergreen.max_tile_pipes = 4;
2842 rdev->config.evergreen.max_simds = 10;
2843 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2844 rdev->config.evergreen.max_gprs = 256;
2845 rdev->config.evergreen.max_threads = 248;
2846 rdev->config.evergreen.max_gs_threads = 32;
2847 rdev->config.evergreen.max_stack_entries = 512;
2848 rdev->config.evergreen.sx_num_of_sets = 4;
2849 rdev->config.evergreen.sx_max_export_size = 256;
2850 rdev->config.evergreen.sx_max_export_pos_size = 64;
2851 rdev->config.evergreen.sx_max_export_smx_size = 192;
2852 rdev->config.evergreen.max_hw_contexts = 8;
2853 rdev->config.evergreen.sq_num_cf_insts = 2;
2854
2855 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2856 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2857 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2858 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
2859 break;
2860 case CHIP_REDWOOD:
2861 rdev->config.evergreen.num_ses = 1;
2862 rdev->config.evergreen.max_pipes = 4;
2863 rdev->config.evergreen.max_tile_pipes = 4;
2864 rdev->config.evergreen.max_simds = 5;
2865 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2866 rdev->config.evergreen.max_gprs = 256;
2867 rdev->config.evergreen.max_threads = 248;
2868 rdev->config.evergreen.max_gs_threads = 32;
2869 rdev->config.evergreen.max_stack_entries = 256;
2870 rdev->config.evergreen.sx_num_of_sets = 4;
2871 rdev->config.evergreen.sx_max_export_size = 256;
2872 rdev->config.evergreen.sx_max_export_pos_size = 64;
2873 rdev->config.evergreen.sx_max_export_smx_size = 192;
2874 rdev->config.evergreen.max_hw_contexts = 8;
2875 rdev->config.evergreen.sq_num_cf_insts = 2;
2876
2877 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2878 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2879 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2880 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
2881 break;
2882 case CHIP_CEDAR:
2883 default:
2884 rdev->config.evergreen.num_ses = 1;
2885 rdev->config.evergreen.max_pipes = 2;
2886 rdev->config.evergreen.max_tile_pipes = 2;
2887 rdev->config.evergreen.max_simds = 2;
2888 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2889 rdev->config.evergreen.max_gprs = 256;
2890 rdev->config.evergreen.max_threads = 192;
2891 rdev->config.evergreen.max_gs_threads = 16;
2892 rdev->config.evergreen.max_stack_entries = 256;
2893 rdev->config.evergreen.sx_num_of_sets = 4;
2894 rdev->config.evergreen.sx_max_export_size = 128;
2895 rdev->config.evergreen.sx_max_export_pos_size = 32;
2896 rdev->config.evergreen.sx_max_export_smx_size = 96;
2897 rdev->config.evergreen.max_hw_contexts = 4;
2898 rdev->config.evergreen.sq_num_cf_insts = 1;
2899
2900 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2901 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2902 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2903 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
2904 break;
2905 case CHIP_PALM:
2906 rdev->config.evergreen.num_ses = 1;
2907 rdev->config.evergreen.max_pipes = 2;
2908 rdev->config.evergreen.max_tile_pipes = 2;
2909 rdev->config.evergreen.max_simds = 2;
2910 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2911 rdev->config.evergreen.max_gprs = 256;
2912 rdev->config.evergreen.max_threads = 192;
2913 rdev->config.evergreen.max_gs_threads = 16;
2914 rdev->config.evergreen.max_stack_entries = 256;
2915 rdev->config.evergreen.sx_num_of_sets = 4;
2916 rdev->config.evergreen.sx_max_export_size = 128;
2917 rdev->config.evergreen.sx_max_export_pos_size = 32;
2918 rdev->config.evergreen.sx_max_export_smx_size = 96;
2919 rdev->config.evergreen.max_hw_contexts = 4;
2920 rdev->config.evergreen.sq_num_cf_insts = 1;
2921
2922 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2923 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2924 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2925 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
2926 break;
2927 case CHIP_SUMO:
2928 rdev->config.evergreen.num_ses = 1;
2929 rdev->config.evergreen.max_pipes = 4;
2930 rdev->config.evergreen.max_tile_pipes = 4;
2931 if (rdev->pdev->device == 0x9648)
2932 rdev->config.evergreen.max_simds = 3;
2933 else if ((rdev->pdev->device == 0x9647) ||
2934 (rdev->pdev->device == 0x964a))
2935 rdev->config.evergreen.max_simds = 4;
2936 else
2937 rdev->config.evergreen.max_simds = 5;
2938 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2939 rdev->config.evergreen.max_gprs = 256;
2940 rdev->config.evergreen.max_threads = 248;
2941 rdev->config.evergreen.max_gs_threads = 32;
2942 rdev->config.evergreen.max_stack_entries = 256;
2943 rdev->config.evergreen.sx_num_of_sets = 4;
2944 rdev->config.evergreen.sx_max_export_size = 256;
2945 rdev->config.evergreen.sx_max_export_pos_size = 64;
2946 rdev->config.evergreen.sx_max_export_smx_size = 192;
2947 rdev->config.evergreen.max_hw_contexts = 8;
2948 rdev->config.evergreen.sq_num_cf_insts = 2;
2949
2950 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2951 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2952 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2953 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
2954 break;
2955 case CHIP_SUMO2:
2956 rdev->config.evergreen.num_ses = 1;
2957 rdev->config.evergreen.max_pipes = 4;
2958 rdev->config.evergreen.max_tile_pipes = 4;
2959 rdev->config.evergreen.max_simds = 2;
2960 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2961 rdev->config.evergreen.max_gprs = 256;
2962 rdev->config.evergreen.max_threads = 248;
2963 rdev->config.evergreen.max_gs_threads = 32;
2964 rdev->config.evergreen.max_stack_entries = 512;
2965 rdev->config.evergreen.sx_num_of_sets = 4;
2966 rdev->config.evergreen.sx_max_export_size = 256;
2967 rdev->config.evergreen.sx_max_export_pos_size = 64;
2968 rdev->config.evergreen.sx_max_export_smx_size = 192;
2969 rdev->config.evergreen.max_hw_contexts = 8;
2970 rdev->config.evergreen.sq_num_cf_insts = 2;
2971
2972 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2973 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2974 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2975 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
2976 break;
2977 case CHIP_BARTS:
2978 rdev->config.evergreen.num_ses = 2;
2979 rdev->config.evergreen.max_pipes = 4;
2980 rdev->config.evergreen.max_tile_pipes = 8;
2981 rdev->config.evergreen.max_simds = 7;
2982 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2983 rdev->config.evergreen.max_gprs = 256;
2984 rdev->config.evergreen.max_threads = 248;
2985 rdev->config.evergreen.max_gs_threads = 32;
2986 rdev->config.evergreen.max_stack_entries = 512;
2987 rdev->config.evergreen.sx_num_of_sets = 4;
2988 rdev->config.evergreen.sx_max_export_size = 256;
2989 rdev->config.evergreen.sx_max_export_pos_size = 64;
2990 rdev->config.evergreen.sx_max_export_smx_size = 192;
2991 rdev->config.evergreen.max_hw_contexts = 8;
2992 rdev->config.evergreen.sq_num_cf_insts = 2;
2993
2994 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2995 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2996 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
2998 break;
2999 case CHIP_TURKS:
3000 rdev->config.evergreen.num_ses = 1;
3001 rdev->config.evergreen.max_pipes = 4;
3002 rdev->config.evergreen.max_tile_pipes = 4;
3003 rdev->config.evergreen.max_simds = 6;
3004 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3005 rdev->config.evergreen.max_gprs = 256;
3006 rdev->config.evergreen.max_threads = 248;
3007 rdev->config.evergreen.max_gs_threads = 32;
3008 rdev->config.evergreen.max_stack_entries = 256;
3009 rdev->config.evergreen.sx_num_of_sets = 4;
3010 rdev->config.evergreen.sx_max_export_size = 256;
3011 rdev->config.evergreen.sx_max_export_pos_size = 64;
3012 rdev->config.evergreen.sx_max_export_smx_size = 192;
3013 rdev->config.evergreen.max_hw_contexts = 8;
3014 rdev->config.evergreen.sq_num_cf_insts = 2;
3015
3016 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3017 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3018 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3019 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3020 break;
3021 case CHIP_CAICOS:
3022 rdev->config.evergreen.num_ses = 1;
3023 rdev->config.evergreen.max_pipes = 2;
3024 rdev->config.evergreen.max_tile_pipes = 2;
3025 rdev->config.evergreen.max_simds = 2;
3026 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3027 rdev->config.evergreen.max_gprs = 256;
3028 rdev->config.evergreen.max_threads = 192;
3029 rdev->config.evergreen.max_gs_threads = 16;
3030 rdev->config.evergreen.max_stack_entries = 256;
3031 rdev->config.evergreen.sx_num_of_sets = 4;
3032 rdev->config.evergreen.sx_max_export_size = 128;
3033 rdev->config.evergreen.sx_max_export_pos_size = 32;
3034 rdev->config.evergreen.sx_max_export_smx_size = 96;
3035 rdev->config.evergreen.max_hw_contexts = 4;
3036 rdev->config.evergreen.sq_num_cf_insts = 1;
3037
3038 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3039 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3040 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3041 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3042 break;
3043 }
3044
3045 /* Initialize HDP */
3046 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3047 WREG32((0x2c14 + j), 0x00000000);
3048 WREG32((0x2c18 + j), 0x00000000);
3049 WREG32((0x2c1c + j), 0x00000000);
3050 WREG32((0x2c20 + j), 0x00000000);
3051 WREG32((0x2c24 + j), 0x00000000);
3052 }
3053
3054 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3055
3056 evergreen_fix_pci_max_read_req_size(rdev);
3057
3058 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3059 if ((rdev->family == CHIP_PALM) ||
3060 (rdev->family == CHIP_SUMO) ||
3061 (rdev->family == CHIP_SUMO2))
3062 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3063 else
3064 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3065
3066 /* setup tiling info dword. gb_addr_config is not adequate since it does
3067 * not have bank info, so create a custom tiling dword.
3068 * bits 3:0 num_pipes
3069 * bits 7:4 num_banks
3070 * bits 11:8 group_size
3071 * bits 15:12 row_size
3072 */
3073 rdev->config.evergreen.tile_config = 0;
3074 switch (rdev->config.evergreen.max_tile_pipes) {
3075 case 1:
3076 default:
3077 rdev->config.evergreen.tile_config |= (0 << 0);
3078 break;
3079 case 2:
3080 rdev->config.evergreen.tile_config |= (1 << 0);
3081 break;
3082 case 4:
3083 rdev->config.evergreen.tile_config |= (2 << 0);
3084 break;
3085 case 8:
3086 rdev->config.evergreen.tile_config |= (3 << 0);
3087 break;
3088 }
3089 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3090 if (rdev->flags & RADEON_IS_IGP)
3091 rdev->config.evergreen.tile_config |= 1 << 4;
3092 else {
3093 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3094 case 0: /* four banks */
3095 rdev->config.evergreen.tile_config |= 0 << 4;
3096 break;
3097 case 1: /* eight banks */
3098 rdev->config.evergreen.tile_config |= 1 << 4;
3099 break;
3100 case 2: /* sixteen banks */
3101 default:
3102 rdev->config.evergreen.tile_config |= 2 << 4;
3103 break;
3104 }
3105 }
3106 rdev->config.evergreen.tile_config |= 0 << 8;
3107 rdev->config.evergreen.tile_config |=
3108 ((gb_addr_config & 0x30000000) >> 28) << 12;
3109
3110 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3111
3112 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3113 u32 efuse_straps_4;
3114 u32 efuse_straps_3;
3115
3116 WREG32(RCU_IND_INDEX, 0x204);
3117 efuse_straps_4 = RREG32(RCU_IND_DATA);
3118 WREG32(RCU_IND_INDEX, 0x203);
3119 efuse_straps_3 = RREG32(RCU_IND_DATA);
3120 tmp = (((efuse_straps_4 & 0xf) << 4) |
3121 ((efuse_straps_3 & 0xf0000000) >> 28));
3122 } else {
3123 tmp = 0;
3124 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3125 u32 rb_disable_bitmap;
3126
3127 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3128 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3129 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3130 tmp <<= 4;
3131 tmp |= rb_disable_bitmap;
3132 }
3133 }
3134 /* enabled rb are just the one not disabled :) */
3135 disabled_rb_mask = tmp;
3136 tmp = 0;
3137 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3138 tmp |= (1 << i);
3139 /* if all the backends are disabled, fix it up here */
3140 if ((disabled_rb_mask & tmp) == tmp) {
3141 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3142 disabled_rb_mask &= ~(1 << i);
3143 }
3144
3145 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3146 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3147
3148 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3149 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3150 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3151 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3152 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3153 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3154 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3155
3156 if ((rdev->config.evergreen.max_backends == 1) &&
3157 (rdev->flags & RADEON_IS_IGP)) {
3158 if ((disabled_rb_mask & 3) == 1) {
3159 /* RB0 disabled, RB1 enabled */
3160 tmp = 0x11111111;
3161 } else {
3162 /* RB1 disabled, RB0 enabled */
3163 tmp = 0x00000000;
3164 }
3165 } else {
3166 tmp = gb_addr_config & NUM_PIPES_MASK;
3167 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3168 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3169 }
3170 WREG32(GB_BACKEND_MAP, tmp);
3171
3172 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3173 WREG32(CGTS_TCC_DISABLE, 0);
3174 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3175 WREG32(CGTS_USER_TCC_DISABLE, 0);
3176
3177 /* set HW defaults for 3D engine */
3178 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3179 ROQ_IB2_START(0x2b)));
3180
3181 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3182
3183 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3184 SYNC_GRADIENT |
3185 SYNC_WALKER |
3186 SYNC_ALIGNER));
3187
3188 sx_debug_1 = RREG32(SX_DEBUG_1);
3189 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3190 WREG32(SX_DEBUG_1, sx_debug_1);
3191
3192
3193 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3194 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3195 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3196 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3197
3198 if (rdev->family <= CHIP_SUMO2)
3199 WREG32(SMX_SAR_CTL0, 0x00010000);
3200
3201 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3202 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3203 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3204
3205 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3206 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3207 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3208
3209 WREG32(VGT_NUM_INSTANCES, 1);
3210 WREG32(SPI_CONFIG_CNTL, 0);
3211 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3212 WREG32(CP_PERFMON_CNTL, 0);
3213
3214 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3215 FETCH_FIFO_HIWATER(0x4) |
3216 DONE_FIFO_HIWATER(0xe0) |
3217 ALU_UPDATE_FIFO_HIWATER(0x8)));
3218
3219 sq_config = RREG32(SQ_CONFIG);
3220 sq_config &= ~(PS_PRIO(3) |
3221 VS_PRIO(3) |
3222 GS_PRIO(3) |
3223 ES_PRIO(3));
3224 sq_config |= (VC_ENABLE |
3225 EXPORT_SRC_C |
3226 PS_PRIO(0) |
3227 VS_PRIO(1) |
3228 GS_PRIO(2) |
3229 ES_PRIO(3));
3230
3231 switch (rdev->family) {
3232 case CHIP_CEDAR:
3233 case CHIP_PALM:
3234 case CHIP_SUMO:
3235 case CHIP_SUMO2:
3236 case CHIP_CAICOS:
3237 /* no vertex cache */
3238 sq_config &= ~VC_ENABLE;
3239 break;
3240 default:
3241 break;
3242 }
3243
3244 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3245
3246 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3247 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3248 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3249 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3250 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3251 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3252 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3253
3254 switch (rdev->family) {
3255 case CHIP_CEDAR:
3256 case CHIP_PALM:
3257 case CHIP_SUMO:
3258 case CHIP_SUMO2:
3259 ps_thread_count = 96;
3260 break;
3261 default:
3262 ps_thread_count = 128;
3263 break;
3264 }
3265
3266 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3267 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3268 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3269 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3270 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3271 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3272
3273 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3274 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3275 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3276 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3277 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3278 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3279
3280 WREG32(SQ_CONFIG, sq_config);
3281 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3282 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3283 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3284 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3285 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3286 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3287 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3288 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3289 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3290 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3291
3292 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3293 FORCE_EOV_MAX_REZ_CNT(255)));
3294
3295 switch (rdev->family) {
3296 case CHIP_CEDAR:
3297 case CHIP_PALM:
3298 case CHIP_SUMO:
3299 case CHIP_SUMO2:
3300 case CHIP_CAICOS:
3301 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3302 break;
3303 default:
3304 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3305 break;
3306 }
3307 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3308 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3309
3310 WREG32(VGT_GS_VERTEX_REUSE, 16);
3311 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3312 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3313
3314 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3315 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3316
3317 WREG32(CB_PERF_CTR0_SEL_0, 0);
3318 WREG32(CB_PERF_CTR0_SEL_1, 0);
3319 WREG32(CB_PERF_CTR1_SEL_0, 0);
3320 WREG32(CB_PERF_CTR1_SEL_1, 0);
3321 WREG32(CB_PERF_CTR2_SEL_0, 0);
3322 WREG32(CB_PERF_CTR2_SEL_1, 0);
3323 WREG32(CB_PERF_CTR3_SEL_0, 0);
3324 WREG32(CB_PERF_CTR3_SEL_1, 0);
3325
3326 /* clear render buffer base addresses */
3327 WREG32(CB_COLOR0_BASE, 0);
3328 WREG32(CB_COLOR1_BASE, 0);
3329 WREG32(CB_COLOR2_BASE, 0);
3330 WREG32(CB_COLOR3_BASE, 0);
3331 WREG32(CB_COLOR4_BASE, 0);
3332 WREG32(CB_COLOR5_BASE, 0);
3333 WREG32(CB_COLOR6_BASE, 0);
3334 WREG32(CB_COLOR7_BASE, 0);
3335 WREG32(CB_COLOR8_BASE, 0);
3336 WREG32(CB_COLOR9_BASE, 0);
3337 WREG32(CB_COLOR10_BASE, 0);
3338 WREG32(CB_COLOR11_BASE, 0);
3339
3340 /* set the shader const cache sizes to 0 */
3341 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3342 WREG32(i, 0);
3343 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3344 WREG32(i, 0);
3345
3346 tmp = RREG32(HDP_MISC_CNTL);
3347 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3348 WREG32(HDP_MISC_CNTL, tmp);
3349
3350 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3351 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3352
3353 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3354
3355 udelay(50);
3356
3357 }
3358
3359 int evergreen_mc_init(struct radeon_device *rdev)
3360 {
3361 u32 tmp;
3362 int chansize, numchan;
3363
3364 /* Get VRAM informations */
3365 rdev->mc.vram_is_ddr = true;
3366 if ((rdev->family == CHIP_PALM) ||
3367 (rdev->family == CHIP_SUMO) ||
3368 (rdev->family == CHIP_SUMO2))
3369 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3370 else
3371 tmp = RREG32(MC_ARB_RAMCFG);
3372 if (tmp & CHANSIZE_OVERRIDE) {
3373 chansize = 16;
3374 } else if (tmp & CHANSIZE_MASK) {
3375 chansize = 64;
3376 } else {
3377 chansize = 32;
3378 }
3379 tmp = RREG32(MC_SHARED_CHMAP);
3380 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3381 case 0:
3382 default:
3383 numchan = 1;
3384 break;
3385 case 1:
3386 numchan = 2;
3387 break;
3388 case 2:
3389 numchan = 4;
3390 break;
3391 case 3:
3392 numchan = 8;
3393 break;
3394 }
3395 rdev->mc.vram_width = numchan * chansize;
3396 /* Could aper size report 0 ? */
3397 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3398 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3399 /* Setup GPU memory space */
3400 if ((rdev->family == CHIP_PALM) ||
3401 (rdev->family == CHIP_SUMO) ||
3402 (rdev->family == CHIP_SUMO2)) {
3403 /* size in bytes on fusion */
3404 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3405 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3406 } else {
3407 /* size in MB on evergreen/cayman/tn */
3408 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3409 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3410 }
3411 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3412 r700_vram_gtt_location(rdev, &rdev->mc);
3413 radeon_update_bandwidth_info(rdev);
3414
3415 return 0;
3416 }
3417
3418 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3419 {
3420 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3421 RREG32(GRBM_STATUS));
3422 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3423 RREG32(GRBM_STATUS_SE0));
3424 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3425 RREG32(GRBM_STATUS_SE1));
3426 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3427 RREG32(SRBM_STATUS));
3428 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3429 RREG32(SRBM_STATUS2));
3430 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3431 RREG32(CP_STALLED_STAT1));
3432 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3433 RREG32(CP_STALLED_STAT2));
3434 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3435 RREG32(CP_BUSY_STAT));
3436 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3437 RREG32(CP_STAT));
3438 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3439 RREG32(DMA_STATUS_REG));
3440 if (rdev->family >= CHIP_CAYMAN) {
3441 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3442 RREG32(DMA_STATUS_REG + 0x800));
3443 }
3444 }
3445
3446 bool evergreen_is_display_hung(struct radeon_device *rdev)
3447 {
3448 u32 crtc_hung = 0;
3449 u32 crtc_status[6];
3450 u32 i, j, tmp;
3451
3452 for (i = 0; i < rdev->num_crtc; i++) {
3453 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3454 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3455 crtc_hung |= (1 << i);
3456 }
3457 }
3458
3459 for (j = 0; j < 10; j++) {
3460 for (i = 0; i < rdev->num_crtc; i++) {
3461 if (crtc_hung & (1 << i)) {
3462 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3463 if (tmp != crtc_status[i])
3464 crtc_hung &= ~(1 << i);
3465 }
3466 }
3467 if (crtc_hung == 0)
3468 return false;
3469 udelay(100);
3470 }
3471
3472 return true;
3473 }
3474
3475 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3476 {
3477 u32 reset_mask = 0;
3478 u32 tmp;
3479
3480 /* GRBM_STATUS */
3481 tmp = RREG32(GRBM_STATUS);
3482 if (tmp & (PA_BUSY | SC_BUSY |
3483 SH_BUSY | SX_BUSY |
3484 TA_BUSY | VGT_BUSY |
3485 DB_BUSY | CB_BUSY |
3486 SPI_BUSY | VGT_BUSY_NO_DMA))
3487 reset_mask |= RADEON_RESET_GFX;
3488
3489 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3490 CP_BUSY | CP_COHERENCY_BUSY))
3491 reset_mask |= RADEON_RESET_CP;
3492
3493 if (tmp & GRBM_EE_BUSY)
3494 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3495
3496 /* DMA_STATUS_REG */
3497 tmp = RREG32(DMA_STATUS_REG);
3498 if (!(tmp & DMA_IDLE))
3499 reset_mask |= RADEON_RESET_DMA;
3500
3501 /* SRBM_STATUS2 */
3502 tmp = RREG32(SRBM_STATUS2);
3503 if (tmp & DMA_BUSY)
3504 reset_mask |= RADEON_RESET_DMA;
3505
3506 /* SRBM_STATUS */
3507 tmp = RREG32(SRBM_STATUS);
3508 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3509 reset_mask |= RADEON_RESET_RLC;
3510
3511 if (tmp & IH_BUSY)
3512 reset_mask |= RADEON_RESET_IH;
3513
3514 if (tmp & SEM_BUSY)
3515 reset_mask |= RADEON_RESET_SEM;
3516
3517 if (tmp & GRBM_RQ_PENDING)
3518 reset_mask |= RADEON_RESET_GRBM;
3519
3520 if (tmp & VMC_BUSY)
3521 reset_mask |= RADEON_RESET_VMC;
3522
3523 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3524 MCC_BUSY | MCD_BUSY))
3525 reset_mask |= RADEON_RESET_MC;
3526
3527 if (evergreen_is_display_hung(rdev))
3528 reset_mask |= RADEON_RESET_DISPLAY;
3529
3530 /* VM_L2_STATUS */
3531 tmp = RREG32(VM_L2_STATUS);
3532 if (tmp & L2_BUSY)
3533 reset_mask |= RADEON_RESET_VMC;
3534
3535 /* Skip MC reset as it's mostly likely not hung, just busy */
3536 if (reset_mask & RADEON_RESET_MC) {
3537 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3538 reset_mask &= ~RADEON_RESET_MC;
3539 }
3540
3541 return reset_mask;
3542 }
3543
3544 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3545 {
3546 struct evergreen_mc_save save;
3547 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3548 u32 tmp;
3549
3550 if (reset_mask == 0)
3551 return;
3552
3553 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3554
3555 evergreen_print_gpu_status_regs(rdev);
3556
3557 /* Disable CP parsing/prefetching */
3558 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3559
3560 if (reset_mask & RADEON_RESET_DMA) {
3561 /* Disable DMA */
3562 tmp = RREG32(DMA_RB_CNTL);
3563 tmp &= ~DMA_RB_ENABLE;
3564 WREG32(DMA_RB_CNTL, tmp);
3565 }
3566
3567 udelay(50);
3568
3569 evergreen_mc_stop(rdev, &save);
3570 if (evergreen_mc_wait_for_idle(rdev)) {
3571 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3572 }
3573
3574 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3575 grbm_soft_reset |= SOFT_RESET_DB |
3576 SOFT_RESET_CB |
3577 SOFT_RESET_PA |
3578 SOFT_RESET_SC |
3579 SOFT_RESET_SPI |
3580 SOFT_RESET_SX |
3581 SOFT_RESET_SH |
3582 SOFT_RESET_TC |
3583 SOFT_RESET_TA |
3584 SOFT_RESET_VC |
3585 SOFT_RESET_VGT;
3586 }
3587
3588 if (reset_mask & RADEON_RESET_CP) {
3589 grbm_soft_reset |= SOFT_RESET_CP |
3590 SOFT_RESET_VGT;
3591
3592 srbm_soft_reset |= SOFT_RESET_GRBM;
3593 }
3594
3595 if (reset_mask & RADEON_RESET_DMA)
3596 srbm_soft_reset |= SOFT_RESET_DMA;
3597
3598 if (reset_mask & RADEON_RESET_DISPLAY)
3599 srbm_soft_reset |= SOFT_RESET_DC;
3600
3601 if (reset_mask & RADEON_RESET_RLC)
3602 srbm_soft_reset |= SOFT_RESET_RLC;
3603
3604 if (reset_mask & RADEON_RESET_SEM)
3605 srbm_soft_reset |= SOFT_RESET_SEM;
3606
3607 if (reset_mask & RADEON_RESET_IH)
3608 srbm_soft_reset |= SOFT_RESET_IH;
3609
3610 if (reset_mask & RADEON_RESET_GRBM)
3611 srbm_soft_reset |= SOFT_RESET_GRBM;
3612
3613 if (reset_mask & RADEON_RESET_VMC)
3614 srbm_soft_reset |= SOFT_RESET_VMC;
3615
3616 if (!(rdev->flags & RADEON_IS_IGP)) {
3617 if (reset_mask & RADEON_RESET_MC)
3618 srbm_soft_reset |= SOFT_RESET_MC;
3619 }
3620
3621 if (grbm_soft_reset) {
3622 tmp = RREG32(GRBM_SOFT_RESET);
3623 tmp |= grbm_soft_reset;
3624 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3625 WREG32(GRBM_SOFT_RESET, tmp);
3626 tmp = RREG32(GRBM_SOFT_RESET);
3627
3628 udelay(50);
3629
3630 tmp &= ~grbm_soft_reset;
3631 WREG32(GRBM_SOFT_RESET, tmp);
3632 tmp = RREG32(GRBM_SOFT_RESET);
3633 }
3634
3635 if (srbm_soft_reset) {
3636 tmp = RREG32(SRBM_SOFT_RESET);
3637 tmp |= srbm_soft_reset;
3638 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3639 WREG32(SRBM_SOFT_RESET, tmp);
3640 tmp = RREG32(SRBM_SOFT_RESET);
3641
3642 udelay(50);
3643
3644 tmp &= ~srbm_soft_reset;
3645 WREG32(SRBM_SOFT_RESET, tmp);
3646 tmp = RREG32(SRBM_SOFT_RESET);
3647 }
3648
3649 /* Wait a little for things to settle down */
3650 udelay(50);
3651
3652 evergreen_mc_resume(rdev, &save);
3653 udelay(50);
3654
3655 evergreen_print_gpu_status_regs(rdev);
3656 }
3657
3658 int evergreen_asic_reset(struct radeon_device *rdev)
3659 {
3660 u32 reset_mask;
3661
3662 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3663
3664 if (reset_mask)
3665 r600_set_bios_scratch_engine_hung(rdev, true);
3666
3667 evergreen_gpu_soft_reset(rdev, reset_mask);
3668
3669 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3670
3671 if (!reset_mask)
3672 r600_set_bios_scratch_engine_hung(rdev, false);
3673
3674 return 0;
3675 }
3676
3677 /**
3678 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3679 *
3680 * @rdev: radeon_device pointer
3681 * @ring: radeon_ring structure holding ring information
3682 *
3683 * Check if the GFX engine is locked up.
3684 * Returns true if the engine appears to be locked up, false if not.
3685 */
3686 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3687 {
3688 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3689
3690 if (!(reset_mask & (RADEON_RESET_GFX |
3691 RADEON_RESET_COMPUTE |
3692 RADEON_RESET_CP))) {
3693 radeon_ring_lockup_update(ring);
3694 return false;
3695 }
3696 /* force CP activities */
3697 radeon_ring_force_activity(rdev, ring);
3698 return radeon_ring_test_lockup(rdev, ring);
3699 }
3700
3701 /**
3702 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3703 *
3704 * @rdev: radeon_device pointer
3705 * @ring: radeon_ring structure holding ring information
3706 *
3707 * Check if the async DMA engine is locked up.
3708 * Returns true if the engine appears to be locked up, false if not.
3709 */
3710 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3711 {
3712 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3713
3714 if (!(reset_mask & RADEON_RESET_DMA)) {
3715 radeon_ring_lockup_update(ring);
3716 return false;
3717 }
3718 /* force ring activities */
3719 radeon_ring_force_activity(rdev, ring);
3720 return radeon_ring_test_lockup(rdev, ring);
3721 }
3722
3723 /* Interrupts */
3724
3725 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
3726 {
3727 if (crtc >= rdev->num_crtc)
3728 return 0;
3729 else
3730 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
3731 }
3732
3733 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
3734 {
3735 u32 tmp;
3736
3737 if (rdev->family >= CHIP_CAYMAN) {
3738 cayman_cp_int_cntl_setup(rdev, 0,
3739 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3740 cayman_cp_int_cntl_setup(rdev, 1, 0);
3741 cayman_cp_int_cntl_setup(rdev, 2, 0);
3742 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
3743 WREG32(CAYMAN_DMA1_CNTL, tmp);
3744 } else
3745 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3746 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3747 WREG32(DMA_CNTL, tmp);
3748 WREG32(GRBM_INT_CNTL, 0);
3749 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3750 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3751 if (rdev->num_crtc >= 4) {
3752 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3753 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
3754 }
3755 if (rdev->num_crtc >= 6) {
3756 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3757 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3758 }
3759
3760 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3761 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3762 if (rdev->num_crtc >= 4) {
3763 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3764 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
3765 }
3766 if (rdev->num_crtc >= 6) {
3767 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3768 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3769 }
3770
3771 /* only one DAC on DCE6 */
3772 if (!ASIC_IS_DCE6(rdev))
3773 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3774 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3775
3776 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3777 WREG32(DC_HPD1_INT_CONTROL, tmp);
3778 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3779 WREG32(DC_HPD2_INT_CONTROL, tmp);
3780 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3781 WREG32(DC_HPD3_INT_CONTROL, tmp);
3782 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3783 WREG32(DC_HPD4_INT_CONTROL, tmp);
3784 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3785 WREG32(DC_HPD5_INT_CONTROL, tmp);
3786 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3787 WREG32(DC_HPD6_INT_CONTROL, tmp);
3788
3789 }
3790
3791 int evergreen_irq_set(struct radeon_device *rdev)
3792 {
3793 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3794 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
3795 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3796 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
3797 u32 grbm_int_cntl = 0;
3798 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
3799 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
3800 u32 dma_cntl, dma_cntl1 = 0;
3801
3802 if (!rdev->irq.installed) {
3803 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3804 return -EINVAL;
3805 }
3806 /* don't enable anything if the ih is disabled */
3807 if (!rdev->ih.enabled) {
3808 r600_disable_interrupts(rdev);
3809 /* force the active interrupt state to all disabled */
3810 evergreen_disable_interrupt_state(rdev);
3811 return 0;
3812 }
3813
3814 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3815 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3816 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3817 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3818 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3819 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3820
3821 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3822 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3823 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3824 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3825 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3826 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3827
3828 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3829
3830 if (rdev->family >= CHIP_CAYMAN) {
3831 /* enable CP interrupts on all rings */
3832 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3833 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3834 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3835 }
3836 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
3837 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
3838 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
3839 }
3840 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
3841 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
3842 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
3843 }
3844 } else {
3845 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3846 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3847 cp_int_cntl |= RB_INT_ENABLE;
3848 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3849 }
3850 }
3851
3852 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3853 DRM_DEBUG("r600_irq_set: sw int dma\n");
3854 dma_cntl |= TRAP_ENABLE;
3855 }
3856
3857 if (rdev->family >= CHIP_CAYMAN) {
3858 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
3859 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
3860 DRM_DEBUG("r600_irq_set: sw int dma1\n");
3861 dma_cntl1 |= TRAP_ENABLE;
3862 }
3863 }
3864
3865 if (rdev->irq.crtc_vblank_int[0] ||
3866 atomic_read(&rdev->irq.pflip[0])) {
3867 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
3868 crtc1 |= VBLANK_INT_MASK;
3869 }
3870 if (rdev->irq.crtc_vblank_int[1] ||
3871 atomic_read(&rdev->irq.pflip[1])) {
3872 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
3873 crtc2 |= VBLANK_INT_MASK;
3874 }
3875 if (rdev->irq.crtc_vblank_int[2] ||
3876 atomic_read(&rdev->irq.pflip[2])) {
3877 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
3878 crtc3 |= VBLANK_INT_MASK;
3879 }
3880 if (rdev->irq.crtc_vblank_int[3] ||
3881 atomic_read(&rdev->irq.pflip[3])) {
3882 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
3883 crtc4 |= VBLANK_INT_MASK;
3884 }
3885 if (rdev->irq.crtc_vblank_int[4] ||
3886 atomic_read(&rdev->irq.pflip[4])) {
3887 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
3888 crtc5 |= VBLANK_INT_MASK;
3889 }
3890 if (rdev->irq.crtc_vblank_int[5] ||
3891 atomic_read(&rdev->irq.pflip[5])) {
3892 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
3893 crtc6 |= VBLANK_INT_MASK;
3894 }
3895 if (rdev->irq.hpd[0]) {
3896 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
3897 hpd1 |= DC_HPDx_INT_EN;
3898 }
3899 if (rdev->irq.hpd[1]) {
3900 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
3901 hpd2 |= DC_HPDx_INT_EN;
3902 }
3903 if (rdev->irq.hpd[2]) {
3904 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
3905 hpd3 |= DC_HPDx_INT_EN;
3906 }
3907 if (rdev->irq.hpd[3]) {
3908 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
3909 hpd4 |= DC_HPDx_INT_EN;
3910 }
3911 if (rdev->irq.hpd[4]) {
3912 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
3913 hpd5 |= DC_HPDx_INT_EN;
3914 }
3915 if (rdev->irq.hpd[5]) {
3916 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
3917 hpd6 |= DC_HPDx_INT_EN;
3918 }
3919 if (rdev->irq.afmt[0]) {
3920 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
3921 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3922 }
3923 if (rdev->irq.afmt[1]) {
3924 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
3925 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3926 }
3927 if (rdev->irq.afmt[2]) {
3928 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
3929 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3930 }
3931 if (rdev->irq.afmt[3]) {
3932 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
3933 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3934 }
3935 if (rdev->irq.afmt[4]) {
3936 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
3937 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3938 }
3939 if (rdev->irq.afmt[5]) {
3940 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
3941 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3942 }
3943
3944 if (rdev->family >= CHIP_CAYMAN) {
3945 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
3946 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
3947 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
3948 } else
3949 WREG32(CP_INT_CNTL, cp_int_cntl);
3950
3951 WREG32(DMA_CNTL, dma_cntl);
3952
3953 if (rdev->family >= CHIP_CAYMAN)
3954 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
3955
3956 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3957
3958 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
3959 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3960 if (rdev->num_crtc >= 4) {
3961 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
3962 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
3963 }
3964 if (rdev->num_crtc >= 6) {
3965 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
3966 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
3967 }
3968
3969 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
3970 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
3971 if (rdev->num_crtc >= 4) {
3972 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
3973 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
3974 }
3975 if (rdev->num_crtc >= 6) {
3976 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
3977 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
3978 }
3979
3980 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3981 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3982 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3983 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3984 WREG32(DC_HPD5_INT_CONTROL, hpd5);
3985 WREG32(DC_HPD6_INT_CONTROL, hpd6);
3986
3987 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
3988 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
3989 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
3990 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
3991 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
3992 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
3993
3994 return 0;
3995 }
3996
3997 static void evergreen_irq_ack(struct radeon_device *rdev)
3998 {
3999 u32 tmp;
4000
4001 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4002 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4003 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4004 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4005 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4006 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4007 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4008 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4009 if (rdev->num_crtc >= 4) {
4010 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4011 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4012 }
4013 if (rdev->num_crtc >= 6) {
4014 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4015 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4016 }
4017
4018 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4019 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4020 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4021 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4022 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4023 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4024
4025 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4026 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4027 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4028 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4029 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4030 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4031 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4032 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4033 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4034 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4035 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4036 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4037
4038 if (rdev->num_crtc >= 4) {
4039 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4040 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4041 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4042 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4043 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4044 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4045 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4046 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4047 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4048 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4049 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4050 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4051 }
4052
4053 if (rdev->num_crtc >= 6) {
4054 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4055 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4056 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4057 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4058 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4059 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4060 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4061 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4062 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4063 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4064 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4065 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4066 }
4067
4068 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4069 tmp = RREG32(DC_HPD1_INT_CONTROL);
4070 tmp |= DC_HPDx_INT_ACK;
4071 WREG32(DC_HPD1_INT_CONTROL, tmp);
4072 }
4073 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4074 tmp = RREG32(DC_HPD2_INT_CONTROL);
4075 tmp |= DC_HPDx_INT_ACK;
4076 WREG32(DC_HPD2_INT_CONTROL, tmp);
4077 }
4078 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4079 tmp = RREG32(DC_HPD3_INT_CONTROL);
4080 tmp |= DC_HPDx_INT_ACK;
4081 WREG32(DC_HPD3_INT_CONTROL, tmp);
4082 }
4083 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4084 tmp = RREG32(DC_HPD4_INT_CONTROL);
4085 tmp |= DC_HPDx_INT_ACK;
4086 WREG32(DC_HPD4_INT_CONTROL, tmp);
4087 }
4088 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4089 tmp = RREG32(DC_HPD5_INT_CONTROL);
4090 tmp |= DC_HPDx_INT_ACK;
4091 WREG32(DC_HPD5_INT_CONTROL, tmp);
4092 }
4093 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4094 tmp = RREG32(DC_HPD5_INT_CONTROL);
4095 tmp |= DC_HPDx_INT_ACK;
4096 WREG32(DC_HPD6_INT_CONTROL, tmp);
4097 }
4098 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4099 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4100 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4101 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4102 }
4103 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4104 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4105 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4106 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4107 }
4108 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4109 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4110 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4111 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4112 }
4113 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4114 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4115 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4116 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4117 }
4118 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4119 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4120 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4121 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4122 }
4123 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4124 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4125 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4126 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4127 }
4128 }
4129
4130 static void evergreen_irq_disable(struct radeon_device *rdev)
4131 {
4132 r600_disable_interrupts(rdev);
4133 /* Wait and acknowledge irq */
4134 mdelay(1);
4135 evergreen_irq_ack(rdev);
4136 evergreen_disable_interrupt_state(rdev);
4137 }
4138
4139 void evergreen_irq_suspend(struct radeon_device *rdev)
4140 {
4141 evergreen_irq_disable(rdev);
4142 r600_rlc_stop(rdev);
4143 }
4144
4145 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4146 {
4147 u32 wptr, tmp;
4148
4149 if (rdev->wb.enabled)
4150 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4151 else
4152 wptr = RREG32(IH_RB_WPTR);
4153
4154 if (wptr & RB_OVERFLOW) {
4155 /* When a ring buffer overflow happen start parsing interrupt
4156 * from the last not overwritten vector (wptr + 16). Hopefully
4157 * this should allow us to catchup.
4158 */
4159 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4160 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4161 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4162 tmp = RREG32(IH_RB_CNTL);
4163 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4164 WREG32(IH_RB_CNTL, tmp);
4165 }
4166 return (wptr & rdev->ih.ptr_mask);
4167 }
4168
4169 int evergreen_irq_process(struct radeon_device *rdev)
4170 {
4171 u32 wptr;
4172 u32 rptr;
4173 u32 src_id, src_data;
4174 u32 ring_index;
4175 bool queue_hotplug = false;
4176 bool queue_hdmi = false;
4177
4178 if (!rdev->ih.enabled || rdev->shutdown)
4179 return IRQ_NONE;
4180
4181 wptr = evergreen_get_ih_wptr(rdev);
4182
4183 restart_ih:
4184 /* is somebody else already processing irqs? */
4185 if (atomic_xchg(&rdev->ih.lock, 1))
4186 return IRQ_NONE;
4187
4188 rptr = rdev->ih.rptr;
4189 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4190
4191 /* Order reading of wptr vs. reading of IH ring data */
4192 rmb();
4193
4194 /* display interrupts */
4195 evergreen_irq_ack(rdev);
4196
4197 while (rptr != wptr) {
4198 /* wptr/rptr are in bytes! */
4199 ring_index = rptr / 4;
4200 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4201 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4202
4203 switch (src_id) {
4204 case 1: /* D1 vblank/vline */
4205 switch (src_data) {
4206 case 0: /* D1 vblank */
4207 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4208 if (rdev->irq.crtc_vblank_int[0]) {
4209 drm_handle_vblank(rdev->ddev, 0);
4210 rdev->pm.vblank_sync = true;
4211 wake_up(&rdev->irq.vblank_queue);
4212 }
4213 if (atomic_read(&rdev->irq.pflip[0]))
4214 radeon_crtc_handle_flip(rdev, 0);
4215 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4216 DRM_DEBUG("IH: D1 vblank\n");
4217 }
4218 break;
4219 case 1: /* D1 vline */
4220 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4221 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4222 DRM_DEBUG("IH: D1 vline\n");
4223 }
4224 break;
4225 default:
4226 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4227 break;
4228 }
4229 break;
4230 case 2: /* D2 vblank/vline */
4231 switch (src_data) {
4232 case 0: /* D2 vblank */
4233 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4234 if (rdev->irq.crtc_vblank_int[1]) {
4235 drm_handle_vblank(rdev->ddev, 1);
4236 rdev->pm.vblank_sync = true;
4237 wake_up(&rdev->irq.vblank_queue);
4238 }
4239 if (atomic_read(&rdev->irq.pflip[1]))
4240 radeon_crtc_handle_flip(rdev, 1);
4241 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4242 DRM_DEBUG("IH: D2 vblank\n");
4243 }
4244 break;
4245 case 1: /* D2 vline */
4246 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4247 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4248 DRM_DEBUG("IH: D2 vline\n");
4249 }
4250 break;
4251 default:
4252 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4253 break;
4254 }
4255 break;
4256 case 3: /* D3 vblank/vline */
4257 switch (src_data) {
4258 case 0: /* D3 vblank */
4259 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4260 if (rdev->irq.crtc_vblank_int[2]) {
4261 drm_handle_vblank(rdev->ddev, 2);
4262 rdev->pm.vblank_sync = true;
4263 wake_up(&rdev->irq.vblank_queue);
4264 }
4265 if (atomic_read(&rdev->irq.pflip[2]))
4266 radeon_crtc_handle_flip(rdev, 2);
4267 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4268 DRM_DEBUG("IH: D3 vblank\n");
4269 }
4270 break;
4271 case 1: /* D3 vline */
4272 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4273 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4274 DRM_DEBUG("IH: D3 vline\n");
4275 }
4276 break;
4277 default:
4278 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4279 break;
4280 }
4281 break;
4282 case 4: /* D4 vblank/vline */
4283 switch (src_data) {
4284 case 0: /* D4 vblank */
4285 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4286 if (rdev->irq.crtc_vblank_int[3]) {
4287 drm_handle_vblank(rdev->ddev, 3);
4288 rdev->pm.vblank_sync = true;
4289 wake_up(&rdev->irq.vblank_queue);
4290 }
4291 if (atomic_read(&rdev->irq.pflip[3]))
4292 radeon_crtc_handle_flip(rdev, 3);
4293 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4294 DRM_DEBUG("IH: D4 vblank\n");
4295 }
4296 break;
4297 case 1: /* D4 vline */
4298 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4299 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4300 DRM_DEBUG("IH: D4 vline\n");
4301 }
4302 break;
4303 default:
4304 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4305 break;
4306 }
4307 break;
4308 case 5: /* D5 vblank/vline */
4309 switch (src_data) {
4310 case 0: /* D5 vblank */
4311 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4312 if (rdev->irq.crtc_vblank_int[4]) {
4313 drm_handle_vblank(rdev->ddev, 4);
4314 rdev->pm.vblank_sync = true;
4315 wake_up(&rdev->irq.vblank_queue);
4316 }
4317 if (atomic_read(&rdev->irq.pflip[4]))
4318 radeon_crtc_handle_flip(rdev, 4);
4319 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4320 DRM_DEBUG("IH: D5 vblank\n");
4321 }
4322 break;
4323 case 1: /* D5 vline */
4324 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4325 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4326 DRM_DEBUG("IH: D5 vline\n");
4327 }
4328 break;
4329 default:
4330 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4331 break;
4332 }
4333 break;
4334 case 6: /* D6 vblank/vline */
4335 switch (src_data) {
4336 case 0: /* D6 vblank */
4337 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4338 if (rdev->irq.crtc_vblank_int[5]) {
4339 drm_handle_vblank(rdev->ddev, 5);
4340 rdev->pm.vblank_sync = true;
4341 wake_up(&rdev->irq.vblank_queue);
4342 }
4343 if (atomic_read(&rdev->irq.pflip[5]))
4344 radeon_crtc_handle_flip(rdev, 5);
4345 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4346 DRM_DEBUG("IH: D6 vblank\n");
4347 }
4348 break;
4349 case 1: /* D6 vline */
4350 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4351 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4352 DRM_DEBUG("IH: D6 vline\n");
4353 }
4354 break;
4355 default:
4356 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4357 break;
4358 }
4359 break;
4360 case 42: /* HPD hotplug */
4361 switch (src_data) {
4362 case 0:
4363 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4364 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4365 queue_hotplug = true;
4366 DRM_DEBUG("IH: HPD1\n");
4367 }
4368 break;
4369 case 1:
4370 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4371 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4372 queue_hotplug = true;
4373 DRM_DEBUG("IH: HPD2\n");
4374 }
4375 break;
4376 case 2:
4377 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4378 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4379 queue_hotplug = true;
4380 DRM_DEBUG("IH: HPD3\n");
4381 }
4382 break;
4383 case 3:
4384 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4385 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4386 queue_hotplug = true;
4387 DRM_DEBUG("IH: HPD4\n");
4388 }
4389 break;
4390 case 4:
4391 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4392 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4393 queue_hotplug = true;
4394 DRM_DEBUG("IH: HPD5\n");
4395 }
4396 break;
4397 case 5:
4398 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4399 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4400 queue_hotplug = true;
4401 DRM_DEBUG("IH: HPD6\n");
4402 }
4403 break;
4404 default:
4405 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4406 break;
4407 }
4408 break;
4409 case 44: /* hdmi */
4410 switch (src_data) {
4411 case 0:
4412 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4413 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4414 queue_hdmi = true;
4415 DRM_DEBUG("IH: HDMI0\n");
4416 }
4417 break;
4418 case 1:
4419 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4420 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4421 queue_hdmi = true;
4422 DRM_DEBUG("IH: HDMI1\n");
4423 }
4424 break;
4425 case 2:
4426 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4427 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4428 queue_hdmi = true;
4429 DRM_DEBUG("IH: HDMI2\n");
4430 }
4431 break;
4432 case 3:
4433 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4434 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4435 queue_hdmi = true;
4436 DRM_DEBUG("IH: HDMI3\n");
4437 }
4438 break;
4439 case 4:
4440 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4441 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4442 queue_hdmi = true;
4443 DRM_DEBUG("IH: HDMI4\n");
4444 }
4445 break;
4446 case 5:
4447 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4448 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4449 queue_hdmi = true;
4450 DRM_DEBUG("IH: HDMI5\n");
4451 }
4452 break;
4453 default:
4454 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4455 break;
4456 }
4457 case 124: /* UVD */
4458 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4459 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4460 break;
4461 case 146:
4462 case 147:
4463 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4464 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4465 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4466 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4467 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4468 /* reset addr and status */
4469 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4470 break;
4471 case 176: /* CP_INT in ring buffer */
4472 case 177: /* CP_INT in IB1 */
4473 case 178: /* CP_INT in IB2 */
4474 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4475 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4476 break;
4477 case 181: /* CP EOP event */
4478 DRM_DEBUG("IH: CP EOP\n");
4479 if (rdev->family >= CHIP_CAYMAN) {
4480 switch (src_data) {
4481 case 0:
4482 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4483 break;
4484 case 1:
4485 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4486 break;
4487 case 2:
4488 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4489 break;
4490 }
4491 } else
4492 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4493 break;
4494 case 224: /* DMA trap event */
4495 DRM_DEBUG("IH: DMA trap\n");
4496 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4497 break;
4498 case 233: /* GUI IDLE */
4499 DRM_DEBUG("IH: GUI idle\n");
4500 break;
4501 case 244: /* DMA trap event */
4502 if (rdev->family >= CHIP_CAYMAN) {
4503 DRM_DEBUG("IH: DMA1 trap\n");
4504 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4505 }
4506 break;
4507 default:
4508 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4509 break;
4510 }
4511
4512 /* wptr/rptr are in bytes! */
4513 rptr += 16;
4514 rptr &= rdev->ih.ptr_mask;
4515 }
4516 if (queue_hotplug)
4517 schedule_work(&rdev->hotplug_work);
4518 if (queue_hdmi)
4519 schedule_work(&rdev->audio_work);
4520 rdev->ih.rptr = rptr;
4521 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4522 atomic_set(&rdev->ih.lock, 0);
4523
4524 /* make sure wptr hasn't changed while processing */
4525 wptr = evergreen_get_ih_wptr(rdev);
4526 if (wptr != rptr)
4527 goto restart_ih;
4528
4529 return IRQ_HANDLED;
4530 }
4531
4532 /**
4533 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4534 *
4535 * @rdev: radeon_device pointer
4536 * @fence: radeon fence object
4537 *
4538 * Add a DMA fence packet to the ring to write
4539 * the fence seq number and DMA trap packet to generate
4540 * an interrupt if needed (evergreen-SI).
4541 */
4542 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4543 struct radeon_fence *fence)
4544 {
4545 struct radeon_ring *ring = &rdev->ring[fence->ring];
4546 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4547 /* write the fence */
4548 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
4549 radeon_ring_write(ring, addr & 0xfffffffc);
4550 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4551 radeon_ring_write(ring, fence->seq);
4552 /* generate an interrupt */
4553 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
4554 /* flush HDP */
4555 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4556 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4557 radeon_ring_write(ring, 1);
4558 }
4559
4560 /**
4561 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4562 *
4563 * @rdev: radeon_device pointer
4564 * @ib: IB object to schedule
4565 *
4566 * Schedule an IB in the DMA ring (evergreen).
4567 */
4568 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4569 struct radeon_ib *ib)
4570 {
4571 struct radeon_ring *ring = &rdev->ring[ib->ring];
4572
4573 if (rdev->wb.enabled) {
4574 u32 next_rptr = ring->wptr + 4;
4575 while ((next_rptr & 7) != 5)
4576 next_rptr++;
4577 next_rptr += 3;
4578 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
4579 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4580 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4581 radeon_ring_write(ring, next_rptr);
4582 }
4583
4584 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4585 * Pad as necessary with NOPs.
4586 */
4587 while ((ring->wptr & 7) != 5)
4588 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4589 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
4590 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4591 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4592
4593 }
4594
4595 /**
4596 * evergreen_copy_dma - copy pages using the DMA engine
4597 *
4598 * @rdev: radeon_device pointer
4599 * @src_offset: src GPU address
4600 * @dst_offset: dst GPU address
4601 * @num_gpu_pages: number of GPU pages to xfer
4602 * @fence: radeon fence object
4603 *
4604 * Copy GPU paging using the DMA engine (evergreen-cayman).
4605 * Used by the radeon ttm implementation to move pages if
4606 * registered as the asic copy callback.
4607 */
4608 int evergreen_copy_dma(struct radeon_device *rdev,
4609 uint64_t src_offset, uint64_t dst_offset,
4610 unsigned num_gpu_pages,
4611 struct radeon_fence **fence)
4612 {
4613 struct radeon_semaphore *sem = NULL;
4614 int ring_index = rdev->asic->copy.dma_ring_index;
4615 struct radeon_ring *ring = &rdev->ring[ring_index];
4616 u32 size_in_dw, cur_size_in_dw;
4617 int i, num_loops;
4618 int r = 0;
4619
4620 r = radeon_semaphore_create(rdev, &sem);
4621 if (r) {
4622 DRM_ERROR("radeon: moving bo (%d).\n", r);
4623 return r;
4624 }
4625
4626 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
4627 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
4628 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
4629 if (r) {
4630 DRM_ERROR("radeon: moving bo (%d).\n", r);
4631 radeon_semaphore_free(rdev, &sem, NULL);
4632 return r;
4633 }
4634
4635 if (radeon_fence_need_sync(*fence, ring->idx)) {
4636 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
4637 ring->idx);
4638 radeon_fence_note_sync(*fence, ring->idx);
4639 } else {
4640 radeon_semaphore_free(rdev, &sem, NULL);
4641 }
4642
4643 for (i = 0; i < num_loops; i++) {
4644 cur_size_in_dw = size_in_dw;
4645 if (cur_size_in_dw > 0xFFFFF)
4646 cur_size_in_dw = 0xFFFFF;
4647 size_in_dw -= cur_size_in_dw;
4648 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
4649 radeon_ring_write(ring, dst_offset & 0xfffffffc);
4650 radeon_ring_write(ring, src_offset & 0xfffffffc);
4651 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
4652 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
4653 src_offset += cur_size_in_dw * 4;
4654 dst_offset += cur_size_in_dw * 4;
4655 }
4656
4657 r = radeon_fence_emit(rdev, fence, ring->idx);
4658 if (r) {
4659 radeon_ring_unlock_undo(rdev, ring);
4660 return r;
4661 }
4662
4663 radeon_ring_unlock_commit(rdev, ring);
4664 radeon_semaphore_free(rdev, &sem, *fence);
4665
4666 return r;
4667 }
4668
4669 static int evergreen_startup(struct radeon_device *rdev)
4670 {
4671 struct radeon_ring *ring;
4672 int r;
4673
4674 /* enable pcie gen2 link */
4675 evergreen_pcie_gen2_enable(rdev);
4676
4677 if (ASIC_IS_DCE5(rdev)) {
4678 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
4679 r = ni_init_microcode(rdev);
4680 if (r) {
4681 DRM_ERROR("Failed to load firmware!\n");
4682 return r;
4683 }
4684 }
4685 r = ni_mc_load_microcode(rdev);
4686 if (r) {
4687 DRM_ERROR("Failed to load MC firmware!\n");
4688 return r;
4689 }
4690 } else {
4691 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
4692 r = r600_init_microcode(rdev);
4693 if (r) {
4694 DRM_ERROR("Failed to load firmware!\n");
4695 return r;
4696 }
4697 }
4698 }
4699
4700 r = r600_vram_scratch_init(rdev);
4701 if (r)
4702 return r;
4703
4704 evergreen_mc_program(rdev);
4705 if (rdev->flags & RADEON_IS_AGP) {
4706 evergreen_agp_enable(rdev);
4707 } else {
4708 r = evergreen_pcie_gart_enable(rdev);
4709 if (r)
4710 return r;
4711 }
4712 evergreen_gpu_init(rdev);
4713
4714 r = evergreen_blit_init(rdev);
4715 if (r) {
4716 r600_blit_fini(rdev);
4717 rdev->asic->copy.copy = NULL;
4718 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
4719 }
4720
4721 /* allocate wb buffer */
4722 r = radeon_wb_init(rdev);
4723 if (r)
4724 return r;
4725
4726 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
4727 if (r) {
4728 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
4729 return r;
4730 }
4731
4732 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
4733 if (r) {
4734 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
4735 return r;
4736 }
4737
4738 r = rv770_uvd_resume(rdev);
4739 if (!r) {
4740 r = radeon_fence_driver_start_ring(rdev,
4741 R600_RING_TYPE_UVD_INDEX);
4742 if (r)
4743 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
4744 }
4745
4746 if (r)
4747 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4748
4749 /* Enable IRQ */
4750 r = r600_irq_init(rdev);
4751 if (r) {
4752 DRM_ERROR("radeon: IH init failed (%d).\n", r);
4753 radeon_irq_kms_fini(rdev);
4754 return r;
4755 }
4756 evergreen_irq_set(rdev);
4757
4758 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4759 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
4760 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
4761 0, 0xfffff, RADEON_CP_PACKET2);
4762 if (r)
4763 return r;
4764
4765 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
4766 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
4767 DMA_RB_RPTR, DMA_RB_WPTR,
4768 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4769 if (r)
4770 return r;
4771
4772 r = evergreen_cp_load_microcode(rdev);
4773 if (r)
4774 return r;
4775 r = evergreen_cp_resume(rdev);
4776 if (r)
4777 return r;
4778 r = r600_dma_resume(rdev);
4779 if (r)
4780 return r;
4781
4782 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4783 if (ring->ring_size) {
4784 r = radeon_ring_init(rdev, ring, ring->ring_size,
4785 R600_WB_UVD_RPTR_OFFSET,
4786 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
4787 0, 0xfffff, RADEON_CP_PACKET2);
4788 if (!r)
4789 r = r600_uvd_init(rdev);
4790
4791 if (r)
4792 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
4793 }
4794
4795 r = radeon_ib_pool_init(rdev);
4796 if (r) {
4797 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
4798 return r;
4799 }
4800
4801 r = r600_audio_init(rdev);
4802 if (r) {
4803 DRM_ERROR("radeon: audio init failed\n");
4804 return r;
4805 }
4806
4807 return 0;
4808 }
4809
4810 int evergreen_resume(struct radeon_device *rdev)
4811 {
4812 int r;
4813
4814 /* reset the asic, the gfx blocks are often in a bad state
4815 * after the driver is unloaded or after a resume
4816 */
4817 if (radeon_asic_reset(rdev))
4818 dev_warn(rdev->dev, "GPU reset failed !\n");
4819 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
4820 * posting will perform necessary task to bring back GPU into good
4821 * shape.
4822 */
4823 /* post card */
4824 atom_asic_init(rdev->mode_info.atom_context);
4825
4826 /* init golden registers */
4827 evergreen_init_golden_registers(rdev);
4828
4829 rdev->accel_working = true;
4830 r = evergreen_startup(rdev);
4831 if (r) {
4832 DRM_ERROR("evergreen startup failed on resume\n");
4833 rdev->accel_working = false;
4834 return r;
4835 }
4836
4837 return r;
4838
4839 }
4840
4841 int evergreen_suspend(struct radeon_device *rdev)
4842 {
4843 r600_audio_fini(rdev);
4844 radeon_uvd_suspend(rdev);
4845 r700_cp_stop(rdev);
4846 r600_dma_stop(rdev);
4847 r600_uvd_rbc_stop(rdev);
4848 evergreen_irq_suspend(rdev);
4849 radeon_wb_disable(rdev);
4850 evergreen_pcie_gart_disable(rdev);
4851
4852 return 0;
4853 }
4854
4855 /* Plan is to move initialization in that function and use
4856 * helper function so that radeon_device_init pretty much
4857 * do nothing more than calling asic specific function. This
4858 * should also allow to remove a bunch of callback function
4859 * like vram_info.
4860 */
4861 int evergreen_init(struct radeon_device *rdev)
4862 {
4863 int r;
4864
4865 /* Read BIOS */
4866 if (!radeon_get_bios(rdev)) {
4867 if (ASIC_IS_AVIVO(rdev))
4868 return -EINVAL;
4869 }
4870 /* Must be an ATOMBIOS */
4871 if (!rdev->is_atom_bios) {
4872 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
4873 return -EINVAL;
4874 }
4875 r = radeon_atombios_init(rdev);
4876 if (r)
4877 return r;
4878 /* reset the asic, the gfx blocks are often in a bad state
4879 * after the driver is unloaded or after a resume
4880 */
4881 if (radeon_asic_reset(rdev))
4882 dev_warn(rdev->dev, "GPU reset failed !\n");
4883 /* Post card if necessary */
4884 if (!radeon_card_posted(rdev)) {
4885 if (!rdev->bios) {
4886 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
4887 return -EINVAL;
4888 }
4889 DRM_INFO("GPU not posted. posting now...\n");
4890 atom_asic_init(rdev->mode_info.atom_context);
4891 }
4892 /* init golden registers */
4893 evergreen_init_golden_registers(rdev);
4894 /* Initialize scratch registers */
4895 r600_scratch_init(rdev);
4896 /* Initialize surface registers */
4897 radeon_surface_init(rdev);
4898 /* Initialize clocks */
4899 radeon_get_clock_info(rdev->ddev);
4900 /* Fence driver */
4901 r = radeon_fence_driver_init(rdev);
4902 if (r)
4903 return r;
4904 /* initialize AGP */
4905 if (rdev->flags & RADEON_IS_AGP) {
4906 r = radeon_agp_init(rdev);
4907 if (r)
4908 radeon_agp_disable(rdev);
4909 }
4910 /* initialize memory controller */
4911 r = evergreen_mc_init(rdev);
4912 if (r)
4913 return r;
4914 /* Memory manager */
4915 r = radeon_bo_init(rdev);
4916 if (r)
4917 return r;
4918
4919 r = radeon_irq_kms_init(rdev);
4920 if (r)
4921 return r;
4922
4923 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
4924 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
4925
4926 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
4927 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
4928
4929 r = radeon_uvd_init(rdev);
4930 if (!r) {
4931 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4932 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
4933 4096);
4934 }
4935
4936 rdev->ih.ring_obj = NULL;
4937 r600_ih_ring_init(rdev, 64 * 1024);
4938
4939 r = r600_pcie_gart_init(rdev);
4940 if (r)
4941 return r;
4942
4943 rdev->accel_working = true;
4944 r = evergreen_startup(rdev);
4945 if (r) {
4946 dev_err(rdev->dev, "disabling GPU acceleration\n");
4947 r700_cp_fini(rdev);
4948 r600_dma_fini(rdev);
4949 r600_irq_fini(rdev);
4950 radeon_wb_fini(rdev);
4951 radeon_ib_pool_fini(rdev);
4952 radeon_irq_kms_fini(rdev);
4953 evergreen_pcie_gart_fini(rdev);
4954 rdev->accel_working = false;
4955 }
4956
4957 /* Don't start up if the MC ucode is missing on BTC parts.
4958 * The default clocks and voltages before the MC ucode
4959 * is loaded are not suffient for advanced operations.
4960 */
4961 if (ASIC_IS_DCE5(rdev)) {
4962 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
4963 DRM_ERROR("radeon: MC ucode required for NI+.\n");
4964 return -EINVAL;
4965 }
4966 }
4967
4968 return 0;
4969 }
4970
4971 void evergreen_fini(struct radeon_device *rdev)
4972 {
4973 r600_audio_fini(rdev);
4974 r600_blit_fini(rdev);
4975 r700_cp_fini(rdev);
4976 r600_dma_fini(rdev);
4977 r600_irq_fini(rdev);
4978 radeon_wb_fini(rdev);
4979 radeon_ib_pool_fini(rdev);
4980 radeon_irq_kms_fini(rdev);
4981 evergreen_pcie_gart_fini(rdev);
4982 radeon_uvd_fini(rdev);
4983 r600_vram_scratch_fini(rdev);
4984 radeon_gem_fini(rdev);
4985 radeon_fence_driver_fini(rdev);
4986 radeon_agp_fini(rdev);
4987 radeon_bo_fini(rdev);
4988 radeon_atombios_fini(rdev);
4989 kfree(rdev->bios);
4990 rdev->bios = NULL;
4991 }
4992
4993 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
4994 {
4995 u32 link_width_cntl, speed_cntl, mask;
4996 int ret;
4997
4998 if (radeon_pcie_gen2 == 0)
4999 return;
5000
5001 if (rdev->flags & RADEON_IS_IGP)
5002 return;
5003
5004 if (!(rdev->flags & RADEON_IS_PCIE))
5005 return;
5006
5007 /* x2 cards have a special sequence */
5008 if (ASIC_IS_X2(rdev))
5009 return;
5010
5011 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
5012 if (ret != 0)
5013 return;
5014
5015 if (!(mask & DRM_PCIE_SPEED_50))
5016 return;
5017
5018 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5019 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5020 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5021 return;
5022 }
5023
5024 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5025
5026 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5027 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5028
5029 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5030 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5031 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5032
5033 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5034 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5035 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5036
5037 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5038 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5039 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5040
5041 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5042 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5043 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5044
5045 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5046 speed_cntl |= LC_GEN2_EN_STRAP;
5047 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5048
5049 } else {
5050 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5051 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5052 if (1)
5053 link_width_cntl |= LC_UPCONFIGURE_DIS;
5054 else
5055 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5056 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5057 }
5058 }
Linux kernel - Deliverables
This page took 0.162309 seconds and 6 git commands to generate.