projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fix UPLL_REF_DIV_MASK definition
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
bcc1c2a1 36
fe251e2f
AD		 37#define EVERGREEN_PFP_UCODE_SIZE 1120
38#define EVERGREEN_PM4_UCODE_SIZE 1376
39
4a15903d
AD		 40static const u32 crtc_offsets[6] =
41{
42 EVERGREEN_CRTC0_REGISTER_OFFSET,
43 EVERGREEN_CRTC1_REGISTER_OFFSET,
44 EVERGREEN_CRTC2_REGISTER_OFFSET,
45 EVERGREEN_CRTC3_REGISTER_OFFSET,
46 EVERGREEN_CRTC4_REGISTER_OFFSET,
47 EVERGREEN_CRTC5_REGISTER_OFFSET
48};
49
bcc1c2a1
AD		 50static void evergreen_gpu_init(struct radeon_device *rdev);
51void evergreen_fini(struct radeon_device *rdev);
b07759bf 52void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1b37078b
AD		 53extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
54 int ring, u32 cp_int_cntl);
bcc1c2a1 55
d4788db3
AD		 56static const u32 evergreen_golden_registers[] =
57{
58 0x3f90, 0xffff0000, 0xff000000,
59 0x9148, 0xffff0000, 0xff000000,
60 0x3f94, 0xffff0000, 0xff000000,
61 0x914c, 0xffff0000, 0xff000000,
62 0x9b7c, 0xffffffff, 0x00000000,
63 0x8a14, 0xffffffff, 0x00000007,
64 0x8b10, 0xffffffff, 0x00000000,
65 0x960c, 0xffffffff, 0x54763210,
66 0x88c4, 0xffffffff, 0x000000c2,
67 0x88d4, 0xffffffff, 0x00000010,
68 0x8974, 0xffffffff, 0x00000000,
69 0xc78, 0x00000080, 0x00000080,
70 0x5eb4, 0xffffffff, 0x00000002,
71 0x5e78, 0xffffffff, 0x001000f0,
72 0x6104, 0x01000300, 0x00000000,
73 0x5bc0, 0x00300000, 0x00000000,
74 0x7030, 0xffffffff, 0x00000011,
75 0x7c30, 0xffffffff, 0x00000011,
76 0x10830, 0xffffffff, 0x00000011,
77 0x11430, 0xffffffff, 0x00000011,
78 0x12030, 0xffffffff, 0x00000011,
79 0x12c30, 0xffffffff, 0x00000011,
80 0xd02c, 0xffffffff, 0x08421000,
81 0x240c, 0xffffffff, 0x00000380,
82 0x8b24, 0xffffffff, 0x00ff0fff,
83 0x28a4c, 0x06000000, 0x06000000,
84 0x10c, 0x00000001, 0x00000001,
85 0x8d00, 0xffffffff, 0x100e4848,
86 0x8d04, 0xffffffff, 0x00164745,
87 0x8c00, 0xffffffff, 0xe4000003,
88 0x8c04, 0xffffffff, 0x40600060,
89 0x8c08, 0xffffffff, 0x001c001c,
90 0x8cf0, 0xffffffff, 0x08e00620,
91 0x8c20, 0xffffffff, 0x00800080,
92 0x8c24, 0xffffffff, 0x00800080,
93 0x8c18, 0xffffffff, 0x20202078,
94 0x8c1c, 0xffffffff, 0x00001010,
95 0x28350, 0xffffffff, 0x00000000,
96 0xa008, 0xffffffff, 0x00010000,
97 0x5cc, 0xffffffff, 0x00000001,
98 0x9508, 0xffffffff, 0x00000002,
99 0x913c, 0x0000000f, 0x0000000a
100};
101
102static const u32 evergreen_golden_registers2[] =
103{
104 0x2f4c, 0xffffffff, 0x00000000,
105 0x54f4, 0xffffffff, 0x00000000,
106 0x54f0, 0xffffffff, 0x00000000,
107 0x5498, 0xffffffff, 0x00000000,
108 0x549c, 0xffffffff, 0x00000000,
109 0x5494, 0xffffffff, 0x00000000,
110 0x53cc, 0xffffffff, 0x00000000,
111 0x53c8, 0xffffffff, 0x00000000,
112 0x53c4, 0xffffffff, 0x00000000,
113 0x53c0, 0xffffffff, 0x00000000,
114 0x53bc, 0xffffffff, 0x00000000,
115 0x53b8, 0xffffffff, 0x00000000,
116 0x53b4, 0xffffffff, 0x00000000,
117 0x53b0, 0xffffffff, 0x00000000
118};
119
120static const u32 cypress_mgcg_init[] =
121{
122 0x802c, 0xffffffff, 0xc0000000,
123 0x5448, 0xffffffff, 0x00000100,
124 0x55e4, 0xffffffff, 0x00000100,
125 0x160c, 0xffffffff, 0x00000100,
126 0x5644, 0xffffffff, 0x00000100,
127 0xc164, 0xffffffff, 0x00000100,
128 0x8a18, 0xffffffff, 0x00000100,
129 0x897c, 0xffffffff, 0x06000100,
130 0x8b28, 0xffffffff, 0x00000100,
131 0x9144, 0xffffffff, 0x00000100,
132 0x9a60, 0xffffffff, 0x00000100,
133 0x9868, 0xffffffff, 0x00000100,
134 0x8d58, 0xffffffff, 0x00000100,
135 0x9510, 0xffffffff, 0x00000100,
136 0x949c, 0xffffffff, 0x00000100,
137 0x9654, 0xffffffff, 0x00000100,
138 0x9030, 0xffffffff, 0x00000100,
139 0x9034, 0xffffffff, 0x00000100,
140 0x9038, 0xffffffff, 0x00000100,
141 0x903c, 0xffffffff, 0x00000100,
142 0x9040, 0xffffffff, 0x00000100,
143 0xa200, 0xffffffff, 0x00000100,
144 0xa204, 0xffffffff, 0x00000100,
145 0xa208, 0xffffffff, 0x00000100,
146 0xa20c, 0xffffffff, 0x00000100,
147 0x971c, 0xffffffff, 0x00000100,
148 0x977c, 0xffffffff, 0x00000100,
149 0x3f80, 0xffffffff, 0x00000100,
150 0xa210, 0xffffffff, 0x00000100,
151 0xa214, 0xffffffff, 0x00000100,
152 0x4d8, 0xffffffff, 0x00000100,
153 0x9784, 0xffffffff, 0x00000100,
154 0x9698, 0xffffffff, 0x00000100,
155 0x4d4, 0xffffffff, 0x00000200,
156 0x30cc, 0xffffffff, 0x00000100,
157 0xd0c0, 0xffffffff, 0xff000100,
158 0x802c, 0xffffffff, 0x40000000,
159 0x915c, 0xffffffff, 0x00010000,
160 0x9160, 0xffffffff, 0x00030002,
161 0x9178, 0xffffffff, 0x00070000,
162 0x917c, 0xffffffff, 0x00030002,
163 0x9180, 0xffffffff, 0x00050004,
164 0x918c, 0xffffffff, 0x00010006,
165 0x9190, 0xffffffff, 0x00090008,
166 0x9194, 0xffffffff, 0x00070000,
167 0x9198, 0xffffffff, 0x00030002,
168 0x919c, 0xffffffff, 0x00050004,
169 0x91a8, 0xffffffff, 0x00010006,
170 0x91ac, 0xffffffff, 0x00090008,
171 0x91b0, 0xffffffff, 0x00070000,
172 0x91b4, 0xffffffff, 0x00030002,
173 0x91b8, 0xffffffff, 0x00050004,
174 0x91c4, 0xffffffff, 0x00010006,
175 0x91c8, 0xffffffff, 0x00090008,
176 0x91cc, 0xffffffff, 0x00070000,
177 0x91d0, 0xffffffff, 0x00030002,
178 0x91d4, 0xffffffff, 0x00050004,
179 0x91e0, 0xffffffff, 0x00010006,
180 0x91e4, 0xffffffff, 0x00090008,
181 0x91e8, 0xffffffff, 0x00000000,
182 0x91ec, 0xffffffff, 0x00070000,
183 0x91f0, 0xffffffff, 0x00030002,
184 0x91f4, 0xffffffff, 0x00050004,
185 0x9200, 0xffffffff, 0x00010006,
186 0x9204, 0xffffffff, 0x00090008,
187 0x9208, 0xffffffff, 0x00070000,
188 0x920c, 0xffffffff, 0x00030002,
189 0x9210, 0xffffffff, 0x00050004,
190 0x921c, 0xffffffff, 0x00010006,
191 0x9220, 0xffffffff, 0x00090008,
192 0x9224, 0xffffffff, 0x00070000,
193 0x9228, 0xffffffff, 0x00030002,
194 0x922c, 0xffffffff, 0x00050004,
195 0x9238, 0xffffffff, 0x00010006,
196 0x923c, 0xffffffff, 0x00090008,
197 0x9240, 0xffffffff, 0x00070000,
198 0x9244, 0xffffffff, 0x00030002,
199 0x9248, 0xffffffff, 0x00050004,
200 0x9254, 0xffffffff, 0x00010006,
201 0x9258, 0xffffffff, 0x00090008,
202 0x925c, 0xffffffff, 0x00070000,
203 0x9260, 0xffffffff, 0x00030002,
204 0x9264, 0xffffffff, 0x00050004,
205 0x9270, 0xffffffff, 0x00010006,
206 0x9274, 0xffffffff, 0x00090008,
207 0x9278, 0xffffffff, 0x00070000,
208 0x927c, 0xffffffff, 0x00030002,
209 0x9280, 0xffffffff, 0x00050004,
210 0x928c, 0xffffffff, 0x00010006,
211 0x9290, 0xffffffff, 0x00090008,
212 0x9294, 0xffffffff, 0x00000000,
213 0x929c, 0xffffffff, 0x00000001,
214 0x802c, 0xffffffff, 0x40010000,
215 0x915c, 0xffffffff, 0x00010000,
216 0x9160, 0xffffffff, 0x00030002,
217 0x9178, 0xffffffff, 0x00070000,
218 0x917c, 0xffffffff, 0x00030002,
219 0x9180, 0xffffffff, 0x00050004,
220 0x918c, 0xffffffff, 0x00010006,
221 0x9190, 0xffffffff, 0x00090008,
222 0x9194, 0xffffffff, 0x00070000,
223 0x9198, 0xffffffff, 0x00030002,
224 0x919c, 0xffffffff, 0x00050004,
225 0x91a8, 0xffffffff, 0x00010006,
226 0x91ac, 0xffffffff, 0x00090008,
227 0x91b0, 0xffffffff, 0x00070000,
228 0x91b4, 0xffffffff, 0x00030002,
229 0x91b8, 0xffffffff, 0x00050004,
230 0x91c4, 0xffffffff, 0x00010006,
231 0x91c8, 0xffffffff, 0x00090008,
232 0x91cc, 0xffffffff, 0x00070000,
233 0x91d0, 0xffffffff, 0x00030002,
234 0x91d4, 0xffffffff, 0x00050004,
235 0x91e0, 0xffffffff, 0x00010006,
236 0x91e4, 0xffffffff, 0x00090008,
237 0x91e8, 0xffffffff, 0x00000000,
238 0x91ec, 0xffffffff, 0x00070000,
239 0x91f0, 0xffffffff, 0x00030002,
240 0x91f4, 0xffffffff, 0x00050004,
241 0x9200, 0xffffffff, 0x00010006,
242 0x9204, 0xffffffff, 0x00090008,
243 0x9208, 0xffffffff, 0x00070000,
244 0x920c, 0xffffffff, 0x00030002,
245 0x9210, 0xffffffff, 0x00050004,
246 0x921c, 0xffffffff, 0x00010006,
247 0x9220, 0xffffffff, 0x00090008,
248 0x9224, 0xffffffff, 0x00070000,
249 0x9228, 0xffffffff, 0x00030002,
250 0x922c, 0xffffffff, 0x00050004,
251 0x9238, 0xffffffff, 0x00010006,
252 0x923c, 0xffffffff, 0x00090008,
253 0x9240, 0xffffffff, 0x00070000,
254 0x9244, 0xffffffff, 0x00030002,
255 0x9248, 0xffffffff, 0x00050004,
256 0x9254, 0xffffffff, 0x00010006,
257 0x9258, 0xffffffff, 0x00090008,
258 0x925c, 0xffffffff, 0x00070000,
259 0x9260, 0xffffffff, 0x00030002,
260 0x9264, 0xffffffff, 0x00050004,
261 0x9270, 0xffffffff, 0x00010006,
262 0x9274, 0xffffffff, 0x00090008,
263 0x9278, 0xffffffff, 0x00070000,
264 0x927c, 0xffffffff, 0x00030002,
265 0x9280, 0xffffffff, 0x00050004,
266 0x928c, 0xffffffff, 0x00010006,
267 0x9290, 0xffffffff, 0x00090008,
268 0x9294, 0xffffffff, 0x00000000,
269 0x929c, 0xffffffff, 0x00000001,
270 0x802c, 0xffffffff, 0xc0000000
271};
272
273static const u32 redwood_mgcg_init[] =
274{
275 0x802c, 0xffffffff, 0xc0000000,
276 0x5448, 0xffffffff, 0x00000100,
277 0x55e4, 0xffffffff, 0x00000100,
278 0x160c, 0xffffffff, 0x00000100,
279 0x5644, 0xffffffff, 0x00000100,
280 0xc164, 0xffffffff, 0x00000100,
281 0x8a18, 0xffffffff, 0x00000100,
282 0x897c, 0xffffffff, 0x06000100,
283 0x8b28, 0xffffffff, 0x00000100,
284 0x9144, 0xffffffff, 0x00000100,
285 0x9a60, 0xffffffff, 0x00000100,
286 0x9868, 0xffffffff, 0x00000100,
287 0x8d58, 0xffffffff, 0x00000100,
288 0x9510, 0xffffffff, 0x00000100,
289 0x949c, 0xffffffff, 0x00000100,
290 0x9654, 0xffffffff, 0x00000100,
291 0x9030, 0xffffffff, 0x00000100,
292 0x9034, 0xffffffff, 0x00000100,
293 0x9038, 0xffffffff, 0x00000100,
294 0x903c, 0xffffffff, 0x00000100,
295 0x9040, 0xffffffff, 0x00000100,
296 0xa200, 0xffffffff, 0x00000100,
297 0xa204, 0xffffffff, 0x00000100,
298 0xa208, 0xffffffff, 0x00000100,
299 0xa20c, 0xffffffff, 0x00000100,
300 0x971c, 0xffffffff, 0x00000100,
301 0x977c, 0xffffffff, 0x00000100,
302 0x3f80, 0xffffffff, 0x00000100,
303 0xa210, 0xffffffff, 0x00000100,
304 0xa214, 0xffffffff, 0x00000100,
305 0x4d8, 0xffffffff, 0x00000100,
306 0x9784, 0xffffffff, 0x00000100,
307 0x9698, 0xffffffff, 0x00000100,
308 0x4d4, 0xffffffff, 0x00000200,
309 0x30cc, 0xffffffff, 0x00000100,
310 0xd0c0, 0xffffffff, 0xff000100,
311 0x802c, 0xffffffff, 0x40000000,
312 0x915c, 0xffffffff, 0x00010000,
313 0x9160, 0xffffffff, 0x00030002,
314 0x9178, 0xffffffff, 0x00070000,
315 0x917c, 0xffffffff, 0x00030002,
316 0x9180, 0xffffffff, 0x00050004,
317 0x918c, 0xffffffff, 0x00010006,
318 0x9190, 0xffffffff, 0x00090008,
319 0x9194, 0xffffffff, 0x00070000,
320 0x9198, 0xffffffff, 0x00030002,
321 0x919c, 0xffffffff, 0x00050004,
322 0x91a8, 0xffffffff, 0x00010006,
323 0x91ac, 0xffffffff, 0x00090008,
324 0x91b0, 0xffffffff, 0x00070000,
325 0x91b4, 0xffffffff, 0x00030002,
326 0x91b8, 0xffffffff, 0x00050004,
327 0x91c4, 0xffffffff, 0x00010006,
328 0x91c8, 0xffffffff, 0x00090008,
329 0x91cc, 0xffffffff, 0x00070000,
330 0x91d0, 0xffffffff, 0x00030002,
331 0x91d4, 0xffffffff, 0x00050004,
332 0x91e0, 0xffffffff, 0x00010006,
333 0x91e4, 0xffffffff, 0x00090008,
334 0x91e8, 0xffffffff, 0x00000000,
335 0x91ec, 0xffffffff, 0x00070000,
336 0x91f0, 0xffffffff, 0x00030002,
337 0x91f4, 0xffffffff, 0x00050004,
338 0x9200, 0xffffffff, 0x00010006,
339 0x9204, 0xffffffff, 0x00090008,
340 0x9294, 0xffffffff, 0x00000000,
341 0x929c, 0xffffffff, 0x00000001,
342 0x802c, 0xffffffff, 0xc0000000
343};
344
345static const u32 cedar_golden_registers[] =
346{
347 0x3f90, 0xffff0000, 0xff000000,
348 0x9148, 0xffff0000, 0xff000000,
349 0x3f94, 0xffff0000, 0xff000000,
350 0x914c, 0xffff0000, 0xff000000,
351 0x9b7c, 0xffffffff, 0x00000000,
352 0x8a14, 0xffffffff, 0x00000007,
353 0x8b10, 0xffffffff, 0x00000000,
354 0x960c, 0xffffffff, 0x54763210,
355 0x88c4, 0xffffffff, 0x000000c2,
356 0x88d4, 0xffffffff, 0x00000000,
357 0x8974, 0xffffffff, 0x00000000,
358 0xc78, 0x00000080, 0x00000080,
359 0x5eb4, 0xffffffff, 0x00000002,
360 0x5e78, 0xffffffff, 0x001000f0,
361 0x6104, 0x01000300, 0x00000000,
362 0x5bc0, 0x00300000, 0x00000000,
363 0x7030, 0xffffffff, 0x00000011,
364 0x7c30, 0xffffffff, 0x00000011,
365 0x10830, 0xffffffff, 0x00000011,
366 0x11430, 0xffffffff, 0x00000011,
367 0xd02c, 0xffffffff, 0x08421000,
368 0x240c, 0xffffffff, 0x00000380,
369 0x8b24, 0xffffffff, 0x00ff0fff,
370 0x28a4c, 0x06000000, 0x06000000,
371 0x10c, 0x00000001, 0x00000001,
372 0x8d00, 0xffffffff, 0x100e4848,
373 0x8d04, 0xffffffff, 0x00164745,
374 0x8c00, 0xffffffff, 0xe4000003,
375 0x8c04, 0xffffffff, 0x40600060,
376 0x8c08, 0xffffffff, 0x001c001c,
377 0x8cf0, 0xffffffff, 0x08e00410,
378 0x8c20, 0xffffffff, 0x00800080,
379 0x8c24, 0xffffffff, 0x00800080,
380 0x8c18, 0xffffffff, 0x20202078,
381 0x8c1c, 0xffffffff, 0x00001010,
382 0x28350, 0xffffffff, 0x00000000,
383 0xa008, 0xffffffff, 0x00010000,
384 0x5cc, 0xffffffff, 0x00000001,
385 0x9508, 0xffffffff, 0x00000002
386};
387
388static const u32 cedar_mgcg_init[] =
389{
390 0x802c, 0xffffffff, 0xc0000000,
391 0x5448, 0xffffffff, 0x00000100,
392 0x55e4, 0xffffffff, 0x00000100,
393 0x160c, 0xffffffff, 0x00000100,
394 0x5644, 0xffffffff, 0x00000100,
395 0xc164, 0xffffffff, 0x00000100,
396 0x8a18, 0xffffffff, 0x00000100,
397 0x897c, 0xffffffff, 0x06000100,
398 0x8b28, 0xffffffff, 0x00000100,
399 0x9144, 0xffffffff, 0x00000100,
400 0x9a60, 0xffffffff, 0x00000100,
401 0x9868, 0xffffffff, 0x00000100,
402 0x8d58, 0xffffffff, 0x00000100,
403 0x9510, 0xffffffff, 0x00000100,
404 0x949c, 0xffffffff, 0x00000100,
405 0x9654, 0xffffffff, 0x00000100,
406 0x9030, 0xffffffff, 0x00000100,
407 0x9034, 0xffffffff, 0x00000100,
408 0x9038, 0xffffffff, 0x00000100,
409 0x903c, 0xffffffff, 0x00000100,
410 0x9040, 0xffffffff, 0x00000100,
411 0xa200, 0xffffffff, 0x00000100,
412 0xa204, 0xffffffff, 0x00000100,
413 0xa208, 0xffffffff, 0x00000100,
414 0xa20c, 0xffffffff, 0x00000100,
415 0x971c, 0xffffffff, 0x00000100,
416 0x977c, 0xffffffff, 0x00000100,
417 0x3f80, 0xffffffff, 0x00000100,
418 0xa210, 0xffffffff, 0x00000100,
419 0xa214, 0xffffffff, 0x00000100,
420 0x4d8, 0xffffffff, 0x00000100,
421 0x9784, 0xffffffff, 0x00000100,
422 0x9698, 0xffffffff, 0x00000100,
423 0x4d4, 0xffffffff, 0x00000200,
424 0x30cc, 0xffffffff, 0x00000100,
425 0xd0c0, 0xffffffff, 0xff000100,
426 0x802c, 0xffffffff, 0x40000000,
427 0x915c, 0xffffffff, 0x00010000,
428 0x9178, 0xffffffff, 0x00050000,
429 0x917c, 0xffffffff, 0x00030002,
430 0x918c, 0xffffffff, 0x00010004,
431 0x9190, 0xffffffff, 0x00070006,
432 0x9194, 0xffffffff, 0x00050000,
433 0x9198, 0xffffffff, 0x00030002,
434 0x91a8, 0xffffffff, 0x00010004,
435 0x91ac, 0xffffffff, 0x00070006,
436 0x91e8, 0xffffffff, 0x00000000,
437 0x9294, 0xffffffff, 0x00000000,
438 0x929c, 0xffffffff, 0x00000001,
439 0x802c, 0xffffffff, 0xc0000000
440};
441
442static const u32 juniper_mgcg_init[] =
443{
444 0x802c, 0xffffffff, 0xc0000000,
445 0x5448, 0xffffffff, 0x00000100,
446 0x55e4, 0xffffffff, 0x00000100,
447 0x160c, 0xffffffff, 0x00000100,
448 0x5644, 0xffffffff, 0x00000100,
449 0xc164, 0xffffffff, 0x00000100,
450 0x8a18, 0xffffffff, 0x00000100,
451 0x897c, 0xffffffff, 0x06000100,
452 0x8b28, 0xffffffff, 0x00000100,
453 0x9144, 0xffffffff, 0x00000100,
454 0x9a60, 0xffffffff, 0x00000100,
455 0x9868, 0xffffffff, 0x00000100,
456 0x8d58, 0xffffffff, 0x00000100,
457 0x9510, 0xffffffff, 0x00000100,
458 0x949c, 0xffffffff, 0x00000100,
459 0x9654, 0xffffffff, 0x00000100,
460 0x9030, 0xffffffff, 0x00000100,
461 0x9034, 0xffffffff, 0x00000100,
462 0x9038, 0xffffffff, 0x00000100,
463 0x903c, 0xffffffff, 0x00000100,
464 0x9040, 0xffffffff, 0x00000100,
465 0xa200, 0xffffffff, 0x00000100,
466 0xa204, 0xffffffff, 0x00000100,
467 0xa208, 0xffffffff, 0x00000100,
468 0xa20c, 0xffffffff, 0x00000100,
469 0x971c, 0xffffffff, 0x00000100,
470 0xd0c0, 0xffffffff, 0xff000100,
471 0x802c, 0xffffffff, 0x40000000,
472 0x915c, 0xffffffff, 0x00010000,
473 0x9160, 0xffffffff, 0x00030002,
474 0x9178, 0xffffffff, 0x00070000,
475 0x917c, 0xffffffff, 0x00030002,
476 0x9180, 0xffffffff, 0x00050004,
477 0x918c, 0xffffffff, 0x00010006,
478 0x9190, 0xffffffff, 0x00090008,
479 0x9194, 0xffffffff, 0x00070000,
480 0x9198, 0xffffffff, 0x00030002,
481 0x919c, 0xffffffff, 0x00050004,
482 0x91a8, 0xffffffff, 0x00010006,
483 0x91ac, 0xffffffff, 0x00090008,
484 0x91b0, 0xffffffff, 0x00070000,
485 0x91b4, 0xffffffff, 0x00030002,
486 0x91b8, 0xffffffff, 0x00050004,
487 0x91c4, 0xffffffff, 0x00010006,
488 0x91c8, 0xffffffff, 0x00090008,
489 0x91cc, 0xffffffff, 0x00070000,
490 0x91d0, 0xffffffff, 0x00030002,
491 0x91d4, 0xffffffff, 0x00050004,
492 0x91e0, 0xffffffff, 0x00010006,
493 0x91e4, 0xffffffff, 0x00090008,
494 0x91e8, 0xffffffff, 0x00000000,
495 0x91ec, 0xffffffff, 0x00070000,
496 0x91f0, 0xffffffff, 0x00030002,
497 0x91f4, 0xffffffff, 0x00050004,
498 0x9200, 0xffffffff, 0x00010006,
499 0x9204, 0xffffffff, 0x00090008,
500 0x9208, 0xffffffff, 0x00070000,
501 0x920c, 0xffffffff, 0x00030002,
502 0x9210, 0xffffffff, 0x00050004,
503 0x921c, 0xffffffff, 0x00010006,
504 0x9220, 0xffffffff, 0x00090008,
505 0x9224, 0xffffffff, 0x00070000,
506 0x9228, 0xffffffff, 0x00030002,
507 0x922c, 0xffffffff, 0x00050004,
508 0x9238, 0xffffffff, 0x00010006,
509 0x923c, 0xffffffff, 0x00090008,
510 0x9240, 0xffffffff, 0x00070000,
511 0x9244, 0xffffffff, 0x00030002,
512 0x9248, 0xffffffff, 0x00050004,
513 0x9254, 0xffffffff, 0x00010006,
514 0x9258, 0xffffffff, 0x00090008,
515 0x925c, 0xffffffff, 0x00070000,
516 0x9260, 0xffffffff, 0x00030002,
517 0x9264, 0xffffffff, 0x00050004,
518 0x9270, 0xffffffff, 0x00010006,
519 0x9274, 0xffffffff, 0x00090008,
520 0x9278, 0xffffffff, 0x00070000,
521 0x927c, 0xffffffff, 0x00030002,
522 0x9280, 0xffffffff, 0x00050004,
523 0x928c, 0xffffffff, 0x00010006,
524 0x9290, 0xffffffff, 0x00090008,
525 0x9294, 0xffffffff, 0x00000000,
526 0x929c, 0xffffffff, 0x00000001,
527 0x802c, 0xffffffff, 0xc0000000,
528 0x977c, 0xffffffff, 0x00000100,
529 0x3f80, 0xffffffff, 0x00000100,
530 0xa210, 0xffffffff, 0x00000100,
531 0xa214, 0xffffffff, 0x00000100,
532 0x4d8, 0xffffffff, 0x00000100,
533 0x9784, 0xffffffff, 0x00000100,
534 0x9698, 0xffffffff, 0x00000100,
535 0x4d4, 0xffffffff, 0x00000200,
536 0x30cc, 0xffffffff, 0x00000100,
537 0x802c, 0xffffffff, 0xc0000000
538};
539
540static const u32 supersumo_golden_registers[] =
541{
542 0x5eb4, 0xffffffff, 0x00000002,
543 0x5cc, 0xffffffff, 0x00000001,
544 0x7030, 0xffffffff, 0x00000011,
545 0x7c30, 0xffffffff, 0x00000011,
546 0x6104, 0x01000300, 0x00000000,
547 0x5bc0, 0x00300000, 0x00000000,
548 0x8c04, 0xffffffff, 0x40600060,
549 0x8c08, 0xffffffff, 0x001c001c,
550 0x8c20, 0xffffffff, 0x00800080,
551 0x8c24, 0xffffffff, 0x00800080,
552 0x8c18, 0xffffffff, 0x20202078,
553 0x8c1c, 0xffffffff, 0x00001010,
554 0x918c, 0xffffffff, 0x00010006,
555 0x91a8, 0xffffffff, 0x00010006,
556 0x91c4, 0xffffffff, 0x00010006,
557 0x91e0, 0xffffffff, 0x00010006,
558 0x9200, 0xffffffff, 0x00010006,
559 0x9150, 0xffffffff, 0x6e944040,
560 0x917c, 0xffffffff, 0x00030002,
561 0x9180, 0xffffffff, 0x00050004,
562 0x9198, 0xffffffff, 0x00030002,
563 0x919c, 0xffffffff, 0x00050004,
564 0x91b4, 0xffffffff, 0x00030002,
565 0x91b8, 0xffffffff, 0x00050004,
566 0x91d0, 0xffffffff, 0x00030002,
567 0x91d4, 0xffffffff, 0x00050004,
568 0x91f0, 0xffffffff, 0x00030002,
569 0x91f4, 0xffffffff, 0x00050004,
570 0x915c, 0xffffffff, 0x00010000,
571 0x9160, 0xffffffff, 0x00030002,
572 0x3f90, 0xffff0000, 0xff000000,
573 0x9178, 0xffffffff, 0x00070000,
574 0x9194, 0xffffffff, 0x00070000,
575 0x91b0, 0xffffffff, 0x00070000,
576 0x91cc, 0xffffffff, 0x00070000,
577 0x91ec, 0xffffffff, 0x00070000,
578 0x9148, 0xffff0000, 0xff000000,
579 0x9190, 0xffffffff, 0x00090008,
580 0x91ac, 0xffffffff, 0x00090008,
581 0x91c8, 0xffffffff, 0x00090008,
582 0x91e4, 0xffffffff, 0x00090008,
583 0x9204, 0xffffffff, 0x00090008,
584 0x3f94, 0xffff0000, 0xff000000,
585 0x914c, 0xffff0000, 0xff000000,
586 0x929c, 0xffffffff, 0x00000001,
587 0x8a18, 0xffffffff, 0x00000100,
588 0x8b28, 0xffffffff, 0x00000100,
589 0x9144, 0xffffffff, 0x00000100,
590 0x5644, 0xffffffff, 0x00000100,
591 0x9b7c, 0xffffffff, 0x00000000,
592 0x8030, 0xffffffff, 0x0000100a,
593 0x8a14, 0xffffffff, 0x00000007,
594 0x8b24, 0xffffffff, 0x00ff0fff,
595 0x8b10, 0xffffffff, 0x00000000,
596 0x28a4c, 0x06000000, 0x06000000,
597 0x4d8, 0xffffffff, 0x00000100,
598 0x913c, 0xffff000f, 0x0100000a,
599 0x960c, 0xffffffff, 0x54763210,
600 0x88c4, 0xffffffff, 0x000000c2,
601 0x88d4, 0xffffffff, 0x00000010,
602 0x8974, 0xffffffff, 0x00000000,
603 0xc78, 0x00000080, 0x00000080,
604 0x5e78, 0xffffffff, 0x001000f0,
605 0xd02c, 0xffffffff, 0x08421000,
606 0xa008, 0xffffffff, 0x00010000,
607 0x8d00, 0xffffffff, 0x100e4848,
608 0x8d04, 0xffffffff, 0x00164745,
609 0x8c00, 0xffffffff, 0xe4000003,
610 0x8cf0, 0x1fffffff, 0x08e00620,
611 0x28350, 0xffffffff, 0x00000000,
612 0x9508, 0xffffffff, 0x00000002
613};
614
615static const u32 sumo_golden_registers[] =
616{
617 0x900c, 0x00ffffff, 0x0017071f,
618 0x8c18, 0xffffffff, 0x10101060,
619 0x8c1c, 0xffffffff, 0x00001010,
620 0x8c30, 0x0000000f, 0x00000005,
621 0x9688, 0x0000000f, 0x00000007
622};
623
624static const u32 wrestler_golden_registers[] =
625{
626 0x5eb4, 0xffffffff, 0x00000002,
627 0x5cc, 0xffffffff, 0x00000001,
628 0x7030, 0xffffffff, 0x00000011,
629 0x7c30, 0xffffffff, 0x00000011,
630 0x6104, 0x01000300, 0x00000000,
631 0x5bc0, 0x00300000, 0x00000000,
632 0x918c, 0xffffffff, 0x00010006,
633 0x91a8, 0xffffffff, 0x00010006,
634 0x9150, 0xffffffff, 0x6e944040,
635 0x917c, 0xffffffff, 0x00030002,
636 0x9198, 0xffffffff, 0x00030002,
637 0x915c, 0xffffffff, 0x00010000,
638 0x3f90, 0xffff0000, 0xff000000,
639 0x9178, 0xffffffff, 0x00070000,
640 0x9194, 0xffffffff, 0x00070000,
641 0x9148, 0xffff0000, 0xff000000,
642 0x9190, 0xffffffff, 0x00090008,
643 0x91ac, 0xffffffff, 0x00090008,
644 0x3f94, 0xffff0000, 0xff000000,
645 0x914c, 0xffff0000, 0xff000000,
646 0x929c, 0xffffffff, 0x00000001,
647 0x8a18, 0xffffffff, 0x00000100,
648 0x8b28, 0xffffffff, 0x00000100,
649 0x9144, 0xffffffff, 0x00000100,
650 0x9b7c, 0xffffffff, 0x00000000,
651 0x8030, 0xffffffff, 0x0000100a,
652 0x8a14, 0xffffffff, 0x00000001,
653 0x8b24, 0xffffffff, 0x00ff0fff,
654 0x8b10, 0xffffffff, 0x00000000,
655 0x28a4c, 0x06000000, 0x06000000,
656 0x4d8, 0xffffffff, 0x00000100,
657 0x913c, 0xffff000f, 0x0100000a,
658 0x960c, 0xffffffff, 0x54763210,
659 0x88c4, 0xffffffff, 0x000000c2,
660 0x88d4, 0xffffffff, 0x00000010,
661 0x8974, 0xffffffff, 0x00000000,
662 0xc78, 0x00000080, 0x00000080,
663 0x5e78, 0xffffffff, 0x001000f0,
664 0xd02c, 0xffffffff, 0x08421000,
665 0xa008, 0xffffffff, 0x00010000,
666 0x8d00, 0xffffffff, 0x100e4848,
667 0x8d04, 0xffffffff, 0x00164745,
668 0x8c00, 0xffffffff, 0xe4000003,
669 0x8cf0, 0x1fffffff, 0x08e00410,
670 0x28350, 0xffffffff, 0x00000000,
671 0x9508, 0xffffffff, 0x00000002,
672 0x900c, 0xffffffff, 0x0017071f,
673 0x8c18, 0xffffffff, 0x10101060,
674 0x8c1c, 0xffffffff, 0x00001010
675};
676
677static const u32 barts_golden_registers[] =
678{
679 0x5eb4, 0xffffffff, 0x00000002,
680 0x5e78, 0x8f311ff1, 0x001000f0,
681 0x3f90, 0xffff0000, 0xff000000,
682 0x9148, 0xffff0000, 0xff000000,
683 0x3f94, 0xffff0000, 0xff000000,
684 0x914c, 0xffff0000, 0xff000000,
685 0xc78, 0x00000080, 0x00000080,
686 0xbd4, 0x70073777, 0x00010001,
687 0xd02c, 0xbfffff1f, 0x08421000,
688 0xd0b8, 0x03773777, 0x02011003,
689 0x5bc0, 0x00200000, 0x50100000,
690 0x98f8, 0x33773777, 0x02011003,
691 0x98fc, 0xffffffff, 0x76543210,
692 0x7030, 0x31000311, 0x00000011,
693 0x2f48, 0x00000007, 0x02011003,
694 0x6b28, 0x00000010, 0x00000012,
695 0x7728, 0x00000010, 0x00000012,
696 0x10328, 0x00000010, 0x00000012,
697 0x10f28, 0x00000010, 0x00000012,
698 0x11b28, 0x00000010, 0x00000012,
699 0x12728, 0x00000010, 0x00000012,
700 0x240c, 0x000007ff, 0x00000380,
701 0x8a14, 0xf000001f, 0x00000007,
702 0x8b24, 0x3fff3fff, 0x00ff0fff,
703 0x8b10, 0x0000ff0f, 0x00000000,
704 0x28a4c, 0x07ffffff, 0x06000000,
705 0x10c, 0x00000001, 0x00010003,
706 0xa02c, 0xffffffff, 0x0000009b,
707 0x913c, 0x0000000f, 0x0100000a,
708 0x8d00, 0xffff7f7f, 0x100e4848,
709 0x8d04, 0x00ffffff, 0x00164745,
710 0x8c00, 0xfffc0003, 0xe4000003,
711 0x8c04, 0xf8ff00ff, 0x40600060,
712 0x8c08, 0x00ff00ff, 0x001c001c,
713 0x8cf0, 0x1fff1fff, 0x08e00620,
714 0x8c20, 0x0fff0fff, 0x00800080,
715 0x8c24, 0x0fff0fff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0x0000ffff, 0x00001010,
718 0x28350, 0x00000f01, 0x00000000,
719 0x9508, 0x3700001f, 0x00000002,
720 0x960c, 0xffffffff, 0x54763210,
721 0x88c4, 0x001f3ae3, 0x000000c2,
722 0x88d4, 0x0000001f, 0x00000010,
723 0x8974, 0xffffffff, 0x00000000
724};
725
726static const u32 turks_golden_registers[] =
727{
728 0x5eb4, 0xffffffff, 0x00000002,
729 0x5e78, 0x8f311ff1, 0x001000f0,
730 0x8c8, 0x00003000, 0x00001070,
731 0x8cc, 0x000fffff, 0x00040035,
732 0x3f90, 0xffff0000, 0xfff00000,
733 0x9148, 0xffff0000, 0xfff00000,
734 0x3f94, 0xffff0000, 0xfff00000,
735 0x914c, 0xffff0000, 0xfff00000,
736 0xc78, 0x00000080, 0x00000080,
737 0xbd4, 0x00073007, 0x00010002,
738 0xd02c, 0xbfffff1f, 0x08421000,
739 0xd0b8, 0x03773777, 0x02010002,
740 0x5bc0, 0x00200000, 0x50100000,
741 0x98f8, 0x33773777, 0x00010002,
742 0x98fc, 0xffffffff, 0x33221100,
743 0x7030, 0x31000311, 0x00000011,
744 0x2f48, 0x33773777, 0x00010002,
745 0x6b28, 0x00000010, 0x00000012,
746 0x7728, 0x00000010, 0x00000012,
747 0x10328, 0x00000010, 0x00000012,
748 0x10f28, 0x00000010, 0x00000012,
749 0x11b28, 0x00000010, 0x00000012,
750 0x12728, 0x00000010, 0x00000012,
751 0x240c, 0x000007ff, 0x00000380,
752 0x8a14, 0xf000001f, 0x00000007,
753 0x8b24, 0x3fff3fff, 0x00ff0fff,
754 0x8b10, 0x0000ff0f, 0x00000000,
755 0x28a4c, 0x07ffffff, 0x06000000,
756 0x10c, 0x00000001, 0x00010003,
757 0xa02c, 0xffffffff, 0x0000009b,
758 0x913c, 0x0000000f, 0x0100000a,
759 0x8d00, 0xffff7f7f, 0x100e4848,
760 0x8d04, 0x00ffffff, 0x00164745,
761 0x8c00, 0xfffc0003, 0xe4000003,
762 0x8c04, 0xf8ff00ff, 0x40600060,
763 0x8c08, 0x00ff00ff, 0x001c001c,
764 0x8cf0, 0x1fff1fff, 0x08e00410,
765 0x8c20, 0x0fff0fff, 0x00800080,
766 0x8c24, 0x0fff0fff, 0x00800080,
767 0x8c18, 0xffffffff, 0x20202078,
768 0x8c1c, 0x0000ffff, 0x00001010,
769 0x28350, 0x00000f01, 0x00000000,
770 0x9508, 0x3700001f, 0x00000002,
771 0x960c, 0xffffffff, 0x54763210,
772 0x88c4, 0x001f3ae3, 0x000000c2,
773 0x88d4, 0x0000001f, 0x00000010,
774 0x8974, 0xffffffff, 0x00000000
775};
776
777static const u32 caicos_golden_registers[] =
778{
779 0x5eb4, 0xffffffff, 0x00000002,
780 0x5e78, 0x8f311ff1, 0x001000f0,
781 0x8c8, 0x00003420, 0x00001450,
782 0x8cc, 0x000fffff, 0x00040035,
783 0x3f90, 0xffff0000, 0xfffc0000,
784 0x9148, 0xffff0000, 0xfffc0000,
785 0x3f94, 0xffff0000, 0xfffc0000,
786 0x914c, 0xffff0000, 0xfffc0000,
787 0xc78, 0x00000080, 0x00000080,
788 0xbd4, 0x00073007, 0x00010001,
789 0xd02c, 0xbfffff1f, 0x08421000,
790 0xd0b8, 0x03773777, 0x02010001,
791 0x5bc0, 0x00200000, 0x50100000,
792 0x98f8, 0x33773777, 0x02010001,
793 0x98fc, 0xffffffff, 0x33221100,
794 0x7030, 0x31000311, 0x00000011,
795 0x2f48, 0x33773777, 0x02010001,
796 0x6b28, 0x00000010, 0x00000012,
797 0x7728, 0x00000010, 0x00000012,
798 0x10328, 0x00000010, 0x00000012,
799 0x10f28, 0x00000010, 0x00000012,
800 0x11b28, 0x00000010, 0x00000012,
801 0x12728, 0x00000010, 0x00000012,
802 0x240c, 0x000007ff, 0x00000380,
803 0x8a14, 0xf000001f, 0x00000001,
804 0x8b24, 0x3fff3fff, 0x00ff0fff,
805 0x8b10, 0x0000ff0f, 0x00000000,
806 0x28a4c, 0x07ffffff, 0x06000000,
807 0x10c, 0x00000001, 0x00010003,
808 0xa02c, 0xffffffff, 0x0000009b,
809 0x913c, 0x0000000f, 0x0100000a,
810 0x8d00, 0xffff7f7f, 0x100e4848,
811 0x8d04, 0x00ffffff, 0x00164745,
812 0x8c00, 0xfffc0003, 0xe4000003,
813 0x8c04, 0xf8ff00ff, 0x40600060,
814 0x8c08, 0x00ff00ff, 0x001c001c,
815 0x8cf0, 0x1fff1fff, 0x08e00410,
816 0x8c20, 0x0fff0fff, 0x00800080,
817 0x8c24, 0x0fff0fff, 0x00800080,
818 0x8c18, 0xffffffff, 0x20202078,
819 0x8c1c, 0x0000ffff, 0x00001010,
820 0x28350, 0x00000f01, 0x00000000,
821 0x9508, 0x3700001f, 0x00000002,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0x001f3ae3, 0x000000c2,
824 0x88d4, 0x0000001f, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000
826};
827
828static void evergreen_init_golden_registers(struct radeon_device *rdev)
829{
830 switch (rdev->family) {
831 case CHIP_CYPRESS:
832 case CHIP_HEMLOCK:
833 radeon_program_register_sequence(rdev,
834 evergreen_golden_registers,
835 (const u32)ARRAY_SIZE(evergreen_golden_registers));
836 radeon_program_register_sequence(rdev,
837 evergreen_golden_registers2,
838 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
839 radeon_program_register_sequence(rdev,
840 cypress_mgcg_init,
841 (const u32)ARRAY_SIZE(cypress_mgcg_init));
842 break;
843 case CHIP_JUNIPER:
844 radeon_program_register_sequence(rdev,
845 evergreen_golden_registers,
846 (const u32)ARRAY_SIZE(evergreen_golden_registers));
847 radeon_program_register_sequence(rdev,
848 evergreen_golden_registers2,
849 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
850 radeon_program_register_sequence(rdev,
851 juniper_mgcg_init,
852 (const u32)ARRAY_SIZE(juniper_mgcg_init));
853 break;
854 case CHIP_REDWOOD:
855 radeon_program_register_sequence(rdev,
856 evergreen_golden_registers,
857 (const u32)ARRAY_SIZE(evergreen_golden_registers));
858 radeon_program_register_sequence(rdev,
859 evergreen_golden_registers2,
860 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
861 radeon_program_register_sequence(rdev,
862 redwood_mgcg_init,
863 (const u32)ARRAY_SIZE(redwood_mgcg_init));
864 break;
865 case CHIP_CEDAR:
866 radeon_program_register_sequence(rdev,
867 cedar_golden_registers,
868 (const u32)ARRAY_SIZE(cedar_golden_registers));
869 radeon_program_register_sequence(rdev,
870 evergreen_golden_registers2,
871 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
872 radeon_program_register_sequence(rdev,
873 cedar_mgcg_init,
874 (const u32)ARRAY_SIZE(cedar_mgcg_init));
875 break;
876 case CHIP_PALM:
877 radeon_program_register_sequence(rdev,
878 wrestler_golden_registers,
879 (const u32)ARRAY_SIZE(wrestler_golden_registers));
880 break;
881 case CHIP_SUMO:
882 radeon_program_register_sequence(rdev,
883 supersumo_golden_registers,
884 (const u32)ARRAY_SIZE(supersumo_golden_registers));
885 break;
886 case CHIP_SUMO2:
887 radeon_program_register_sequence(rdev,
888 supersumo_golden_registers,
889 (const u32)ARRAY_SIZE(supersumo_golden_registers));
890 radeon_program_register_sequence(rdev,
891 sumo_golden_registers,
892 (const u32)ARRAY_SIZE(sumo_golden_registers));
893 break;
894 case CHIP_BARTS:
895 radeon_program_register_sequence(rdev,
896 barts_golden_registers,
897 (const u32)ARRAY_SIZE(barts_golden_registers));
898 break;
899 case CHIP_TURKS:
900 radeon_program_register_sequence(rdev,
901 turks_golden_registers,
902 (const u32)ARRAY_SIZE(turks_golden_registers));
903 break;
904 case CHIP_CAICOS:
905 radeon_program_register_sequence(rdev,
906 caicos_golden_registers,
907 (const u32)ARRAY_SIZE(caicos_golden_registers));
908 break;
909 default:
910 break;
911 }
912}
913
285484e2
JG		 914void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
915 unsigned *bankh, unsigned *mtaspect,
916 unsigned *tile_split)
917{
918 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
919 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
920 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
921 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
922 switch (*bankw) {
923 default:
924 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
925 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
926 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
927 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
928 }
929 switch (*bankh) {
930 default:
931 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
932 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
933 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
934 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
935 }
936 switch (*mtaspect) {
937 default:
938 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
939 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
940 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
941 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
942 }
943}
944
23d33ba3
AD		 945static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
946 u32 cntl_reg, u32 status_reg)
947{
948 int r, i;
949 struct atom_clock_dividers dividers;
950
951 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
952 clock, false, &dividers);
953 if (r)
954 return r;
955
956 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
957
958 for (i = 0; i < 100; i++) {
959 if (RREG32(status_reg) & DCLK_STATUS)
960 break;
961 mdelay(10);
962 }
963 if (i == 100)
964 return -ETIMEDOUT;
965
966 return 0;
967}
968
969int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
970{
971 int r = 0;
972 u32 cg_scratch = RREG32(CG_SCRATCH1);
973
974 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
975 if (r)
976 goto done;
977 cg_scratch &= 0xffff0000;
978 cg_scratch |= vclk / 100; /* Mhz */
979
980 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
981 if (r)
982 goto done;
983 cg_scratch &= 0x0000ffff;
984 cg_scratch |= (dclk / 100) << 16; /* Mhz */
985
986done:
987 WREG32(CG_SCRATCH1, cg_scratch);
988
989 return r;
990}
991
a8b4925c
AD		 992static int evergreen_uvd_calc_post_div(unsigned target_freq,
993 unsigned vco_freq,
994 unsigned *div)
995{
996 /* target larger than vco frequency ? */
997 if (vco_freq < target_freq)
998 return -1; /* forget it */
999
1000 /* Fclk = Fvco / PDIV */
1001 *div = vco_freq / target_freq;
1002
1003 /* we alway need a frequency less than or equal the target */
1004 if ((vco_freq / *div) > target_freq)
1005 *div += 1;
1006
1007 /* dividers above 5 must be even */
1008 if (*div > 5 && *div % 2)
1009 *div += 1;
1010
1011 /* out of range ? */
1012 if (*div >= 128)
1013 return -1; /* forget it */
1014
1015 return vco_freq / *div;
1016}
1017
1018static int evergreen_uvd_send_upll_ctlreq(struct radeon_device *rdev)
1019{
1020 unsigned i;
1021
1022 /* assert UPLL_CTLREQ */
1023 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1024
1025 /* wait for CTLACK and CTLACK2 to get asserted */
1026 for (i = 0; i < 100; ++i) {
1027 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1028 if ((RREG32(CG_UPLL_FUNC_CNTL) & mask) == mask)
1029 break;
1030 mdelay(10);
1031 }
1032 if (i == 100)
1033 return -ETIMEDOUT;
1034
1035 /* deassert UPLL_CTLREQ */
1036 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1037
1038 return 0;
1039}
1040
1041int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1042{
1043 /* start off with something large */
1044 int optimal_diff_score = 0x7FFFFFF;
1045 unsigned optimal_fb_div = 0, optimal_vclk_div = 0;
1046 unsigned optimal_dclk_div = 0, optimal_vco_freq = 0;
1047 unsigned vco_freq;
1048 int r;
1049
4ed10835
CK		 1050 /* bypass vclk and dclk with bclk */
1051 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1052 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1053 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1054
1055 /* put PLL in bypass mode */
1056 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1057
1058 if (!vclk || !dclk) {
1059 /* keep the Bypass mode, put PLL to sleep */
1060 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1061 return 0;
1062 }
1063
a8b4925c
AD		 1064 /* loop through vco from low to high */
1065 for (vco_freq = 125000; vco_freq <= 250000; vco_freq += 100) {
1066 unsigned fb_div = vco_freq / rdev->clock.spll.reference_freq * 16384;
1067 int calc_clk, diff_score, diff_vclk, diff_dclk;
1068 unsigned vclk_div, dclk_div;
1069
1070 /* fb div out of range ? */
1071 if (fb_div > 0x03FFFFFF)
1072 break; /* it can oly get worse */
1073
1074 /* calc vclk with current vco freq. */
1075 calc_clk = evergreen_uvd_calc_post_div(vclk, vco_freq, &vclk_div);
1076 if (calc_clk == -1)
1077 break; /* vco is too big, it has to stop. */
1078 diff_vclk = vclk - calc_clk;
1079
1080 /* calc dclk with current vco freq. */
1081 calc_clk = evergreen_uvd_calc_post_div(dclk, vco_freq, &dclk_div);
1082 if (calc_clk == -1)
1083 break; /* vco is too big, it has to stop. */
1084 diff_dclk = dclk - calc_clk;
1085
1086 /* determine if this vco setting is better than current optimal settings */
1087 diff_score = abs(diff_vclk) + abs(diff_dclk);
1088 if (diff_score < optimal_diff_score) {
1089 optimal_fb_div = fb_div;
1090 optimal_vclk_div = vclk_div;
1091 optimal_dclk_div = dclk_div;
1092 optimal_vco_freq = vco_freq;
1093 optimal_diff_score = diff_score;
1094 if (optimal_diff_score == 0)
1095 break; /* it can't get better than this */
1096 }
1097 }
1098
1099 /* set VCO_MODE to 1 */
1100 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1101
1102 /* toggle UPLL_SLEEP to 1 then back to 0 */
1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1104 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1105
1106 /* deassert UPLL_RESET */
1107 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1108
1109 mdelay(1);
1110
a8b4925c
AD		 1111 r = evergreen_uvd_send_upll_ctlreq(rdev);
1112 if (r)
1113 return r;
1114
1115 /* assert UPLL_RESET again */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1117
1118 /* disable spread spectrum. */
1119 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1120
1121 /* set feedback divider */
1122 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(optimal_fb_div), ~UPLL_FB_DIV_MASK);
1123
1124 /* set ref divider to 0 */
1125 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1126
1127 if (optimal_vco_freq < 187500)
1128 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1129 else
1130 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1131
1132 /* set PDIV_A and PDIV_B */
1133 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1134 UPLL_PDIV_A(optimal_vclk_div) | UPLL_PDIV_B(optimal_dclk_div),
1135 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1136
1137 /* give the PLL some time to settle */
1138 mdelay(15);
1139
1140 /* deassert PLL_RESET */
1141 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1142
1143 mdelay(15);
1144
1145 /* switch from bypass mode to normal mode */
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1147
1148 r = evergreen_uvd_send_upll_ctlreq(rdev);
1149 if (r)
1150 return r;
1151
1152 /* switch VCLK and DCLK selection */
1153 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1154 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1155 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1156
1157 mdelay(100);
1158
1159 return 0;
1160}
1161
d054ac16
AD		 1162void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1163{
1164 u16 ctl, v;
32195aec 1165 int err;
d054ac16 1166
32195aec 1167 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1168 if (err)
1169 return;
1170
1171 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1172
1173 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1174 * to avoid hangs or perfomance issues
1175 */
1176 if ((v == 0) || (v == 6) || (v == 7)) {
1177 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1178 ctl |= (2 << 12);
32195aec 1179 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1180 }
1181}
1182
10257a6d
AD		 1183static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1184{
1185 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1186 return true;
1187 else
1188 return false;
1189}
1190
1191static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1192{
1193 u32 pos1, pos2;
1194
1195 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1196 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1197
1198 if (pos1 != pos2)
1199 return true;
1200 else
1201 return false;
1202}
1203
377edc8b
AD		 1204/**
1205 * dce4_wait_for_vblank - vblank wait asic callback.
1206 *
1207 * @rdev: radeon_device pointer
1208 * @crtc: crtc to wait for vblank on
1209 *
1210 * Wait for vblank on the requested crtc (evergreen+).
1211 */
3ae19b75
AD		 1212void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1213{
10257a6d 1214 unsigned i = 0;
3ae19b75 1215
4a15903d
AD		 1216 if (crtc >= rdev->num_crtc)
1217 return;
1218
10257a6d
AD		 1219 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1220 return;
1221
1222 /* depending on when we hit vblank, we may be close to active; if so,
1223 * wait for another frame.
1224 */
1225 while (dce4_is_in_vblank(rdev, crtc)) {
1226 if (i++ % 100 == 0) {
1227 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1228 break;
3ae19b75 1229 }
10257a6d
AD		 1230 }
1231
1232 while (!dce4_is_in_vblank(rdev, crtc)) {
1233 if (i++ % 100 == 0) {
1234 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1235 break;
3ae19b75
AD		 1236 }
1237 }
1238}
1239
377edc8b
AD		 1240/**
1241 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1242 *
1243 * @rdev: radeon_device pointer
1244 * @crtc: crtc to prepare for pageflip on
1245 *
1246 * Pre-pageflip callback (evergreen+).
1247 * Enables the pageflip irq (vblank irq).
1248 */
6f34be50
AD		 1249void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1250{
6f34be50
AD		 1251 /* enable the pflip int */
1252 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1253}
1254
377edc8b
AD		 1255/**
1256 * evergreen_post_page_flip - pos-pageflip callback.
1257 *
1258 * @rdev: radeon_device pointer
1259 * @crtc: crtc to cleanup pageflip on
1260 *
1261 * Post-pageflip callback (evergreen+).
1262 * Disables the pageflip irq (vblank irq).
1263 */
6f34be50
AD		 1264void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1265{
1266 /* disable the pflip int */
1267 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1268}
1269
377edc8b
AD		 1270/**
1271 * evergreen_page_flip - pageflip callback.
1272 *
1273 * @rdev: radeon_device pointer
1274 * @crtc_id: crtc to cleanup pageflip on
1275 * @crtc_base: new address of the crtc (GPU MC address)
1276 *
1277 * Does the actual pageflip (evergreen+).
1278 * During vblank we take the crtc lock and wait for the update_pending
1279 * bit to go high, when it does, we release the lock, and allow the
1280 * double buffered update to take place.
1281 * Returns the current update pending status.
1282 */
6f34be50
AD		 1283u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1284{
1285 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1286 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1287 int i;
6f34be50
AD		 1288
1289 /* Lock the graphics update lock */
1290 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1291 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1292
1293 /* update the scanout addresses */
1294 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1295 upper_32_bits(crtc_base));
1296 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1297 (u32)crtc_base);
1298
1299 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1300 upper_32_bits(crtc_base));
1301 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1302 (u32)crtc_base);
1303
1304 /* Wait for update_pending to go high. */
f6496479
AD		 1305 for (i = 0; i < rdev->usec_timeout; i++) {
1306 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1307 break;
1308 udelay(1);
1309 }
6f34be50
AD		 1310 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1311
1312 /* Unlock the lock, so double-buffering can take place inside vblank */
1313 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1314 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1315
1316 /* Return current update_pending status: */
1317 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1318}
1319
21a8122a 1320/* get temperature in millidegrees */
20d391d7 1321int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1322{
1c88d74f
AD		 1323 u32 temp, toffset;
1324 int actual_temp = 0;
67b3f823
AD		 1325
1326 if (rdev->family == CHIP_JUNIPER) {
1327 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1328 TOFFSET_SHIFT;
1329 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1330 TS0_ADC_DOUT_SHIFT;
1331
1332 if (toffset & 0x100)
1333 actual_temp = temp / 2 - (0x200 - toffset);
1334 else
1335 actual_temp = temp / 2 + toffset;
1336
1337 actual_temp = actual_temp * 1000;
1338
1339 } else {
1340 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1341 ASIC_T_SHIFT;
1342
1343 if (temp & 0x400)
1344 actual_temp = -256;
1345 else if (temp & 0x200)
1346 actual_temp = 255;
1347 else if (temp & 0x100) {
1348 actual_temp = temp & 0x1ff;
1349 actual_temp |= ~0x1ff;
1350 } else
1351 actual_temp = temp & 0xff;
1352
1353 actual_temp = (actual_temp * 1000) / 2;
1354 }
21a8122a 1355
67b3f823 1356 return actual_temp;
21a8122a
AD		 1357}
1358
20d391d7 1359int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1360{
1361 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1362 int actual_temp = temp - 49;
e33df25f
AD		 1363
1364 return actual_temp * 1000;
1365}
1366
377edc8b
AD		 1367/**
1368 * sumo_pm_init_profile - Initialize power profiles callback.
1369 *
1370 * @rdev: radeon_device pointer
1371 *
1372 * Initialize the power states used in profile mode
1373 * (sumo, trinity, SI).
1374 * Used for profile mode only.
1375 */
a4c9e2ee
AD		 1376void sumo_pm_init_profile(struct radeon_device *rdev)
1377{
1378 int idx;
1379
1380 /* default */
1381 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1382 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1383 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1384 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1385
1386 /* low,mid sh/mh */
1387 if (rdev->flags & RADEON_IS_MOBILITY)
1388 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1389 else
1390 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1391
1392 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1393 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1394 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1395 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1396
1397 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1398 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1400 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1401
1402 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1403 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1406
1407 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1408 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1411
1412 /* high sh/mh */
1413 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1414 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1415 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1417 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1418 rdev->pm.power_state[idx].num_clock_modes - 1;
1419
1420 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1424 rdev->pm.power_state[idx].num_clock_modes - 1;
1425}
1426
27810fb2
AD		 1427/**
1428 * btc_pm_init_profile - Initialize power profiles callback.
1429 *
1430 * @rdev: radeon_device pointer
1431 *
1432 * Initialize the power states used in profile mode
1433 * (BTC, cayman).
1434 * Used for profile mode only.
1435 */
1436void btc_pm_init_profile(struct radeon_device *rdev)
1437{
1438 int idx;
1439
1440 /* default */
1441 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1442 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1443 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1444 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1445 /* starting with BTC, there is one state that is used for both
1446 * MH and SH. Difference is that we always use the high clock index for
1447 * mclk.
1448 */
1449 if (rdev->flags & RADEON_IS_MOBILITY)
1450 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1451 else
1452 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1453 /* low sh */
1454 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1457 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1458 /* mid sh */
1459 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1463 /* high sh */
1464 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1468 /* low mh */
1469 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1473 /* mid mh */
1474 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1477 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1478 /* high mh */
1479 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1480 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1483}
1484
377edc8b
AD		 1485/**
1486 * evergreen_pm_misc - set additional pm hw parameters callback.
1487 *
1488 * @rdev: radeon_device pointer
1489 *
1490 * Set non-clock parameters associated with a power state
1491 * (voltage, etc.) (evergreen+).
1492 */
49e02b73
AD		 1493void evergreen_pm_misc(struct radeon_device *rdev)
1494{
a081a9d6
RM		 1495 int req_ps_idx = rdev->pm.requested_power_state_index;
1496 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1497 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1498 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1499
2feea49a 1500 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 1501 /* 0xff01 is a flag rather then an actual voltage */
1502 if (voltage->voltage == 0xff01)
1503 return;
2feea49a 1504 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1505 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1506 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1507 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1508 }
7ae764b1
AD		 1509
1510 /* starting with BTC, there is one state that is used for both
1511 * MH and SH. Difference is that we always use the high clock index for
1512 * mclk and vddci.
1513 */
1514 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1515 (rdev->family >= CHIP_BARTS) &&
1516 rdev->pm.active_crtc_count &&
1517 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1518 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1519 voltage = &rdev->pm.power_state[req_ps_idx].
1520 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1521
a377e187
AD		 1522 /* 0xff01 is a flag rather then an actual voltage */
1523 if (voltage->vddci == 0xff01)
1524 return;
2feea49a
AD		 1525 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1526 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1527 rdev->pm.current_vddci = voltage->vddci;
1528 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1529 }
1530 }
49e02b73
AD		 1531}
1532
377edc8b
AD		 1533/**
1534 * evergreen_pm_prepare - pre-power state change callback.
1535 *
1536 * @rdev: radeon_device pointer
1537 *
1538 * Prepare for a power state change (evergreen+).
1539 */
49e02b73
AD		 1540void evergreen_pm_prepare(struct radeon_device *rdev)
1541{
1542 struct drm_device *ddev = rdev->ddev;
1543 struct drm_crtc *crtc;
1544 struct radeon_crtc *radeon_crtc;
1545 u32 tmp;
1546
1547 /* disable any active CRTCs */
1548 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1549 radeon_crtc = to_radeon_crtc(crtc);
1550 if (radeon_crtc->enabled) {
1551 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1552 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1553 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1554 }
1555 }
1556}
1557
377edc8b
AD		 1558/**
1559 * evergreen_pm_finish - post-power state change callback.
1560 *
1561 * @rdev: radeon_device pointer
1562 *
1563 * Clean up after a power state change (evergreen+).
1564 */
49e02b73
AD		 1565void evergreen_pm_finish(struct radeon_device *rdev)
1566{
1567 struct drm_device *ddev = rdev->ddev;
1568 struct drm_crtc *crtc;
1569 struct radeon_crtc *radeon_crtc;
1570 u32 tmp;
1571
1572 /* enable any active CRTCs */
1573 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1574 radeon_crtc = to_radeon_crtc(crtc);
1575 if (radeon_crtc->enabled) {
1576 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1577 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1578 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1579 }
1580 }
1581}
1582
377edc8b
AD		 1583/**
1584 * evergreen_hpd_sense - hpd sense callback.
1585 *
1586 * @rdev: radeon_device pointer
1587 * @hpd: hpd (hotplug detect) pin
1588 *
1589 * Checks if a digital monitor is connected (evergreen+).
1590 * Returns true if connected, false if not connected.
1591 */
bcc1c2a1
AD		 1592bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1593{
1594 bool connected = false;
0ca2ab52
AD		 1595
1596 switch (hpd) {
1597 case RADEON_HPD_1:
1598 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1599 connected = true;
1600 break;
1601 case RADEON_HPD_2:
1602 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1603 connected = true;
1604 break;
1605 case RADEON_HPD_3:
1606 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1607 connected = true;
1608 break;
1609 case RADEON_HPD_4:
1610 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1611 connected = true;
1612 break;
1613 case RADEON_HPD_5:
1614 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1615 connected = true;
1616 break;
1617 case RADEON_HPD_6:
1618 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1619 connected = true;
1620 break;
1621 default:
1622 break;
1623 }
1624
bcc1c2a1
AD		 1625 return connected;
1626}
1627
377edc8b
AD		 1628/**
1629 * evergreen_hpd_set_polarity - hpd set polarity callback.
1630 *
1631 * @rdev: radeon_device pointer
1632 * @hpd: hpd (hotplug detect) pin
1633 *
1634 * Set the polarity of the hpd pin (evergreen+).
1635 */
bcc1c2a1
AD		 1636void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1637 enum radeon_hpd_id hpd)
1638{
0ca2ab52
AD		 1639 u32 tmp;
1640 bool connected = evergreen_hpd_sense(rdev, hpd);
1641
1642 switch (hpd) {
1643 case RADEON_HPD_1:
1644 tmp = RREG32(DC_HPD1_INT_CONTROL);
1645 if (connected)
1646 tmp &= ~DC_HPDx_INT_POLARITY;
1647 else
1648 tmp |= DC_HPDx_INT_POLARITY;
1649 WREG32(DC_HPD1_INT_CONTROL, tmp);
1650 break;
1651 case RADEON_HPD_2:
1652 tmp = RREG32(DC_HPD2_INT_CONTROL);
1653 if (connected)
1654 tmp &= ~DC_HPDx_INT_POLARITY;
1655 else
1656 tmp |= DC_HPDx_INT_POLARITY;
1657 WREG32(DC_HPD2_INT_CONTROL, tmp);
1658 break;
1659 case RADEON_HPD_3:
1660 tmp = RREG32(DC_HPD3_INT_CONTROL);
1661 if (connected)
1662 tmp &= ~DC_HPDx_INT_POLARITY;
1663 else
1664 tmp |= DC_HPDx_INT_POLARITY;
1665 WREG32(DC_HPD3_INT_CONTROL, tmp);
1666 break;
1667 case RADEON_HPD_4:
1668 tmp = RREG32(DC_HPD4_INT_CONTROL);
1669 if (connected)
1670 tmp &= ~DC_HPDx_INT_POLARITY;
1671 else
1672 tmp |= DC_HPDx_INT_POLARITY;
1673 WREG32(DC_HPD4_INT_CONTROL, tmp);
1674 break;
1675 case RADEON_HPD_5:
1676 tmp = RREG32(DC_HPD5_INT_CONTROL);
1677 if (connected)
1678 tmp &= ~DC_HPDx_INT_POLARITY;
1679 else
1680 tmp |= DC_HPDx_INT_POLARITY;
1681 WREG32(DC_HPD5_INT_CONTROL, tmp);
1682 break;
1683 case RADEON_HPD_6:
1684 tmp = RREG32(DC_HPD6_INT_CONTROL);
1685 if (connected)
1686 tmp &= ~DC_HPDx_INT_POLARITY;
1687 else
1688 tmp |= DC_HPDx_INT_POLARITY;
1689 WREG32(DC_HPD6_INT_CONTROL, tmp);
1690 break;
1691 default:
1692 break;
1693 }
bcc1c2a1
AD		 1694}
1695
377edc8b
AD		 1696/**
1697 * evergreen_hpd_init - hpd setup callback.
1698 *
1699 * @rdev: radeon_device pointer
1700 *
1701 * Setup the hpd pins used by the card (evergreen+).
1702 * Enable the pin, set the polarity, and enable the hpd interrupts.
1703 */
bcc1c2a1
AD		 1704void evergreen_hpd_init(struct radeon_device *rdev)
1705{
0ca2ab52
AD		 1706 struct drm_device *dev = rdev->ddev;
1707 struct drm_connector *connector;
fb98257a 1708 unsigned enabled = 0;
0ca2ab52
AD		 1709 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1710 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1711
0ca2ab52
AD		 1712 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1713 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1714
1715 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1716 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1717 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1718 * aux dp channel on imac and help (but not completely fix)
1719 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1720 * also avoid interrupt storms during dpms.
1721 */
1722 continue;
1723 }
0ca2ab52
AD		 1724 switch (radeon_connector->hpd.hpd) {
1725 case RADEON_HPD_1:
1726 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1727 break;
1728 case RADEON_HPD_2:
1729 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1730 break;
1731 case RADEON_HPD_3:
1732 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1733 break;
1734 case RADEON_HPD_4:
1735 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1736 break;
1737 case RADEON_HPD_5:
1738 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1739 break;
1740 case RADEON_HPD_6:
1741 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1742 break;
1743 default:
1744 break;
1745 }
64912e99 1746 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1747 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1748 }
fb98257a 1749 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1750}
1751
377edc8b
AD		 1752/**
1753 * evergreen_hpd_fini - hpd tear down callback.
1754 *
1755 * @rdev: radeon_device pointer
1756 *
1757 * Tear down the hpd pins used by the card (evergreen+).
1758 * Disable the hpd interrupts.
1759 */
0ca2ab52 1760void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1761{
0ca2ab52
AD		 1762 struct drm_device *dev = rdev->ddev;
1763 struct drm_connector *connector;
fb98257a 1764 unsigned disabled = 0;
0ca2ab52
AD		 1765
1766 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1767 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1768 switch (radeon_connector->hpd.hpd) {
1769 case RADEON_HPD_1:
1770 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1771 break;
1772 case RADEON_HPD_2:
1773 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1774 break;
1775 case RADEON_HPD_3:
1776 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1777 break;
1778 case RADEON_HPD_4:
1779 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1780 break;
1781 case RADEON_HPD_5:
1782 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1783 break;
1784 case RADEON_HPD_6:
1785 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1786 break;
1787 default:
1788 break;
1789 }
fb98257a 1790 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1791 }
fb98257a 1792 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1793}
1794
f9d9c362
AD		 1795/* watermark setup */
1796
1797static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1798 struct radeon_crtc *radeon_crtc,
1799 struct drm_display_mode *mode,
1800 struct drm_display_mode *other_mode)
1801{
12dfc843 1802 u32 tmp;
f9d9c362
AD		 1803 /*
1804 * Line Buffer Setup
1805 * There are 3 line buffers, each one shared by 2 display controllers.
1806 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1807 * the display controllers. The paritioning is done via one of four
1808 * preset allocations specified in bits 2:0:
1809 * first display controller
1810 * 0 - first half of lb (3840 * 2)
1811 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1812 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1813 * 3 - first 1/4 of lb (1920 * 2)
1814 * second display controller
1815 * 4 - second half of lb (3840 * 2)
1816 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1817 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1818 * 7 - last 1/4 of lb (1920 * 2)
1819 */
12dfc843
AD		 1820 /* this can get tricky if we have two large displays on a paired group
1821 * of crtcs. Ideally for multiple large displays we'd assign them to
1822 * non-linked crtcs for maximum line buffer allocation.
1823 */
1824 if (radeon_crtc->base.enabled && mode) {
1825 if (other_mode)
f9d9c362 1826 tmp = 0; /* 1/2 */
12dfc843
AD		 1827 else
1828 tmp = 2; /* whole */
1829 } else
1830 tmp = 0;
f9d9c362
AD		 1831
1832 /* second controller of the pair uses second half of the lb */
1833 if (radeon_crtc->crtc_id % 2)
1834 tmp += 4;
1835 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1836
12dfc843
AD		 1837 if (radeon_crtc->base.enabled && mode) {
1838 switch (tmp) {
1839 case 0:
1840 case 4:
1841 default:
1842 if (ASIC_IS_DCE5(rdev))
1843 return 4096 * 2;
1844 else
1845 return 3840 * 2;
1846 case 1:
1847 case 5:
1848 if (ASIC_IS_DCE5(rdev))
1849 return 6144 * 2;
1850 else
1851 return 5760 * 2;
1852 case 2:
1853 case 6:
1854 if (ASIC_IS_DCE5(rdev))
1855 return 8192 * 2;
1856 else
1857 return 7680 * 2;
1858 case 3:
1859 case 7:
1860 if (ASIC_IS_DCE5(rdev))
1861 return 2048 * 2;
1862 else
1863 return 1920 * 2;
1864 }
f9d9c362 1865 }
12dfc843
AD		 1866
1867 /* controller not enabled, so no lb used */
1868 return 0;
f9d9c362
AD		 1869}
1870
ca7db22b 1871u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1872{
1873 u32 tmp = RREG32(MC_SHARED_CHMAP);
1874
1875 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1876 case 0:
1877 default:
1878 return 1;
1879 case 1:
1880 return 2;
1881 case 2:
1882 return 4;
1883 case 3:
1884 return 8;
1885 }
1886}
1887
1888struct evergreen_wm_params {
1889 u32 dram_channels; /* number of dram channels */
1890 u32 yclk; /* bandwidth per dram data pin in kHz */
1891 u32 sclk; /* engine clock in kHz */
1892 u32 disp_clk; /* display clock in kHz */
1893 u32 src_width; /* viewport width */
1894 u32 active_time; /* active display time in ns */
1895 u32 blank_time; /* blank time in ns */
1896 bool interlaced; /* mode is interlaced */
1897 fixed20_12 vsc; /* vertical scale ratio */
1898 u32 num_heads; /* number of active crtcs */
1899 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1900 u32 lb_size; /* line buffer allocated to pipe */
1901 u32 vtaps; /* vertical scaler taps */
1902};
1903
1904static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1905{
1906 /* Calculate DRAM Bandwidth and the part allocated to display. */
1907 fixed20_12 dram_efficiency; /* 0.7 */
1908 fixed20_12 yclk, dram_channels, bandwidth;
1909 fixed20_12 a;
1910
1911 a.full = dfixed_const(1000);
1912 yclk.full = dfixed_const(wm->yclk);
1913 yclk.full = dfixed_div(yclk, a);
1914 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1915 a.full = dfixed_const(10);
1916 dram_efficiency.full = dfixed_const(7);
1917 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1918 bandwidth.full = dfixed_mul(dram_channels, yclk);
1919 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1920
1921 return dfixed_trunc(bandwidth);
1922}
1923
1924static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1925{
1926 /* Calculate DRAM Bandwidth and the part allocated to display. */
1927 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1928 fixed20_12 yclk, dram_channels, bandwidth;
1929 fixed20_12 a;
1930
1931 a.full = dfixed_const(1000);
1932 yclk.full = dfixed_const(wm->yclk);
1933 yclk.full = dfixed_div(yclk, a);
1934 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1935 a.full = dfixed_const(10);
1936 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1937 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1938 bandwidth.full = dfixed_mul(dram_channels, yclk);
1939 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1940
1941 return dfixed_trunc(bandwidth);
1942}
1943
1944static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1945{
1946 /* Calculate the display Data return Bandwidth */
1947 fixed20_12 return_efficiency; /* 0.8 */
1948 fixed20_12 sclk, bandwidth;
1949 fixed20_12 a;
1950
1951 a.full = dfixed_const(1000);
1952 sclk.full = dfixed_const(wm->sclk);
1953 sclk.full = dfixed_div(sclk, a);
1954 a.full = dfixed_const(10);
1955 return_efficiency.full = dfixed_const(8);
1956 return_efficiency.full = dfixed_div(return_efficiency, a);
1957 a.full = dfixed_const(32);
1958 bandwidth.full = dfixed_mul(a, sclk);
1959 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1960
1961 return dfixed_trunc(bandwidth);
1962}
1963
1964static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1965{
1966 /* Calculate the DMIF Request Bandwidth */
1967 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1968 fixed20_12 disp_clk, bandwidth;
1969 fixed20_12 a;
1970
1971 a.full = dfixed_const(1000);
1972 disp_clk.full = dfixed_const(wm->disp_clk);
1973 disp_clk.full = dfixed_div(disp_clk, a);
1974 a.full = dfixed_const(10);
1975 disp_clk_request_efficiency.full = dfixed_const(8);
1976 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1977 a.full = dfixed_const(32);
1978 bandwidth.full = dfixed_mul(a, disp_clk);
1979 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1980
1981 return dfixed_trunc(bandwidth);
1982}
1983
1984static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1985{
1986 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1987 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1988 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1989 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1990
1991 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1992}
1993
1994static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
1995{
1996 /* Calculate the display mode Average Bandwidth
1997 * DisplayMode should contain the source and destination dimensions,
1998 * timing, etc.
1999 */
2000 fixed20_12 bpp;
2001 fixed20_12 line_time;
2002 fixed20_12 src_width;
2003 fixed20_12 bandwidth;
2004 fixed20_12 a;
2005
2006 a.full = dfixed_const(1000);
2007 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2008 line_time.full = dfixed_div(line_time, a);
2009 bpp.full = dfixed_const(wm->bytes_per_pixel);
2010 src_width.full = dfixed_const(wm->src_width);
2011 bandwidth.full = dfixed_mul(src_width, bpp);
2012 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2013 bandwidth.full = dfixed_div(bandwidth, line_time);
2014
2015 return dfixed_trunc(bandwidth);
2016}
2017
2018static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2019{
2020 /* First calcualte the latency in ns */
2021 u32 mc_latency = 2000; /* 2000 ns. */
2022 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2023 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2024 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2025 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2026 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2027 (wm->num_heads * cursor_line_pair_return_time);
2028 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2029 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2030 fixed20_12 a, b, c;
2031
2032 if (wm->num_heads == 0)
2033 return 0;
2034
2035 a.full = dfixed_const(2);
2036 b.full = dfixed_const(1);
2037 if ((wm->vsc.full > a.full) ||
2038 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2039 (wm->vtaps >= 5) ||
2040 ((wm->vsc.full >= a.full) && wm->interlaced))
2041 max_src_lines_per_dst_line = 4;
2042 else
2043 max_src_lines_per_dst_line = 2;
2044
2045 a.full = dfixed_const(available_bandwidth);
2046 b.full = dfixed_const(wm->num_heads);
2047 a.full = dfixed_div(a, b);
2048
2049 b.full = dfixed_const(1000);
2050 c.full = dfixed_const(wm->disp_clk);
2051 b.full = dfixed_div(c, b);
2052 c.full = dfixed_const(wm->bytes_per_pixel);
2053 b.full = dfixed_mul(b, c);
2054
2055 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2056
2057 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2058 b.full = dfixed_const(1000);
2059 c.full = dfixed_const(lb_fill_bw);
2060 b.full = dfixed_div(c, b);
2061 a.full = dfixed_div(a, b);
2062 line_fill_time = dfixed_trunc(a);
2063
2064 if (line_fill_time < wm->active_time)
2065 return latency;
2066 else
2067 return latency + (line_fill_time - wm->active_time);
2068
2069}
2070
2071static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2072{
2073 if (evergreen_average_bandwidth(wm) <=
2074 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2075 return true;
2076 else
2077 return false;
2078};
2079
2080static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2081{
2082 if (evergreen_average_bandwidth(wm) <=
2083 (evergreen_available_bandwidth(wm) / wm->num_heads))
2084 return true;
2085 else
2086 return false;
2087};
2088
2089static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2090{
2091 u32 lb_partitions = wm->lb_size / wm->src_width;
2092 u32 line_time = wm->active_time + wm->blank_time;
2093 u32 latency_tolerant_lines;
2094 u32 latency_hiding;
2095 fixed20_12 a;
2096
2097 a.full = dfixed_const(1);
2098 if (wm->vsc.full > a.full)
2099 latency_tolerant_lines = 1;
2100 else {
2101 if (lb_partitions <= (wm->vtaps + 1))
2102 latency_tolerant_lines = 1;
2103 else
2104 latency_tolerant_lines = 2;
2105 }
2106
2107 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2108
2109 if (evergreen_latency_watermark(wm) <= latency_hiding)
2110 return true;
2111 else
2112 return false;
2113}
2114
2115static void evergreen_program_watermarks(struct radeon_device *rdev,
2116 struct radeon_crtc *radeon_crtc,
2117 u32 lb_size, u32 num_heads)
2118{
2119 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2120 struct evergreen_wm_params wm;
2121 u32 pixel_period;
2122 u32 line_time = 0;
2123 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2124 u32 priority_a_mark = 0, priority_b_mark = 0;
2125 u32 priority_a_cnt = PRIORITY_OFF;
2126 u32 priority_b_cnt = PRIORITY_OFF;
2127 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2128 u32 tmp, arb_control3;
2129 fixed20_12 a, b, c;
2130
2131 if (radeon_crtc->base.enabled && num_heads && mode) {
2132 pixel_period = 1000000 / (u32)mode->clock;
2133 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2134 priority_a_cnt = 0;
2135 priority_b_cnt = 0;
2136
2137 wm.yclk = rdev->pm.current_mclk * 10;
2138 wm.sclk = rdev->pm.current_sclk * 10;
2139 wm.disp_clk = mode->clock;
2140 wm.src_width = mode->crtc_hdisplay;
2141 wm.active_time = mode->crtc_hdisplay * pixel_period;
2142 wm.blank_time = line_time - wm.active_time;
2143 wm.interlaced = false;
2144 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2145 wm.interlaced = true;
2146 wm.vsc = radeon_crtc->vsc;
2147 wm.vtaps = 1;
2148 if (radeon_crtc->rmx_type != RMX_OFF)
2149 wm.vtaps = 2;
2150 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
2151 wm.lb_size = lb_size;
2152 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
2153 wm.num_heads = num_heads;
2154
2155 /* set for high clocks */
2156 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
2157 /* set for low clocks */
2158 /* wm.yclk = low clk; wm.sclk = low clk */
2159 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
2160
2161 /* possibly force display priority to high */
2162 /* should really do this at mode validation time... */
2163 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
2164 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
2165 !evergreen_check_latency_hiding(&wm) ||
2166 (rdev->disp_priority == 2)) {
92bdfd4a 2167 DRM_DEBUG_KMS("force priority to high\n");
f9d9c362
AD		 2168 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2169 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2170 }
2171
2172 a.full = dfixed_const(1000);
2173 b.full = dfixed_const(mode->clock);
2174 b.full = dfixed_div(b, a);
2175 c.full = dfixed_const(latency_watermark_a);
2176 c.full = dfixed_mul(c, b);
2177 c.full = dfixed_mul(c, radeon_crtc->hsc);
2178 c.full = dfixed_div(c, a);
2179 a.full = dfixed_const(16);
2180 c.full = dfixed_div(c, a);
2181 priority_a_mark = dfixed_trunc(c);
2182 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2183
2184 a.full = dfixed_const(1000);
2185 b.full = dfixed_const(mode->clock);
2186 b.full = dfixed_div(b, a);
2187 c.full = dfixed_const(latency_watermark_b);
2188 c.full = dfixed_mul(c, b);
2189 c.full = dfixed_mul(c, radeon_crtc->hsc);
2190 c.full = dfixed_div(c, a);
2191 a.full = dfixed_const(16);
2192 c.full = dfixed_div(c, a);
2193 priority_b_mark = dfixed_trunc(c);
2194 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2195 }
2196
2197 /* select wm A */
2198 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2199 tmp = arb_control3;
2200 tmp &= ~LATENCY_WATERMARK_MASK(3);
2201 tmp |= LATENCY_WATERMARK_MASK(1);
2202 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2203 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2204 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2205 LATENCY_HIGH_WATERMARK(line_time)));
2206 /* select wm B */
2207 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2208 tmp &= ~LATENCY_WATERMARK_MASK(3);
2209 tmp |= LATENCY_WATERMARK_MASK(2);
2210 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2211 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2212 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2213 LATENCY_HIGH_WATERMARK(line_time)));
2214 /* restore original selection */
2215 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2216
2217 /* write the priority marks */
2218 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2219 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2220
2221}
2222
377edc8b
AD		 2223/**
2224 * evergreen_bandwidth_update - update display watermarks callback.
2225 *
2226 * @rdev: radeon_device pointer
2227 *
2228 * Update the display watermarks based on the requested mode(s)
2229 * (evergreen+).
2230 */
0ca2ab52 2231void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2232{
f9d9c362
AD		 2233 struct drm_display_mode *mode0 = NULL;
2234 struct drm_display_mode *mode1 = NULL;
2235 u32 num_heads = 0, lb_size;
2236 int i;
2237
2238 radeon_update_display_priority(rdev);
2239
2240 for (i = 0; i < rdev->num_crtc; i++) {
2241 if (rdev->mode_info.crtcs[i]->base.enabled)
2242 num_heads++;
2243 }
2244 for (i = 0; i < rdev->num_crtc; i += 2) {
2245 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2246 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2247 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2248 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2249 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2250 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2251 }
bcc1c2a1
AD		 2252}
2253
377edc8b
AD		 2254/**
2255 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2256 *
2257 * @rdev: radeon_device pointer
2258 *
2259 * Wait for the MC (memory controller) to be idle.
2260 * (evergreen+).
2261 * Returns 0 if the MC is idle, -1 if not.
2262 */
b9952a8a 2263int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2264{
2265 unsigned i;
2266 u32 tmp;
2267
2268 for (i = 0; i < rdev->usec_timeout; i++) {
2269 /* read MC_STATUS */
2270 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2271 if (!tmp)
2272 return 0;
2273 udelay(1);
2274 }
2275 return -1;
2276}
2277
2278/*
2279 * GART
2280 */
0fcdb61e
AD		 2281void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2282{
2283 unsigned i;
2284 u32 tmp;
2285
6f2f48a9
AD		 2286 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2287
0fcdb61e
AD		 2288 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2289 for (i = 0; i < rdev->usec_timeout; i++) {
2290 /* read MC_STATUS */
2291 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2292 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2293 if (tmp == 2) {
2294 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2295 return;
2296 }
2297 if (tmp) {
2298 return;
2299 }
2300 udelay(1);
2301 }
2302}
2303
1109ca09 2304static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2305{
2306 u32 tmp;
0fcdb61e 2307 int r;
bcc1c2a1 2308
c9a1be96 2309 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2310 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2311 return -EINVAL;
2312 }
2313 r = radeon_gart_table_vram_pin(rdev);
2314 if (r)
2315 return r;
82568565 2316 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2317 /* Setup L2 cache */
2318 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2319 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2320 EFFECTIVE_L2_QUEUE_SIZE(7));
2321 WREG32(VM_L2_CNTL2, 0);
2322 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2323 /* Setup TLB control */
2324 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2325 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2326 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2327 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2328 if (rdev->flags & RADEON_IS_IGP) {
2329 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2330 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2331 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2332 } else {
2333 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2334 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2335 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2336 if ((rdev->family == CHIP_JUNIPER) ||
2337 (rdev->family == CHIP_CYPRESS) ||
2338 (rdev->family == CHIP_HEMLOCK) ||
2339 (rdev->family == CHIP_BARTS))
2340 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2341 }
bcc1c2a1
AD		 2342 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2343 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2344 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2345 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2346 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2347 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2348 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2349 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2350 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2351 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2352 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2353 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2354
0fcdb61e 2355 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2356 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2357 (unsigned)(rdev->mc.gtt_size >> 20),
2358 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2359 rdev->gart.ready = true;
2360 return 0;
2361}
2362
1109ca09 2363static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2364{
2365 u32 tmp;
bcc1c2a1
AD		 2366
2367 /* Disable all tables */
0fcdb61e
AD		 2368 WREG32(VM_CONTEXT0_CNTL, 0);
2369 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2370
2371 /* Setup L2 cache */
2372 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2373 EFFECTIVE_L2_QUEUE_SIZE(7));
2374 WREG32(VM_L2_CNTL2, 0);
2375 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2376 /* Setup TLB control */
2377 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2378 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2379 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2380 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2381 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2382 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2383 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2384 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2385 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2386}
2387
1109ca09 2388static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2389{
2390 evergreen_pcie_gart_disable(rdev);
2391 radeon_gart_table_vram_free(rdev);
2392 radeon_gart_fini(rdev);
2393}
2394
2395
1109ca09 2396static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2397{
2398 u32 tmp;
bcc1c2a1
AD		 2399
2400 /* Setup L2 cache */
2401 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2402 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2403 EFFECTIVE_L2_QUEUE_SIZE(7));
2404 WREG32(VM_L2_CNTL2, 0);
2405 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2406 /* Setup TLB control */
2407 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2408 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2409 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2410 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2411 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2412 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2413 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2414 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2415 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2416 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2417 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2418 WREG32(VM_CONTEXT0_CNTL, 0);
2419 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2420}
2421
b9952a8a 2422void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2423{
62444b74
AD		 2424 u32 crtc_enabled, tmp, frame_count, blackout;
2425 int i, j;
2426
bcc1c2a1
AD		 2427 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2428 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2429
62444b74 2430 /* disable VGA render */
bcc1c2a1 2431 WREG32(VGA_RENDER_CONTROL, 0);
62444b74
AD		 2432 /* blank the display controllers */
2433 for (i = 0; i < rdev->num_crtc; i++) {
2434 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2435 if (crtc_enabled) {
2436 save->crtc_enabled[i] = true;
2437 if (ASIC_IS_DCE6(rdev)) {
2438 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2439 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2440 radeon_wait_for_vblank(rdev, i);
abf1457b 2441 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2442 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2443 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2444 }
2445 } else {
2446 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2447 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2448 radeon_wait_for_vblank(rdev, i);
abf1457b 2449 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2450 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2451 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2452 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2453 }
2454 }
2455 /* wait for the next frame */
2456 frame_count = radeon_get_vblank_counter(rdev, i);
2457 for (j = 0; j < rdev->usec_timeout; j++) {
2458 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2459 break;
2460 udelay(1);
2461 }
abf1457b
AD		 2462
2463 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2464 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2465 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2466 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2467 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2468 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2469 save->crtc_enabled[i] = false;
2470 /* ***** */
804cc4a0
AD		 2471 } else {
2472 save->crtc_enabled[i] = false;
62444b74 2473 }
18007401 2474 }
bcc1c2a1 2475
62444b74
AD		 2476 radeon_mc_wait_for_idle(rdev);
2477
2478 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2479 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2480 /* Block CPU access */
2481 WREG32(BIF_FB_EN, 0);
2482 /* blackout the MC */
2483 blackout &= ~BLACKOUT_MODE_MASK;
2484 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2485 }
ed39fadd
AD		 2486 /* wait for the MC to settle */
2487 udelay(100);
968c0166
AD		 2488
2489 /* lock double buffered regs */
2490 for (i = 0; i < rdev->num_crtc; i++) {
2491 if (save->crtc_enabled[i]) {
2492 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2493 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2494 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2495 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2496 }
2497 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2498 if (!(tmp & 1)) {
2499 tmp |= 1;
2500 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2501 }
2502 }
2503 }
bcc1c2a1
AD		 2504}
2505
b9952a8a 2506void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2507{
62444b74
AD		 2508 u32 tmp, frame_count;
2509 int i, j;
18007401 2510
62444b74
AD		 2511 /* update crtc base addresses */
2512 for (i = 0; i < rdev->num_crtc; i++) {
2513 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2514 upper_32_bits(rdev->mc.vram_start));
62444b74 2515 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2516 upper_32_bits(rdev->mc.vram_start));
62444b74 2517 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2518 (u32)rdev->mc.vram_start);
62444b74 2519 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2520 (u32)rdev->mc.vram_start);
2521 }
bcc1c2a1
AD		 2522 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2523 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
62444b74 2524
968c0166
AD		 2525 /* unlock regs and wait for update */
2526 for (i = 0; i < rdev->num_crtc; i++) {
2527 if (save->crtc_enabled[i]) {
2528 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2529 if ((tmp & 0x3) != 0) {
2530 tmp &= ~0x3;
2531 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2532 }
2533 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2534 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2535 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2536 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2537 }
2538 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2539 if (tmp & 1) {
2540 tmp &= ~1;
2541 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2542 }
2543 for (j = 0; j < rdev->usec_timeout; j++) {
2544 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2545 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2546 break;
2547 udelay(1);
2548 }
2549 }
2550 }
2551
62444b74
AD		 2552 /* unblackout the MC */
2553 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2554 tmp &= ~BLACKOUT_MODE_MASK;
2555 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2556 /* allow CPU access */
2557 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2558
2559 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2560 if (save->crtc_enabled[i]) {
62444b74
AD		 2561 if (ASIC_IS_DCE6(rdev)) {
2562 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2563 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2564 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2565 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2566 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2567 } else {
2568 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2569 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2570 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2571 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2572 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2573 }
2574 /* wait for the next frame */
2575 frame_count = radeon_get_vblank_counter(rdev, i);
2576 for (j = 0; j < rdev->usec_timeout; j++) {
2577 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2578 break;
2579 udelay(1);
2580 }
2581 }
2582 }
2583 /* Unlock vga access */
bcc1c2a1
AD		 2584 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2585 mdelay(1);
bcc1c2a1
AD		 2586 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2587}
2588
755d819e 2589void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2590{
2591 struct evergreen_mc_save save;
2592 u32 tmp;
2593 int i, j;
2594
2595 /* Initialize HDP */
2596 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2597 WREG32((0x2c14 + j), 0x00000000);
2598 WREG32((0x2c18 + j), 0x00000000);
2599 WREG32((0x2c1c + j), 0x00000000);
2600 WREG32((0x2c20 + j), 0x00000000);
2601 WREG32((0x2c24 + j), 0x00000000);
2602 }
2603 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2604
2605 evergreen_mc_stop(rdev, &save);
2606 if (evergreen_mc_wait_for_idle(rdev)) {
2607 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2608 }
2609 /* Lockout access through VGA aperture*/
2610 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2611 /* Update configuration */
2612 if (rdev->flags & RADEON_IS_AGP) {
2613 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2614 /* VRAM before AGP */
2615 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2616 rdev->mc.vram_start >> 12);
2617 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2618 rdev->mc.gtt_end >> 12);
2619 } else {
2620 /* VRAM after AGP */
2621 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2622 rdev->mc.gtt_start >> 12);
2623 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2624 rdev->mc.vram_end >> 12);
2625 }
2626 } else {
2627 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2628 rdev->mc.vram_start >> 12);
2629 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2630 rdev->mc.vram_end >> 12);
2631 }
3b9832f6 2632 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2633 /* llano/ontario only */
2634 if ((rdev->family == CHIP_PALM) ||
2635 (rdev->family == CHIP_SUMO) ||
2636 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2637 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2638 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2639 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2640 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2641 }
bcc1c2a1
AD		 2642 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2643 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2644 WREG32(MC_VM_FB_LOCATION, tmp);
2645 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2646 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2647 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2648 if (rdev->flags & RADEON_IS_AGP) {
2649 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2650 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2651 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2652 } else {
2653 WREG32(MC_VM_AGP_BASE, 0);
2654 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2655 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2656 }
2657 if (evergreen_mc_wait_for_idle(rdev)) {
2658 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2659 }
2660 evergreen_mc_resume(rdev, &save);
2661 /* we need to own VRAM, so turn off the VGA renderer here
2662 * to stop it overwriting our objects */
2663 rv515_vga_render_disable(rdev);
2664}
2665
bcc1c2a1
AD		 2666/*
2667 * CP.
2668 */
12920591
AD		 2669void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2670{
876dc9f3 2671 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2672 u32 next_rptr;
7b1f2485 2673
12920591 2674 /* set to DX10/11 mode */
e32eb50d
CK		 2675 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2676 radeon_ring_write(ring, 1);
45df6803
CK		 2677
2678 if (ring->rptr_save_reg) {
89d35807 2679 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2680 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2681 radeon_ring_write(ring, ((ring->rptr_save_reg -
2682 PACKET3_SET_CONFIG_REG_START) >> 2));
2683 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2684 } else if (rdev->wb.enabled) {
2685 next_rptr = ring->wptr + 5 + 4;
2686 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2687 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2688 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2689 radeon_ring_write(ring, next_rptr);
2690 radeon_ring_write(ring, 0);
45df6803
CK		 2691 }
2692
e32eb50d
CK		 2693 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2694 radeon_ring_write(ring,
0f234f5f
AD		 2695#ifdef __BIG_ENDIAN
2696 (2 << 0) |
2697#endif
2698 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2699 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2700 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2701}
2702
bcc1c2a1
AD		 2703
2704static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2705{
fe251e2f
AD		 2706 const __be32 *fw_data;
2707 int i;
2708
2709 if (!rdev->me_fw || !rdev->pfp_fw)
2710 return -EINVAL;
bcc1c2a1 2711
fe251e2f 2712 r700_cp_stop(rdev);
0f234f5f
AD		 2713 WREG32(CP_RB_CNTL,
2714#ifdef __BIG_ENDIAN
2715 BUF_SWAP_32BIT |
2716#endif
2717 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2718
2719 fw_data = (const __be32 *)rdev->pfp_fw->data;
2720 WREG32(CP_PFP_UCODE_ADDR, 0);
2721 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2722 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2723 WREG32(CP_PFP_UCODE_ADDR, 0);
2724
2725 fw_data = (const __be32 *)rdev->me_fw->data;
2726 WREG32(CP_ME_RAM_WADDR, 0);
2727 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2728 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2729
2730 WREG32(CP_PFP_UCODE_ADDR, 0);
2731 WREG32(CP_ME_RAM_WADDR, 0);
2732 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2733 return 0;
2734}
2735
7e7b41d2
AD		 2736static int evergreen_cp_start(struct radeon_device *rdev)
2737{
e32eb50d 2738 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2739 int r, i;
7e7b41d2
AD		 2740 uint32_t cp_me;
2741
e32eb50d 2742 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2743 if (r) {
2744 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2745 return r;
2746 }
e32eb50d
CK		 2747 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2748 radeon_ring_write(ring, 0x1);
2749 radeon_ring_write(ring, 0x0);
2750 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2751 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2752 radeon_ring_write(ring, 0);
2753 radeon_ring_write(ring, 0);
2754 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2755
2756 cp_me = 0xff;
2757 WREG32(CP_ME_CNTL, cp_me);
2758
e32eb50d 2759 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2760 if (r) {
2761 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2762 return r;
2763 }
2281a378
AD		 2764
2765 /* setup clear context state */
e32eb50d
CK		 2766 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2767 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2768
2769 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2770 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2771
e32eb50d
CK		 2772 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2773 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2774
2775 /* set clear context state */
e32eb50d
CK		 2776 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2777 radeon_ring_write(ring, 0);
2281a378
AD		 2778
2779 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2780 radeon_ring_write(ring, 0xc0026f00);
2781 radeon_ring_write(ring, 0x00000000);
2782 radeon_ring_write(ring, 0x00000000);
2783 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2784
2785 /* Clear consts */
e32eb50d
CK		 2786 radeon_ring_write(ring, 0xc0036f00);
2787 radeon_ring_write(ring, 0x00000bc4);
2788 radeon_ring_write(ring, 0xffffffff);
2789 radeon_ring_write(ring, 0xffffffff);
2790 radeon_ring_write(ring, 0xffffffff);
2281a378 2791
e32eb50d
CK		 2792 radeon_ring_write(ring, 0xc0026900);
2793 radeon_ring_write(ring, 0x00000316);
2794 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2795 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2796
e32eb50d 2797 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2798
2799 return 0;
2800}
2801
1109ca09 2802static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2803{
e32eb50d 2804 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2805 u32 tmp;
2806 u32 rb_bufsz;
2807 int r;
2808
2809 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2810 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2811 SOFT_RESET_PA |
2812 SOFT_RESET_SH |
2813 SOFT_RESET_VGT |
a49a50da 2814 SOFT_RESET_SPI |
fe251e2f
AD		 2815 SOFT_RESET_SX));
2816 RREG32(GRBM_SOFT_RESET);
2817 mdelay(15);
2818 WREG32(GRBM_SOFT_RESET, 0);
2819 RREG32(GRBM_SOFT_RESET);
2820
2821 /* Set ring buffer size */
e32eb50d 2822 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2823 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2824#ifdef __BIG_ENDIAN
2825 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2826#endif
fe251e2f 2827 WREG32(CP_RB_CNTL, tmp);
15d3332f 2828 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2829 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2830
2831 /* Set the write pointer delay */
2832 WREG32(CP_RB_WPTR_DELAY, 0);
2833
2834 /* Initialize the ring buffer's read and write pointers */
2835 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2836 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2837 ring->wptr = 0;
2838 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2839
48fc7f7e 2840 /* set the wb address whether it's enabled or not */
0f234f5f 2841 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2842 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2843 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2844 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2845
2846 if (rdev->wb.enabled)
2847 WREG32(SCRATCH_UMSK, 0xff);
2848 else {
2849 tmp |= RB_NO_UPDATE;
2850 WREG32(SCRATCH_UMSK, 0);
2851 }
2852
fe251e2f
AD		 2853 mdelay(1);
2854 WREG32(CP_RB_CNTL, tmp);
2855
e32eb50d 2856 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2857 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2858
e32eb50d 2859 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2860
7e7b41d2 2861 evergreen_cp_start(rdev);
e32eb50d 2862 ring->ready = true;
f712812e 2863 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2864 if (r) {
e32eb50d 2865 ring->ready = false;
fe251e2f
AD		 2866 return r;
2867 }
2868 return 0;
2869}
bcc1c2a1
AD		 2870
2871/*
2872 * Core functions
2873 */
bcc1c2a1
AD		 2874static void evergreen_gpu_init(struct radeon_device *rdev)
2875{
416a2bd2 2876 u32 gb_addr_config;
32fcdbf4 2877 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2878 u32 sx_debug_1;
2879 u32 smx_dc_ctl0;
2880 u32 sq_config;
2881 u32 sq_lds_resource_mgmt;
2882 u32 sq_gpr_resource_mgmt_1;
2883 u32 sq_gpr_resource_mgmt_2;
2884 u32 sq_gpr_resource_mgmt_3;
2885 u32 sq_thread_resource_mgmt;
2886 u32 sq_thread_resource_mgmt_2;
2887 u32 sq_stack_resource_mgmt_1;
2888 u32 sq_stack_resource_mgmt_2;
2889 u32 sq_stack_resource_mgmt_3;
2890 u32 vgt_cache_invalidation;
f25a5c63 2891 u32 hdp_host_path_cntl, tmp;
416a2bd2 2892 u32 disabled_rb_mask;
32fcdbf4
AD		 2893 int i, j, num_shader_engines, ps_thread_count;
2894
2895 switch (rdev->family) {
2896 case CHIP_CYPRESS:
2897 case CHIP_HEMLOCK:
2898 rdev->config.evergreen.num_ses = 2;
2899 rdev->config.evergreen.max_pipes = 4;
2900 rdev->config.evergreen.max_tile_pipes = 8;
2901 rdev->config.evergreen.max_simds = 10;
2902 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2903 rdev->config.evergreen.max_gprs = 256;
2904 rdev->config.evergreen.max_threads = 248;
2905 rdev->config.evergreen.max_gs_threads = 32;
2906 rdev->config.evergreen.max_stack_entries = 512;
2907 rdev->config.evergreen.sx_num_of_sets = 4;
2908 rdev->config.evergreen.sx_max_export_size = 256;
2909 rdev->config.evergreen.sx_max_export_pos_size = 64;
2910 rdev->config.evergreen.sx_max_export_smx_size = 192;
2911 rdev->config.evergreen.max_hw_contexts = 8;
2912 rdev->config.evergreen.sq_num_cf_insts = 2;
2913
2914 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2915 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2916 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2917 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2918 break;
2919 case CHIP_JUNIPER:
2920 rdev->config.evergreen.num_ses = 1;
2921 rdev->config.evergreen.max_pipes = 4;
2922 rdev->config.evergreen.max_tile_pipes = 4;
2923 rdev->config.evergreen.max_simds = 10;
2924 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2925 rdev->config.evergreen.max_gprs = 256;
2926 rdev->config.evergreen.max_threads = 248;
2927 rdev->config.evergreen.max_gs_threads = 32;
2928 rdev->config.evergreen.max_stack_entries = 512;
2929 rdev->config.evergreen.sx_num_of_sets = 4;
2930 rdev->config.evergreen.sx_max_export_size = 256;
2931 rdev->config.evergreen.sx_max_export_pos_size = 64;
2932 rdev->config.evergreen.sx_max_export_smx_size = 192;
2933 rdev->config.evergreen.max_hw_contexts = 8;
2934 rdev->config.evergreen.sq_num_cf_insts = 2;
2935
2936 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2937 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2938 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2939 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2940 break;
2941 case CHIP_REDWOOD:
2942 rdev->config.evergreen.num_ses = 1;
2943 rdev->config.evergreen.max_pipes = 4;
2944 rdev->config.evergreen.max_tile_pipes = 4;
2945 rdev->config.evergreen.max_simds = 5;
2946 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2947 rdev->config.evergreen.max_gprs = 256;
2948 rdev->config.evergreen.max_threads = 248;
2949 rdev->config.evergreen.max_gs_threads = 32;
2950 rdev->config.evergreen.max_stack_entries = 256;
2951 rdev->config.evergreen.sx_num_of_sets = 4;
2952 rdev->config.evergreen.sx_max_export_size = 256;
2953 rdev->config.evergreen.sx_max_export_pos_size = 64;
2954 rdev->config.evergreen.sx_max_export_smx_size = 192;
2955 rdev->config.evergreen.max_hw_contexts = 8;
2956 rdev->config.evergreen.sq_num_cf_insts = 2;
2957
2958 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2959 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2960 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2961 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2962 break;
2963 case CHIP_CEDAR:
2964 default:
2965 rdev->config.evergreen.num_ses = 1;
2966 rdev->config.evergreen.max_pipes = 2;
2967 rdev->config.evergreen.max_tile_pipes = 2;
2968 rdev->config.evergreen.max_simds = 2;
2969 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2970 rdev->config.evergreen.max_gprs = 256;
2971 rdev->config.evergreen.max_threads = 192;
2972 rdev->config.evergreen.max_gs_threads = 16;
2973 rdev->config.evergreen.max_stack_entries = 256;
2974 rdev->config.evergreen.sx_num_of_sets = 4;
2975 rdev->config.evergreen.sx_max_export_size = 128;
2976 rdev->config.evergreen.sx_max_export_pos_size = 32;
2977 rdev->config.evergreen.sx_max_export_smx_size = 96;
2978 rdev->config.evergreen.max_hw_contexts = 4;
2979 rdev->config.evergreen.sq_num_cf_insts = 1;
2980
d5e455e4
AD		 2981 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2982 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2983 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2984 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 2985 break;
2986 case CHIP_PALM:
2987 rdev->config.evergreen.num_ses = 1;
2988 rdev->config.evergreen.max_pipes = 2;
2989 rdev->config.evergreen.max_tile_pipes = 2;
2990 rdev->config.evergreen.max_simds = 2;
2991 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2992 rdev->config.evergreen.max_gprs = 256;
2993 rdev->config.evergreen.max_threads = 192;
2994 rdev->config.evergreen.max_gs_threads = 16;
2995 rdev->config.evergreen.max_stack_entries = 256;
2996 rdev->config.evergreen.sx_num_of_sets = 4;
2997 rdev->config.evergreen.sx_max_export_size = 128;
2998 rdev->config.evergreen.sx_max_export_pos_size = 32;
2999 rdev->config.evergreen.sx_max_export_smx_size = 96;
3000 rdev->config.evergreen.max_hw_contexts = 4;
3001 rdev->config.evergreen.sq_num_cf_insts = 1;
3002
d5c5a72f
AD		 3003 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3004 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3005 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3006 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3007 break;
3008 case CHIP_SUMO:
3009 rdev->config.evergreen.num_ses = 1;
3010 rdev->config.evergreen.max_pipes = 4;
bd25f078 3011 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3012 if (rdev->pdev->device == 0x9648)
3013 rdev->config.evergreen.max_simds = 3;
3014 else if ((rdev->pdev->device == 0x9647) ||
3015 (rdev->pdev->device == 0x964a))
3016 rdev->config.evergreen.max_simds = 4;
3017 else
3018 rdev->config.evergreen.max_simds = 5;
3019 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3020 rdev->config.evergreen.max_gprs = 256;
3021 rdev->config.evergreen.max_threads = 248;
3022 rdev->config.evergreen.max_gs_threads = 32;
3023 rdev->config.evergreen.max_stack_entries = 256;
3024 rdev->config.evergreen.sx_num_of_sets = 4;
3025 rdev->config.evergreen.sx_max_export_size = 256;
3026 rdev->config.evergreen.sx_max_export_pos_size = 64;
3027 rdev->config.evergreen.sx_max_export_smx_size = 192;
3028 rdev->config.evergreen.max_hw_contexts = 8;
3029 rdev->config.evergreen.sq_num_cf_insts = 2;
3030
3031 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3032 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3033 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3034 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3035 break;
3036 case CHIP_SUMO2:
3037 rdev->config.evergreen.num_ses = 1;
3038 rdev->config.evergreen.max_pipes = 4;
3039 rdev->config.evergreen.max_tile_pipes = 4;
3040 rdev->config.evergreen.max_simds = 2;
3041 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3042 rdev->config.evergreen.max_gprs = 256;
3043 rdev->config.evergreen.max_threads = 248;
3044 rdev->config.evergreen.max_gs_threads = 32;
3045 rdev->config.evergreen.max_stack_entries = 512;
3046 rdev->config.evergreen.sx_num_of_sets = 4;
3047 rdev->config.evergreen.sx_max_export_size = 256;
3048 rdev->config.evergreen.sx_max_export_pos_size = 64;
3049 rdev->config.evergreen.sx_max_export_smx_size = 192;
3050 rdev->config.evergreen.max_hw_contexts = 8;
3051 rdev->config.evergreen.sq_num_cf_insts = 2;
3052
adb68fa2
AD		 3053 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3054 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3055 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3056 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3057 break;
3058 case CHIP_BARTS:
3059 rdev->config.evergreen.num_ses = 2;
3060 rdev->config.evergreen.max_pipes = 4;
3061 rdev->config.evergreen.max_tile_pipes = 8;
3062 rdev->config.evergreen.max_simds = 7;
3063 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3064 rdev->config.evergreen.max_gprs = 256;
3065 rdev->config.evergreen.max_threads = 248;
3066 rdev->config.evergreen.max_gs_threads = 32;
3067 rdev->config.evergreen.max_stack_entries = 512;
3068 rdev->config.evergreen.sx_num_of_sets = 4;
3069 rdev->config.evergreen.sx_max_export_size = 256;
3070 rdev->config.evergreen.sx_max_export_pos_size = 64;
3071 rdev->config.evergreen.sx_max_export_smx_size = 192;
3072 rdev->config.evergreen.max_hw_contexts = 8;
3073 rdev->config.evergreen.sq_num_cf_insts = 2;
3074
3075 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3076 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3077 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3078 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3079 break;
3080 case CHIP_TURKS:
3081 rdev->config.evergreen.num_ses = 1;
3082 rdev->config.evergreen.max_pipes = 4;
3083 rdev->config.evergreen.max_tile_pipes = 4;
3084 rdev->config.evergreen.max_simds = 6;
3085 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3086 rdev->config.evergreen.max_gprs = 256;
3087 rdev->config.evergreen.max_threads = 248;
3088 rdev->config.evergreen.max_gs_threads = 32;
3089 rdev->config.evergreen.max_stack_entries = 256;
3090 rdev->config.evergreen.sx_num_of_sets = 4;
3091 rdev->config.evergreen.sx_max_export_size = 256;
3092 rdev->config.evergreen.sx_max_export_pos_size = 64;
3093 rdev->config.evergreen.sx_max_export_smx_size = 192;
3094 rdev->config.evergreen.max_hw_contexts = 8;
3095 rdev->config.evergreen.sq_num_cf_insts = 2;
3096
3097 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3098 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3099 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3100 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3101 break;
3102 case CHIP_CAICOS:
3103 rdev->config.evergreen.num_ses = 1;
bd25f078 3104 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3105 rdev->config.evergreen.max_tile_pipes = 2;
3106 rdev->config.evergreen.max_simds = 2;
3107 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3108 rdev->config.evergreen.max_gprs = 256;
3109 rdev->config.evergreen.max_threads = 192;
3110 rdev->config.evergreen.max_gs_threads = 16;
3111 rdev->config.evergreen.max_stack_entries = 256;
3112 rdev->config.evergreen.sx_num_of_sets = 4;
3113 rdev->config.evergreen.sx_max_export_size = 128;
3114 rdev->config.evergreen.sx_max_export_pos_size = 32;
3115 rdev->config.evergreen.sx_max_export_smx_size = 96;
3116 rdev->config.evergreen.max_hw_contexts = 4;
3117 rdev->config.evergreen.sq_num_cf_insts = 1;
3118
32fcdbf4
AD		 3119 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3120 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3121 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3122 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3123 break;
3124 }
3125
3126 /* Initialize HDP */
3127 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3128 WREG32((0x2c14 + j), 0x00000000);
3129 WREG32((0x2c18 + j), 0x00000000);
3130 WREG32((0x2c1c + j), 0x00000000);
3131 WREG32((0x2c20 + j), 0x00000000);
3132 WREG32((0x2c24 + j), 0x00000000);
3133 }
3134
3135 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3136
d054ac16
AD		 3137 evergreen_fix_pci_max_read_req_size(rdev);
3138
32fcdbf4 3139 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3140 if ((rdev->family == CHIP_PALM) ||
3141 (rdev->family == CHIP_SUMO) ||
3142 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3143 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3144 else
3145 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3146
1aa52bd3
AD		 3147 /* setup tiling info dword. gb_addr_config is not adequate since it does
3148 * not have bank info, so create a custom tiling dword.
3149 * bits 3:0 num_pipes
3150 * bits 7:4 num_banks
3151 * bits 11:8 group_size
3152 * bits 15:12 row_size
3153 */
3154 rdev->config.evergreen.tile_config = 0;
3155 switch (rdev->config.evergreen.max_tile_pipes) {
3156 case 1:
3157 default:
3158 rdev->config.evergreen.tile_config |= (0 << 0);
3159 break;
3160 case 2:
3161 rdev->config.evergreen.tile_config |= (1 << 0);
3162 break;
3163 case 4:
3164 rdev->config.evergreen.tile_config |= (2 << 0);
3165 break;
3166 case 8:
3167 rdev->config.evergreen.tile_config |= (3 << 0);
3168 break;
3169 }
d698a34d 3170 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3171 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3172 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3173 else {
c8d15edc
AD		 3174 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3175 case 0: /* four banks */
29d65406 3176 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3177 break;
3178 case 1: /* eight banks */
3179 rdev->config.evergreen.tile_config |= 1 << 4;
3180 break;
3181 case 2: /* sixteen banks */
3182 default:
3183 rdev->config.evergreen.tile_config |= 2 << 4;
3184 break;
3185 }
29d65406 3186 }
416a2bd2 3187 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3188 rdev->config.evergreen.tile_config |=
3189 ((gb_addr_config & 0x30000000) >> 28) << 12;
3190
416a2bd2 3191 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3192
416a2bd2
AD		 3193 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3194 u32 efuse_straps_4;
3195 u32 efuse_straps_3;
32fcdbf4 3196
416a2bd2
AD		 3197 WREG32(RCU_IND_INDEX, 0x204);
3198 efuse_straps_4 = RREG32(RCU_IND_DATA);
3199 WREG32(RCU_IND_INDEX, 0x203);
3200 efuse_straps_3 = RREG32(RCU_IND_DATA);
3201 tmp = (((efuse_straps_4 & 0xf) << 4) |
3202 ((efuse_straps_3 & 0xf0000000) >> 28));
3203 } else {
3204 tmp = 0;
3205 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3206 u32 rb_disable_bitmap;
3207
3208 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3209 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3210 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3211 tmp <<= 4;
3212 tmp |= rb_disable_bitmap;
32fcdbf4 3213 }
416a2bd2
AD		 3214 }
3215 /* enabled rb are just the one not disabled :) */
3216 disabled_rb_mask = tmp;
cedb655a
AD		 3217 tmp = 0;
3218 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3219 tmp |= (1 << i);
3220 /* if all the backends are disabled, fix it up here */
3221 if ((disabled_rb_mask & tmp) == tmp) {
3222 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3223 disabled_rb_mask &= ~(1 << i);
3224 }
32fcdbf4 3225
416a2bd2
AD		 3226 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3227 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3228
416a2bd2
AD		 3229 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3230 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3231 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3232 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3233 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3234 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3235 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3236
f7eb9730
AD		 3237 if ((rdev->config.evergreen.max_backends == 1) &&
3238 (rdev->flags & RADEON_IS_IGP)) {
3239 if ((disabled_rb_mask & 3) == 1) {
3240 /* RB0 disabled, RB1 enabled */
3241 tmp = 0x11111111;
3242 } else {
3243 /* RB1 disabled, RB0 enabled */
3244 tmp = 0x00000000;
3245 }
3246 } else {
3247 tmp = gb_addr_config & NUM_PIPES_MASK;
3248 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3249 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3250 }
416a2bd2 3251 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3252
3253 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3254 WREG32(CGTS_TCC_DISABLE, 0);
3255 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3256 WREG32(CGTS_USER_TCC_DISABLE, 0);
3257
3258 /* set HW defaults for 3D engine */
3259 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3260 ROQ_IB2_START(0x2b)));
3261
3262 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3263
3264 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3265 SYNC_GRADIENT |
3266 SYNC_WALKER |
3267 SYNC_ALIGNER));
3268
3269 sx_debug_1 = RREG32(SX_DEBUG_1);
3270 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3271 WREG32(SX_DEBUG_1, sx_debug_1);
3272
3273
3274 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3275 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3276 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3277 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3278
b866d133
AD		 3279 if (rdev->family <= CHIP_SUMO2)
3280 WREG32(SMX_SAR_CTL0, 0x00010000);
3281
32fcdbf4
AD		 3282 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3283 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3284 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3285
3286 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3287 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3288 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3289
3290 WREG32(VGT_NUM_INSTANCES, 1);
3291 WREG32(SPI_CONFIG_CNTL, 0);
3292 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3293 WREG32(CP_PERFMON_CNTL, 0);
3294
3295 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3296 FETCH_FIFO_HIWATER(0x4) |
3297 DONE_FIFO_HIWATER(0xe0) |
3298 ALU_UPDATE_FIFO_HIWATER(0x8)));
3299
3300 sq_config = RREG32(SQ_CONFIG);
3301 sq_config &= ~(PS_PRIO(3) |
3302 VS_PRIO(3) |
3303 GS_PRIO(3) |
3304 ES_PRIO(3));
3305 sq_config |= (VC_ENABLE |
3306 EXPORT_SRC_C |
3307 PS_PRIO(0) |
3308 VS_PRIO(1) |
3309 GS_PRIO(2) |
3310 ES_PRIO(3));
3311
d5e455e4
AD		 3312 switch (rdev->family) {
3313 case CHIP_CEDAR:
3314 case CHIP_PALM:
d5c5a72f
AD		 3315 case CHIP_SUMO:
3316 case CHIP_SUMO2:
adb68fa2 3317 case CHIP_CAICOS:
32fcdbf4
AD		 3318 /* no vertex cache */
3319 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3320 break;
3321 default:
3322 break;
3323 }
32fcdbf4
AD		 3324
3325 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3326
3327 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3328 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3329 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3330 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3331 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3332 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3333 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3334
d5e455e4
AD		 3335 switch (rdev->family) {
3336 case CHIP_CEDAR:
3337 case CHIP_PALM:
d5c5a72f
AD		 3338 case CHIP_SUMO:
3339 case CHIP_SUMO2:
32fcdbf4 3340 ps_thread_count = 96;
d5e455e4
AD		 3341 break;
3342 default:
32fcdbf4 3343 ps_thread_count = 128;
d5e455e4
AD		 3344 break;
3345 }
32fcdbf4
AD		 3346
3347 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3348 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3349 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3350 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3351 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3352 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3353
3354 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3355 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3356 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3357 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3358 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3359 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3360
3361 WREG32(SQ_CONFIG, sq_config);
3362 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3363 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3364 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3365 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3366 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3367 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3368 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3369 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3370 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3371 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3372
3373 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3374 FORCE_EOV_MAX_REZ_CNT(255)));
3375
d5e455e4
AD		 3376 switch (rdev->family) {
3377 case CHIP_CEDAR:
3378 case CHIP_PALM:
d5c5a72f
AD		 3379 case CHIP_SUMO:
3380 case CHIP_SUMO2:
adb68fa2 3381 case CHIP_CAICOS:
32fcdbf4 3382 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3383 break;
3384 default:
32fcdbf4 3385 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3386 break;
3387 }
32fcdbf4
AD		 3388 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3389 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3390
3391 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3392 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3393 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3394
60a4a3e0
AD		 3395 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3396 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3397
32fcdbf4
AD		 3398 WREG32(CB_PERF_CTR0_SEL_0, 0);
3399 WREG32(CB_PERF_CTR0_SEL_1, 0);
3400 WREG32(CB_PERF_CTR1_SEL_0, 0);
3401 WREG32(CB_PERF_CTR1_SEL_1, 0);
3402 WREG32(CB_PERF_CTR2_SEL_0, 0);
3403 WREG32(CB_PERF_CTR2_SEL_1, 0);
3404 WREG32(CB_PERF_CTR3_SEL_0, 0);
3405 WREG32(CB_PERF_CTR3_SEL_1, 0);
3406
60a4a3e0
AD		 3407 /* clear render buffer base addresses */
3408 WREG32(CB_COLOR0_BASE, 0);
3409 WREG32(CB_COLOR1_BASE, 0);
3410 WREG32(CB_COLOR2_BASE, 0);
3411 WREG32(CB_COLOR3_BASE, 0);
3412 WREG32(CB_COLOR4_BASE, 0);
3413 WREG32(CB_COLOR5_BASE, 0);
3414 WREG32(CB_COLOR6_BASE, 0);
3415 WREG32(CB_COLOR7_BASE, 0);
3416 WREG32(CB_COLOR8_BASE, 0);
3417 WREG32(CB_COLOR9_BASE, 0);
3418 WREG32(CB_COLOR10_BASE, 0);
3419 WREG32(CB_COLOR11_BASE, 0);
3420
3421 /* set the shader const cache sizes to 0 */
3422 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3423 WREG32(i, 0);
3424 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3425 WREG32(i, 0);
3426
f25a5c63
AD		 3427 tmp = RREG32(HDP_MISC_CNTL);
3428 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3429 WREG32(HDP_MISC_CNTL, tmp);
3430
32fcdbf4
AD		 3431 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3432 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3433
3434 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3435
3436 udelay(50);
3437
bcc1c2a1
AD		 3438}
3439
3440int evergreen_mc_init(struct radeon_device *rdev)
3441{
bcc1c2a1
AD		 3442 u32 tmp;
3443 int chansize, numchan;
bcc1c2a1
AD		 3444
3445 /* Get VRAM informations */
3446 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3447 if ((rdev->family == CHIP_PALM) ||
3448 (rdev->family == CHIP_SUMO) ||
3449 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3450 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3451 else
3452 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3453 if (tmp & CHANSIZE_OVERRIDE) {
3454 chansize = 16;
3455 } else if (tmp & CHANSIZE_MASK) {
3456 chansize = 64;
3457 } else {
3458 chansize = 32;
3459 }
3460 tmp = RREG32(MC_SHARED_CHMAP);
3461 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3462 case 0:
3463 default:
3464 numchan = 1;
3465 break;
3466 case 1:
3467 numchan = 2;
3468 break;
3469 case 2:
3470 numchan = 4;
3471 break;
3472 case 3:
3473 numchan = 8;
3474 break;
3475 }
3476 rdev->mc.vram_width = numchan * chansize;
3477 /* Could aper size report 0 ? */
01d73a69
JC		 3478 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3479 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3480 /* Setup GPU memory space */
05b3ef69
AD		 3481 if ((rdev->family == CHIP_PALM) ||
3482 (rdev->family == CHIP_SUMO) ||
3483 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3484 /* size in bytes on fusion */
3485 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3486 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3487 } else {
05b3ef69 3488 /* size in MB on evergreen/cayman/tn */
6eb18f8b
AD		 3489 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3490 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3491 }
51e5fcd3 3492 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3493 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3494 radeon_update_bandwidth_info(rdev);
3495
bcc1c2a1
AD		 3496 return 0;
3497}
d594e46a 3498
187e3593 3499void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3500{
64c56e8c 3501 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3502 RREG32(GRBM_STATUS));
64c56e8c 3503 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3504 RREG32(GRBM_STATUS_SE0));
64c56e8c 3505 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3506 RREG32(GRBM_STATUS_SE1));
64c56e8c 3507 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3508 RREG32(SRBM_STATUS));
a65a4369
AD		 3509 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3510 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3511 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3512 RREG32(CP_STALLED_STAT1));
3513 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3514 RREG32(CP_STALLED_STAT2));
3515 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3516 RREG32(CP_BUSY_STAT));
3517 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3518 RREG32(CP_STAT));
eaaa6983
JG		 3519 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3520 RREG32(DMA_STATUS_REG));
168757ea
AD		 3521 if (rdev->family >= CHIP_CAYMAN) {
3522 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3523 RREG32(DMA_STATUS_REG + 0x800));
3524 }
0ecebb9e
AD		 3525}
3526
168757ea 3527bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3528{
a65a4369
AD		 3529 u32 crtc_hung = 0;
3530 u32 crtc_status[6];
3531 u32 i, j, tmp;
3532
3533 for (i = 0; i < rdev->num_crtc; i++) {
3534 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3535 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3536 crtc_hung |= (1 << i);
3537 }
3538 }
3539
3540 for (j = 0; j < 10; j++) {
3541 for (i = 0; i < rdev->num_crtc; i++) {
3542 if (crtc_hung & (1 << i)) {
3543 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3544 if (tmp != crtc_status[i])
3545 crtc_hung &= ~(1 << i);
3546 }
3547 }
3548 if (crtc_hung == 0)
3549 return false;
3550 udelay(100);
3551 }
3552
3553 return true;
3554}
3555
3556static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3557{
3558 u32 reset_mask = 0;
b7630473 3559 u32 tmp;
0ecebb9e 3560
a65a4369
AD		 3561 /* GRBM_STATUS */
3562 tmp = RREG32(GRBM_STATUS);
3563 if (tmp & (PA_BUSY | SC_BUSY |
3564 SH_BUSY | SX_BUSY |
3565 TA_BUSY | VGT_BUSY |
3566 DB_BUSY | CB_BUSY |
3567 SPI_BUSY | VGT_BUSY_NO_DMA))
3568 reset_mask |= RADEON_RESET_GFX;
3569
3570 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3571 CP_BUSY | CP_COHERENCY_BUSY))
3572 reset_mask |= RADEON_RESET_CP;
3573
3574 if (tmp & GRBM_EE_BUSY)
3575 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3576
a65a4369
AD		 3577 /* DMA_STATUS_REG */
3578 tmp = RREG32(DMA_STATUS_REG);
3579 if (!(tmp & DMA_IDLE))
3580 reset_mask |= RADEON_RESET_DMA;
3581
3582 /* SRBM_STATUS2 */
3583 tmp = RREG32(SRBM_STATUS2);
3584 if (tmp & DMA_BUSY)
3585 reset_mask |= RADEON_RESET_DMA;
3586
3587 /* SRBM_STATUS */
3588 tmp = RREG32(SRBM_STATUS);
3589 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3590 reset_mask |= RADEON_RESET_RLC;
3591
3592 if (tmp & IH_BUSY)
3593 reset_mask |= RADEON_RESET_IH;
3594
3595 if (tmp & SEM_BUSY)
3596 reset_mask |= RADEON_RESET_SEM;
3597
3598 if (tmp & GRBM_RQ_PENDING)
3599 reset_mask |= RADEON_RESET_GRBM;
3600
3601 if (tmp & VMC_BUSY)
3602 reset_mask |= RADEON_RESET_VMC;
3603
3604 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3605 MCC_BUSY | MCD_BUSY))
3606 reset_mask |= RADEON_RESET_MC;
3607
3608 if (evergreen_is_display_hung(rdev))
3609 reset_mask |= RADEON_RESET_DISPLAY;
3610
3611 /* VM_L2_STATUS */
3612 tmp = RREG32(VM_L2_STATUS);
3613 if (tmp & L2_BUSY)
3614 reset_mask |= RADEON_RESET_VMC;
3615
d808fc88
AD		 3616 /* Skip MC reset as it's mostly likely not hung, just busy */
3617 if (reset_mask & RADEON_RESET_MC) {
3618 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3619 reset_mask &= ~RADEON_RESET_MC;
3620 }
3621
a65a4369
AD		 3622 return reset_mask;
3623}
3624
3625static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3626{
3627 struct evergreen_mc_save save;
3628 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3629 u32 tmp;
19fc42ed 3630
0ecebb9e 3631 if (reset_mask == 0)
a65a4369 3632 return;
0ecebb9e
AD		 3633
3634 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3635
b7630473
AD		 3636 evergreen_print_gpu_status_regs(rdev);
3637
b7630473
AD		 3638 /* Disable CP parsing/prefetching */
3639 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3640
3641 if (reset_mask & RADEON_RESET_DMA) {
3642 /* Disable DMA */
3643 tmp = RREG32(DMA_RB_CNTL);
3644 tmp &= ~DMA_RB_ENABLE;
3645 WREG32(DMA_RB_CNTL, tmp);
3646 }
3647
b21b6e7a
AD		 3648 udelay(50);
3649
3650 evergreen_mc_stop(rdev, &save);
3651 if (evergreen_mc_wait_for_idle(rdev)) {
3652 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3653 }
3654
b7630473
AD		 3655 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3656 grbm_soft_reset |= SOFT_RESET_DB |
3657 SOFT_RESET_CB |
3658 SOFT_RESET_PA |
3659 SOFT_RESET_SC |
3660 SOFT_RESET_SPI |
3661 SOFT_RESET_SX |
3662 SOFT_RESET_SH |
3663 SOFT_RESET_TC |
3664 SOFT_RESET_TA |
3665 SOFT_RESET_VC |
3666 SOFT_RESET_VGT;
3667 }
3668
3669 if (reset_mask & RADEON_RESET_CP) {
3670 grbm_soft_reset |= SOFT_RESET_CP |
3671 SOFT_RESET_VGT;
3672
3673 srbm_soft_reset |= SOFT_RESET_GRBM;
3674 }
0ecebb9e
AD		 3675
3676 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3677 srbm_soft_reset |= SOFT_RESET_DMA;
3678
a65a4369
AD		 3679 if (reset_mask & RADEON_RESET_DISPLAY)
3680 srbm_soft_reset |= SOFT_RESET_DC;
3681
3682 if (reset_mask & RADEON_RESET_RLC)
3683 srbm_soft_reset |= SOFT_RESET_RLC;
3684
3685 if (reset_mask & RADEON_RESET_SEM)
3686 srbm_soft_reset |= SOFT_RESET_SEM;
3687
3688 if (reset_mask & RADEON_RESET_IH)
3689 srbm_soft_reset |= SOFT_RESET_IH;
3690
3691 if (reset_mask & RADEON_RESET_GRBM)
3692 srbm_soft_reset |= SOFT_RESET_GRBM;
3693
3694 if (reset_mask & RADEON_RESET_VMC)
3695 srbm_soft_reset |= SOFT_RESET_VMC;
3696
24178ec4
AD		 3697 if (!(rdev->flags & RADEON_IS_IGP)) {
3698 if (reset_mask & RADEON_RESET_MC)
3699 srbm_soft_reset |= SOFT_RESET_MC;
3700 }
a65a4369 3701
b7630473
AD		 3702 if (grbm_soft_reset) {
3703 tmp = RREG32(GRBM_SOFT_RESET);
3704 tmp |= grbm_soft_reset;
3705 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3706 WREG32(GRBM_SOFT_RESET, tmp);
3707 tmp = RREG32(GRBM_SOFT_RESET);
3708
3709 udelay(50);
3710
3711 tmp &= ~grbm_soft_reset;
3712 WREG32(GRBM_SOFT_RESET, tmp);
3713 tmp = RREG32(GRBM_SOFT_RESET);
3714 }
3715
3716 if (srbm_soft_reset) {
3717 tmp = RREG32(SRBM_SOFT_RESET);
3718 tmp |= srbm_soft_reset;
3719 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3720 WREG32(SRBM_SOFT_RESET, tmp);
3721 tmp = RREG32(SRBM_SOFT_RESET);
3722
3723 udelay(50);
3724
3725 tmp &= ~srbm_soft_reset;
3726 WREG32(SRBM_SOFT_RESET, tmp);
3727 tmp = RREG32(SRBM_SOFT_RESET);
3728 }
0ecebb9e
AD		 3729
3730 /* Wait a little for things to settle down */
3731 udelay(50);
3732
747943ea 3733 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3734 udelay(50);
3735
b7630473 3736 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3737}
3738
a2d07b74 3739int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3740{
a65a4369
AD		 3741 u32 reset_mask;
3742
3743 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3744
3745 if (reset_mask)
3746 r600_set_bios_scratch_engine_hung(rdev, true);
3747
3748 evergreen_gpu_soft_reset(rdev, reset_mask);
3749
3750 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3751
3752 if (!reset_mask)
3753 r600_set_bios_scratch_engine_hung(rdev, false);
3754
3755 return 0;
747943ea
AD		 3756}
3757
123bc183
AD		 3758/**
3759 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3760 *
3761 * @rdev: radeon_device pointer
3762 * @ring: radeon_ring structure holding ring information
3763 *
3764 * Check if the GFX engine is locked up.
3765 * Returns true if the engine appears to be locked up, false if not.
3766 */
3767bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3768{
3769 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3770
3771 if (!(reset_mask & (RADEON_RESET_GFX |
3772 RADEON_RESET_COMPUTE |
3773 RADEON_RESET_CP))) {
3774 radeon_ring_lockup_update(ring);
3775 return false;
3776 }
3777 /* force CP activities */
3778 radeon_ring_force_activity(rdev, ring);
3779 return radeon_ring_test_lockup(rdev, ring);
3780}
3781
3782/**
3783 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3784 *
3785 * @rdev: radeon_device pointer
3786 * @ring: radeon_ring structure holding ring information
3787 *
3788 * Check if the async DMA engine is locked up.
3789 * Returns true if the engine appears to be locked up, false if not.
3790 */
3791bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3792{
3793 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3794
3795 if (!(reset_mask & RADEON_RESET_DMA)) {
3796 radeon_ring_lockup_update(ring);
3797 return false;
3798 }
3799 /* force ring activities */
3800 radeon_ring_force_activity(rdev, ring);
3801 return radeon_ring_test_lockup(rdev, ring);
3802}
3803
45f9a39b
AD		 3804/* Interrupts */
3805
3806u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
3807{
46437057 3808 if (crtc >= rdev->num_crtc)
45f9a39b 3809 return 0;
46437057
AD		 3810 else
3811 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 3812}
3813
3814void evergreen_disable_interrupt_state(struct radeon_device *rdev)
3815{
3816 u32 tmp;
3817
1b37078b
AD		 3818 if (rdev->family >= CHIP_CAYMAN) {
3819 cayman_cp_int_cntl_setup(rdev, 0,
3820 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3821 cayman_cp_int_cntl_setup(rdev, 1, 0);
3822 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 3823 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
3824 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 3825 } else
3826 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 3827 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3828 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 3829 WREG32(GRBM_INT_CNTL, 0);
3830 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3831 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 3832 if (rdev->num_crtc >= 4) {
18007401
AD		 3833 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3834 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 3835 }
3836 if (rdev->num_crtc >= 6) {
18007401
AD		 3837 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3838 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3839 }
45f9a39b
AD		 3840
3841 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3842 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 3843 if (rdev->num_crtc >= 4) {
18007401
AD		 3844 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3845 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 3846 }
3847 if (rdev->num_crtc >= 6) {
18007401
AD		 3848 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3849 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3850 }
45f9a39b 3851
05b3ef69
AD		 3852 /* only one DAC on DCE6 */
3853 if (!ASIC_IS_DCE6(rdev))
3854 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 3855 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3856
3857 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3858 WREG32(DC_HPD1_INT_CONTROL, tmp);
3859 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3860 WREG32(DC_HPD2_INT_CONTROL, tmp);
3861 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3862 WREG32(DC_HPD3_INT_CONTROL, tmp);
3863 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3864 WREG32(DC_HPD4_INT_CONTROL, tmp);
3865 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3866 WREG32(DC_HPD5_INT_CONTROL, tmp);
3867 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3868 WREG32(DC_HPD6_INT_CONTROL, tmp);
3869
3870}
3871
3872int evergreen_irq_set(struct radeon_device *rdev)
3873{
3874 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 3875 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 3876 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3877 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 3878 u32 grbm_int_cntl = 0;
6f34be50 3879 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 3880 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 3881 u32 dma_cntl, dma_cntl1 = 0;
45f9a39b
AD		 3882
3883 if (!rdev->irq.installed) {
fce7d61b 3884 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 3885 return -EINVAL;
3886 }
3887 /* don't enable anything if the ih is disabled */
3888 if (!rdev->ih.enabled) {
3889 r600_disable_interrupts(rdev);
3890 /* force the active interrupt state to all disabled */
3891 evergreen_disable_interrupt_state(rdev);
3892 return 0;
3893 }
3894
3895 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3896 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3897 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3898 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3899 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3900 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3901
f122c610
AD		 3902 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3903 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3904 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3905 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3906 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3907 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3908
233d1ad5
AD		 3909 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3910
1b37078b
AD		 3911 if (rdev->family >= CHIP_CAYMAN) {
3912 /* enable CP interrupts on all rings */
736fc37f 3913 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 3914 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3915 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3916 }
736fc37f 3917 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 3918 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
3919 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
3920 }
736fc37f 3921 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 3922 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
3923 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
3924 }
3925 } else {
736fc37f 3926 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 3927 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3928 cp_int_cntl |= RB_INT_ENABLE;
3929 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3930 }
45f9a39b 3931 }
1b37078b 3932
233d1ad5
AD		 3933 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3934 DRM_DEBUG("r600_irq_set: sw int dma\n");
3935 dma_cntl |= TRAP_ENABLE;
3936 }
3937
f60cbd11
AD		 3938 if (rdev->family >= CHIP_CAYMAN) {
3939 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
3940 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
3941 DRM_DEBUG("r600_irq_set: sw int dma1\n");
3942 dma_cntl1 |= TRAP_ENABLE;
3943 }
3944 }
3945
6f34be50 3946 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 3947 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 3948 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
3949 crtc1 |= VBLANK_INT_MASK;
3950 }
6f34be50 3951 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 3952 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 3953 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
3954 crtc2 |= VBLANK_INT_MASK;
3955 }
6f34be50 3956 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 3957 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 3958 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
3959 crtc3 |= VBLANK_INT_MASK;
3960 }
6f34be50 3961 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 3962 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 3963 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
3964 crtc4 |= VBLANK_INT_MASK;
3965 }
6f34be50 3966 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 3967 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 3968 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
3969 crtc5 |= VBLANK_INT_MASK;
3970 }
6f34be50 3971 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 3972 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 3973 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
3974 crtc6 |= VBLANK_INT_MASK;
3975 }
3976 if (rdev->irq.hpd[0]) {
3977 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
3978 hpd1 |= DC_HPDx_INT_EN;
3979 }
3980 if (rdev->irq.hpd[1]) {
3981 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
3982 hpd2 |= DC_HPDx_INT_EN;
3983 }
3984 if (rdev->irq.hpd[2]) {
3985 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
3986 hpd3 |= DC_HPDx_INT_EN;
3987 }
3988 if (rdev->irq.hpd[3]) {
3989 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
3990 hpd4 |= DC_HPDx_INT_EN;
3991 }
3992 if (rdev->irq.hpd[4]) {
3993 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
3994 hpd5 |= DC_HPDx_INT_EN;
3995 }
3996 if (rdev->irq.hpd[5]) {
3997 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
3998 hpd6 |= DC_HPDx_INT_EN;
3999 }
f122c610
AD		 4000 if (rdev->irq.afmt[0]) {
4001 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4002 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4003 }
4004 if (rdev->irq.afmt[1]) {
4005 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4006 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4007 }
4008 if (rdev->irq.afmt[2]) {
4009 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4010 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4011 }
4012 if (rdev->irq.afmt[3]) {
4013 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4014 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4015 }
4016 if (rdev->irq.afmt[4]) {
4017 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4018 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4019 }
4020 if (rdev->irq.afmt[5]) {
4021 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4022 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4023 }
45f9a39b 4024
1b37078b
AD		 4025 if (rdev->family >= CHIP_CAYMAN) {
4026 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4027 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4028 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4029 } else
4030 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4031
4032 WREG32(DMA_CNTL, dma_cntl);
4033
f60cbd11
AD		 4034 if (rdev->family >= CHIP_CAYMAN)
4035 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4036
2031f77c 4037 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4038
4039 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4040 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4041 if (rdev->num_crtc >= 4) {
18007401
AD		 4042 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4043 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4044 }
4045 if (rdev->num_crtc >= 6) {
18007401
AD		 4046 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4047 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4048 }
45f9a39b 4049
6f34be50
AD		 4050 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4051 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4052 if (rdev->num_crtc >= 4) {
4053 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4054 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4055 }
4056 if (rdev->num_crtc >= 6) {
4057 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4058 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4059 }
6f34be50 4060
45f9a39b
AD		 4061 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4062 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4063 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4064 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4065 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4066 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4067
f122c610
AD		 4068 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4069 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4070 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4071 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4072 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4073 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4074
bcc1c2a1
AD		 4075 return 0;
4076}
4077
cbdd4501 4078static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4079{
4080 u32 tmp;
4081
6f34be50
AD		 4082 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4083 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4084 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4085 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4086 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4087 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4088 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4089 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4090 if (rdev->num_crtc >= 4) {
4091 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4092 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4093 }
4094 if (rdev->num_crtc >= 6) {
4095 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4096 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4097 }
6f34be50 4098
f122c610
AD		 4099 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4100 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4101 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4102 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4103 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4104 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4105
6f34be50
AD		 4106 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4107 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4108 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4109 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4110 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4111 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4112 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4113 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4114 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4115 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4116 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4117 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4118
b7eff394
AD		 4119 if (rdev->num_crtc >= 4) {
4120 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4121 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4122 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4123 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4124 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4125 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4126 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4127 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4128 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4129 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4130 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4131 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4132 }
4133
4134 if (rdev->num_crtc >= 6) {
4135 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4136 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4137 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4138 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4139 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4140 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4141 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4142 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4143 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4144 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4145 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4146 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4147 }
45f9a39b 4148
6f34be50 4149 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4150 tmp = RREG32(DC_HPD1_INT_CONTROL);
4151 tmp |= DC_HPDx_INT_ACK;
4152 WREG32(DC_HPD1_INT_CONTROL, tmp);
4153 }
6f34be50 4154 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4155 tmp = RREG32(DC_HPD2_INT_CONTROL);
4156 tmp |= DC_HPDx_INT_ACK;
4157 WREG32(DC_HPD2_INT_CONTROL, tmp);
4158 }
6f34be50 4159 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4160 tmp = RREG32(DC_HPD3_INT_CONTROL);
4161 tmp |= DC_HPDx_INT_ACK;
4162 WREG32(DC_HPD3_INT_CONTROL, tmp);
4163 }
6f34be50 4164 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4165 tmp = RREG32(DC_HPD4_INT_CONTROL);
4166 tmp |= DC_HPDx_INT_ACK;
4167 WREG32(DC_HPD4_INT_CONTROL, tmp);
4168 }
6f34be50 4169 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4170 tmp = RREG32(DC_HPD5_INT_CONTROL);
4171 tmp |= DC_HPDx_INT_ACK;
4172 WREG32(DC_HPD5_INT_CONTROL, tmp);
4173 }
6f34be50 4174 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4175 tmp = RREG32(DC_HPD5_INT_CONTROL);
4176 tmp |= DC_HPDx_INT_ACK;
4177 WREG32(DC_HPD6_INT_CONTROL, tmp);
4178 }
f122c610
AD		 4179 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4180 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4181 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4182 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4183 }
4184 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4185 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4186 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4187 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4188 }
4189 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4190 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4191 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4192 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4193 }
4194 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4195 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4196 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4197 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4198 }
4199 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4200 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4201 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4202 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4203 }
4204 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4205 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4206 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4207 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4208 }
45f9a39b
AD		 4209}
4210
1109ca09 4211static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4212{
45f9a39b
AD		 4213 r600_disable_interrupts(rdev);
4214 /* Wait and acknowledge irq */
4215 mdelay(1);
6f34be50 4216 evergreen_irq_ack(rdev);
45f9a39b
AD		 4217 evergreen_disable_interrupt_state(rdev);
4218}
4219
755d819e 4220void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4221{
4222 evergreen_irq_disable(rdev);
4223 r600_rlc_stop(rdev);
4224}
4225
cbdd4501 4226static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4227{
4228 u32 wptr, tmp;
4229
724c80e1 4230 if (rdev->wb.enabled)
204ae24d 4231 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4232 else
4233 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4234
4235 if (wptr & RB_OVERFLOW) {
4236 /* When a ring buffer overflow happen start parsing interrupt
4237 * from the last not overwritten vector (wptr + 16). Hopefully
4238 * this should allow us to catchup.
4239 */
4240 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4241 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4242 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4243 tmp = RREG32(IH_RB_CNTL);
4244 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4245 WREG32(IH_RB_CNTL, tmp);
4246 }
4247 return (wptr & rdev->ih.ptr_mask);
4248}
4249
4250int evergreen_irq_process(struct radeon_device *rdev)
4251{
682f1a54
DA		 4252 u32 wptr;
4253 u32 rptr;
45f9a39b
AD		 4254 u32 src_id, src_data;
4255 u32 ring_index;
45f9a39b 4256 bool queue_hotplug = false;
f122c610 4257 bool queue_hdmi = false;
45f9a39b 4258
682f1a54 4259 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4260 return IRQ_NONE;
4261
682f1a54 4262 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4263
4264restart_ih:
4265 /* is somebody else already processing irqs? */
4266 if (atomic_xchg(&rdev->ih.lock, 1))
4267 return IRQ_NONE;
4268
682f1a54
DA		 4269 rptr = rdev->ih.rptr;
4270 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4271
964f6645
BH		 4272 /* Order reading of wptr vs. reading of IH ring data */
4273 rmb();
4274
45f9a39b 4275 /* display interrupts */
6f34be50 4276 evergreen_irq_ack(rdev);
45f9a39b 4277
45f9a39b
AD		 4278 while (rptr != wptr) {
4279 /* wptr/rptr are in bytes! */
4280 ring_index = rptr / 4;
0f234f5f
AD		 4281 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4282 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4283
4284 switch (src_id) {
4285 case 1: /* D1 vblank/vline */
4286 switch (src_data) {
4287 case 0: /* D1 vblank */
6f34be50 4288 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4289 if (rdev->irq.crtc_vblank_int[0]) {
4290 drm_handle_vblank(rdev->ddev, 0);
4291 rdev->pm.vblank_sync = true;
4292 wake_up(&rdev->irq.vblank_queue);
4293 }
736fc37f 4294 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4295 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4296 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4297 DRM_DEBUG("IH: D1 vblank\n");
4298 }
4299 break;
4300 case 1: /* D1 vline */
6f34be50
AD		 4301 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4302 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4303 DRM_DEBUG("IH: D1 vline\n");
4304 }
4305 break;
4306 default:
4307 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4308 break;
4309 }
4310 break;
4311 case 2: /* D2 vblank/vline */
4312 switch (src_data) {
4313 case 0: /* D2 vblank */
6f34be50 4314 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4315 if (rdev->irq.crtc_vblank_int[1]) {
4316 drm_handle_vblank(rdev->ddev, 1);
4317 rdev->pm.vblank_sync = true;
4318 wake_up(&rdev->irq.vblank_queue);
4319 }
736fc37f 4320 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4321 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4322 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4323 DRM_DEBUG("IH: D2 vblank\n");
4324 }
4325 break;
4326 case 1: /* D2 vline */
6f34be50
AD		 4327 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4328 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4329 DRM_DEBUG("IH: D2 vline\n");
4330 }
4331 break;
4332 default:
4333 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4334 break;
4335 }
4336 break;
4337 case 3: /* D3 vblank/vline */
4338 switch (src_data) {
4339 case 0: /* D3 vblank */
6f34be50
AD		 4340 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4341 if (rdev->irq.crtc_vblank_int[2]) {
4342 drm_handle_vblank(rdev->ddev, 2);
4343 rdev->pm.vblank_sync = true;
4344 wake_up(&rdev->irq.vblank_queue);
4345 }
736fc37f 4346 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4347 radeon_crtc_handle_flip(rdev, 2);
4348 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4349 DRM_DEBUG("IH: D3 vblank\n");
4350 }
4351 break;
4352 case 1: /* D3 vline */
6f34be50
AD		 4353 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4354 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4355 DRM_DEBUG("IH: D3 vline\n");
4356 }
4357 break;
4358 default:
4359 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4360 break;
4361 }
4362 break;
4363 case 4: /* D4 vblank/vline */
4364 switch (src_data) {
4365 case 0: /* D4 vblank */
6f34be50
AD		 4366 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4367 if (rdev->irq.crtc_vblank_int[3]) {
4368 drm_handle_vblank(rdev->ddev, 3);
4369 rdev->pm.vblank_sync = true;
4370 wake_up(&rdev->irq.vblank_queue);
4371 }
736fc37f 4372 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4373 radeon_crtc_handle_flip(rdev, 3);
4374 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4375 DRM_DEBUG("IH: D4 vblank\n");
4376 }
4377 break;
4378 case 1: /* D4 vline */
6f34be50
AD		 4379 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4380 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4381 DRM_DEBUG("IH: D4 vline\n");
4382 }
4383 break;
4384 default:
4385 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4386 break;
4387 }
4388 break;
4389 case 5: /* D5 vblank/vline */
4390 switch (src_data) {
4391 case 0: /* D5 vblank */
6f34be50
AD		 4392 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4393 if (rdev->irq.crtc_vblank_int[4]) {
4394 drm_handle_vblank(rdev->ddev, 4);
4395 rdev->pm.vblank_sync = true;
4396 wake_up(&rdev->irq.vblank_queue);
4397 }
736fc37f 4398 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4399 radeon_crtc_handle_flip(rdev, 4);
4400 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4401 DRM_DEBUG("IH: D5 vblank\n");
4402 }
4403 break;
4404 case 1: /* D5 vline */
6f34be50
AD		 4405 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4406 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4407 DRM_DEBUG("IH: D5 vline\n");
4408 }
4409 break;
4410 default:
4411 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4412 break;
4413 }
4414 break;
4415 case 6: /* D6 vblank/vline */
4416 switch (src_data) {
4417 case 0: /* D6 vblank */
6f34be50
AD		 4418 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4419 if (rdev->irq.crtc_vblank_int[5]) {
4420 drm_handle_vblank(rdev->ddev, 5);
4421 rdev->pm.vblank_sync = true;
4422 wake_up(&rdev->irq.vblank_queue);
4423 }
736fc37f 4424 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4425 radeon_crtc_handle_flip(rdev, 5);
4426 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4427 DRM_DEBUG("IH: D6 vblank\n");
4428 }
4429 break;
4430 case 1: /* D6 vline */
6f34be50
AD		 4431 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4432 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4433 DRM_DEBUG("IH: D6 vline\n");
4434 }
4435 break;
4436 default:
4437 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4438 break;
4439 }
4440 break;
4441 case 42: /* HPD hotplug */
4442 switch (src_data) {
4443 case 0:
6f34be50
AD		 4444 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4445 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4446 queue_hotplug = true;
4447 DRM_DEBUG("IH: HPD1\n");
4448 }
4449 break;
4450 case 1:
6f34be50
AD		 4451 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4452 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4453 queue_hotplug = true;
4454 DRM_DEBUG("IH: HPD2\n");
4455 }
4456 break;
4457 case 2:
6f34be50
AD		 4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4459 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4460 queue_hotplug = true;
4461 DRM_DEBUG("IH: HPD3\n");
4462 }
4463 break;
4464 case 3:
6f34be50
AD		 4465 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4466 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4467 queue_hotplug = true;
4468 DRM_DEBUG("IH: HPD4\n");
4469 }
4470 break;
4471 case 4:
6f34be50
AD		 4472 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4473 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4474 queue_hotplug = true;
4475 DRM_DEBUG("IH: HPD5\n");
4476 }
4477 break;
4478 case 5:
6f34be50
AD		 4479 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4480 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4481 queue_hotplug = true;
4482 DRM_DEBUG("IH: HPD6\n");
4483 }
4484 break;
4485 default:
4486 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4487 break;
4488 }
4489 break;
f122c610
AD		 4490 case 44: /* hdmi */
4491 switch (src_data) {
4492 case 0:
4493 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4494 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4495 queue_hdmi = true;
4496 DRM_DEBUG("IH: HDMI0\n");
4497 }
4498 break;
4499 case 1:
4500 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4501 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4502 queue_hdmi = true;
4503 DRM_DEBUG("IH: HDMI1\n");
4504 }
4505 break;
4506 case 2:
4507 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4508 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4509 queue_hdmi = true;
4510 DRM_DEBUG("IH: HDMI2\n");
4511 }
4512 break;
4513 case 3:
4514 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4515 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4516 queue_hdmi = true;
4517 DRM_DEBUG("IH: HDMI3\n");
4518 }
4519 break;
4520 case 4:
4521 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4522 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4523 queue_hdmi = true;
4524 DRM_DEBUG("IH: HDMI4\n");
4525 }
4526 break;
4527 case 5:
4528 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4529 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4530 queue_hdmi = true;
4531 DRM_DEBUG("IH: HDMI5\n");
4532 }
4533 break;
4534 default:
4535 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4536 break;
4537 }
f2ba57b5
CK		 4538 case 124: /* UVD */
4539 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4540 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4541 break;
ae133a11
CK		 4542 case 146:
4543 case 147:
4544 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4545 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4546 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4547 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4548 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4549 /* reset addr and status */
4550 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4551 break;
45f9a39b
AD		 4552 case 176: /* CP_INT in ring buffer */
4553 case 177: /* CP_INT in IB1 */
4554 case 178: /* CP_INT in IB2 */
4555 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4556 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4557 break;
4558 case 181: /* CP EOP event */
4559 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4560 if (rdev->family >= CHIP_CAYMAN) {
4561 switch (src_data) {
4562 case 0:
4563 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4564 break;
4565 case 1:
4566 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4567 break;
4568 case 2:
4569 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4570 break;
4571 }
4572 } else
4573 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4574 break;
233d1ad5
AD		 4575 case 224: /* DMA trap event */
4576 DRM_DEBUG("IH: DMA trap\n");
4577 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4578 break;
2031f77c 4579 case 233: /* GUI IDLE */
303c805c 4580 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4581 break;
f60cbd11
AD		 4582 case 244: /* DMA trap event */
4583 if (rdev->family >= CHIP_CAYMAN) {
4584 DRM_DEBUG("IH: DMA1 trap\n");
4585 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4586 }
4587 break;
45f9a39b
AD		 4588 default:
4589 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4590 break;
4591 }
4592
4593 /* wptr/rptr are in bytes! */
4594 rptr += 16;
4595 rptr &= rdev->ih.ptr_mask;
4596 }
45f9a39b 4597 if (queue_hotplug)
32c87fca 4598 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4599 if (queue_hdmi)
4600 schedule_work(&rdev->audio_work);
45f9a39b
AD		 4601 rdev->ih.rptr = rptr;
4602 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4603 atomic_set(&rdev->ih.lock, 0);
4604
4605 /* make sure wptr hasn't changed while processing */
4606 wptr = evergreen_get_ih_wptr(rdev);
4607 if (wptr != rptr)
4608 goto restart_ih;
4609
45f9a39b
AD		 4610 return IRQ_HANDLED;
4611}
4612
233d1ad5
AD		 4613/**
4614 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4615 *
4616 * @rdev: radeon_device pointer
4617 * @fence: radeon fence object
4618 *
4619 * Add a DMA fence packet to the ring to write
4620 * the fence seq number and DMA trap packet to generate
4621 * an interrupt if needed (evergreen-SI).
4622 */
4623void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4624 struct radeon_fence *fence)
4625{
4626 struct radeon_ring *ring = &rdev->ring[fence->ring];
4627 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4628 /* write the fence */
0fcb6155 4629 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4630 radeon_ring_write(ring, addr & 0xfffffffc);
4631 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4632 radeon_ring_write(ring, fence->seq);
4633 /* generate an interrupt */
0fcb6155 4634 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4635 /* flush HDP */
0fcb6155 4636 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4637 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4638 radeon_ring_write(ring, 1);
4639}
4640
4641/**
4642 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4643 *
4644 * @rdev: radeon_device pointer
4645 * @ib: IB object to schedule
4646 *
4647 * Schedule an IB in the DMA ring (evergreen).
4648 */
4649void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4650 struct radeon_ib *ib)
4651{
4652 struct radeon_ring *ring = &rdev->ring[ib->ring];
4653
4654 if (rdev->wb.enabled) {
4655 u32 next_rptr = ring->wptr + 4;
4656 while ((next_rptr & 7) != 5)
4657 next_rptr++;
4658 next_rptr += 3;
0fcb6155 4659 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 4660 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4661 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4662 radeon_ring_write(ring, next_rptr);
4663 }
4664
4665 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4666 * Pad as necessary with NOPs.
4667 */
4668 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 4669 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4670 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 4671 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4672 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4673
4674}
4675
4676/**
4677 * evergreen_copy_dma - copy pages using the DMA engine
4678 *
4679 * @rdev: radeon_device pointer
4680 * @src_offset: src GPU address
4681 * @dst_offset: dst GPU address
4682 * @num_gpu_pages: number of GPU pages to xfer
4683 * @fence: radeon fence object
4684 *
4685 * Copy GPU paging using the DMA engine (evergreen-cayman).
4686 * Used by the radeon ttm implementation to move pages if
4687 * registered as the asic copy callback.
4688 */
4689int evergreen_copy_dma(struct radeon_device *rdev,
4690 uint64_t src_offset, uint64_t dst_offset,
4691 unsigned num_gpu_pages,
4692 struct radeon_fence **fence)
4693{
4694 struct radeon_semaphore *sem = NULL;
4695 int ring_index = rdev->asic->copy.dma_ring_index;
4696 struct radeon_ring *ring = &rdev->ring[ring_index];
4697 u32 size_in_dw, cur_size_in_dw;
4698 int i, num_loops;
4699 int r = 0;
4700
4701 r = radeon_semaphore_create(rdev, &sem);
4702 if (r) {
4703 DRM_ERROR("radeon: moving bo (%d).\n", r);
4704 return r;
4705 }
4706
4707 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
4708 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
4709 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
4710 if (r) {
4711 DRM_ERROR("radeon: moving bo (%d).\n", r);
4712 radeon_semaphore_free(rdev, &sem, NULL);
4713 return r;
4714 }
4715
4716 if (radeon_fence_need_sync(*fence, ring->idx)) {
4717 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
4718 ring->idx);
4719 radeon_fence_note_sync(*fence, ring->idx);
4720 } else {
4721 radeon_semaphore_free(rdev, &sem, NULL);
4722 }
4723
4724 for (i = 0; i < num_loops; i++) {
4725 cur_size_in_dw = size_in_dw;
4726 if (cur_size_in_dw > 0xFFFFF)
4727 cur_size_in_dw = 0xFFFFF;
4728 size_in_dw -= cur_size_in_dw;
0fcb6155 4729 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 4730 radeon_ring_write(ring, dst_offset & 0xfffffffc);
4731 radeon_ring_write(ring, src_offset & 0xfffffffc);
4732 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
4733 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
4734 src_offset += cur_size_in_dw * 4;
4735 dst_offset += cur_size_in_dw * 4;
4736 }
4737
4738 r = radeon_fence_emit(rdev, fence, ring->idx);
4739 if (r) {
4740 radeon_ring_unlock_undo(rdev, ring);
4741 return r;
4742 }
4743
4744 radeon_ring_unlock_commit(rdev, ring);
4745 radeon_semaphore_free(rdev, &sem, *fence);
4746
4747 return r;
4748}
4749
bcc1c2a1
AD		 4750static int evergreen_startup(struct radeon_device *rdev)
4751{
f2ba57b5 4752 struct radeon_ring *ring;
bcc1c2a1
AD		 4753 int r;
4754
9e46a48d 4755 /* enable pcie gen2 link */
cd54033a 4756 evergreen_pcie_gen2_enable(rdev);
9e46a48d 4757
0af62b01
AD		 4758 if (ASIC_IS_DCE5(rdev)) {
4759 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
4760 r = ni_init_microcode(rdev);
4761 if (r) {
4762 DRM_ERROR("Failed to load firmware!\n");
4763 return r;
4764 }
4765 }
755d819e 4766 r = ni_mc_load_microcode(rdev);
bcc1c2a1 4767 if (r) {
0af62b01 4768 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 4769 return r;
4770 }
0af62b01
AD		 4771 } else {
4772 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
4773 r = r600_init_microcode(rdev);
4774 if (r) {
4775 DRM_ERROR("Failed to load firmware!\n");
4776 return r;
4777 }
4778 }
bcc1c2a1 4779 }
fe251e2f 4780
16cdf04d
AD		 4781 r = r600_vram_scratch_init(rdev);
4782 if (r)
4783 return r;
4784
bcc1c2a1 4785 evergreen_mc_program(rdev);
bcc1c2a1 4786 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 4787 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 4788 } else {
4789 r = evergreen_pcie_gart_enable(rdev);
4790 if (r)
4791 return r;
4792 }
bcc1c2a1 4793 evergreen_gpu_init(rdev);
bcc1c2a1 4794
d7ccd8fc 4795 r = evergreen_blit_init(rdev);
bcc1c2a1 4796 if (r) {
fb3d9e97 4797 r600_blit_fini(rdev);
27cd7769 4798 rdev->asic->copy.copy = NULL;
d7ccd8fc 4799 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 4800 }
4801
724c80e1
AD		 4802 /* allocate wb buffer */
4803 r = radeon_wb_init(rdev);
4804 if (r)
4805 return r;
4806
30eb77f4
JG		 4807 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
4808 if (r) {
4809 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
4810 return r;
4811 }
4812
233d1ad5
AD		 4813 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
4814 if (r) {
4815 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
4816 return r;
4817 }
4818
f2ba57b5
CK		 4819 r = rv770_uvd_resume(rdev);
4820 if (!r) {
4821 r = radeon_fence_driver_start_ring(rdev,
4822 R600_RING_TYPE_UVD_INDEX);
4823 if (r)
4824 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
4825 }
4826
4827 if (r)
4828 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4829
bcc1c2a1
AD		 4830 /* Enable IRQ */
4831 r = r600_irq_init(rdev);
4832 if (r) {
4833 DRM_ERROR("radeon: IH init failed (%d).\n", r);
4834 radeon_irq_kms_fini(rdev);
4835 return r;
4836 }
45f9a39b 4837 evergreen_irq_set(rdev);
bcc1c2a1 4838
f2ba57b5 4839 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 4840 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 4841 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
4842 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 4843 if (r)
4844 return r;
233d1ad5
AD		 4845
4846 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
4847 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
4848 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 4849 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 4850 if (r)
4851 return r;
4852
bcc1c2a1
AD		 4853 r = evergreen_cp_load_microcode(rdev);
4854 if (r)
4855 return r;
fe251e2f 4856 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 4857 if (r)
4858 return r;
4859 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 4860 if (r)
4861 return r;
fe251e2f 4862
f2ba57b5
CK		 4863 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4864 if (ring->ring_size) {
4865 r = radeon_ring_init(rdev, ring, ring->ring_size,
4866 R600_WB_UVD_RPTR_OFFSET,
4867 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
4868 0, 0xfffff, RADEON_CP_PACKET2);
4869 if (!r)
4870 r = r600_uvd_init(rdev);
4871
4872 if (r)
4873 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
4874 }
4875
2898c348
CK		 4876 r = radeon_ib_pool_init(rdev);
4877 if (r) {
4878 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 4879 return r;
2898c348 4880 }
b15ba512 4881
69d2ae57
RM		 4882 r = r600_audio_init(rdev);
4883 if (r) {
4884 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 4885 return r;
4886 }
4887
bcc1c2a1
AD		 4888 return 0;
4889}
4890
4891int evergreen_resume(struct radeon_device *rdev)
4892{
4893 int r;
4894
86f5c9ed
AD		 4895 /* reset the asic, the gfx blocks are often in a bad state
4896 * after the driver is unloaded or after a resume
4897 */
4898 if (radeon_asic_reset(rdev))
4899 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 4900 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
4901 * posting will perform necessary task to bring back GPU into good
4902 * shape.
4903 */
4904 /* post card */
4905 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 4906
d4788db3
AD		 4907 /* init golden registers */
4908 evergreen_init_golden_registers(rdev);
4909
b15ba512 4910 rdev->accel_working = true;
bcc1c2a1
AD		 4911 r = evergreen_startup(rdev);
4912 if (r) {
755d819e 4913 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 4914 rdev->accel_working = false;
bcc1c2a1
AD		 4915 return r;
4916 }
fe251e2f 4917
bcc1c2a1
AD		 4918 return r;
4919
4920}
4921
4922int evergreen_suspend(struct radeon_device *rdev)
4923{
69d2ae57 4924 r600_audio_fini(rdev);
f2ba57b5 4925 radeon_uvd_suspend(rdev);
bcc1c2a1 4926 r700_cp_stop(rdev);
233d1ad5 4927 r600_dma_stop(rdev);
f2ba57b5 4928 r600_uvd_rbc_stop(rdev);
45f9a39b 4929 evergreen_irq_suspend(rdev);
724c80e1 4930 radeon_wb_disable(rdev);
bcc1c2a1 4931 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 4932
4933 return 0;
4934}
4935
bcc1c2a1
AD		 4936/* Plan is to move initialization in that function and use
4937 * helper function so that radeon_device_init pretty much
4938 * do nothing more than calling asic specific function. This
4939 * should also allow to remove a bunch of callback function
4940 * like vram_info.
4941 */
4942int evergreen_init(struct radeon_device *rdev)
4943{
4944 int r;
4945
bcc1c2a1
AD		 4946 /* Read BIOS */
4947 if (!radeon_get_bios(rdev)) {
4948 if (ASIC_IS_AVIVO(rdev))
4949 return -EINVAL;
4950 }
4951 /* Must be an ATOMBIOS */
4952 if (!rdev->is_atom_bios) {
755d819e 4953 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 4954 return -EINVAL;
4955 }
4956 r = radeon_atombios_init(rdev);
4957 if (r)
4958 return r;
86f5c9ed
AD		 4959 /* reset the asic, the gfx blocks are often in a bad state
4960 * after the driver is unloaded or after a resume
4961 */
4962 if (radeon_asic_reset(rdev))
4963 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 4964 /* Post card if necessary */
fd909c37 4965 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 4966 if (!rdev->bios) {
4967 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
4968 return -EINVAL;
4969 }
4970 DRM_INFO("GPU not posted. posting now...\n");
4971 atom_asic_init(rdev->mode_info.atom_context);
4972 }
d4788db3
AD		 4973 /* init golden registers */
4974 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 4975 /* Initialize scratch registers */
4976 r600_scratch_init(rdev);
4977 /* Initialize surface registers */
4978 radeon_surface_init(rdev);
4979 /* Initialize clocks */
4980 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 4981 /* Fence driver */
4982 r = radeon_fence_driver_init(rdev);
4983 if (r)
4984 return r;
d594e46a
JG		 4985 /* initialize AGP */
4986 if (rdev->flags & RADEON_IS_AGP) {
4987 r = radeon_agp_init(rdev);
4988 if (r)
4989 radeon_agp_disable(rdev);
4990 }
4991 /* initialize memory controller */
bcc1c2a1
AD		 4992 r = evergreen_mc_init(rdev);
4993 if (r)
4994 return r;
4995 /* Memory manager */
4996 r = radeon_bo_init(rdev);
4997 if (r)
4998 return r;
45f9a39b 4999
bcc1c2a1
AD		 5000 r = radeon_irq_kms_init(rdev);
5001 if (r)
5002 return r;
5003
e32eb50d
CK		 5004 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5005 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5006
233d1ad5
AD		 5007 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5008 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5009
f2ba57b5
CK		 5010 r = radeon_uvd_init(rdev);
5011 if (!r) {
5012 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5013 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5014 4096);
5015 }
5016
bcc1c2a1
AD		 5017 rdev->ih.ring_obj = NULL;
5018 r600_ih_ring_init(rdev, 64 * 1024);
5019
5020 r = r600_pcie_gart_init(rdev);
5021 if (r)
5022 return r;
0fcdb61e 5023
148a03bc 5024 rdev->accel_working = true;
bcc1c2a1
AD		 5025 r = evergreen_startup(rdev);
5026 if (r) {
fe251e2f
AD		 5027 dev_err(rdev->dev, "disabling GPU acceleration\n");
5028 r700_cp_fini(rdev);
233d1ad5 5029 r600_dma_fini(rdev);
fe251e2f 5030 r600_irq_fini(rdev);
724c80e1 5031 radeon_wb_fini(rdev);
2898c348 5032 radeon_ib_pool_fini(rdev);
fe251e2f 5033 radeon_irq_kms_fini(rdev);
0fcdb61e 5034 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5035 rdev->accel_working = false;
5036 }
77e00f2e
AD		 5037
5038 /* Don't start up if the MC ucode is missing on BTC parts.
5039 * The default clocks and voltages before the MC ucode
5040 * is loaded are not suffient for advanced operations.
5041 */
5042 if (ASIC_IS_DCE5(rdev)) {
5043 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5044 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5045 return -EINVAL;
5046 }
5047 }
5048
bcc1c2a1
AD		 5049 return 0;
5050}
5051
5052void evergreen_fini(struct radeon_device *rdev)
5053{
69d2ae57 5054 r600_audio_fini(rdev);
fb3d9e97 5055 r600_blit_fini(rdev);
45f9a39b 5056 r700_cp_fini(rdev);
233d1ad5 5057 r600_dma_fini(rdev);
bcc1c2a1 5058 r600_irq_fini(rdev);
724c80e1 5059 radeon_wb_fini(rdev);
2898c348 5060 radeon_ib_pool_fini(rdev);
bcc1c2a1 5061 radeon_irq_kms_fini(rdev);
bcc1c2a1 5062 evergreen_pcie_gart_fini(rdev);
f2ba57b5 5063 radeon_uvd_fini(rdev);
16cdf04d 5064 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5065 radeon_gem_fini(rdev);
5066 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5067 radeon_agp_fini(rdev);
5068 radeon_bo_fini(rdev);
5069 radeon_atombios_fini(rdev);
5070 kfree(rdev->bios);
5071 rdev->bios = NULL;
bcc1c2a1 5072}
9e46a48d 5073
b07759bf 5074void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5075{
197bbb3d
DA		 5076 u32 link_width_cntl, speed_cntl, mask;
5077 int ret;
9e46a48d 5078
d42dd579
AD		 5079 if (radeon_pcie_gen2 == 0)
5080 return;
5081
9e46a48d
AD		 5082 if (rdev->flags & RADEON_IS_IGP)
5083 return;
5084
5085 if (!(rdev->flags & RADEON_IS_PCIE))
5086 return;
5087
5088 /* x2 cards have a special sequence */
5089 if (ASIC_IS_X2(rdev))
5090 return;
5091
197bbb3d
DA		 5092 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
5093 if (ret != 0)
5094 return;
5095
5096 if (!(mask & DRM_PCIE_SPEED_50))
5097 return;
5098
492d2b61 5099 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5100 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5101 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5102 return;
5103 }
5104
197bbb3d
DA		 5105 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5106
9e46a48d
AD		 5107 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5108 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5109
492d2b61 5110 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5111 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5112 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5113
492d2b61 5114 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5115 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5116 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5117
492d2b61 5118 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5119 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5120 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5121
492d2b61 5122 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5123 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5124 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5125
492d2b61 5126 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5127 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5128 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5129
5130 } else {
492d2b61 5131 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5132 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5133 if (1)
5134 link_width_cntl |= LC_UPCONFIGURE_DIS;
5135 else
5136 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5137 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5138 }
5139}
Linux kernel - Deliverables
This page took 0.492225 seconds and 5 git commands to generate.