projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'range-macro' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[deliverable/linux.git] / drivers / gpu / drm / radeon / si.c
1 /*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "sid.h"
32 #include "atom.h"
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
36
37
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
68
69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
70 static void si_program_aspm(struct radeon_device *rdev);
71 extern void sumo_rlc_fini(struct radeon_device *rdev);
72 extern int sumo_rlc_init(struct radeon_device *rdev);
73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
81 extern void si_dma_vm_set_page(struct radeon_device *rdev,
82 struct radeon_ib *ib,
83 uint64_t pe,
84 uint64_t addr, unsigned count,
85 uint32_t incr, uint32_t flags);
86 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
87 bool enable);
88 static void si_fini_pg(struct radeon_device *rdev);
89 static void si_fini_cg(struct radeon_device *rdev);
90 static void si_rlc_stop(struct radeon_device *rdev);
91
92 static const u32 verde_rlc_save_restore_register_list[] =
93 {
94 (0x8000 << 16) | (0x98f4 >> 2),
95 0x00000000,
96 (0x8040 << 16) | (0x98f4 >> 2),
97 0x00000000,
98 (0x8000 << 16) | (0xe80 >> 2),
99 0x00000000,
100 (0x8040 << 16) | (0xe80 >> 2),
101 0x00000000,
102 (0x8000 << 16) | (0x89bc >> 2),
103 0x00000000,
104 (0x8040 << 16) | (0x89bc >> 2),
105 0x00000000,
106 (0x8000 << 16) | (0x8c1c >> 2),
107 0x00000000,
108 (0x8040 << 16) | (0x8c1c >> 2),
109 0x00000000,
110 (0x9c00 << 16) | (0x98f0 >> 2),
111 0x00000000,
112 (0x9c00 << 16) | (0xe7c >> 2),
113 0x00000000,
114 (0x8000 << 16) | (0x9148 >> 2),
115 0x00000000,
116 (0x8040 << 16) | (0x9148 >> 2),
117 0x00000000,
118 (0x9c00 << 16) | (0x9150 >> 2),
119 0x00000000,
120 (0x9c00 << 16) | (0x897c >> 2),
121 0x00000000,
122 (0x9c00 << 16) | (0x8d8c >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0xac54 >> 2),
125 0X00000000,
126 0x3,
127 (0x9c00 << 16) | (0x98f8 >> 2),
128 0x00000000,
129 (0x9c00 << 16) | (0x9910 >> 2),
130 0x00000000,
131 (0x9c00 << 16) | (0x9914 >> 2),
132 0x00000000,
133 (0x9c00 << 16) | (0x9918 >> 2),
134 0x00000000,
135 (0x9c00 << 16) | (0x991c >> 2),
136 0x00000000,
137 (0x9c00 << 16) | (0x9920 >> 2),
138 0x00000000,
139 (0x9c00 << 16) | (0x9924 >> 2),
140 0x00000000,
141 (0x9c00 << 16) | (0x9928 >> 2),
142 0x00000000,
143 (0x9c00 << 16) | (0x992c >> 2),
144 0x00000000,
145 (0x9c00 << 16) | (0x9930 >> 2),
146 0x00000000,
147 (0x9c00 << 16) | (0x9934 >> 2),
148 0x00000000,
149 (0x9c00 << 16) | (0x9938 >> 2),
150 0x00000000,
151 (0x9c00 << 16) | (0x993c >> 2),
152 0x00000000,
153 (0x9c00 << 16) | (0x9940 >> 2),
154 0x00000000,
155 (0x9c00 << 16) | (0x9944 >> 2),
156 0x00000000,
157 (0x9c00 << 16) | (0x9948 >> 2),
158 0x00000000,
159 (0x9c00 << 16) | (0x994c >> 2),
160 0x00000000,
161 (0x9c00 << 16) | (0x9950 >> 2),
162 0x00000000,
163 (0x9c00 << 16) | (0x9954 >> 2),
164 0x00000000,
165 (0x9c00 << 16) | (0x9958 >> 2),
166 0x00000000,
167 (0x9c00 << 16) | (0x995c >> 2),
168 0x00000000,
169 (0x9c00 << 16) | (0x9960 >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9964 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x9968 >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x996c >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x9970 >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x9974 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x9978 >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x997c >> 2),
184 0x00000000,
185 (0x9c00 << 16) | (0x9980 >> 2),
186 0x00000000,
187 (0x9c00 << 16) | (0x9984 >> 2),
188 0x00000000,
189 (0x9c00 << 16) | (0x9988 >> 2),
190 0x00000000,
191 (0x9c00 << 16) | (0x998c >> 2),
192 0x00000000,
193 (0x9c00 << 16) | (0x8c00 >> 2),
194 0x00000000,
195 (0x9c00 << 16) | (0x8c14 >> 2),
196 0x00000000,
197 (0x9c00 << 16) | (0x8c04 >> 2),
198 0x00000000,
199 (0x9c00 << 16) | (0x8c08 >> 2),
200 0x00000000,
201 (0x8000 << 16) | (0x9b7c >> 2),
202 0x00000000,
203 (0x8040 << 16) | (0x9b7c >> 2),
204 0x00000000,
205 (0x8000 << 16) | (0xe84 >> 2),
206 0x00000000,
207 (0x8040 << 16) | (0xe84 >> 2),
208 0x00000000,
209 (0x8000 << 16) | (0x89c0 >> 2),
210 0x00000000,
211 (0x8040 << 16) | (0x89c0 >> 2),
212 0x00000000,
213 (0x8000 << 16) | (0x914c >> 2),
214 0x00000000,
215 (0x8040 << 16) | (0x914c >> 2),
216 0x00000000,
217 (0x8000 << 16) | (0x8c20 >> 2),
218 0x00000000,
219 (0x8040 << 16) | (0x8c20 >> 2),
220 0x00000000,
221 (0x8000 << 16) | (0x9354 >> 2),
222 0x00000000,
223 (0x8040 << 16) | (0x9354 >> 2),
224 0x00000000,
225 (0x9c00 << 16) | (0x9060 >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9364 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x9100 >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x913c >> 2),
232 0x00000000,
233 (0x8000 << 16) | (0x90e0 >> 2),
234 0x00000000,
235 (0x8000 << 16) | (0x90e4 >> 2),
236 0x00000000,
237 (0x8000 << 16) | (0x90e8 >> 2),
238 0x00000000,
239 (0x8040 << 16) | (0x90e0 >> 2),
240 0x00000000,
241 (0x8040 << 16) | (0x90e4 >> 2),
242 0x00000000,
243 (0x8040 << 16) | (0x90e8 >> 2),
244 0x00000000,
245 (0x9c00 << 16) | (0x8bcc >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x8b24 >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0x88c4 >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0x8e50 >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0x8c0c >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0x8e58 >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0x8e5c >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0x9508 >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0x950c >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0x9494 >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0xac0c >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0xac10 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0xac14 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0xae00 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0xac08 >> 2),
274 0x00000000,
275 (0x9c00 << 16) | (0x88d4 >> 2),
276 0x00000000,
277 (0x9c00 << 16) | (0x88c8 >> 2),
278 0x00000000,
279 (0x9c00 << 16) | (0x88cc >> 2),
280 0x00000000,
281 (0x9c00 << 16) | (0x89b0 >> 2),
282 0x00000000,
283 (0x9c00 << 16) | (0x8b10 >> 2),
284 0x00000000,
285 (0x9c00 << 16) | (0x8a14 >> 2),
286 0x00000000,
287 (0x9c00 << 16) | (0x9830 >> 2),
288 0x00000000,
289 (0x9c00 << 16) | (0x9834 >> 2),
290 0x00000000,
291 (0x9c00 << 16) | (0x9838 >> 2),
292 0x00000000,
293 (0x9c00 << 16) | (0x9a10 >> 2),
294 0x00000000,
295 (0x8000 << 16) | (0x9870 >> 2),
296 0x00000000,
297 (0x8000 << 16) | (0x9874 >> 2),
298 0x00000000,
299 (0x8001 << 16) | (0x9870 >> 2),
300 0x00000000,
301 (0x8001 << 16) | (0x9874 >> 2),
302 0x00000000,
303 (0x8040 << 16) | (0x9870 >> 2),
304 0x00000000,
305 (0x8040 << 16) | (0x9874 >> 2),
306 0x00000000,
307 (0x8041 << 16) | (0x9870 >> 2),
308 0x00000000,
309 (0x8041 << 16) | (0x9874 >> 2),
310 0x00000000,
311 0x00000000
312 };
313
314 static const u32 tahiti_golden_rlc_registers[] =
315 {
316 0xc424, 0xffffffff, 0x00601005,
317 0xc47c, 0xffffffff, 0x10104040,
318 0xc488, 0xffffffff, 0x0100000a,
319 0xc314, 0xffffffff, 0x00000800,
320 0xc30c, 0xffffffff, 0x800000f4,
321 0xf4a8, 0xffffffff, 0x00000000
322 };
323
324 static const u32 tahiti_golden_registers[] =
325 {
326 0x9a10, 0x00010000, 0x00018208,
327 0x9830, 0xffffffff, 0x00000000,
328 0x9834, 0xf00fffff, 0x00000400,
329 0x9838, 0x0002021c, 0x00020200,
330 0xc78, 0x00000080, 0x00000000,
331 0xd030, 0x000300c0, 0x00800040,
332 0xd830, 0x000300c0, 0x00800040,
333 0x5bb0, 0x000000f0, 0x00000070,
334 0x5bc0, 0x00200000, 0x50100000,
335 0x7030, 0x31000311, 0x00000011,
336 0x277c, 0x00000003, 0x000007ff,
337 0x240c, 0x000007ff, 0x00000000,
338 0x8a14, 0xf000001f, 0x00000007,
339 0x8b24, 0xffffffff, 0x00ffffff,
340 0x8b10, 0x0000ff0f, 0x00000000,
341 0x28a4c, 0x07ffffff, 0x4e000000,
342 0x28350, 0x3f3f3fff, 0x2a00126a,
343 0x30, 0x000000ff, 0x0040,
344 0x34, 0x00000040, 0x00004040,
345 0x9100, 0x07ffffff, 0x03000000,
346 0x8e88, 0x01ff1f3f, 0x00000000,
347 0x8e84, 0x01ff1f3f, 0x00000000,
348 0x9060, 0x0000007f, 0x00000020,
349 0x9508, 0x00010000, 0x00010000,
350 0xac14, 0x00000200, 0x000002fb,
351 0xac10, 0xffffffff, 0x0000543b,
352 0xac0c, 0xffffffff, 0xa9210876,
353 0x88d0, 0xffffffff, 0x000fff40,
354 0x88d4, 0x0000001f, 0x00000010,
355 0x1410, 0x20000000, 0x20fffed8,
356 0x15c0, 0x000c0fc0, 0x000c0400
357 };
358
359 static const u32 tahiti_golden_registers2[] =
360 {
361 0xc64, 0x00000001, 0x00000001
362 };
363
364 static const u32 pitcairn_golden_rlc_registers[] =
365 {
366 0xc424, 0xffffffff, 0x00601004,
367 0xc47c, 0xffffffff, 0x10102020,
368 0xc488, 0xffffffff, 0x01000020,
369 0xc314, 0xffffffff, 0x00000800,
370 0xc30c, 0xffffffff, 0x800000a4
371 };
372
373 static const u32 pitcairn_golden_registers[] =
374 {
375 0x9a10, 0x00010000, 0x00018208,
376 0x9830, 0xffffffff, 0x00000000,
377 0x9834, 0xf00fffff, 0x00000400,
378 0x9838, 0x0002021c, 0x00020200,
379 0xc78, 0x00000080, 0x00000000,
380 0xd030, 0x000300c0, 0x00800040,
381 0xd830, 0x000300c0, 0x00800040,
382 0x5bb0, 0x000000f0, 0x00000070,
383 0x5bc0, 0x00200000, 0x50100000,
384 0x7030, 0x31000311, 0x00000011,
385 0x2ae4, 0x00073ffe, 0x000022a2,
386 0x240c, 0x000007ff, 0x00000000,
387 0x8a14, 0xf000001f, 0x00000007,
388 0x8b24, 0xffffffff, 0x00ffffff,
389 0x8b10, 0x0000ff0f, 0x00000000,
390 0x28a4c, 0x07ffffff, 0x4e000000,
391 0x28350, 0x3f3f3fff, 0x2a00126a,
392 0x30, 0x000000ff, 0x0040,
393 0x34, 0x00000040, 0x00004040,
394 0x9100, 0x07ffffff, 0x03000000,
395 0x9060, 0x0000007f, 0x00000020,
396 0x9508, 0x00010000, 0x00010000,
397 0xac14, 0x000003ff, 0x000000f7,
398 0xac10, 0xffffffff, 0x00000000,
399 0xac0c, 0xffffffff, 0x32761054,
400 0x88d4, 0x0000001f, 0x00000010,
401 0x15c0, 0x000c0fc0, 0x000c0400
402 };
403
404 static const u32 verde_golden_rlc_registers[] =
405 {
406 0xc424, 0xffffffff, 0x033f1005,
407 0xc47c, 0xffffffff, 0x10808020,
408 0xc488, 0xffffffff, 0x00800008,
409 0xc314, 0xffffffff, 0x00001000,
410 0xc30c, 0xffffffff, 0x80010014
411 };
412
413 static const u32 verde_golden_registers[] =
414 {
415 0x9a10, 0x00010000, 0x00018208,
416 0x9830, 0xffffffff, 0x00000000,
417 0x9834, 0xf00fffff, 0x00000400,
418 0x9838, 0x0002021c, 0x00020200,
419 0xc78, 0x00000080, 0x00000000,
420 0xd030, 0x000300c0, 0x00800040,
421 0xd030, 0x000300c0, 0x00800040,
422 0xd830, 0x000300c0, 0x00800040,
423 0xd830, 0x000300c0, 0x00800040,
424 0x5bb0, 0x000000f0, 0x00000070,
425 0x5bc0, 0x00200000, 0x50100000,
426 0x7030, 0x31000311, 0x00000011,
427 0x2ae4, 0x00073ffe, 0x000022a2,
428 0x2ae4, 0x00073ffe, 0x000022a2,
429 0x2ae4, 0x00073ffe, 0x000022a2,
430 0x240c, 0x000007ff, 0x00000000,
431 0x240c, 0x000007ff, 0x00000000,
432 0x240c, 0x000007ff, 0x00000000,
433 0x8a14, 0xf000001f, 0x00000007,
434 0x8a14, 0xf000001f, 0x00000007,
435 0x8a14, 0xf000001f, 0x00000007,
436 0x8b24, 0xffffffff, 0x00ffffff,
437 0x8b10, 0x0000ff0f, 0x00000000,
438 0x28a4c, 0x07ffffff, 0x4e000000,
439 0x28350, 0x3f3f3fff, 0x0000124a,
440 0x28350, 0x3f3f3fff, 0x0000124a,
441 0x28350, 0x3f3f3fff, 0x0000124a,
442 0x30, 0x000000ff, 0x0040,
443 0x34, 0x00000040, 0x00004040,
444 0x9100, 0x07ffffff, 0x03000000,
445 0x9100, 0x07ffffff, 0x03000000,
446 0x8e88, 0x01ff1f3f, 0x00000000,
447 0x8e88, 0x01ff1f3f, 0x00000000,
448 0x8e88, 0x01ff1f3f, 0x00000000,
449 0x8e84, 0x01ff1f3f, 0x00000000,
450 0x8e84, 0x01ff1f3f, 0x00000000,
451 0x8e84, 0x01ff1f3f, 0x00000000,
452 0x9060, 0x0000007f, 0x00000020,
453 0x9508, 0x00010000, 0x00010000,
454 0xac14, 0x000003ff, 0x00000003,
455 0xac14, 0x000003ff, 0x00000003,
456 0xac14, 0x000003ff, 0x00000003,
457 0xac10, 0xffffffff, 0x00000000,
458 0xac10, 0xffffffff, 0x00000000,
459 0xac10, 0xffffffff, 0x00000000,
460 0xac0c, 0xffffffff, 0x00001032,
461 0xac0c, 0xffffffff, 0x00001032,
462 0xac0c, 0xffffffff, 0x00001032,
463 0x88d4, 0x0000001f, 0x00000010,
464 0x88d4, 0x0000001f, 0x00000010,
465 0x88d4, 0x0000001f, 0x00000010,
466 0x15c0, 0x000c0fc0, 0x000c0400
467 };
468
469 static const u32 oland_golden_rlc_registers[] =
470 {
471 0xc424, 0xffffffff, 0x00601005,
472 0xc47c, 0xffffffff, 0x10104040,
473 0xc488, 0xffffffff, 0x0100000a,
474 0xc314, 0xffffffff, 0x00000800,
475 0xc30c, 0xffffffff, 0x800000f4
476 };
477
478 static const u32 oland_golden_registers[] =
479 {
480 0x9a10, 0x00010000, 0x00018208,
481 0x9830, 0xffffffff, 0x00000000,
482 0x9834, 0xf00fffff, 0x00000400,
483 0x9838, 0x0002021c, 0x00020200,
484 0xc78, 0x00000080, 0x00000000,
485 0xd030, 0x000300c0, 0x00800040,
486 0xd830, 0x000300c0, 0x00800040,
487 0x5bb0, 0x000000f0, 0x00000070,
488 0x5bc0, 0x00200000, 0x50100000,
489 0x7030, 0x31000311, 0x00000011,
490 0x2ae4, 0x00073ffe, 0x000022a2,
491 0x240c, 0x000007ff, 0x00000000,
492 0x8a14, 0xf000001f, 0x00000007,
493 0x8b24, 0xffffffff, 0x00ffffff,
494 0x8b10, 0x0000ff0f, 0x00000000,
495 0x28a4c, 0x07ffffff, 0x4e000000,
496 0x28350, 0x3f3f3fff, 0x00000082,
497 0x30, 0x000000ff, 0x0040,
498 0x34, 0x00000040, 0x00004040,
499 0x9100, 0x07ffffff, 0x03000000,
500 0x9060, 0x0000007f, 0x00000020,
501 0x9508, 0x00010000, 0x00010000,
502 0xac14, 0x000003ff, 0x000000f3,
503 0xac10, 0xffffffff, 0x00000000,
504 0xac0c, 0xffffffff, 0x00003210,
505 0x88d4, 0x0000001f, 0x00000010,
506 0x15c0, 0x000c0fc0, 0x000c0400
507 };
508
509 static const u32 hainan_golden_registers[] =
510 {
511 0x9a10, 0x00010000, 0x00018208,
512 0x9830, 0xffffffff, 0x00000000,
513 0x9834, 0xf00fffff, 0x00000400,
514 0x9838, 0x0002021c, 0x00020200,
515 0xd0c0, 0xff000fff, 0x00000100,
516 0xd030, 0x000300c0, 0x00800040,
517 0xd8c0, 0xff000fff, 0x00000100,
518 0xd830, 0x000300c0, 0x00800040,
519 0x2ae4, 0x00073ffe, 0x000022a2,
520 0x240c, 0x000007ff, 0x00000000,
521 0x8a14, 0xf000001f, 0x00000007,
522 0x8b24, 0xffffffff, 0x00ffffff,
523 0x8b10, 0x0000ff0f, 0x00000000,
524 0x28a4c, 0x07ffffff, 0x4e000000,
525 0x28350, 0x3f3f3fff, 0x00000000,
526 0x30, 0x000000ff, 0x0040,
527 0x34, 0x00000040, 0x00004040,
528 0x9100, 0x03e00000, 0x03600000,
529 0x9060, 0x0000007f, 0x00000020,
530 0x9508, 0x00010000, 0x00010000,
531 0xac14, 0x000003ff, 0x000000f1,
532 0xac10, 0xffffffff, 0x00000000,
533 0xac0c, 0xffffffff, 0x00003210,
534 0x88d4, 0x0000001f, 0x00000010,
535 0x15c0, 0x000c0fc0, 0x000c0400
536 };
537
538 static const u32 hainan_golden_registers2[] =
539 {
540 0x98f8, 0xffffffff, 0x02010001
541 };
542
543 static const u32 tahiti_mgcg_cgcg_init[] =
544 {
545 0xc400, 0xffffffff, 0xfffffffc,
546 0x802c, 0xffffffff, 0xe0000000,
547 0x9a60, 0xffffffff, 0x00000100,
548 0x92a4, 0xffffffff, 0x00000100,
549 0xc164, 0xffffffff, 0x00000100,
550 0x9774, 0xffffffff, 0x00000100,
551 0x8984, 0xffffffff, 0x06000100,
552 0x8a18, 0xffffffff, 0x00000100,
553 0x92a0, 0xffffffff, 0x00000100,
554 0xc380, 0xffffffff, 0x00000100,
555 0x8b28, 0xffffffff, 0x00000100,
556 0x9144, 0xffffffff, 0x00000100,
557 0x8d88, 0xffffffff, 0x00000100,
558 0x8d8c, 0xffffffff, 0x00000100,
559 0x9030, 0xffffffff, 0x00000100,
560 0x9034, 0xffffffff, 0x00000100,
561 0x9038, 0xffffffff, 0x00000100,
562 0x903c, 0xffffffff, 0x00000100,
563 0xad80, 0xffffffff, 0x00000100,
564 0xac54, 0xffffffff, 0x00000100,
565 0x897c, 0xffffffff, 0x06000100,
566 0x9868, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0xaf04, 0xffffffff, 0x00000100,
569 0xae04, 0xffffffff, 0x00000100,
570 0x949c, 0xffffffff, 0x00000100,
571 0x802c, 0xffffffff, 0xe0000000,
572 0x9160, 0xffffffff, 0x00010000,
573 0x9164, 0xffffffff, 0x00030002,
574 0x9168, 0xffffffff, 0x00040007,
575 0x916c, 0xffffffff, 0x00060005,
576 0x9170, 0xffffffff, 0x00090008,
577 0x9174, 0xffffffff, 0x00020001,
578 0x9178, 0xffffffff, 0x00040003,
579 0x917c, 0xffffffff, 0x00000007,
580 0x9180, 0xffffffff, 0x00060005,
581 0x9184, 0xffffffff, 0x00090008,
582 0x9188, 0xffffffff, 0x00030002,
583 0x918c, 0xffffffff, 0x00050004,
584 0x9190, 0xffffffff, 0x00000008,
585 0x9194, 0xffffffff, 0x00070006,
586 0x9198, 0xffffffff, 0x000a0009,
587 0x919c, 0xffffffff, 0x00040003,
588 0x91a0, 0xffffffff, 0x00060005,
589 0x91a4, 0xffffffff, 0x00000009,
590 0x91a8, 0xffffffff, 0x00080007,
591 0x91ac, 0xffffffff, 0x000b000a,
592 0x91b0, 0xffffffff, 0x00050004,
593 0x91b4, 0xffffffff, 0x00070006,
594 0x91b8, 0xffffffff, 0x0008000b,
595 0x91bc, 0xffffffff, 0x000a0009,
596 0x91c0, 0xffffffff, 0x000d000c,
597 0x91c4, 0xffffffff, 0x00060005,
598 0x91c8, 0xffffffff, 0x00080007,
599 0x91cc, 0xffffffff, 0x0000000b,
600 0x91d0, 0xffffffff, 0x000a0009,
601 0x91d4, 0xffffffff, 0x000d000c,
602 0x91d8, 0xffffffff, 0x00070006,
603 0x91dc, 0xffffffff, 0x00090008,
604 0x91e0, 0xffffffff, 0x0000000c,
605 0x91e4, 0xffffffff, 0x000b000a,
606 0x91e8, 0xffffffff, 0x000e000d,
607 0x91ec, 0xffffffff, 0x00080007,
608 0x91f0, 0xffffffff, 0x000a0009,
609 0x91f4, 0xffffffff, 0x0000000d,
610 0x91f8, 0xffffffff, 0x000c000b,
611 0x91fc, 0xffffffff, 0x000f000e,
612 0x9200, 0xffffffff, 0x00090008,
613 0x9204, 0xffffffff, 0x000b000a,
614 0x9208, 0xffffffff, 0x000c000f,
615 0x920c, 0xffffffff, 0x000e000d,
616 0x9210, 0xffffffff, 0x00110010,
617 0x9214, 0xffffffff, 0x000a0009,
618 0x9218, 0xffffffff, 0x000c000b,
619 0x921c, 0xffffffff, 0x0000000f,
620 0x9220, 0xffffffff, 0x000e000d,
621 0x9224, 0xffffffff, 0x00110010,
622 0x9228, 0xffffffff, 0x000b000a,
623 0x922c, 0xffffffff, 0x000d000c,
624 0x9230, 0xffffffff, 0x00000010,
625 0x9234, 0xffffffff, 0x000f000e,
626 0x9238, 0xffffffff, 0x00120011,
627 0x923c, 0xffffffff, 0x000c000b,
628 0x9240, 0xffffffff, 0x000e000d,
629 0x9244, 0xffffffff, 0x00000011,
630 0x9248, 0xffffffff, 0x0010000f,
631 0x924c, 0xffffffff, 0x00130012,
632 0x9250, 0xffffffff, 0x000d000c,
633 0x9254, 0xffffffff, 0x000f000e,
634 0x9258, 0xffffffff, 0x00100013,
635 0x925c, 0xffffffff, 0x00120011,
636 0x9260, 0xffffffff, 0x00150014,
637 0x9264, 0xffffffff, 0x000e000d,
638 0x9268, 0xffffffff, 0x0010000f,
639 0x926c, 0xffffffff, 0x00000013,
640 0x9270, 0xffffffff, 0x00120011,
641 0x9274, 0xffffffff, 0x00150014,
642 0x9278, 0xffffffff, 0x000f000e,
643 0x927c, 0xffffffff, 0x00110010,
644 0x9280, 0xffffffff, 0x00000014,
645 0x9284, 0xffffffff, 0x00130012,
646 0x9288, 0xffffffff, 0x00160015,
647 0x928c, 0xffffffff, 0x0010000f,
648 0x9290, 0xffffffff, 0x00120011,
649 0x9294, 0xffffffff, 0x00000015,
650 0x9298, 0xffffffff, 0x00140013,
651 0x929c, 0xffffffff, 0x00170016,
652 0x9150, 0xffffffff, 0x96940200,
653 0x8708, 0xffffffff, 0x00900100,
654 0xc478, 0xffffffff, 0x00000080,
655 0xc404, 0xffffffff, 0x0020003f,
656 0x30, 0xffffffff, 0x0000001c,
657 0x34, 0x000f0000, 0x000f0000,
658 0x160c, 0xffffffff, 0x00000100,
659 0x1024, 0xffffffff, 0x00000100,
660 0x102c, 0x00000101, 0x00000000,
661 0x20a8, 0xffffffff, 0x00000104,
662 0x264c, 0x000c0000, 0x000c0000,
663 0x2648, 0x000c0000, 0x000c0000,
664 0x55e4, 0xff000fff, 0x00000100,
665 0x55e8, 0x00000001, 0x00000001,
666 0x2f50, 0x00000001, 0x00000001,
667 0x30cc, 0xc0000fff, 0x00000104,
668 0xc1e4, 0x00000001, 0x00000001,
669 0xd0c0, 0xfffffff0, 0x00000100,
670 0xd8c0, 0xfffffff0, 0x00000100
671 };
672
673 static const u32 pitcairn_mgcg_cgcg_init[] =
674 {
675 0xc400, 0xffffffff, 0xfffffffc,
676 0x802c, 0xffffffff, 0xe0000000,
677 0x9a60, 0xffffffff, 0x00000100,
678 0x92a4, 0xffffffff, 0x00000100,
679 0xc164, 0xffffffff, 0x00000100,
680 0x9774, 0xffffffff, 0x00000100,
681 0x8984, 0xffffffff, 0x06000100,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x92a0, 0xffffffff, 0x00000100,
684 0xc380, 0xffffffff, 0x00000100,
685 0x8b28, 0xffffffff, 0x00000100,
686 0x9144, 0xffffffff, 0x00000100,
687 0x8d88, 0xffffffff, 0x00000100,
688 0x8d8c, 0xffffffff, 0x00000100,
689 0x9030, 0xffffffff, 0x00000100,
690 0x9034, 0xffffffff, 0x00000100,
691 0x9038, 0xffffffff, 0x00000100,
692 0x903c, 0xffffffff, 0x00000100,
693 0xad80, 0xffffffff, 0x00000100,
694 0xac54, 0xffffffff, 0x00000100,
695 0x897c, 0xffffffff, 0x06000100,
696 0x9868, 0xffffffff, 0x00000100,
697 0x9510, 0xffffffff, 0x00000100,
698 0xaf04, 0xffffffff, 0x00000100,
699 0xae04, 0xffffffff, 0x00000100,
700 0x949c, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xe0000000,
702 0x9160, 0xffffffff, 0x00010000,
703 0x9164, 0xffffffff, 0x00030002,
704 0x9168, 0xffffffff, 0x00040007,
705 0x916c, 0xffffffff, 0x00060005,
706 0x9170, 0xffffffff, 0x00090008,
707 0x9174, 0xffffffff, 0x00020001,
708 0x9178, 0xffffffff, 0x00040003,
709 0x917c, 0xffffffff, 0x00000007,
710 0x9180, 0xffffffff, 0x00060005,
711 0x9184, 0xffffffff, 0x00090008,
712 0x9188, 0xffffffff, 0x00030002,
713 0x918c, 0xffffffff, 0x00050004,
714 0x9190, 0xffffffff, 0x00000008,
715 0x9194, 0xffffffff, 0x00070006,
716 0x9198, 0xffffffff, 0x000a0009,
717 0x919c, 0xffffffff, 0x00040003,
718 0x91a0, 0xffffffff, 0x00060005,
719 0x91a4, 0xffffffff, 0x00000009,
720 0x91a8, 0xffffffff, 0x00080007,
721 0x91ac, 0xffffffff, 0x000b000a,
722 0x91b0, 0xffffffff, 0x00050004,
723 0x91b4, 0xffffffff, 0x00070006,
724 0x91b8, 0xffffffff, 0x0008000b,
725 0x91bc, 0xffffffff, 0x000a0009,
726 0x91c0, 0xffffffff, 0x000d000c,
727 0x9200, 0xffffffff, 0x00090008,
728 0x9204, 0xffffffff, 0x000b000a,
729 0x9208, 0xffffffff, 0x000c000f,
730 0x920c, 0xffffffff, 0x000e000d,
731 0x9210, 0xffffffff, 0x00110010,
732 0x9214, 0xffffffff, 0x000a0009,
733 0x9218, 0xffffffff, 0x000c000b,
734 0x921c, 0xffffffff, 0x0000000f,
735 0x9220, 0xffffffff, 0x000e000d,
736 0x9224, 0xffffffff, 0x00110010,
737 0x9228, 0xffffffff, 0x000b000a,
738 0x922c, 0xffffffff, 0x000d000c,
739 0x9230, 0xffffffff, 0x00000010,
740 0x9234, 0xffffffff, 0x000f000e,
741 0x9238, 0xffffffff, 0x00120011,
742 0x923c, 0xffffffff, 0x000c000b,
743 0x9240, 0xffffffff, 0x000e000d,
744 0x9244, 0xffffffff, 0x00000011,
745 0x9248, 0xffffffff, 0x0010000f,
746 0x924c, 0xffffffff, 0x00130012,
747 0x9250, 0xffffffff, 0x000d000c,
748 0x9254, 0xffffffff, 0x000f000e,
749 0x9258, 0xffffffff, 0x00100013,
750 0x925c, 0xffffffff, 0x00120011,
751 0x9260, 0xffffffff, 0x00150014,
752 0x9150, 0xffffffff, 0x96940200,
753 0x8708, 0xffffffff, 0x00900100,
754 0xc478, 0xffffffff, 0x00000080,
755 0xc404, 0xffffffff, 0x0020003f,
756 0x30, 0xffffffff, 0x0000001c,
757 0x34, 0x000f0000, 0x000f0000,
758 0x160c, 0xffffffff, 0x00000100,
759 0x1024, 0xffffffff, 0x00000100,
760 0x102c, 0x00000101, 0x00000000,
761 0x20a8, 0xffffffff, 0x00000104,
762 0x55e4, 0xff000fff, 0x00000100,
763 0x55e8, 0x00000001, 0x00000001,
764 0x2f50, 0x00000001, 0x00000001,
765 0x30cc, 0xc0000fff, 0x00000104,
766 0xc1e4, 0x00000001, 0x00000001,
767 0xd0c0, 0xfffffff0, 0x00000100,
768 0xd8c0, 0xfffffff0, 0x00000100
769 };
770
771 static const u32 verde_mgcg_cgcg_init[] =
772 {
773 0xc400, 0xffffffff, 0xfffffffc,
774 0x802c, 0xffffffff, 0xe0000000,
775 0x9a60, 0xffffffff, 0x00000100,
776 0x92a4, 0xffffffff, 0x00000100,
777 0xc164, 0xffffffff, 0x00000100,
778 0x9774, 0xffffffff, 0x00000100,
779 0x8984, 0xffffffff, 0x06000100,
780 0x8a18, 0xffffffff, 0x00000100,
781 0x92a0, 0xffffffff, 0x00000100,
782 0xc380, 0xffffffff, 0x00000100,
783 0x8b28, 0xffffffff, 0x00000100,
784 0x9144, 0xffffffff, 0x00000100,
785 0x8d88, 0xffffffff, 0x00000100,
786 0x8d8c, 0xffffffff, 0x00000100,
787 0x9030, 0xffffffff, 0x00000100,
788 0x9034, 0xffffffff, 0x00000100,
789 0x9038, 0xffffffff, 0x00000100,
790 0x903c, 0xffffffff, 0x00000100,
791 0xad80, 0xffffffff, 0x00000100,
792 0xac54, 0xffffffff, 0x00000100,
793 0x897c, 0xffffffff, 0x06000100,
794 0x9868, 0xffffffff, 0x00000100,
795 0x9510, 0xffffffff, 0x00000100,
796 0xaf04, 0xffffffff, 0x00000100,
797 0xae04, 0xffffffff, 0x00000100,
798 0x949c, 0xffffffff, 0x00000100,
799 0x802c, 0xffffffff, 0xe0000000,
800 0x9160, 0xffffffff, 0x00010000,
801 0x9164, 0xffffffff, 0x00030002,
802 0x9168, 0xffffffff, 0x00040007,
803 0x916c, 0xffffffff, 0x00060005,
804 0x9170, 0xffffffff, 0x00090008,
805 0x9174, 0xffffffff, 0x00020001,
806 0x9178, 0xffffffff, 0x00040003,
807 0x917c, 0xffffffff, 0x00000007,
808 0x9180, 0xffffffff, 0x00060005,
809 0x9184, 0xffffffff, 0x00090008,
810 0x9188, 0xffffffff, 0x00030002,
811 0x918c, 0xffffffff, 0x00050004,
812 0x9190, 0xffffffff, 0x00000008,
813 0x9194, 0xffffffff, 0x00070006,
814 0x9198, 0xffffffff, 0x000a0009,
815 0x919c, 0xffffffff, 0x00040003,
816 0x91a0, 0xffffffff, 0x00060005,
817 0x91a4, 0xffffffff, 0x00000009,
818 0x91a8, 0xffffffff, 0x00080007,
819 0x91ac, 0xffffffff, 0x000b000a,
820 0x91b0, 0xffffffff, 0x00050004,
821 0x91b4, 0xffffffff, 0x00070006,
822 0x91b8, 0xffffffff, 0x0008000b,
823 0x91bc, 0xffffffff, 0x000a0009,
824 0x91c0, 0xffffffff, 0x000d000c,
825 0x9200, 0xffffffff, 0x00090008,
826 0x9204, 0xffffffff, 0x000b000a,
827 0x9208, 0xffffffff, 0x000c000f,
828 0x920c, 0xffffffff, 0x000e000d,
829 0x9210, 0xffffffff, 0x00110010,
830 0x9214, 0xffffffff, 0x000a0009,
831 0x9218, 0xffffffff, 0x000c000b,
832 0x921c, 0xffffffff, 0x0000000f,
833 0x9220, 0xffffffff, 0x000e000d,
834 0x9224, 0xffffffff, 0x00110010,
835 0x9228, 0xffffffff, 0x000b000a,
836 0x922c, 0xffffffff, 0x000d000c,
837 0x9230, 0xffffffff, 0x00000010,
838 0x9234, 0xffffffff, 0x000f000e,
839 0x9238, 0xffffffff, 0x00120011,
840 0x923c, 0xffffffff, 0x000c000b,
841 0x9240, 0xffffffff, 0x000e000d,
842 0x9244, 0xffffffff, 0x00000011,
843 0x9248, 0xffffffff, 0x0010000f,
844 0x924c, 0xffffffff, 0x00130012,
845 0x9250, 0xffffffff, 0x000d000c,
846 0x9254, 0xffffffff, 0x000f000e,
847 0x9258, 0xffffffff, 0x00100013,
848 0x925c, 0xffffffff, 0x00120011,
849 0x9260, 0xffffffff, 0x00150014,
850 0x9150, 0xffffffff, 0x96940200,
851 0x8708, 0xffffffff, 0x00900100,
852 0xc478, 0xffffffff, 0x00000080,
853 0xc404, 0xffffffff, 0x0020003f,
854 0x30, 0xffffffff, 0x0000001c,
855 0x34, 0x000f0000, 0x000f0000,
856 0x160c, 0xffffffff, 0x00000100,
857 0x1024, 0xffffffff, 0x00000100,
858 0x102c, 0x00000101, 0x00000000,
859 0x20a8, 0xffffffff, 0x00000104,
860 0x264c, 0x000c0000, 0x000c0000,
861 0x2648, 0x000c0000, 0x000c0000,
862 0x55e4, 0xff000fff, 0x00000100,
863 0x55e8, 0x00000001, 0x00000001,
864 0x2f50, 0x00000001, 0x00000001,
865 0x30cc, 0xc0000fff, 0x00000104,
866 0xc1e4, 0x00000001, 0x00000001,
867 0xd0c0, 0xfffffff0, 0x00000100,
868 0xd8c0, 0xfffffff0, 0x00000100
869 };
870
871 static const u32 oland_mgcg_cgcg_init[] =
872 {
873 0xc400, 0xffffffff, 0xfffffffc,
874 0x802c, 0xffffffff, 0xe0000000,
875 0x9a60, 0xffffffff, 0x00000100,
876 0x92a4, 0xffffffff, 0x00000100,
877 0xc164, 0xffffffff, 0x00000100,
878 0x9774, 0xffffffff, 0x00000100,
879 0x8984, 0xffffffff, 0x06000100,
880 0x8a18, 0xffffffff, 0x00000100,
881 0x92a0, 0xffffffff, 0x00000100,
882 0xc380, 0xffffffff, 0x00000100,
883 0x8b28, 0xffffffff, 0x00000100,
884 0x9144, 0xffffffff, 0x00000100,
885 0x8d88, 0xffffffff, 0x00000100,
886 0x8d8c, 0xffffffff, 0x00000100,
887 0x9030, 0xffffffff, 0x00000100,
888 0x9034, 0xffffffff, 0x00000100,
889 0x9038, 0xffffffff, 0x00000100,
890 0x903c, 0xffffffff, 0x00000100,
891 0xad80, 0xffffffff, 0x00000100,
892 0xac54, 0xffffffff, 0x00000100,
893 0x897c, 0xffffffff, 0x06000100,
894 0x9868, 0xffffffff, 0x00000100,
895 0x9510, 0xffffffff, 0x00000100,
896 0xaf04, 0xffffffff, 0x00000100,
897 0xae04, 0xffffffff, 0x00000100,
898 0x949c, 0xffffffff, 0x00000100,
899 0x802c, 0xffffffff, 0xe0000000,
900 0x9160, 0xffffffff, 0x00010000,
901 0x9164, 0xffffffff, 0x00030002,
902 0x9168, 0xffffffff, 0x00040007,
903 0x916c, 0xffffffff, 0x00060005,
904 0x9170, 0xffffffff, 0x00090008,
905 0x9174, 0xffffffff, 0x00020001,
906 0x9178, 0xffffffff, 0x00040003,
907 0x917c, 0xffffffff, 0x00000007,
908 0x9180, 0xffffffff, 0x00060005,
909 0x9184, 0xffffffff, 0x00090008,
910 0x9188, 0xffffffff, 0x00030002,
911 0x918c, 0xffffffff, 0x00050004,
912 0x9190, 0xffffffff, 0x00000008,
913 0x9194, 0xffffffff, 0x00070006,
914 0x9198, 0xffffffff, 0x000a0009,
915 0x919c, 0xffffffff, 0x00040003,
916 0x91a0, 0xffffffff, 0x00060005,
917 0x91a4, 0xffffffff, 0x00000009,
918 0x91a8, 0xffffffff, 0x00080007,
919 0x91ac, 0xffffffff, 0x000b000a,
920 0x91b0, 0xffffffff, 0x00050004,
921 0x91b4, 0xffffffff, 0x00070006,
922 0x91b8, 0xffffffff, 0x0008000b,
923 0x91bc, 0xffffffff, 0x000a0009,
924 0x91c0, 0xffffffff, 0x000d000c,
925 0x91c4, 0xffffffff, 0x00060005,
926 0x91c8, 0xffffffff, 0x00080007,
927 0x91cc, 0xffffffff, 0x0000000b,
928 0x91d0, 0xffffffff, 0x000a0009,
929 0x91d4, 0xffffffff, 0x000d000c,
930 0x9150, 0xffffffff, 0x96940200,
931 0x8708, 0xffffffff, 0x00900100,
932 0xc478, 0xffffffff, 0x00000080,
933 0xc404, 0xffffffff, 0x0020003f,
934 0x30, 0xffffffff, 0x0000001c,
935 0x34, 0x000f0000, 0x000f0000,
936 0x160c, 0xffffffff, 0x00000100,
937 0x1024, 0xffffffff, 0x00000100,
938 0x102c, 0x00000101, 0x00000000,
939 0x20a8, 0xffffffff, 0x00000104,
940 0x264c, 0x000c0000, 0x000c0000,
941 0x2648, 0x000c0000, 0x000c0000,
942 0x55e4, 0xff000fff, 0x00000100,
943 0x55e8, 0x00000001, 0x00000001,
944 0x2f50, 0x00000001, 0x00000001,
945 0x30cc, 0xc0000fff, 0x00000104,
946 0xc1e4, 0x00000001, 0x00000001,
947 0xd0c0, 0xfffffff0, 0x00000100,
948 0xd8c0, 0xfffffff0, 0x00000100
949 };
950
951 static const u32 hainan_mgcg_cgcg_init[] =
952 {
953 0xc400, 0xffffffff, 0xfffffffc,
954 0x802c, 0xffffffff, 0xe0000000,
955 0x9a60, 0xffffffff, 0x00000100,
956 0x92a4, 0xffffffff, 0x00000100,
957 0xc164, 0xffffffff, 0x00000100,
958 0x9774, 0xffffffff, 0x00000100,
959 0x8984, 0xffffffff, 0x06000100,
960 0x8a18, 0xffffffff, 0x00000100,
961 0x92a0, 0xffffffff, 0x00000100,
962 0xc380, 0xffffffff, 0x00000100,
963 0x8b28, 0xffffffff, 0x00000100,
964 0x9144, 0xffffffff, 0x00000100,
965 0x8d88, 0xffffffff, 0x00000100,
966 0x8d8c, 0xffffffff, 0x00000100,
967 0x9030, 0xffffffff, 0x00000100,
968 0x9034, 0xffffffff, 0x00000100,
969 0x9038, 0xffffffff, 0x00000100,
970 0x903c, 0xffffffff, 0x00000100,
971 0xad80, 0xffffffff, 0x00000100,
972 0xac54, 0xffffffff, 0x00000100,
973 0x897c, 0xffffffff, 0x06000100,
974 0x9868, 0xffffffff, 0x00000100,
975 0x9510, 0xffffffff, 0x00000100,
976 0xaf04, 0xffffffff, 0x00000100,
977 0xae04, 0xffffffff, 0x00000100,
978 0x949c, 0xffffffff, 0x00000100,
979 0x802c, 0xffffffff, 0xe0000000,
980 0x9160, 0xffffffff, 0x00010000,
981 0x9164, 0xffffffff, 0x00030002,
982 0x9168, 0xffffffff, 0x00040007,
983 0x916c, 0xffffffff, 0x00060005,
984 0x9170, 0xffffffff, 0x00090008,
985 0x9174, 0xffffffff, 0x00020001,
986 0x9178, 0xffffffff, 0x00040003,
987 0x917c, 0xffffffff, 0x00000007,
988 0x9180, 0xffffffff, 0x00060005,
989 0x9184, 0xffffffff, 0x00090008,
990 0x9188, 0xffffffff, 0x00030002,
991 0x918c, 0xffffffff, 0x00050004,
992 0x9190, 0xffffffff, 0x00000008,
993 0x9194, 0xffffffff, 0x00070006,
994 0x9198, 0xffffffff, 0x000a0009,
995 0x919c, 0xffffffff, 0x00040003,
996 0x91a0, 0xffffffff, 0x00060005,
997 0x91a4, 0xffffffff, 0x00000009,
998 0x91a8, 0xffffffff, 0x00080007,
999 0x91ac, 0xffffffff, 0x000b000a,
1000 0x91b0, 0xffffffff, 0x00050004,
1001 0x91b4, 0xffffffff, 0x00070006,
1002 0x91b8, 0xffffffff, 0x0008000b,
1003 0x91bc, 0xffffffff, 0x000a0009,
1004 0x91c0, 0xffffffff, 0x000d000c,
1005 0x91c4, 0xffffffff, 0x00060005,
1006 0x91c8, 0xffffffff, 0x00080007,
1007 0x91cc, 0xffffffff, 0x0000000b,
1008 0x91d0, 0xffffffff, 0x000a0009,
1009 0x91d4, 0xffffffff, 0x000d000c,
1010 0x9150, 0xffffffff, 0x96940200,
1011 0x8708, 0xffffffff, 0x00900100,
1012 0xc478, 0xffffffff, 0x00000080,
1013 0xc404, 0xffffffff, 0x0020003f,
1014 0x30, 0xffffffff, 0x0000001c,
1015 0x34, 0x000f0000, 0x000f0000,
1016 0x160c, 0xffffffff, 0x00000100,
1017 0x1024, 0xffffffff, 0x00000100,
1018 0x20a8, 0xffffffff, 0x00000104,
1019 0x264c, 0x000c0000, 0x000c0000,
1020 0x2648, 0x000c0000, 0x000c0000,
1021 0x2f50, 0x00000001, 0x00000001,
1022 0x30cc, 0xc0000fff, 0x00000104,
1023 0xc1e4, 0x00000001, 0x00000001,
1024 0xd0c0, 0xfffffff0, 0x00000100,
1025 0xd8c0, 0xfffffff0, 0x00000100
1026 };
1027
1028 static u32 verde_pg_init[] =
1029 {
1030 0x353c, 0xffffffff, 0x40000,
1031 0x3538, 0xffffffff, 0x200010ff,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x7007,
1038 0x3538, 0xffffffff, 0x300010ff,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x400000,
1045 0x3538, 0xffffffff, 0x100010ff,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x120200,
1052 0x3538, 0xffffffff, 0x500010ff,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x1e1e16,
1059 0x3538, 0xffffffff, 0x600010ff,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x171f1e,
1066 0x3538, 0xffffffff, 0x700010ff,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x353c, 0xffffffff, 0x0,
1071 0x353c, 0xffffffff, 0x0,
1072 0x353c, 0xffffffff, 0x0,
1073 0x3538, 0xffffffff, 0x9ff,
1074 0x3500, 0xffffffff, 0x0,
1075 0x3504, 0xffffffff, 0x10000800,
1076 0x3504, 0xffffffff, 0xf,
1077 0x3504, 0xffffffff, 0xf,
1078 0x3500, 0xffffffff, 0x4,
1079 0x3504, 0xffffffff, 0x1000051e,
1080 0x3504, 0xffffffff, 0xffff,
1081 0x3504, 0xffffffff, 0xffff,
1082 0x3500, 0xffffffff, 0x8,
1083 0x3504, 0xffffffff, 0x80500,
1084 0x3500, 0xffffffff, 0x12,
1085 0x3504, 0xffffffff, 0x9050c,
1086 0x3500, 0xffffffff, 0x1d,
1087 0x3504, 0xffffffff, 0xb052c,
1088 0x3500, 0xffffffff, 0x2a,
1089 0x3504, 0xffffffff, 0x1053e,
1090 0x3500, 0xffffffff, 0x2d,
1091 0x3504, 0xffffffff, 0x10546,
1092 0x3500, 0xffffffff, 0x30,
1093 0x3504, 0xffffffff, 0xa054e,
1094 0x3500, 0xffffffff, 0x3c,
1095 0x3504, 0xffffffff, 0x1055f,
1096 0x3500, 0xffffffff, 0x3f,
1097 0x3504, 0xffffffff, 0x10567,
1098 0x3500, 0xffffffff, 0x42,
1099 0x3504, 0xffffffff, 0x1056f,
1100 0x3500, 0xffffffff, 0x45,
1101 0x3504, 0xffffffff, 0x10572,
1102 0x3500, 0xffffffff, 0x48,
1103 0x3504, 0xffffffff, 0x20575,
1104 0x3500, 0xffffffff, 0x4c,
1105 0x3504, 0xffffffff, 0x190801,
1106 0x3500, 0xffffffff, 0x67,
1107 0x3504, 0xffffffff, 0x1082a,
1108 0x3500, 0xffffffff, 0x6a,
1109 0x3504, 0xffffffff, 0x1b082d,
1110 0x3500, 0xffffffff, 0x87,
1111 0x3504, 0xffffffff, 0x310851,
1112 0x3500, 0xffffffff, 0xba,
1113 0x3504, 0xffffffff, 0x891,
1114 0x3500, 0xffffffff, 0xbc,
1115 0x3504, 0xffffffff, 0x893,
1116 0x3500, 0xffffffff, 0xbe,
1117 0x3504, 0xffffffff, 0x20895,
1118 0x3500, 0xffffffff, 0xc2,
1119 0x3504, 0xffffffff, 0x20899,
1120 0x3500, 0xffffffff, 0xc6,
1121 0x3504, 0xffffffff, 0x2089d,
1122 0x3500, 0xffffffff, 0xca,
1123 0x3504, 0xffffffff, 0x8a1,
1124 0x3500, 0xffffffff, 0xcc,
1125 0x3504, 0xffffffff, 0x8a3,
1126 0x3500, 0xffffffff, 0xce,
1127 0x3504, 0xffffffff, 0x308a5,
1128 0x3500, 0xffffffff, 0xd3,
1129 0x3504, 0xffffffff, 0x6d08cd,
1130 0x3500, 0xffffffff, 0x142,
1131 0x3504, 0xffffffff, 0x2000095a,
1132 0x3504, 0xffffffff, 0x1,
1133 0x3500, 0xffffffff, 0x144,
1134 0x3504, 0xffffffff, 0x301f095b,
1135 0x3500, 0xffffffff, 0x165,
1136 0x3504, 0xffffffff, 0xc094d,
1137 0x3500, 0xffffffff, 0x173,
1138 0x3504, 0xffffffff, 0xf096d,
1139 0x3500, 0xffffffff, 0x184,
1140 0x3504, 0xffffffff, 0x15097f,
1141 0x3500, 0xffffffff, 0x19b,
1142 0x3504, 0xffffffff, 0xc0998,
1143 0x3500, 0xffffffff, 0x1a9,
1144 0x3504, 0xffffffff, 0x409a7,
1145 0x3500, 0xffffffff, 0x1af,
1146 0x3504, 0xffffffff, 0xcdc,
1147 0x3500, 0xffffffff, 0x1b1,
1148 0x3504, 0xffffffff, 0x800,
1149 0x3508, 0xffffffff, 0x6c9b2000,
1150 0x3510, 0xfc00, 0x2000,
1151 0x3544, 0xffffffff, 0xfc0,
1152 0x28d4, 0x00000100, 0x100
1153 };
1154
1155 static void si_init_golden_registers(struct radeon_device *rdev)
1156 {
1157 switch (rdev->family) {
1158 case CHIP_TAHITI:
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_golden_rlc_registers,
1164 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_mgcg_cgcg_init,
1167 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1168 radeon_program_register_sequence(rdev,
1169 tahiti_golden_registers2,
1170 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1171 break;
1172 case CHIP_PITCAIRN:
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_golden_rlc_registers,
1178 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1179 radeon_program_register_sequence(rdev,
1180 pitcairn_mgcg_cgcg_init,
1181 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1182 break;
1183 case CHIP_VERDE:
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_golden_rlc_registers,
1189 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1190 radeon_program_register_sequence(rdev,
1191 verde_mgcg_cgcg_init,
1192 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1193 radeon_program_register_sequence(rdev,
1194 verde_pg_init,
1195 (const u32)ARRAY_SIZE(verde_pg_init));
1196 break;
1197 case CHIP_OLAND:
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_golden_rlc_registers,
1203 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1204 radeon_program_register_sequence(rdev,
1205 oland_mgcg_cgcg_init,
1206 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1207 break;
1208 case CHIP_HAINAN:
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers));
1212 radeon_program_register_sequence(rdev,
1213 hainan_golden_registers2,
1214 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1215 radeon_program_register_sequence(rdev,
1216 hainan_mgcg_cgcg_init,
1217 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1218 break;
1219 default:
1220 break;
1221 }
1222 }
1223
1224 #define PCIE_BUS_CLK 10000
1225 #define TCLK (PCIE_BUS_CLK / 10)
1226
1227 /**
1228 * si_get_xclk - get the xclk
1229 *
1230 * @rdev: radeon_device pointer
1231 *
1232 * Returns the reference clock used by the gfx engine
1233 * (SI).
1234 */
1235 u32 si_get_xclk(struct radeon_device *rdev)
1236 {
1237 u32 reference_clock = rdev->clock.spll.reference_freq;
1238 u32 tmp;
1239
1240 tmp = RREG32(CG_CLKPIN_CNTL_2);
1241 if (tmp & MUX_TCLK_TO_XCLK)
1242 return TCLK;
1243
1244 tmp = RREG32(CG_CLKPIN_CNTL);
1245 if (tmp & XTALIN_DIVIDE)
1246 return reference_clock / 4;
1247
1248 return reference_clock;
1249 }
1250
1251 /* get temperature in millidegrees */
1252 int si_get_temp(struct radeon_device *rdev)
1253 {
1254 u32 temp;
1255 int actual_temp = 0;
1256
1257 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1258 CTF_TEMP_SHIFT;
1259
1260 if (temp & 0x200)
1261 actual_temp = 255;
1262 else
1263 actual_temp = temp & 0x1ff;
1264
1265 actual_temp = (actual_temp * 1000);
1266
1267 return actual_temp;
1268 }
1269
1270 #define TAHITI_IO_MC_REGS_SIZE 36
1271
1272 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1273 {0x0000006f, 0x03044000},
1274 {0x00000070, 0x0480c018},
1275 {0x00000071, 0x00000040},
1276 {0x00000072, 0x01000000},
1277 {0x00000074, 0x000000ff},
1278 {0x00000075, 0x00143400},
1279 {0x00000076, 0x08ec0800},
1280 {0x00000077, 0x040000cc},
1281 {0x00000079, 0x00000000},
1282 {0x0000007a, 0x21000409},
1283 {0x0000007c, 0x00000000},
1284 {0x0000007d, 0xe8000000},
1285 {0x0000007e, 0x044408a8},
1286 {0x0000007f, 0x00000003},
1287 {0x00000080, 0x00000000},
1288 {0x00000081, 0x01000000},
1289 {0x00000082, 0x02000000},
1290 {0x00000083, 0x00000000},
1291 {0x00000084, 0xe3f3e4f4},
1292 {0x00000085, 0x00052024},
1293 {0x00000087, 0x00000000},
1294 {0x00000088, 0x66036603},
1295 {0x00000089, 0x01000000},
1296 {0x0000008b, 0x1c0a0000},
1297 {0x0000008c, 0xff010000},
1298 {0x0000008e, 0xffffefff},
1299 {0x0000008f, 0xfff3efff},
1300 {0x00000090, 0xfff3efbf},
1301 {0x00000094, 0x00101101},
1302 {0x00000095, 0x00000fff},
1303 {0x00000096, 0x00116fff},
1304 {0x00000097, 0x60010000},
1305 {0x00000098, 0x10010000},
1306 {0x00000099, 0x00006000},
1307 {0x0000009a, 0x00001000},
1308 {0x0000009f, 0x00a77400}
1309 };
1310
1311 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1312 {0x0000006f, 0x03044000},
1313 {0x00000070, 0x0480c018},
1314 {0x00000071, 0x00000040},
1315 {0x00000072, 0x01000000},
1316 {0x00000074, 0x000000ff},
1317 {0x00000075, 0x00143400},
1318 {0x00000076, 0x08ec0800},
1319 {0x00000077, 0x040000cc},
1320 {0x00000079, 0x00000000},
1321 {0x0000007a, 0x21000409},
1322 {0x0000007c, 0x00000000},
1323 {0x0000007d, 0xe8000000},
1324 {0x0000007e, 0x044408a8},
1325 {0x0000007f, 0x00000003},
1326 {0x00000080, 0x00000000},
1327 {0x00000081, 0x01000000},
1328 {0x00000082, 0x02000000},
1329 {0x00000083, 0x00000000},
1330 {0x00000084, 0xe3f3e4f4},
1331 {0x00000085, 0x00052024},
1332 {0x00000087, 0x00000000},
1333 {0x00000088, 0x66036603},
1334 {0x00000089, 0x01000000},
1335 {0x0000008b, 0x1c0a0000},
1336 {0x0000008c, 0xff010000},
1337 {0x0000008e, 0xffffefff},
1338 {0x0000008f, 0xfff3efff},
1339 {0x00000090, 0xfff3efbf},
1340 {0x00000094, 0x00101101},
1341 {0x00000095, 0x00000fff},
1342 {0x00000096, 0x00116fff},
1343 {0x00000097, 0x60010000},
1344 {0x00000098, 0x10010000},
1345 {0x00000099, 0x00006000},
1346 {0x0000009a, 0x00001000},
1347 {0x0000009f, 0x00a47400}
1348 };
1349
1350 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1351 {0x0000006f, 0x03044000},
1352 {0x00000070, 0x0480c018},
1353 {0x00000071, 0x00000040},
1354 {0x00000072, 0x01000000},
1355 {0x00000074, 0x000000ff},
1356 {0x00000075, 0x00143400},
1357 {0x00000076, 0x08ec0800},
1358 {0x00000077, 0x040000cc},
1359 {0x00000079, 0x00000000},
1360 {0x0000007a, 0x21000409},
1361 {0x0000007c, 0x00000000},
1362 {0x0000007d, 0xe8000000},
1363 {0x0000007e, 0x044408a8},
1364 {0x0000007f, 0x00000003},
1365 {0x00000080, 0x00000000},
1366 {0x00000081, 0x01000000},
1367 {0x00000082, 0x02000000},
1368 {0x00000083, 0x00000000},
1369 {0x00000084, 0xe3f3e4f4},
1370 {0x00000085, 0x00052024},
1371 {0x00000087, 0x00000000},
1372 {0x00000088, 0x66036603},
1373 {0x00000089, 0x01000000},
1374 {0x0000008b, 0x1c0a0000},
1375 {0x0000008c, 0xff010000},
1376 {0x0000008e, 0xffffefff},
1377 {0x0000008f, 0xfff3efff},
1378 {0x00000090, 0xfff3efbf},
1379 {0x00000094, 0x00101101},
1380 {0x00000095, 0x00000fff},
1381 {0x00000096, 0x00116fff},
1382 {0x00000097, 0x60010000},
1383 {0x00000098, 0x10010000},
1384 {0x00000099, 0x00006000},
1385 {0x0000009a, 0x00001000},
1386 {0x0000009f, 0x00a37400}
1387 };
1388
1389 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1390 {0x0000006f, 0x03044000},
1391 {0x00000070, 0x0480c018},
1392 {0x00000071, 0x00000040},
1393 {0x00000072, 0x01000000},
1394 {0x00000074, 0x000000ff},
1395 {0x00000075, 0x00143400},
1396 {0x00000076, 0x08ec0800},
1397 {0x00000077, 0x040000cc},
1398 {0x00000079, 0x00000000},
1399 {0x0000007a, 0x21000409},
1400 {0x0000007c, 0x00000000},
1401 {0x0000007d, 0xe8000000},
1402 {0x0000007e, 0x044408a8},
1403 {0x0000007f, 0x00000003},
1404 {0x00000080, 0x00000000},
1405 {0x00000081, 0x01000000},
1406 {0x00000082, 0x02000000},
1407 {0x00000083, 0x00000000},
1408 {0x00000084, 0xe3f3e4f4},
1409 {0x00000085, 0x00052024},
1410 {0x00000087, 0x00000000},
1411 {0x00000088, 0x66036603},
1412 {0x00000089, 0x01000000},
1413 {0x0000008b, 0x1c0a0000},
1414 {0x0000008c, 0xff010000},
1415 {0x0000008e, 0xffffefff},
1416 {0x0000008f, 0xfff3efff},
1417 {0x00000090, 0xfff3efbf},
1418 {0x00000094, 0x00101101},
1419 {0x00000095, 0x00000fff},
1420 {0x00000096, 0x00116fff},
1421 {0x00000097, 0x60010000},
1422 {0x00000098, 0x10010000},
1423 {0x00000099, 0x00006000},
1424 {0x0000009a, 0x00001000},
1425 {0x0000009f, 0x00a17730}
1426 };
1427
1428 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1429 {0x0000006f, 0x03044000},
1430 {0x00000070, 0x0480c018},
1431 {0x00000071, 0x00000040},
1432 {0x00000072, 0x01000000},
1433 {0x00000074, 0x000000ff},
1434 {0x00000075, 0x00143400},
1435 {0x00000076, 0x08ec0800},
1436 {0x00000077, 0x040000cc},
1437 {0x00000079, 0x00000000},
1438 {0x0000007a, 0x21000409},
1439 {0x0000007c, 0x00000000},
1440 {0x0000007d, 0xe8000000},
1441 {0x0000007e, 0x044408a8},
1442 {0x0000007f, 0x00000003},
1443 {0x00000080, 0x00000000},
1444 {0x00000081, 0x01000000},
1445 {0x00000082, 0x02000000},
1446 {0x00000083, 0x00000000},
1447 {0x00000084, 0xe3f3e4f4},
1448 {0x00000085, 0x00052024},
1449 {0x00000087, 0x00000000},
1450 {0x00000088, 0x66036603},
1451 {0x00000089, 0x01000000},
1452 {0x0000008b, 0x1c0a0000},
1453 {0x0000008c, 0xff010000},
1454 {0x0000008e, 0xffffefff},
1455 {0x0000008f, 0xfff3efff},
1456 {0x00000090, 0xfff3efbf},
1457 {0x00000094, 0x00101101},
1458 {0x00000095, 0x00000fff},
1459 {0x00000096, 0x00116fff},
1460 {0x00000097, 0x60010000},
1461 {0x00000098, 0x10010000},
1462 {0x00000099, 0x00006000},
1463 {0x0000009a, 0x00001000},
1464 {0x0000009f, 0x00a07730}
1465 };
1466
1467 /* ucode loading */
1468 static int si_mc_load_microcode(struct radeon_device *rdev)
1469 {
1470 const __be32 *fw_data;
1471 u32 running, blackout = 0;
1472 u32 *io_mc_regs;
1473 int i, ucode_size, regs_size;
1474
1475 if (!rdev->mc_fw)
1476 return -EINVAL;
1477
1478 switch (rdev->family) {
1479 case CHIP_TAHITI:
1480 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
1484 case CHIP_PITCAIRN:
1485 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1486 ucode_size = SI_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 break;
1489 case CHIP_VERDE:
1490 default:
1491 io_mc_regs = (u32 *)&verde_io_mc_regs;
1492 ucode_size = SI_MC_UCODE_SIZE;
1493 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494 break;
1495 case CHIP_OLAND:
1496 io_mc_regs = (u32 *)&oland_io_mc_regs;
1497 ucode_size = OLAND_MC_UCODE_SIZE;
1498 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499 break;
1500 case CHIP_HAINAN:
1501 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1502 ucode_size = OLAND_MC_UCODE_SIZE;
1503 regs_size = TAHITI_IO_MC_REGS_SIZE;
1504 break;
1505 }
1506
1507 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1508
1509 if (running == 0) {
1510 if (running) {
1511 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1512 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1513 }
1514
1515 /* reset the engine and set to writable */
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1517 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1518
1519 /* load mc io regs */
1520 for (i = 0; i < regs_size; i++) {
1521 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1522 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1523 }
1524 /* load the MC ucode */
1525 fw_data = (const __be32 *)rdev->mc_fw->data;
1526 for (i = 0; i < ucode_size; i++)
1527 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1528
1529 /* put the engine back into the active state */
1530 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1531 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1532 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1533
1534 /* wait for training to complete */
1535 for (i = 0; i < rdev->usec_timeout; i++) {
1536 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1537 break;
1538 udelay(1);
1539 }
1540 for (i = 0; i < rdev->usec_timeout; i++) {
1541 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1542 break;
1543 udelay(1);
1544 }
1545
1546 if (running)
1547 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1548 }
1549
1550 return 0;
1551 }
1552
1553 static int si_init_microcode(struct radeon_device *rdev)
1554 {
1555 const char *chip_name;
1556 const char *rlc_chip_name;
1557 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1558 size_t smc_req_size;
1559 char fw_name[30];
1560 int err;
1561
1562 DRM_DEBUG("\n");
1563
1564 switch (rdev->family) {
1565 case CHIP_TAHITI:
1566 chip_name = "TAHITI";
1567 rlc_chip_name = "TAHITI";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1574 break;
1575 case CHIP_PITCAIRN:
1576 chip_name = "PITCAIRN";
1577 rlc_chip_name = "PITCAIRN";
1578 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1579 me_req_size = SI_PM4_UCODE_SIZE * 4;
1580 ce_req_size = SI_CE_UCODE_SIZE * 4;
1581 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1582 mc_req_size = SI_MC_UCODE_SIZE * 4;
1583 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1584 break;
1585 case CHIP_VERDE:
1586 chip_name = "VERDE";
1587 rlc_chip_name = "VERDE";
1588 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1589 me_req_size = SI_PM4_UCODE_SIZE * 4;
1590 ce_req_size = SI_CE_UCODE_SIZE * 4;
1591 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1592 mc_req_size = SI_MC_UCODE_SIZE * 4;
1593 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1594 break;
1595 case CHIP_OLAND:
1596 chip_name = "OLAND";
1597 rlc_chip_name = "OLAND";
1598 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1602 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1603 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1604 break;
1605 case CHIP_HAINAN:
1606 chip_name = "HAINAN";
1607 rlc_chip_name = "HAINAN";
1608 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1609 me_req_size = SI_PM4_UCODE_SIZE * 4;
1610 ce_req_size = SI_CE_UCODE_SIZE * 4;
1611 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1612 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1613 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1614 break;
1615 default: BUG();
1616 }
1617
1618 DRM_INFO("Loading %s Microcode\n", chip_name);
1619
1620 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1621 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1622 if (err)
1623 goto out;
1624 if (rdev->pfp_fw->size != pfp_req_size) {
1625 printk(KERN_ERR
1626 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627 rdev->pfp_fw->size, fw_name);
1628 err = -EINVAL;
1629 goto out;
1630 }
1631
1632 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1633 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1634 if (err)
1635 goto out;
1636 if (rdev->me_fw->size != me_req_size) {
1637 printk(KERN_ERR
1638 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1639 rdev->me_fw->size, fw_name);
1640 err = -EINVAL;
1641 }
1642
1643 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1644 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1645 if (err)
1646 goto out;
1647 if (rdev->ce_fw->size != ce_req_size) {
1648 printk(KERN_ERR
1649 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1650 rdev->ce_fw->size, fw_name);
1651 err = -EINVAL;
1652 }
1653
1654 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1655 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1656 if (err)
1657 goto out;
1658 if (rdev->rlc_fw->size != rlc_req_size) {
1659 printk(KERN_ERR
1660 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1661 rdev->rlc_fw->size, fw_name);
1662 err = -EINVAL;
1663 }
1664
1665 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1666 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1667 if (err)
1668 goto out;
1669 if (rdev->mc_fw->size != mc_req_size) {
1670 printk(KERN_ERR
1671 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1672 rdev->mc_fw->size, fw_name);
1673 err = -EINVAL;
1674 }
1675
1676 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1677 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1678 if (err) {
1679 printk(KERN_ERR
1680 "smc: error loading firmware \"%s\"\n",
1681 fw_name);
1682 release_firmware(rdev->smc_fw);
1683 rdev->smc_fw = NULL;
1684 } else if (rdev->smc_fw->size != smc_req_size) {
1685 printk(KERN_ERR
1686 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1687 rdev->smc_fw->size, fw_name);
1688 err = -EINVAL;
1689 }
1690
1691 out:
1692 if (err) {
1693 if (err != -EINVAL)
1694 printk(KERN_ERR
1695 "si_cp: Failed to load firmware \"%s\"\n",
1696 fw_name);
1697 release_firmware(rdev->pfp_fw);
1698 rdev->pfp_fw = NULL;
1699 release_firmware(rdev->me_fw);
1700 rdev->me_fw = NULL;
1701 release_firmware(rdev->ce_fw);
1702 rdev->ce_fw = NULL;
1703 release_firmware(rdev->rlc_fw);
1704 rdev->rlc_fw = NULL;
1705 release_firmware(rdev->mc_fw);
1706 rdev->mc_fw = NULL;
1707 release_firmware(rdev->smc_fw);
1708 rdev->smc_fw = NULL;
1709 }
1710 return err;
1711 }
1712
1713 /* watermark setup */
1714 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1715 struct radeon_crtc *radeon_crtc,
1716 struct drm_display_mode *mode,
1717 struct drm_display_mode *other_mode)
1718 {
1719 u32 tmp, buffer_alloc, i;
1720 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1721 /*
1722 * Line Buffer Setup
1723 * There are 3 line buffers, each one shared by 2 display controllers.
1724 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1725 * the display controllers. The paritioning is done via one of four
1726 * preset allocations specified in bits 21:20:
1727 * 0 - half lb
1728 * 2 - whole lb, other crtc must be disabled
1729 */
1730 /* this can get tricky if we have two large displays on a paired group
1731 * of crtcs. Ideally for multiple large displays we'd assign them to
1732 * non-linked crtcs for maximum line buffer allocation.
1733 */
1734 if (radeon_crtc->base.enabled && mode) {
1735 if (other_mode) {
1736 tmp = 0; /* 1/2 */
1737 buffer_alloc = 1;
1738 } else {
1739 tmp = 2; /* whole */
1740 buffer_alloc = 2;
1741 }
1742 } else {
1743 tmp = 0;
1744 buffer_alloc = 0;
1745 }
1746
1747 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1748 DC_LB_MEMORY_CONFIG(tmp));
1749
1750 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1751 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1752 for (i = 0; i < rdev->usec_timeout; i++) {
1753 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1754 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1755 break;
1756 udelay(1);
1757 }
1758
1759 if (radeon_crtc->base.enabled && mode) {
1760 switch (tmp) {
1761 case 0:
1762 default:
1763 return 4096 * 2;
1764 case 2:
1765 return 8192 * 2;
1766 }
1767 }
1768
1769 /* controller not enabled, so no lb used */
1770 return 0;
1771 }
1772
1773 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1774 {
1775 u32 tmp = RREG32(MC_SHARED_CHMAP);
1776
1777 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1778 case 0:
1779 default:
1780 return 1;
1781 case 1:
1782 return 2;
1783 case 2:
1784 return 4;
1785 case 3:
1786 return 8;
1787 case 4:
1788 return 3;
1789 case 5:
1790 return 6;
1791 case 6:
1792 return 10;
1793 case 7:
1794 return 12;
1795 case 8:
1796 return 16;
1797 }
1798 }
1799
1800 struct dce6_wm_params {
1801 u32 dram_channels; /* number of dram channels */
1802 u32 yclk; /* bandwidth per dram data pin in kHz */
1803 u32 sclk; /* engine clock in kHz */
1804 u32 disp_clk; /* display clock in kHz */
1805 u32 src_width; /* viewport width */
1806 u32 active_time; /* active display time in ns */
1807 u32 blank_time; /* blank time in ns */
1808 bool interlaced; /* mode is interlaced */
1809 fixed20_12 vsc; /* vertical scale ratio */
1810 u32 num_heads; /* number of active crtcs */
1811 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1812 u32 lb_size; /* line buffer allocated to pipe */
1813 u32 vtaps; /* vertical scaler taps */
1814 };
1815
1816 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1817 {
1818 /* Calculate raw DRAM Bandwidth */
1819 fixed20_12 dram_efficiency; /* 0.7 */
1820 fixed20_12 yclk, dram_channels, bandwidth;
1821 fixed20_12 a;
1822
1823 a.full = dfixed_const(1000);
1824 yclk.full = dfixed_const(wm->yclk);
1825 yclk.full = dfixed_div(yclk, a);
1826 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1827 a.full = dfixed_const(10);
1828 dram_efficiency.full = dfixed_const(7);
1829 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1830 bandwidth.full = dfixed_mul(dram_channels, yclk);
1831 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1832
1833 return dfixed_trunc(bandwidth);
1834 }
1835
1836 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1837 {
1838 /* Calculate DRAM Bandwidth and the part allocated to display. */
1839 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1840 fixed20_12 yclk, dram_channels, bandwidth;
1841 fixed20_12 a;
1842
1843 a.full = dfixed_const(1000);
1844 yclk.full = dfixed_const(wm->yclk);
1845 yclk.full = dfixed_div(yclk, a);
1846 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1847 a.full = dfixed_const(10);
1848 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1849 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1850 bandwidth.full = dfixed_mul(dram_channels, yclk);
1851 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1852
1853 return dfixed_trunc(bandwidth);
1854 }
1855
1856 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1857 {
1858 /* Calculate the display Data return Bandwidth */
1859 fixed20_12 return_efficiency; /* 0.8 */
1860 fixed20_12 sclk, bandwidth;
1861 fixed20_12 a;
1862
1863 a.full = dfixed_const(1000);
1864 sclk.full = dfixed_const(wm->sclk);
1865 sclk.full = dfixed_div(sclk, a);
1866 a.full = dfixed_const(10);
1867 return_efficiency.full = dfixed_const(8);
1868 return_efficiency.full = dfixed_div(return_efficiency, a);
1869 a.full = dfixed_const(32);
1870 bandwidth.full = dfixed_mul(a, sclk);
1871 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1872
1873 return dfixed_trunc(bandwidth);
1874 }
1875
1876 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1877 {
1878 return 32;
1879 }
1880
1881 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1882 {
1883 /* Calculate the DMIF Request Bandwidth */
1884 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1885 fixed20_12 disp_clk, sclk, bandwidth;
1886 fixed20_12 a, b1, b2;
1887 u32 min_bandwidth;
1888
1889 a.full = dfixed_const(1000);
1890 disp_clk.full = dfixed_const(wm->disp_clk);
1891 disp_clk.full = dfixed_div(disp_clk, a);
1892 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1893 b1.full = dfixed_mul(a, disp_clk);
1894
1895 a.full = dfixed_const(1000);
1896 sclk.full = dfixed_const(wm->sclk);
1897 sclk.full = dfixed_div(sclk, a);
1898 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1899 b2.full = dfixed_mul(a, sclk);
1900
1901 a.full = dfixed_const(10);
1902 disp_clk_request_efficiency.full = dfixed_const(8);
1903 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1904
1905 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1906
1907 a.full = dfixed_const(min_bandwidth);
1908 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1909
1910 return dfixed_trunc(bandwidth);
1911 }
1912
1913 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1914 {
1915 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1916 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1917 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1918 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1919
1920 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1921 }
1922
1923 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1924 {
1925 /* Calculate the display mode Average Bandwidth
1926 * DisplayMode should contain the source and destination dimensions,
1927 * timing, etc.
1928 */
1929 fixed20_12 bpp;
1930 fixed20_12 line_time;
1931 fixed20_12 src_width;
1932 fixed20_12 bandwidth;
1933 fixed20_12 a;
1934
1935 a.full = dfixed_const(1000);
1936 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1937 line_time.full = dfixed_div(line_time, a);
1938 bpp.full = dfixed_const(wm->bytes_per_pixel);
1939 src_width.full = dfixed_const(wm->src_width);
1940 bandwidth.full = dfixed_mul(src_width, bpp);
1941 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1942 bandwidth.full = dfixed_div(bandwidth, line_time);
1943
1944 return dfixed_trunc(bandwidth);
1945 }
1946
1947 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1948 {
1949 /* First calcualte the latency in ns */
1950 u32 mc_latency = 2000; /* 2000 ns. */
1951 u32 available_bandwidth = dce6_available_bandwidth(wm);
1952 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1953 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1954 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1955 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1956 (wm->num_heads * cursor_line_pair_return_time);
1957 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1958 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1959 u32 tmp, dmif_size = 12288;
1960 fixed20_12 a, b, c;
1961
1962 if (wm->num_heads == 0)
1963 return 0;
1964
1965 a.full = dfixed_const(2);
1966 b.full = dfixed_const(1);
1967 if ((wm->vsc.full > a.full) ||
1968 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1969 (wm->vtaps >= 5) ||
1970 ((wm->vsc.full >= a.full) && wm->interlaced))
1971 max_src_lines_per_dst_line = 4;
1972 else
1973 max_src_lines_per_dst_line = 2;
1974
1975 a.full = dfixed_const(available_bandwidth);
1976 b.full = dfixed_const(wm->num_heads);
1977 a.full = dfixed_div(a, b);
1978
1979 b.full = dfixed_const(mc_latency + 512);
1980 c.full = dfixed_const(wm->disp_clk);
1981 b.full = dfixed_div(b, c);
1982
1983 c.full = dfixed_const(dmif_size);
1984 b.full = dfixed_div(c, b);
1985
1986 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1987
1988 b.full = dfixed_const(1000);
1989 c.full = dfixed_const(wm->disp_clk);
1990 b.full = dfixed_div(c, b);
1991 c.full = dfixed_const(wm->bytes_per_pixel);
1992 b.full = dfixed_mul(b, c);
1993
1994 lb_fill_bw = min(tmp, dfixed_trunc(b));
1995
1996 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1997 b.full = dfixed_const(1000);
1998 c.full = dfixed_const(lb_fill_bw);
1999 b.full = dfixed_div(c, b);
2000 a.full = dfixed_div(a, b);
2001 line_fill_time = dfixed_trunc(a);
2002
2003 if (line_fill_time < wm->active_time)
2004 return latency;
2005 else
2006 return latency + (line_fill_time - wm->active_time);
2007
2008 }
2009
2010 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2011 {
2012 if (dce6_average_bandwidth(wm) <=
2013 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2014 return true;
2015 else
2016 return false;
2017 };
2018
2019 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2020 {
2021 if (dce6_average_bandwidth(wm) <=
2022 (dce6_available_bandwidth(wm) / wm->num_heads))
2023 return true;
2024 else
2025 return false;
2026 };
2027
2028 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2029 {
2030 u32 lb_partitions = wm->lb_size / wm->src_width;
2031 u32 line_time = wm->active_time + wm->blank_time;
2032 u32 latency_tolerant_lines;
2033 u32 latency_hiding;
2034 fixed20_12 a;
2035
2036 a.full = dfixed_const(1);
2037 if (wm->vsc.full > a.full)
2038 latency_tolerant_lines = 1;
2039 else {
2040 if (lb_partitions <= (wm->vtaps + 1))
2041 latency_tolerant_lines = 1;
2042 else
2043 latency_tolerant_lines = 2;
2044 }
2045
2046 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2047
2048 if (dce6_latency_watermark(wm) <= latency_hiding)
2049 return true;
2050 else
2051 return false;
2052 }
2053
2054 static void dce6_program_watermarks(struct radeon_device *rdev,
2055 struct radeon_crtc *radeon_crtc,
2056 u32 lb_size, u32 num_heads)
2057 {
2058 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2059 struct dce6_wm_params wm_low, wm_high;
2060 u32 dram_channels;
2061 u32 pixel_period;
2062 u32 line_time = 0;
2063 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2064 u32 priority_a_mark = 0, priority_b_mark = 0;
2065 u32 priority_a_cnt = PRIORITY_OFF;
2066 u32 priority_b_cnt = PRIORITY_OFF;
2067 u32 tmp, arb_control3;
2068 fixed20_12 a, b, c;
2069
2070 if (radeon_crtc->base.enabled && num_heads && mode) {
2071 pixel_period = 1000000 / (u32)mode->clock;
2072 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2073 priority_a_cnt = 0;
2074 priority_b_cnt = 0;
2075
2076 if (rdev->family == CHIP_ARUBA)
2077 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2078 else
2079 dram_channels = si_get_number_of_dram_channels(rdev);
2080
2081 /* watermark for high clocks */
2082 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2083 wm_high.yclk =
2084 radeon_dpm_get_mclk(rdev, false) * 10;
2085 wm_high.sclk =
2086 radeon_dpm_get_sclk(rdev, false) * 10;
2087 } else {
2088 wm_high.yclk = rdev->pm.current_mclk * 10;
2089 wm_high.sclk = rdev->pm.current_sclk * 10;
2090 }
2091
2092 wm_high.disp_clk = mode->clock;
2093 wm_high.src_width = mode->crtc_hdisplay;
2094 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2095 wm_high.blank_time = line_time - wm_high.active_time;
2096 wm_high.interlaced = false;
2097 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2098 wm_high.interlaced = true;
2099 wm_high.vsc = radeon_crtc->vsc;
2100 wm_high.vtaps = 1;
2101 if (radeon_crtc->rmx_type != RMX_OFF)
2102 wm_high.vtaps = 2;
2103 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2104 wm_high.lb_size = lb_size;
2105 wm_high.dram_channels = dram_channels;
2106 wm_high.num_heads = num_heads;
2107
2108 /* watermark for low clocks */
2109 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2110 wm_low.yclk =
2111 radeon_dpm_get_mclk(rdev, true) * 10;
2112 wm_low.sclk =
2113 radeon_dpm_get_sclk(rdev, true) * 10;
2114 } else {
2115 wm_low.yclk = rdev->pm.current_mclk * 10;
2116 wm_low.sclk = rdev->pm.current_sclk * 10;
2117 }
2118
2119 wm_low.disp_clk = mode->clock;
2120 wm_low.src_width = mode->crtc_hdisplay;
2121 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2122 wm_low.blank_time = line_time - wm_low.active_time;
2123 wm_low.interlaced = false;
2124 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2125 wm_low.interlaced = true;
2126 wm_low.vsc = radeon_crtc->vsc;
2127 wm_low.vtaps = 1;
2128 if (radeon_crtc->rmx_type != RMX_OFF)
2129 wm_low.vtaps = 2;
2130 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2131 wm_low.lb_size = lb_size;
2132 wm_low.dram_channels = dram_channels;
2133 wm_low.num_heads = num_heads;
2134
2135 /* set for high clocks */
2136 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2137 /* set for low clocks */
2138 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2139
2140 /* possibly force display priority to high */
2141 /* should really do this at mode validation time... */
2142 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2143 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2144 !dce6_check_latency_hiding(&wm_high) ||
2145 (rdev->disp_priority == 2)) {
2146 DRM_DEBUG_KMS("force priority to high\n");
2147 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2148 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2149 }
2150 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2151 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2152 !dce6_check_latency_hiding(&wm_low) ||
2153 (rdev->disp_priority == 2)) {
2154 DRM_DEBUG_KMS("force priority to high\n");
2155 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2156 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2157 }
2158
2159 a.full = dfixed_const(1000);
2160 b.full = dfixed_const(mode->clock);
2161 b.full = dfixed_div(b, a);
2162 c.full = dfixed_const(latency_watermark_a);
2163 c.full = dfixed_mul(c, b);
2164 c.full = dfixed_mul(c, radeon_crtc->hsc);
2165 c.full = dfixed_div(c, a);
2166 a.full = dfixed_const(16);
2167 c.full = dfixed_div(c, a);
2168 priority_a_mark = dfixed_trunc(c);
2169 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2170
2171 a.full = dfixed_const(1000);
2172 b.full = dfixed_const(mode->clock);
2173 b.full = dfixed_div(b, a);
2174 c.full = dfixed_const(latency_watermark_b);
2175 c.full = dfixed_mul(c, b);
2176 c.full = dfixed_mul(c, radeon_crtc->hsc);
2177 c.full = dfixed_div(c, a);
2178 a.full = dfixed_const(16);
2179 c.full = dfixed_div(c, a);
2180 priority_b_mark = dfixed_trunc(c);
2181 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2182 }
2183
2184 /* select wm A */
2185 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2186 tmp = arb_control3;
2187 tmp &= ~LATENCY_WATERMARK_MASK(3);
2188 tmp |= LATENCY_WATERMARK_MASK(1);
2189 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2190 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2191 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2192 LATENCY_HIGH_WATERMARK(line_time)));
2193 /* select wm B */
2194 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2195 tmp &= ~LATENCY_WATERMARK_MASK(3);
2196 tmp |= LATENCY_WATERMARK_MASK(2);
2197 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2198 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2199 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2200 LATENCY_HIGH_WATERMARK(line_time)));
2201 /* restore original selection */
2202 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2203
2204 /* write the priority marks */
2205 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2206 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2207
2208 /* save values for DPM */
2209 radeon_crtc->line_time = line_time;
2210 radeon_crtc->wm_high = latency_watermark_a;
2211 radeon_crtc->wm_low = latency_watermark_b;
2212 }
2213
2214 void dce6_bandwidth_update(struct radeon_device *rdev)
2215 {
2216 struct drm_display_mode *mode0 = NULL;
2217 struct drm_display_mode *mode1 = NULL;
2218 u32 num_heads = 0, lb_size;
2219 int i;
2220
2221 radeon_update_display_priority(rdev);
2222
2223 for (i = 0; i < rdev->num_crtc; i++) {
2224 if (rdev->mode_info.crtcs[i]->base.enabled)
2225 num_heads++;
2226 }
2227 for (i = 0; i < rdev->num_crtc; i += 2) {
2228 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2229 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2230 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2231 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2232 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2233 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2234 }
2235 }
2236
2237 /*
2238 * Core functions
2239 */
2240 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2241 {
2242 const u32 num_tile_mode_states = 32;
2243 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2244
2245 switch (rdev->config.si.mem_row_size_in_kb) {
2246 case 1:
2247 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2248 break;
2249 case 2:
2250 default:
2251 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2252 break;
2253 case 4:
2254 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2255 break;
2256 }
2257
2258 if ((rdev->family == CHIP_TAHITI) ||
2259 (rdev->family == CHIP_PITCAIRN)) {
2260 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2261 switch (reg_offset) {
2262 case 0: /* non-AA compressed depth or any compressed stencil */
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2264 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2266 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2267 NUM_BANKS(ADDR_SURF_16_BANK) |
2268 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2271 break;
2272 case 1: /* 2xAA/4xAA compressed depth only */
2273 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2274 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2276 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2277 NUM_BANKS(ADDR_SURF_16_BANK) |
2278 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2279 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2280 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2281 break;
2282 case 2: /* 8xAA compressed depth only */
2283 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2284 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2286 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2287 NUM_BANKS(ADDR_SURF_16_BANK) |
2288 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2291 break;
2292 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2293 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2294 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2295 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2296 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2297 NUM_BANKS(ADDR_SURF_16_BANK) |
2298 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2301 break;
2302 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2303 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2304 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2305 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2306 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2307 NUM_BANKS(ADDR_SURF_16_BANK) |
2308 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2309 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2310 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2311 break;
2312 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2313 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2314 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2315 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2316 TILE_SPLIT(split_equal_to_row_size) |
2317 NUM_BANKS(ADDR_SURF_16_BANK) |
2318 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2321 break;
2322 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2324 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2325 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2326 TILE_SPLIT(split_equal_to_row_size) |
2327 NUM_BANKS(ADDR_SURF_16_BANK) |
2328 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2329 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2330 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2331 break;
2332 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2334 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2336 TILE_SPLIT(split_equal_to_row_size) |
2337 NUM_BANKS(ADDR_SURF_16_BANK) |
2338 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2339 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2340 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2341 break;
2342 case 8: /* 1D and 1D Array Surfaces */
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2344 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2346 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2347 NUM_BANKS(ADDR_SURF_16_BANK) |
2348 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2349 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2350 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2351 break;
2352 case 9: /* Displayable maps. */
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2355 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2356 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2357 NUM_BANKS(ADDR_SURF_16_BANK) |
2358 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2361 break;
2362 case 10: /* Display 8bpp. */
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2364 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2365 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2366 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2367 NUM_BANKS(ADDR_SURF_16_BANK) |
2368 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2371 break;
2372 case 11: /* Display 16bpp. */
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2376 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2377 NUM_BANKS(ADDR_SURF_16_BANK) |
2378 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2381 break;
2382 case 12: /* Display 32bpp. */
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2384 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2386 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2387 NUM_BANKS(ADDR_SURF_16_BANK) |
2388 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2391 break;
2392 case 13: /* Thin. */
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2394 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2396 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2397 NUM_BANKS(ADDR_SURF_16_BANK) |
2398 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2401 break;
2402 case 14: /* Thin 8 bpp. */
2403 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2404 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2405 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2406 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2407 NUM_BANKS(ADDR_SURF_16_BANK) |
2408 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2411 break;
2412 case 15: /* Thin 16 bpp. */
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2416 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2417 NUM_BANKS(ADDR_SURF_16_BANK) |
2418 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2421 break;
2422 case 16: /* Thin 32 bpp. */
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2424 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2425 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2426 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2427 NUM_BANKS(ADDR_SURF_16_BANK) |
2428 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2431 break;
2432 case 17: /* Thin 64 bpp. */
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2434 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2436 TILE_SPLIT(split_equal_to_row_size) |
2437 NUM_BANKS(ADDR_SURF_16_BANK) |
2438 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2441 break;
2442 case 21: /* 8 bpp PRT. */
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2447 NUM_BANKS(ADDR_SURF_16_BANK) |
2448 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2451 break;
2452 case 22: /* 16 bpp PRT */
2453 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2454 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2455 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2456 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2457 NUM_BANKS(ADDR_SURF_16_BANK) |
2458 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2460 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2461 break;
2462 case 23: /* 32 bpp PRT */
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2467 NUM_BANKS(ADDR_SURF_16_BANK) |
2468 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2471 break;
2472 case 24: /* 64 bpp PRT */
2473 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2474 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2475 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2476 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2477 NUM_BANKS(ADDR_SURF_16_BANK) |
2478 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2480 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2481 break;
2482 case 25: /* 128 bpp PRT */
2483 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2484 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2485 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2486 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2487 NUM_BANKS(ADDR_SURF_8_BANK) |
2488 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2491 break;
2492 default:
2493 gb_tile_moden = 0;
2494 break;
2495 }
2496 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2497 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2498 }
2499 } else if ((rdev->family == CHIP_VERDE) ||
2500 (rdev->family == CHIP_OLAND) ||
2501 (rdev->family == CHIP_HAINAN)) {
2502 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2503 switch (reg_offset) {
2504 case 0: /* non-AA compressed depth or any compressed stencil */
2505 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2513 break;
2514 case 1: /* 2xAA/4xAA compressed depth only */
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2518 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2519 NUM_BANKS(ADDR_SURF_16_BANK) |
2520 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2523 break;
2524 case 2: /* 8xAA compressed depth only */
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2529 NUM_BANKS(ADDR_SURF_16_BANK) |
2530 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2531 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2532 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2533 break;
2534 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2538 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2539 NUM_BANKS(ADDR_SURF_16_BANK) |
2540 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2542 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2543 break;
2544 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2546 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2549 NUM_BANKS(ADDR_SURF_16_BANK) |
2550 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2551 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2552 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2553 break;
2554 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2556 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2557 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2558 TILE_SPLIT(split_equal_to_row_size) |
2559 NUM_BANKS(ADDR_SURF_16_BANK) |
2560 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2561 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2562 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2563 break;
2564 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2566 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 TILE_SPLIT(split_equal_to_row_size) |
2569 NUM_BANKS(ADDR_SURF_16_BANK) |
2570 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2571 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2572 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2573 break;
2574 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(split_equal_to_row_size) |
2579 NUM_BANKS(ADDR_SURF_16_BANK) |
2580 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2581 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2582 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2583 break;
2584 case 8: /* 1D and 1D Array Surfaces */
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2586 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2589 NUM_BANKS(ADDR_SURF_16_BANK) |
2590 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2591 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2592 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2593 break;
2594 case 9: /* Displayable maps. */
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603 break;
2604 case 10: /* Display 8bpp. */
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2609 NUM_BANKS(ADDR_SURF_16_BANK) |
2610 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2613 break;
2614 case 11: /* Display 16bpp. */
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2616 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2618 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2619 NUM_BANKS(ADDR_SURF_16_BANK) |
2620 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2623 break;
2624 case 12: /* Display 32bpp. */
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2626 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2628 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2629 NUM_BANKS(ADDR_SURF_16_BANK) |
2630 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2633 break;
2634 case 13: /* Thin. */
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2636 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2637 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2638 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2639 NUM_BANKS(ADDR_SURF_16_BANK) |
2640 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2643 break;
2644 case 14: /* Thin 8 bpp. */
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2646 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2649 NUM_BANKS(ADDR_SURF_16_BANK) |
2650 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2651 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2652 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2653 break;
2654 case 15: /* Thin 16 bpp. */
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2659 NUM_BANKS(ADDR_SURF_16_BANK) |
2660 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2663 break;
2664 case 16: /* Thin 32 bpp. */
2665 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2669 NUM_BANKS(ADDR_SURF_16_BANK) |
2670 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2671 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2672 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2673 break;
2674 case 17: /* Thin 64 bpp. */
2675 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2678 TILE_SPLIT(split_equal_to_row_size) |
2679 NUM_BANKS(ADDR_SURF_16_BANK) |
2680 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2683 break;
2684 case 21: /* 8 bpp PRT. */
2685 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2693 break;
2694 case 22: /* 16 bpp PRT */
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2698 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2699 NUM_BANKS(ADDR_SURF_16_BANK) |
2700 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2701 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2702 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2703 break;
2704 case 23: /* 32 bpp PRT */
2705 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2707 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2708 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2709 NUM_BANKS(ADDR_SURF_16_BANK) |
2710 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2713 break;
2714 case 24: /* 64 bpp PRT */
2715 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2716 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2717 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2718 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2719 NUM_BANKS(ADDR_SURF_16_BANK) |
2720 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2723 break;
2724 case 25: /* 128 bpp PRT */
2725 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2726 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2727 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2728 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2729 NUM_BANKS(ADDR_SURF_8_BANK) |
2730 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2731 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2732 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2733 break;
2734 default:
2735 gb_tile_moden = 0;
2736 break;
2737 }
2738 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2739 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2740 }
2741 } else
2742 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2743 }
2744
2745 static void si_select_se_sh(struct radeon_device *rdev,
2746 u32 se_num, u32 sh_num)
2747 {
2748 u32 data = INSTANCE_BROADCAST_WRITES;
2749
2750 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2751 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2752 else if (se_num == 0xffffffff)
2753 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2754 else if (sh_num == 0xffffffff)
2755 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2756 else
2757 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2758 WREG32(GRBM_GFX_INDEX, data);
2759 }
2760
2761 static u32 si_create_bitmask(u32 bit_width)
2762 {
2763 u32 i, mask = 0;
2764
2765 for (i = 0; i < bit_width; i++) {
2766 mask <<= 1;
2767 mask |= 1;
2768 }
2769 return mask;
2770 }
2771
2772 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2773 {
2774 u32 data, mask;
2775
2776 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2777 if (data & 1)
2778 data &= INACTIVE_CUS_MASK;
2779 else
2780 data = 0;
2781 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2782
2783 data >>= INACTIVE_CUS_SHIFT;
2784
2785 mask = si_create_bitmask(cu_per_sh);
2786
2787 return ~data & mask;
2788 }
2789
2790 static void si_setup_spi(struct radeon_device *rdev,
2791 u32 se_num, u32 sh_per_se,
2792 u32 cu_per_sh)
2793 {
2794 int i, j, k;
2795 u32 data, mask, active_cu;
2796
2797 for (i = 0; i < se_num; i++) {
2798 for (j = 0; j < sh_per_se; j++) {
2799 si_select_se_sh(rdev, i, j);
2800 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2801 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2802
2803 mask = 1;
2804 for (k = 0; k < 16; k++) {
2805 mask <<= k;
2806 if (active_cu & mask) {
2807 data &= ~mask;
2808 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2809 break;
2810 }
2811 }
2812 }
2813 }
2814 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2815 }
2816
2817 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2818 u32 max_rb_num, u32 se_num,
2819 u32 sh_per_se)
2820 {
2821 u32 data, mask;
2822
2823 data = RREG32(CC_RB_BACKEND_DISABLE);
2824 if (data & 1)
2825 data &= BACKEND_DISABLE_MASK;
2826 else
2827 data = 0;
2828 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2829
2830 data >>= BACKEND_DISABLE_SHIFT;
2831
2832 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2833
2834 return data & mask;
2835 }
2836
2837 static void si_setup_rb(struct radeon_device *rdev,
2838 u32 se_num, u32 sh_per_se,
2839 u32 max_rb_num)
2840 {
2841 int i, j;
2842 u32 data, mask;
2843 u32 disabled_rbs = 0;
2844 u32 enabled_rbs = 0;
2845
2846 for (i = 0; i < se_num; i++) {
2847 for (j = 0; j < sh_per_se; j++) {
2848 si_select_se_sh(rdev, i, j);
2849 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2850 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2851 }
2852 }
2853 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2854
2855 mask = 1;
2856 for (i = 0; i < max_rb_num; i++) {
2857 if (!(disabled_rbs & mask))
2858 enabled_rbs |= mask;
2859 mask <<= 1;
2860 }
2861
2862 for (i = 0; i < se_num; i++) {
2863 si_select_se_sh(rdev, i, 0xffffffff);
2864 data = 0;
2865 for (j = 0; j < sh_per_se; j++) {
2866 switch (enabled_rbs & 3) {
2867 case 1:
2868 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2869 break;
2870 case 2:
2871 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2872 break;
2873 case 3:
2874 default:
2875 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2876 break;
2877 }
2878 enabled_rbs >>= 2;
2879 }
2880 WREG32(PA_SC_RASTER_CONFIG, data);
2881 }
2882 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2883 }
2884
2885 static void si_gpu_init(struct radeon_device *rdev)
2886 {
2887 u32 gb_addr_config = 0;
2888 u32 mc_shared_chmap, mc_arb_ramcfg;
2889 u32 sx_debug_1;
2890 u32 hdp_host_path_cntl;
2891 u32 tmp;
2892 int i, j;
2893
2894 switch (rdev->family) {
2895 case CHIP_TAHITI:
2896 rdev->config.si.max_shader_engines = 2;
2897 rdev->config.si.max_tile_pipes = 12;
2898 rdev->config.si.max_cu_per_sh = 8;
2899 rdev->config.si.max_sh_per_se = 2;
2900 rdev->config.si.max_backends_per_se = 4;
2901 rdev->config.si.max_texture_channel_caches = 12;
2902 rdev->config.si.max_gprs = 256;
2903 rdev->config.si.max_gs_threads = 32;
2904 rdev->config.si.max_hw_contexts = 8;
2905
2906 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2907 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2908 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2909 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2910 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2911 break;
2912 case CHIP_PITCAIRN:
2913 rdev->config.si.max_shader_engines = 2;
2914 rdev->config.si.max_tile_pipes = 8;
2915 rdev->config.si.max_cu_per_sh = 5;
2916 rdev->config.si.max_sh_per_se = 2;
2917 rdev->config.si.max_backends_per_se = 4;
2918 rdev->config.si.max_texture_channel_caches = 8;
2919 rdev->config.si.max_gprs = 256;
2920 rdev->config.si.max_gs_threads = 32;
2921 rdev->config.si.max_hw_contexts = 8;
2922
2923 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2924 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2925 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2926 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2927 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2928 break;
2929 case CHIP_VERDE:
2930 default:
2931 rdev->config.si.max_shader_engines = 1;
2932 rdev->config.si.max_tile_pipes = 4;
2933 rdev->config.si.max_cu_per_sh = 5;
2934 rdev->config.si.max_sh_per_se = 2;
2935 rdev->config.si.max_backends_per_se = 4;
2936 rdev->config.si.max_texture_channel_caches = 4;
2937 rdev->config.si.max_gprs = 256;
2938 rdev->config.si.max_gs_threads = 32;
2939 rdev->config.si.max_hw_contexts = 8;
2940
2941 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2942 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2943 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2944 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2945 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2946 break;
2947 case CHIP_OLAND:
2948 rdev->config.si.max_shader_engines = 1;
2949 rdev->config.si.max_tile_pipes = 4;
2950 rdev->config.si.max_cu_per_sh = 6;
2951 rdev->config.si.max_sh_per_se = 1;
2952 rdev->config.si.max_backends_per_se = 2;
2953 rdev->config.si.max_texture_channel_caches = 4;
2954 rdev->config.si.max_gprs = 256;
2955 rdev->config.si.max_gs_threads = 16;
2956 rdev->config.si.max_hw_contexts = 8;
2957
2958 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2959 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2960 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2961 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2962 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2963 break;
2964 case CHIP_HAINAN:
2965 rdev->config.si.max_shader_engines = 1;
2966 rdev->config.si.max_tile_pipes = 4;
2967 rdev->config.si.max_cu_per_sh = 5;
2968 rdev->config.si.max_sh_per_se = 1;
2969 rdev->config.si.max_backends_per_se = 1;
2970 rdev->config.si.max_texture_channel_caches = 2;
2971 rdev->config.si.max_gprs = 256;
2972 rdev->config.si.max_gs_threads = 16;
2973 rdev->config.si.max_hw_contexts = 8;
2974
2975 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2976 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2977 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2980 break;
2981 }
2982
2983 /* Initialize HDP */
2984 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2985 WREG32((0x2c14 + j), 0x00000000);
2986 WREG32((0x2c18 + j), 0x00000000);
2987 WREG32((0x2c1c + j), 0x00000000);
2988 WREG32((0x2c20 + j), 0x00000000);
2989 WREG32((0x2c24 + j), 0x00000000);
2990 }
2991
2992 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2993
2994 evergreen_fix_pci_max_read_req_size(rdev);
2995
2996 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2997
2998 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2999 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3000
3001 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3002 rdev->config.si.mem_max_burst_length_bytes = 256;
3003 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3004 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3005 if (rdev->config.si.mem_row_size_in_kb > 4)
3006 rdev->config.si.mem_row_size_in_kb = 4;
3007 /* XXX use MC settings? */
3008 rdev->config.si.shader_engine_tile_size = 32;
3009 rdev->config.si.num_gpus = 1;
3010 rdev->config.si.multi_gpu_tile_size = 64;
3011
3012 /* fix up row size */
3013 gb_addr_config &= ~ROW_SIZE_MASK;
3014 switch (rdev->config.si.mem_row_size_in_kb) {
3015 case 1:
3016 default:
3017 gb_addr_config |= ROW_SIZE(0);
3018 break;
3019 case 2:
3020 gb_addr_config |= ROW_SIZE(1);
3021 break;
3022 case 4:
3023 gb_addr_config |= ROW_SIZE(2);
3024 break;
3025 }
3026
3027 /* setup tiling info dword. gb_addr_config is not adequate since it does
3028 * not have bank info, so create a custom tiling dword.
3029 * bits 3:0 num_pipes
3030 * bits 7:4 num_banks
3031 * bits 11:8 group_size
3032 * bits 15:12 row_size
3033 */
3034 rdev->config.si.tile_config = 0;
3035 switch (rdev->config.si.num_tile_pipes) {
3036 case 1:
3037 rdev->config.si.tile_config |= (0 << 0);
3038 break;
3039 case 2:
3040 rdev->config.si.tile_config |= (1 << 0);
3041 break;
3042 case 4:
3043 rdev->config.si.tile_config |= (2 << 0);
3044 break;
3045 case 8:
3046 default:
3047 /* XXX what about 12? */
3048 rdev->config.si.tile_config |= (3 << 0);
3049 break;
3050 }
3051 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3052 case 0: /* four banks */
3053 rdev->config.si.tile_config |= 0 << 4;
3054 break;
3055 case 1: /* eight banks */
3056 rdev->config.si.tile_config |= 1 << 4;
3057 break;
3058 case 2: /* sixteen banks */
3059 default:
3060 rdev->config.si.tile_config |= 2 << 4;
3061 break;
3062 }
3063 rdev->config.si.tile_config |=
3064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3065 rdev->config.si.tile_config |=
3066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3067
3068 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3069 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3070 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3071 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3072 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3073 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3074 if (rdev->has_uvd) {
3075 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3076 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3077 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3078 }
3079
3080 si_tiling_mode_table_init(rdev);
3081
3082 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3083 rdev->config.si.max_sh_per_se,
3084 rdev->config.si.max_backends_per_se);
3085
3086 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3087 rdev->config.si.max_sh_per_se,
3088 rdev->config.si.max_cu_per_sh);
3089
3090
3091 /* set HW defaults for 3D engine */
3092 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3093 ROQ_IB2_START(0x2b)));
3094 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3095
3096 sx_debug_1 = RREG32(SX_DEBUG_1);
3097 WREG32(SX_DEBUG_1, sx_debug_1);
3098
3099 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3100
3101 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3102 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3103 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3104 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3105
3106 WREG32(VGT_NUM_INSTANCES, 1);
3107
3108 WREG32(CP_PERFMON_CNTL, 0);
3109
3110 WREG32(SQ_CONFIG, 0);
3111
3112 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3113 FORCE_EOV_MAX_REZ_CNT(255)));
3114
3115 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3116 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3117
3118 WREG32(VGT_GS_VERTEX_REUSE, 16);
3119 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3120
3121 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3122 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3123 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3124 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3125 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3126 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3127 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3128 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3129
3130 tmp = RREG32(HDP_MISC_CNTL);
3131 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3132 WREG32(HDP_MISC_CNTL, tmp);
3133
3134 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3135 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3136
3137 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3138
3139 udelay(50);
3140 }
3141
3142 /*
3143 * GPU scratch registers helpers function.
3144 */
3145 static void si_scratch_init(struct radeon_device *rdev)
3146 {
3147 int i;
3148
3149 rdev->scratch.num_reg = 7;
3150 rdev->scratch.reg_base = SCRATCH_REG0;
3151 for (i = 0; i < rdev->scratch.num_reg; i++) {
3152 rdev->scratch.free[i] = true;
3153 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3154 }
3155 }
3156
3157 void si_fence_ring_emit(struct radeon_device *rdev,
3158 struct radeon_fence *fence)
3159 {
3160 struct radeon_ring *ring = &rdev->ring[fence->ring];
3161 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3162
3163 /* flush read cache over gart */
3164 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3165 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3166 radeon_ring_write(ring, 0);
3167 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3168 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3169 PACKET3_TC_ACTION_ENA |
3170 PACKET3_SH_KCACHE_ACTION_ENA |
3171 PACKET3_SH_ICACHE_ACTION_ENA);
3172 radeon_ring_write(ring, 0xFFFFFFFF);
3173 radeon_ring_write(ring, 0);
3174 radeon_ring_write(ring, 10); /* poll interval */
3175 /* EVENT_WRITE_EOP - flush caches, send int */
3176 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3177 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3178 radeon_ring_write(ring, addr & 0xffffffff);
3179 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3180 radeon_ring_write(ring, fence->seq);
3181 radeon_ring_write(ring, 0);
3182 }
3183
3184 /*
3185 * IB stuff
3186 */
3187 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3188 {
3189 struct radeon_ring *ring = &rdev->ring[ib->ring];
3190 u32 header;
3191
3192 if (ib->is_const_ib) {
3193 /* set switch buffer packet before const IB */
3194 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3195 radeon_ring_write(ring, 0);
3196
3197 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3198 } else {
3199 u32 next_rptr;
3200 if (ring->rptr_save_reg) {
3201 next_rptr = ring->wptr + 3 + 4 + 8;
3202 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3203 radeon_ring_write(ring, ((ring->rptr_save_reg -
3204 PACKET3_SET_CONFIG_REG_START) >> 2));
3205 radeon_ring_write(ring, next_rptr);
3206 } else if (rdev->wb.enabled) {
3207 next_rptr = ring->wptr + 5 + 4 + 8;
3208 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3209 radeon_ring_write(ring, (1 << 8));
3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3212 radeon_ring_write(ring, next_rptr);
3213 }
3214
3215 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3216 }
3217
3218 radeon_ring_write(ring, header);
3219 radeon_ring_write(ring,
3220 #ifdef __BIG_ENDIAN
3221 (2 << 0) |
3222 #endif
3223 (ib->gpu_addr & 0xFFFFFFFC));
3224 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3225 radeon_ring_write(ring, ib->length_dw |
3226 (ib->vm ? (ib->vm->id << 24) : 0));
3227
3228 if (!ib->is_const_ib) {
3229 /* flush read cache over gart for this vmid */
3230 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3231 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3232 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3233 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3234 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3235 PACKET3_TC_ACTION_ENA |
3236 PACKET3_SH_KCACHE_ACTION_ENA |
3237 PACKET3_SH_ICACHE_ACTION_ENA);
3238 radeon_ring_write(ring, 0xFFFFFFFF);
3239 radeon_ring_write(ring, 0);
3240 radeon_ring_write(ring, 10); /* poll interval */
3241 }
3242 }
3243
3244 /*
3245 * CP.
3246 */
3247 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3248 {
3249 if (enable)
3250 WREG32(CP_ME_CNTL, 0);
3251 else {
3252 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3253 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3254 WREG32(SCRATCH_UMSK, 0);
3255 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3256 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3257 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3258 }
3259 udelay(50);
3260 }
3261
3262 static int si_cp_load_microcode(struct radeon_device *rdev)
3263 {
3264 const __be32 *fw_data;
3265 int i;
3266
3267 if (!rdev->me_fw || !rdev->pfp_fw)
3268 return -EINVAL;
3269
3270 si_cp_enable(rdev, false);
3271
3272 /* PFP */
3273 fw_data = (const __be32 *)rdev->pfp_fw->data;
3274 WREG32(CP_PFP_UCODE_ADDR, 0);
3275 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3276 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3277 WREG32(CP_PFP_UCODE_ADDR, 0);
3278
3279 /* CE */
3280 fw_data = (const __be32 *)rdev->ce_fw->data;
3281 WREG32(CP_CE_UCODE_ADDR, 0);
3282 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3283 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3284 WREG32(CP_CE_UCODE_ADDR, 0);
3285
3286 /* ME */
3287 fw_data = (const __be32 *)rdev->me_fw->data;
3288 WREG32(CP_ME_RAM_WADDR, 0);
3289 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3290 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3291 WREG32(CP_ME_RAM_WADDR, 0);
3292
3293 WREG32(CP_PFP_UCODE_ADDR, 0);
3294 WREG32(CP_CE_UCODE_ADDR, 0);
3295 WREG32(CP_ME_RAM_WADDR, 0);
3296 WREG32(CP_ME_RAM_RADDR, 0);
3297 return 0;
3298 }
3299
3300 static int si_cp_start(struct radeon_device *rdev)
3301 {
3302 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3303 int r, i;
3304
3305 r = radeon_ring_lock(rdev, ring, 7 + 4);
3306 if (r) {
3307 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3308 return r;
3309 }
3310 /* init the CP */
3311 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3312 radeon_ring_write(ring, 0x1);
3313 radeon_ring_write(ring, 0x0);
3314 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3315 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3316 radeon_ring_write(ring, 0);
3317 radeon_ring_write(ring, 0);
3318
3319 /* init the CE partitions */
3320 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3321 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3322 radeon_ring_write(ring, 0xc000);
3323 radeon_ring_write(ring, 0xe000);
3324 radeon_ring_unlock_commit(rdev, ring);
3325
3326 si_cp_enable(rdev, true);
3327
3328 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3329 if (r) {
3330 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3331 return r;
3332 }
3333
3334 /* setup clear context state */
3335 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3336 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3337
3338 for (i = 0; i < si_default_size; i++)
3339 radeon_ring_write(ring, si_default_state[i]);
3340
3341 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3342 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3343
3344 /* set clear context state */
3345 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3346 radeon_ring_write(ring, 0);
3347
3348 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3349 radeon_ring_write(ring, 0x00000316);
3350 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3351 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3352
3353 radeon_ring_unlock_commit(rdev, ring);
3354
3355 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3356 ring = &rdev->ring[i];
3357 r = radeon_ring_lock(rdev, ring, 2);
3358
3359 /* clear the compute context state */
3360 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3361 radeon_ring_write(ring, 0);
3362
3363 radeon_ring_unlock_commit(rdev, ring);
3364 }
3365
3366 return 0;
3367 }
3368
3369 static void si_cp_fini(struct radeon_device *rdev)
3370 {
3371 struct radeon_ring *ring;
3372 si_cp_enable(rdev, false);
3373
3374 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3375 radeon_ring_fini(rdev, ring);
3376 radeon_scratch_free(rdev, ring->rptr_save_reg);
3377
3378 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3379 radeon_ring_fini(rdev, ring);
3380 radeon_scratch_free(rdev, ring->rptr_save_reg);
3381
3382 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3383 radeon_ring_fini(rdev, ring);
3384 radeon_scratch_free(rdev, ring->rptr_save_reg);
3385 }
3386
3387 static int si_cp_resume(struct radeon_device *rdev)
3388 {
3389 struct radeon_ring *ring;
3390 u32 tmp;
3391 u32 rb_bufsz;
3392 int r;
3393
3394 si_enable_gui_idle_interrupt(rdev, false);
3395
3396 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3397 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3398
3399 /* Set the write pointer delay */
3400 WREG32(CP_RB_WPTR_DELAY, 0);
3401
3402 WREG32(CP_DEBUG, 0);
3403 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3404
3405 /* ring 0 - compute and gfx */
3406 /* Set ring buffer size */
3407 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3408 rb_bufsz = order_base_2(ring->ring_size / 8);
3409 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3410 #ifdef __BIG_ENDIAN
3411 tmp |= BUF_SWAP_32BIT;
3412 #endif
3413 WREG32(CP_RB0_CNTL, tmp);
3414
3415 /* Initialize the ring buffer's read and write pointers */
3416 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3417 ring->wptr = 0;
3418 WREG32(CP_RB0_WPTR, ring->wptr);
3419
3420 /* set the wb address whether it's enabled or not */
3421 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3422 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3423
3424 if (rdev->wb.enabled)
3425 WREG32(SCRATCH_UMSK, 0xff);
3426 else {
3427 tmp |= RB_NO_UPDATE;
3428 WREG32(SCRATCH_UMSK, 0);
3429 }
3430
3431 mdelay(1);
3432 WREG32(CP_RB0_CNTL, tmp);
3433
3434 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3435
3436 ring->rptr = RREG32(CP_RB0_RPTR);
3437
3438 /* ring1 - compute only */
3439 /* Set ring buffer size */
3440 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3441 rb_bufsz = order_base_2(ring->ring_size / 8);
3442 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3443 #ifdef __BIG_ENDIAN
3444 tmp |= BUF_SWAP_32BIT;
3445 #endif
3446 WREG32(CP_RB1_CNTL, tmp);
3447
3448 /* Initialize the ring buffer's read and write pointers */
3449 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3450 ring->wptr = 0;
3451 WREG32(CP_RB1_WPTR, ring->wptr);
3452
3453 /* set the wb address whether it's enabled or not */
3454 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3455 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3456
3457 mdelay(1);
3458 WREG32(CP_RB1_CNTL, tmp);
3459
3460 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3461
3462 ring->rptr = RREG32(CP_RB1_RPTR);
3463
3464 /* ring2 - compute only */
3465 /* Set ring buffer size */
3466 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3467 rb_bufsz = order_base_2(ring->ring_size / 8);
3468 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3469 #ifdef __BIG_ENDIAN
3470 tmp |= BUF_SWAP_32BIT;
3471 #endif
3472 WREG32(CP_RB2_CNTL, tmp);
3473
3474 /* Initialize the ring buffer's read and write pointers */
3475 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3476 ring->wptr = 0;
3477 WREG32(CP_RB2_WPTR, ring->wptr);
3478
3479 /* set the wb address whether it's enabled or not */
3480 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3481 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3482
3483 mdelay(1);
3484 WREG32(CP_RB2_CNTL, tmp);
3485
3486 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3487
3488 ring->rptr = RREG32(CP_RB2_RPTR);
3489
3490 /* start the rings */
3491 si_cp_start(rdev);
3492 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3494 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3495 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3496 if (r) {
3497 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3498 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3499 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3500 return r;
3501 }
3502 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3503 if (r) {
3504 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3505 }
3506 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3507 if (r) {
3508 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3509 }
3510
3511 si_enable_gui_idle_interrupt(rdev, true);
3512
3513 return 0;
3514 }
3515
3516 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3517 {
3518 u32 reset_mask = 0;
3519 u32 tmp;
3520
3521 /* GRBM_STATUS */
3522 tmp = RREG32(GRBM_STATUS);
3523 if (tmp & (PA_BUSY | SC_BUSY |
3524 BCI_BUSY | SX_BUSY |
3525 TA_BUSY | VGT_BUSY |
3526 DB_BUSY | CB_BUSY |
3527 GDS_BUSY | SPI_BUSY |
3528 IA_BUSY | IA_BUSY_NO_DMA))
3529 reset_mask |= RADEON_RESET_GFX;
3530
3531 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3532 CP_BUSY | CP_COHERENCY_BUSY))
3533 reset_mask |= RADEON_RESET_CP;
3534
3535 if (tmp & GRBM_EE_BUSY)
3536 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3537
3538 /* GRBM_STATUS2 */
3539 tmp = RREG32(GRBM_STATUS2);
3540 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3541 reset_mask |= RADEON_RESET_RLC;
3542
3543 /* DMA_STATUS_REG 0 */
3544 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3545 if (!(tmp & DMA_IDLE))
3546 reset_mask |= RADEON_RESET_DMA;
3547
3548 /* DMA_STATUS_REG 1 */
3549 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3550 if (!(tmp & DMA_IDLE))
3551 reset_mask |= RADEON_RESET_DMA1;
3552
3553 /* SRBM_STATUS2 */
3554 tmp = RREG32(SRBM_STATUS2);
3555 if (tmp & DMA_BUSY)
3556 reset_mask |= RADEON_RESET_DMA;
3557
3558 if (tmp & DMA1_BUSY)
3559 reset_mask |= RADEON_RESET_DMA1;
3560
3561 /* SRBM_STATUS */
3562 tmp = RREG32(SRBM_STATUS);
3563
3564 if (tmp & IH_BUSY)
3565 reset_mask |= RADEON_RESET_IH;
3566
3567 if (tmp & SEM_BUSY)
3568 reset_mask |= RADEON_RESET_SEM;
3569
3570 if (tmp & GRBM_RQ_PENDING)
3571 reset_mask |= RADEON_RESET_GRBM;
3572
3573 if (tmp & VMC_BUSY)
3574 reset_mask |= RADEON_RESET_VMC;
3575
3576 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3577 MCC_BUSY | MCD_BUSY))
3578 reset_mask |= RADEON_RESET_MC;
3579
3580 if (evergreen_is_display_hung(rdev))
3581 reset_mask |= RADEON_RESET_DISPLAY;
3582
3583 /* VM_L2_STATUS */
3584 tmp = RREG32(VM_L2_STATUS);
3585 if (tmp & L2_BUSY)
3586 reset_mask |= RADEON_RESET_VMC;
3587
3588 /* Skip MC reset as it's mostly likely not hung, just busy */
3589 if (reset_mask & RADEON_RESET_MC) {
3590 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3591 reset_mask &= ~RADEON_RESET_MC;
3592 }
3593
3594 return reset_mask;
3595 }
3596
3597 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3598 {
3599 struct evergreen_mc_save save;
3600 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3601 u32 tmp;
3602
3603 if (reset_mask == 0)
3604 return;
3605
3606 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3607
3608 evergreen_print_gpu_status_regs(rdev);
3609 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3610 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3611 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3612 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3613
3614 /* disable PG/CG */
3615 si_fini_pg(rdev);
3616 si_fini_cg(rdev);
3617
3618 /* stop the rlc */
3619 si_rlc_stop(rdev);
3620
3621 /* Disable CP parsing/prefetching */
3622 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3623
3624 if (reset_mask & RADEON_RESET_DMA) {
3625 /* dma0 */
3626 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3627 tmp &= ~DMA_RB_ENABLE;
3628 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3629 }
3630 if (reset_mask & RADEON_RESET_DMA1) {
3631 /* dma1 */
3632 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3633 tmp &= ~DMA_RB_ENABLE;
3634 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3635 }
3636
3637 udelay(50);
3638
3639 evergreen_mc_stop(rdev, &save);
3640 if (evergreen_mc_wait_for_idle(rdev)) {
3641 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3642 }
3643
3644 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3645 grbm_soft_reset = SOFT_RESET_CB |
3646 SOFT_RESET_DB |
3647 SOFT_RESET_GDS |
3648 SOFT_RESET_PA |
3649 SOFT_RESET_SC |
3650 SOFT_RESET_BCI |
3651 SOFT_RESET_SPI |
3652 SOFT_RESET_SX |
3653 SOFT_RESET_TC |
3654 SOFT_RESET_TA |
3655 SOFT_RESET_VGT |
3656 SOFT_RESET_IA;
3657 }
3658
3659 if (reset_mask & RADEON_RESET_CP) {
3660 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3661
3662 srbm_soft_reset |= SOFT_RESET_GRBM;
3663 }
3664
3665 if (reset_mask & RADEON_RESET_DMA)
3666 srbm_soft_reset |= SOFT_RESET_DMA;
3667
3668 if (reset_mask & RADEON_RESET_DMA1)
3669 srbm_soft_reset |= SOFT_RESET_DMA1;
3670
3671 if (reset_mask & RADEON_RESET_DISPLAY)
3672 srbm_soft_reset |= SOFT_RESET_DC;
3673
3674 if (reset_mask & RADEON_RESET_RLC)
3675 grbm_soft_reset |= SOFT_RESET_RLC;
3676
3677 if (reset_mask & RADEON_RESET_SEM)
3678 srbm_soft_reset |= SOFT_RESET_SEM;
3679
3680 if (reset_mask & RADEON_RESET_IH)
3681 srbm_soft_reset |= SOFT_RESET_IH;
3682
3683 if (reset_mask & RADEON_RESET_GRBM)
3684 srbm_soft_reset |= SOFT_RESET_GRBM;
3685
3686 if (reset_mask & RADEON_RESET_VMC)
3687 srbm_soft_reset |= SOFT_RESET_VMC;
3688
3689 if (reset_mask & RADEON_RESET_MC)
3690 srbm_soft_reset |= SOFT_RESET_MC;
3691
3692 if (grbm_soft_reset) {
3693 tmp = RREG32(GRBM_SOFT_RESET);
3694 tmp |= grbm_soft_reset;
3695 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3696 WREG32(GRBM_SOFT_RESET, tmp);
3697 tmp = RREG32(GRBM_SOFT_RESET);
3698
3699 udelay(50);
3700
3701 tmp &= ~grbm_soft_reset;
3702 WREG32(GRBM_SOFT_RESET, tmp);
3703 tmp = RREG32(GRBM_SOFT_RESET);
3704 }
3705
3706 if (srbm_soft_reset) {
3707 tmp = RREG32(SRBM_SOFT_RESET);
3708 tmp |= srbm_soft_reset;
3709 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3710 WREG32(SRBM_SOFT_RESET, tmp);
3711 tmp = RREG32(SRBM_SOFT_RESET);
3712
3713 udelay(50);
3714
3715 tmp &= ~srbm_soft_reset;
3716 WREG32(SRBM_SOFT_RESET, tmp);
3717 tmp = RREG32(SRBM_SOFT_RESET);
3718 }
3719
3720 /* Wait a little for things to settle down */
3721 udelay(50);
3722
3723 evergreen_mc_resume(rdev, &save);
3724 udelay(50);
3725
3726 evergreen_print_gpu_status_regs(rdev);
3727 }
3728
3729 int si_asic_reset(struct radeon_device *rdev)
3730 {
3731 u32 reset_mask;
3732
3733 reset_mask = si_gpu_check_soft_reset(rdev);
3734
3735 if (reset_mask)
3736 r600_set_bios_scratch_engine_hung(rdev, true);
3737
3738 si_gpu_soft_reset(rdev, reset_mask);
3739
3740 reset_mask = si_gpu_check_soft_reset(rdev);
3741
3742 if (!reset_mask)
3743 r600_set_bios_scratch_engine_hung(rdev, false);
3744
3745 return 0;
3746 }
3747
3748 /**
3749 * si_gfx_is_lockup - Check if the GFX engine is locked up
3750 *
3751 * @rdev: radeon_device pointer
3752 * @ring: radeon_ring structure holding ring information
3753 *
3754 * Check if the GFX engine is locked up.
3755 * Returns true if the engine appears to be locked up, false if not.
3756 */
3757 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3758 {
3759 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3760
3761 if (!(reset_mask & (RADEON_RESET_GFX |
3762 RADEON_RESET_COMPUTE |
3763 RADEON_RESET_CP))) {
3764 radeon_ring_lockup_update(ring);
3765 return false;
3766 }
3767 /* force CP activities */
3768 radeon_ring_force_activity(rdev, ring);
3769 return radeon_ring_test_lockup(rdev, ring);
3770 }
3771
3772 /* MC */
3773 static void si_mc_program(struct radeon_device *rdev)
3774 {
3775 struct evergreen_mc_save save;
3776 u32 tmp;
3777 int i, j;
3778
3779 /* Initialize HDP */
3780 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3781 WREG32((0x2c14 + j), 0x00000000);
3782 WREG32((0x2c18 + j), 0x00000000);
3783 WREG32((0x2c1c + j), 0x00000000);
3784 WREG32((0x2c20 + j), 0x00000000);
3785 WREG32((0x2c24 + j), 0x00000000);
3786 }
3787 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3788
3789 evergreen_mc_stop(rdev, &save);
3790 if (radeon_mc_wait_for_idle(rdev)) {
3791 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3792 }
3793 if (!ASIC_IS_NODCE(rdev))
3794 /* Lockout access through VGA aperture*/
3795 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3796 /* Update configuration */
3797 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3798 rdev->mc.vram_start >> 12);
3799 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3800 rdev->mc.vram_end >> 12);
3801 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3802 rdev->vram_scratch.gpu_addr >> 12);
3803 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3804 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3805 WREG32(MC_VM_FB_LOCATION, tmp);
3806 /* XXX double check these! */
3807 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3808 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3809 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3810 WREG32(MC_VM_AGP_BASE, 0);
3811 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3812 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3813 if (radeon_mc_wait_for_idle(rdev)) {
3814 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3815 }
3816 evergreen_mc_resume(rdev, &save);
3817 if (!ASIC_IS_NODCE(rdev)) {
3818 /* we need to own VRAM, so turn off the VGA renderer here
3819 * to stop it overwriting our objects */
3820 rv515_vga_render_disable(rdev);
3821 }
3822 }
3823
3824 void si_vram_gtt_location(struct radeon_device *rdev,
3825 struct radeon_mc *mc)
3826 {
3827 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3828 /* leave room for at least 1024M GTT */
3829 dev_warn(rdev->dev, "limiting VRAM\n");
3830 mc->real_vram_size = 0xFFC0000000ULL;
3831 mc->mc_vram_size = 0xFFC0000000ULL;
3832 }
3833 radeon_vram_location(rdev, &rdev->mc, 0);
3834 rdev->mc.gtt_base_align = 0;
3835 radeon_gtt_location(rdev, mc);
3836 }
3837
3838 static int si_mc_init(struct radeon_device *rdev)
3839 {
3840 u32 tmp;
3841 int chansize, numchan;
3842
3843 /* Get VRAM informations */
3844 rdev->mc.vram_is_ddr = true;
3845 tmp = RREG32(MC_ARB_RAMCFG);
3846 if (tmp & CHANSIZE_OVERRIDE) {
3847 chansize = 16;
3848 } else if (tmp & CHANSIZE_MASK) {
3849 chansize = 64;
3850 } else {
3851 chansize = 32;
3852 }
3853 tmp = RREG32(MC_SHARED_CHMAP);
3854 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3855 case 0:
3856 default:
3857 numchan = 1;
3858 break;
3859 case 1:
3860 numchan = 2;
3861 break;
3862 case 2:
3863 numchan = 4;
3864 break;
3865 case 3:
3866 numchan = 8;
3867 break;
3868 case 4:
3869 numchan = 3;
3870 break;
3871 case 5:
3872 numchan = 6;
3873 break;
3874 case 6:
3875 numchan = 10;
3876 break;
3877 case 7:
3878 numchan = 12;
3879 break;
3880 case 8:
3881 numchan = 16;
3882 break;
3883 }
3884 rdev->mc.vram_width = numchan * chansize;
3885 /* Could aper size report 0 ? */
3886 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3887 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3888 /* size in MB on si */
3889 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3890 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3891 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3892 si_vram_gtt_location(rdev, &rdev->mc);
3893 radeon_update_bandwidth_info(rdev);
3894
3895 return 0;
3896 }
3897
3898 /*
3899 * GART
3900 */
3901 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3902 {
3903 /* flush hdp cache */
3904 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3905
3906 /* bits 0-15 are the VM contexts0-15 */
3907 WREG32(VM_INVALIDATE_REQUEST, 1);
3908 }
3909
3910 static int si_pcie_gart_enable(struct radeon_device *rdev)
3911 {
3912 int r, i;
3913
3914 if (rdev->gart.robj == NULL) {
3915 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3916 return -EINVAL;
3917 }
3918 r = radeon_gart_table_vram_pin(rdev);
3919 if (r)
3920 return r;
3921 radeon_gart_restore(rdev);
3922 /* Setup TLB control */
3923 WREG32(MC_VM_MX_L1_TLB_CNTL,
3924 (0xA << 7) |
3925 ENABLE_L1_TLB |
3926 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3927 ENABLE_ADVANCED_DRIVER_MODEL |
3928 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3929 /* Setup L2 cache */
3930 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3931 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3932 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3933 EFFECTIVE_L2_QUEUE_SIZE(7) |
3934 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3935 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3936 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3937 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3938 /* setup context0 */
3939 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3940 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3941 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3942 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3943 (u32)(rdev->dummy_page.addr >> 12));
3944 WREG32(VM_CONTEXT0_CNTL2, 0);
3945 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3946 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3947
3948 WREG32(0x15D4, 0);
3949 WREG32(0x15D8, 0);
3950 WREG32(0x15DC, 0);
3951
3952 /* empty context1-15 */
3953 /* set vm size, must be a multiple of 4 */
3954 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3955 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3956 /* Assign the pt base to something valid for now; the pts used for
3957 * the VMs are determined by the application and setup and assigned
3958 * on the fly in the vm part of radeon_gart.c
3959 */
3960 for (i = 1; i < 16; i++) {
3961 if (i < 8)
3962 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3963 rdev->gart.table_addr >> 12);
3964 else
3965 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3966 rdev->gart.table_addr >> 12);
3967 }
3968
3969 /* enable context1-15 */
3970 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3971 (u32)(rdev->dummy_page.addr >> 12));
3972 WREG32(VM_CONTEXT1_CNTL2, 4);
3973 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3974 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3975 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3976 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3977 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3978 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3979 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3980 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3981 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3982 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3983 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3984 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3985 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3986
3987 si_pcie_gart_tlb_flush(rdev);
3988 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3989 (unsigned)(rdev->mc.gtt_size >> 20),
3990 (unsigned long long)rdev->gart.table_addr);
3991 rdev->gart.ready = true;
3992 return 0;
3993 }
3994
3995 static void si_pcie_gart_disable(struct radeon_device *rdev)
3996 {
3997 /* Disable all tables */
3998 WREG32(VM_CONTEXT0_CNTL, 0);
3999 WREG32(VM_CONTEXT1_CNTL, 0);
4000 /* Setup TLB control */
4001 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4002 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4003 /* Setup L2 cache */
4004 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4005 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4006 EFFECTIVE_L2_QUEUE_SIZE(7) |
4007 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4008 WREG32(VM_L2_CNTL2, 0);
4009 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4010 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4011 radeon_gart_table_vram_unpin(rdev);
4012 }
4013
4014 static void si_pcie_gart_fini(struct radeon_device *rdev)
4015 {
4016 si_pcie_gart_disable(rdev);
4017 radeon_gart_table_vram_free(rdev);
4018 radeon_gart_fini(rdev);
4019 }
4020
4021 /* vm parser */
4022 static bool si_vm_reg_valid(u32 reg)
4023 {
4024 /* context regs are fine */
4025 if (reg >= 0x28000)
4026 return true;
4027
4028 /* check config regs */
4029 switch (reg) {
4030 case GRBM_GFX_INDEX:
4031 case CP_STRMOUT_CNTL:
4032 case VGT_VTX_VECT_EJECT_REG:
4033 case VGT_CACHE_INVALIDATION:
4034 case VGT_ESGS_RING_SIZE:
4035 case VGT_GSVS_RING_SIZE:
4036 case VGT_GS_VERTEX_REUSE:
4037 case VGT_PRIMITIVE_TYPE:
4038 case VGT_INDEX_TYPE:
4039 case VGT_NUM_INDICES:
4040 case VGT_NUM_INSTANCES:
4041 case VGT_TF_RING_SIZE:
4042 case VGT_HS_OFFCHIP_PARAM:
4043 case VGT_TF_MEMORY_BASE:
4044 case PA_CL_ENHANCE:
4045 case PA_SU_LINE_STIPPLE_VALUE:
4046 case PA_SC_LINE_STIPPLE_STATE:
4047 case PA_SC_ENHANCE:
4048 case SQC_CACHES:
4049 case SPI_STATIC_THREAD_MGMT_1:
4050 case SPI_STATIC_THREAD_MGMT_2:
4051 case SPI_STATIC_THREAD_MGMT_3:
4052 case SPI_PS_MAX_WAVE_ID:
4053 case SPI_CONFIG_CNTL:
4054 case SPI_CONFIG_CNTL_1:
4055 case TA_CNTL_AUX:
4056 return true;
4057 default:
4058 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4059 return false;
4060 }
4061 }
4062
4063 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4064 u32 *ib, struct radeon_cs_packet *pkt)
4065 {
4066 switch (pkt->opcode) {
4067 case PACKET3_NOP:
4068 case PACKET3_SET_BASE:
4069 case PACKET3_SET_CE_DE_COUNTERS:
4070 case PACKET3_LOAD_CONST_RAM:
4071 case PACKET3_WRITE_CONST_RAM:
4072 case PACKET3_WRITE_CONST_RAM_OFFSET:
4073 case PACKET3_DUMP_CONST_RAM:
4074 case PACKET3_INCREMENT_CE_COUNTER:
4075 case PACKET3_WAIT_ON_DE_COUNTER:
4076 case PACKET3_CE_WRITE:
4077 break;
4078 default:
4079 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4080 return -EINVAL;
4081 }
4082 return 0;
4083 }
4084
4085 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4086 {
4087 u32 start_reg, reg, i;
4088 u32 command = ib[idx + 4];
4089 u32 info = ib[idx + 1];
4090 u32 idx_value = ib[idx];
4091 if (command & PACKET3_CP_DMA_CMD_SAS) {
4092 /* src address space is register */
4093 if (((info & 0x60000000) >> 29) == 0) {
4094 start_reg = idx_value << 2;
4095 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4096 reg = start_reg;
4097 if (!si_vm_reg_valid(reg)) {
4098 DRM_ERROR("CP DMA Bad SRC register\n");
4099 return -EINVAL;
4100 }
4101 } else {
4102 for (i = 0; i < (command & 0x1fffff); i++) {
4103 reg = start_reg + (4 * i);
4104 if (!si_vm_reg_valid(reg)) {
4105 DRM_ERROR("CP DMA Bad SRC register\n");
4106 return -EINVAL;
4107 }
4108 }
4109 }
4110 }
4111 }
4112 if (command & PACKET3_CP_DMA_CMD_DAS) {
4113 /* dst address space is register */
4114 if (((info & 0x00300000) >> 20) == 0) {
4115 start_reg = ib[idx + 2];
4116 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4117 reg = start_reg;
4118 if (!si_vm_reg_valid(reg)) {
4119 DRM_ERROR("CP DMA Bad DST register\n");
4120 return -EINVAL;
4121 }
4122 } else {
4123 for (i = 0; i < (command & 0x1fffff); i++) {
4124 reg = start_reg + (4 * i);
4125 if (!si_vm_reg_valid(reg)) {
4126 DRM_ERROR("CP DMA Bad DST register\n");
4127 return -EINVAL;
4128 }
4129 }
4130 }
4131 }
4132 }
4133 return 0;
4134 }
4135
4136 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4137 u32 *ib, struct radeon_cs_packet *pkt)
4138 {
4139 int r;
4140 u32 idx = pkt->idx + 1;
4141 u32 idx_value = ib[idx];
4142 u32 start_reg, end_reg, reg, i;
4143
4144 switch (pkt->opcode) {
4145 case PACKET3_NOP:
4146 case PACKET3_SET_BASE:
4147 case PACKET3_CLEAR_STATE:
4148 case PACKET3_INDEX_BUFFER_SIZE:
4149 case PACKET3_DISPATCH_DIRECT:
4150 case PACKET3_DISPATCH_INDIRECT:
4151 case PACKET3_ALLOC_GDS:
4152 case PACKET3_WRITE_GDS_RAM:
4153 case PACKET3_ATOMIC_GDS:
4154 case PACKET3_ATOMIC:
4155 case PACKET3_OCCLUSION_QUERY:
4156 case PACKET3_SET_PREDICATION:
4157 case PACKET3_COND_EXEC:
4158 case PACKET3_PRED_EXEC:
4159 case PACKET3_DRAW_INDIRECT:
4160 case PACKET3_DRAW_INDEX_INDIRECT:
4161 case PACKET3_INDEX_BASE:
4162 case PACKET3_DRAW_INDEX_2:
4163 case PACKET3_CONTEXT_CONTROL:
4164 case PACKET3_INDEX_TYPE:
4165 case PACKET3_DRAW_INDIRECT_MULTI:
4166 case PACKET3_DRAW_INDEX_AUTO:
4167 case PACKET3_DRAW_INDEX_IMMD:
4168 case PACKET3_NUM_INSTANCES:
4169 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4170 case PACKET3_STRMOUT_BUFFER_UPDATE:
4171 case PACKET3_DRAW_INDEX_OFFSET_2:
4172 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4173 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4174 case PACKET3_MPEG_INDEX:
4175 case PACKET3_WAIT_REG_MEM:
4176 case PACKET3_MEM_WRITE:
4177 case PACKET3_PFP_SYNC_ME:
4178 case PACKET3_SURFACE_SYNC:
4179 case PACKET3_EVENT_WRITE:
4180 case PACKET3_EVENT_WRITE_EOP:
4181 case PACKET3_EVENT_WRITE_EOS:
4182 case PACKET3_SET_CONTEXT_REG:
4183 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4184 case PACKET3_SET_SH_REG:
4185 case PACKET3_SET_SH_REG_OFFSET:
4186 case PACKET3_INCREMENT_DE_COUNTER:
4187 case PACKET3_WAIT_ON_CE_COUNTER:
4188 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4189 case PACKET3_ME_WRITE:
4190 break;
4191 case PACKET3_COPY_DATA:
4192 if ((idx_value & 0xf00) == 0) {
4193 reg = ib[idx + 3] * 4;
4194 if (!si_vm_reg_valid(reg))
4195 return -EINVAL;
4196 }
4197 break;
4198 case PACKET3_WRITE_DATA:
4199 if ((idx_value & 0xf00) == 0) {
4200 start_reg = ib[idx + 1] * 4;
4201 if (idx_value & 0x10000) {
4202 if (!si_vm_reg_valid(start_reg))
4203 return -EINVAL;
4204 } else {
4205 for (i = 0; i < (pkt->count - 2); i++) {
4206 reg = start_reg + (4 * i);
4207 if (!si_vm_reg_valid(reg))
4208 return -EINVAL;
4209 }
4210 }
4211 }
4212 break;
4213 case PACKET3_COND_WRITE:
4214 if (idx_value & 0x100) {
4215 reg = ib[idx + 5] * 4;
4216 if (!si_vm_reg_valid(reg))
4217 return -EINVAL;
4218 }
4219 break;
4220 case PACKET3_COPY_DW:
4221 if (idx_value & 0x2) {
4222 reg = ib[idx + 3] * 4;
4223 if (!si_vm_reg_valid(reg))
4224 return -EINVAL;
4225 }
4226 break;
4227 case PACKET3_SET_CONFIG_REG:
4228 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4229 end_reg = 4 * pkt->count + start_reg - 4;
4230 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4231 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4232 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4233 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4234 return -EINVAL;
4235 }
4236 for (i = 0; i < pkt->count; i++) {
4237 reg = start_reg + (4 * i);
4238 if (!si_vm_reg_valid(reg))
4239 return -EINVAL;
4240 }
4241 break;
4242 case PACKET3_CP_DMA:
4243 r = si_vm_packet3_cp_dma_check(ib, idx);
4244 if (r)
4245 return r;
4246 break;
4247 default:
4248 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4249 return -EINVAL;
4250 }
4251 return 0;
4252 }
4253
4254 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4255 u32 *ib, struct radeon_cs_packet *pkt)
4256 {
4257 int r;
4258 u32 idx = pkt->idx + 1;
4259 u32 idx_value = ib[idx];
4260 u32 start_reg, reg, i;
4261
4262 switch (pkt->opcode) {
4263 case PACKET3_NOP:
4264 case PACKET3_SET_BASE:
4265 case PACKET3_CLEAR_STATE:
4266 case PACKET3_DISPATCH_DIRECT:
4267 case PACKET3_DISPATCH_INDIRECT:
4268 case PACKET3_ALLOC_GDS:
4269 case PACKET3_WRITE_GDS_RAM:
4270 case PACKET3_ATOMIC_GDS:
4271 case PACKET3_ATOMIC:
4272 case PACKET3_OCCLUSION_QUERY:
4273 case PACKET3_SET_PREDICATION:
4274 case PACKET3_COND_EXEC:
4275 case PACKET3_PRED_EXEC:
4276 case PACKET3_CONTEXT_CONTROL:
4277 case PACKET3_STRMOUT_BUFFER_UPDATE:
4278 case PACKET3_WAIT_REG_MEM:
4279 case PACKET3_MEM_WRITE:
4280 case PACKET3_PFP_SYNC_ME:
4281 case PACKET3_SURFACE_SYNC:
4282 case PACKET3_EVENT_WRITE:
4283 case PACKET3_EVENT_WRITE_EOP:
4284 case PACKET3_EVENT_WRITE_EOS:
4285 case PACKET3_SET_CONTEXT_REG:
4286 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4287 case PACKET3_SET_SH_REG:
4288 case PACKET3_SET_SH_REG_OFFSET:
4289 case PACKET3_INCREMENT_DE_COUNTER:
4290 case PACKET3_WAIT_ON_CE_COUNTER:
4291 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4292 case PACKET3_ME_WRITE:
4293 break;
4294 case PACKET3_COPY_DATA:
4295 if ((idx_value & 0xf00) == 0) {
4296 reg = ib[idx + 3] * 4;
4297 if (!si_vm_reg_valid(reg))
4298 return -EINVAL;
4299 }
4300 break;
4301 case PACKET3_WRITE_DATA:
4302 if ((idx_value & 0xf00) == 0) {
4303 start_reg = ib[idx + 1] * 4;
4304 if (idx_value & 0x10000) {
4305 if (!si_vm_reg_valid(start_reg))
4306 return -EINVAL;
4307 } else {
4308 for (i = 0; i < (pkt->count - 2); i++) {
4309 reg = start_reg + (4 * i);
4310 if (!si_vm_reg_valid(reg))
4311 return -EINVAL;
4312 }
4313 }
4314 }
4315 break;
4316 case PACKET3_COND_WRITE:
4317 if (idx_value & 0x100) {
4318 reg = ib[idx + 5] * 4;
4319 if (!si_vm_reg_valid(reg))
4320 return -EINVAL;
4321 }
4322 break;
4323 case PACKET3_COPY_DW:
4324 if (idx_value & 0x2) {
4325 reg = ib[idx + 3] * 4;
4326 if (!si_vm_reg_valid(reg))
4327 return -EINVAL;
4328 }
4329 break;
4330 case PACKET3_CP_DMA:
4331 r = si_vm_packet3_cp_dma_check(ib, idx);
4332 if (r)
4333 return r;
4334 break;
4335 default:
4336 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4337 return -EINVAL;
4338 }
4339 return 0;
4340 }
4341
4342 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4343 {
4344 int ret = 0;
4345 u32 idx = 0;
4346 struct radeon_cs_packet pkt;
4347
4348 do {
4349 pkt.idx = idx;
4350 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4351 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4352 pkt.one_reg_wr = 0;
4353 switch (pkt.type) {
4354 case RADEON_PACKET_TYPE0:
4355 dev_err(rdev->dev, "Packet0 not allowed!\n");
4356 ret = -EINVAL;
4357 break;
4358 case RADEON_PACKET_TYPE2:
4359 idx += 1;
4360 break;
4361 case RADEON_PACKET_TYPE3:
4362 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4363 if (ib->is_const_ib)
4364 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4365 else {
4366 switch (ib->ring) {
4367 case RADEON_RING_TYPE_GFX_INDEX:
4368 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4369 break;
4370 case CAYMAN_RING_TYPE_CP1_INDEX:
4371 case CAYMAN_RING_TYPE_CP2_INDEX:
4372 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4373 break;
4374 default:
4375 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4376 ret = -EINVAL;
4377 break;
4378 }
4379 }
4380 idx += pkt.count + 2;
4381 break;
4382 default:
4383 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4384 ret = -EINVAL;
4385 break;
4386 }
4387 if (ret)
4388 break;
4389 } while (idx < ib->length_dw);
4390
4391 return ret;
4392 }
4393
4394 /*
4395 * vm
4396 */
4397 int si_vm_init(struct radeon_device *rdev)
4398 {
4399 /* number of VMs */
4400 rdev->vm_manager.nvm = 16;
4401 /* base offset of vram pages */
4402 rdev->vm_manager.vram_base_offset = 0;
4403
4404 return 0;
4405 }
4406
4407 void si_vm_fini(struct radeon_device *rdev)
4408 {
4409 }
4410
4411 /**
4412 * si_vm_decode_fault - print human readable fault info
4413 *
4414 * @rdev: radeon_device pointer
4415 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4416 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4417 *
4418 * Print human readable fault information (SI).
4419 */
4420 static void si_vm_decode_fault(struct radeon_device *rdev,
4421 u32 status, u32 addr)
4422 {
4423 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4424 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4425 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4426 char *block;
4427
4428 if (rdev->family == CHIP_TAHITI) {
4429 switch (mc_id) {
4430 case 160:
4431 case 144:
4432 case 96:
4433 case 80:
4434 case 224:
4435 case 208:
4436 case 32:
4437 case 16:
4438 block = "CB";
4439 break;
4440 case 161:
4441 case 145:
4442 case 97:
4443 case 81:
4444 case 225:
4445 case 209:
4446 case 33:
4447 case 17:
4448 block = "CB_FMASK";
4449 break;
4450 case 162:
4451 case 146:
4452 case 98:
4453 case 82:
4454 case 226:
4455 case 210:
4456 case 34:
4457 case 18:
4458 block = "CB_CMASK";
4459 break;
4460 case 163:
4461 case 147:
4462 case 99:
4463 case 83:
4464 case 227:
4465 case 211:
4466 case 35:
4467 case 19:
4468 block = "CB_IMMED";
4469 break;
4470 case 164:
4471 case 148:
4472 case 100:
4473 case 84:
4474 case 228:
4475 case 212:
4476 case 36:
4477 case 20:
4478 block = "DB";
4479 break;
4480 case 165:
4481 case 149:
4482 case 101:
4483 case 85:
4484 case 229:
4485 case 213:
4486 case 37:
4487 case 21:
4488 block = "DB_HTILE";
4489 break;
4490 case 167:
4491 case 151:
4492 case 103:
4493 case 87:
4494 case 231:
4495 case 215:
4496 case 39:
4497 case 23:
4498 block = "DB_STEN";
4499 break;
4500 case 72:
4501 case 68:
4502 case 64:
4503 case 8:
4504 case 4:
4505 case 0:
4506 case 136:
4507 case 132:
4508 case 128:
4509 case 200:
4510 case 196:
4511 case 192:
4512 block = "TC";
4513 break;
4514 case 112:
4515 case 48:
4516 block = "CP";
4517 break;
4518 case 49:
4519 case 177:
4520 case 50:
4521 case 178:
4522 block = "SH";
4523 break;
4524 case 53:
4525 case 190:
4526 block = "VGT";
4527 break;
4528 case 117:
4529 block = "IH";
4530 break;
4531 case 51:
4532 case 115:
4533 block = "RLC";
4534 break;
4535 case 119:
4536 case 183:
4537 block = "DMA0";
4538 break;
4539 case 61:
4540 block = "DMA1";
4541 break;
4542 case 248:
4543 case 120:
4544 block = "HDP";
4545 break;
4546 default:
4547 block = "unknown";
4548 break;
4549 }
4550 } else {
4551 switch (mc_id) {
4552 case 32:
4553 case 16:
4554 case 96:
4555 case 80:
4556 case 160:
4557 case 144:
4558 case 224:
4559 case 208:
4560 block = "CB";
4561 break;
4562 case 33:
4563 case 17:
4564 case 97:
4565 case 81:
4566 case 161:
4567 case 145:
4568 case 225:
4569 case 209:
4570 block = "CB_FMASK";
4571 break;
4572 case 34:
4573 case 18:
4574 case 98:
4575 case 82:
4576 case 162:
4577 case 146:
4578 case 226:
4579 case 210:
4580 block = "CB_CMASK";
4581 break;
4582 case 35:
4583 case 19:
4584 case 99:
4585 case 83:
4586 case 163:
4587 case 147:
4588 case 227:
4589 case 211:
4590 block = "CB_IMMED";
4591 break;
4592 case 36:
4593 case 20:
4594 case 100:
4595 case 84:
4596 case 164:
4597 case 148:
4598 case 228:
4599 case 212:
4600 block = "DB";
4601 break;
4602 case 37:
4603 case 21:
4604 case 101:
4605 case 85:
4606 case 165:
4607 case 149:
4608 case 229:
4609 case 213:
4610 block = "DB_HTILE";
4611 break;
4612 case 39:
4613 case 23:
4614 case 103:
4615 case 87:
4616 case 167:
4617 case 151:
4618 case 231:
4619 case 215:
4620 block = "DB_STEN";
4621 break;
4622 case 72:
4623 case 68:
4624 case 8:
4625 case 4:
4626 case 136:
4627 case 132:
4628 case 200:
4629 case 196:
4630 block = "TC";
4631 break;
4632 case 112:
4633 case 48:
4634 block = "CP";
4635 break;
4636 case 49:
4637 case 177:
4638 case 50:
4639 case 178:
4640 block = "SH";
4641 break;
4642 case 53:
4643 block = "VGT";
4644 break;
4645 case 117:
4646 block = "IH";
4647 break;
4648 case 51:
4649 case 115:
4650 block = "RLC";
4651 break;
4652 case 119:
4653 case 183:
4654 block = "DMA0";
4655 break;
4656 case 61:
4657 block = "DMA1";
4658 break;
4659 case 248:
4660 case 120:
4661 block = "HDP";
4662 break;
4663 default:
4664 block = "unknown";
4665 break;
4666 }
4667 }
4668
4669 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4670 protections, vmid, addr,
4671 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4672 block, mc_id);
4673 }
4674
4675 /**
4676 * si_vm_set_page - update the page tables using the CP
4677 *
4678 * @rdev: radeon_device pointer
4679 * @ib: indirect buffer to fill with commands
4680 * @pe: addr of the page entry
4681 * @addr: dst addr to write into pe
4682 * @count: number of page entries to update
4683 * @incr: increase next addr by incr bytes
4684 * @flags: access flags
4685 *
4686 * Update the page tables using the CP (SI).
4687 */
4688 void si_vm_set_page(struct radeon_device *rdev,
4689 struct radeon_ib *ib,
4690 uint64_t pe,
4691 uint64_t addr, unsigned count,
4692 uint32_t incr, uint32_t flags)
4693 {
4694 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4695 uint64_t value;
4696 unsigned ndw;
4697
4698 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4699 while (count) {
4700 ndw = 2 + count * 2;
4701 if (ndw > 0x3FFE)
4702 ndw = 0x3FFE;
4703
4704 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4705 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4706 WRITE_DATA_DST_SEL(1));
4707 ib->ptr[ib->length_dw++] = pe;
4708 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4709 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4710 if (flags & RADEON_VM_PAGE_SYSTEM) {
4711 value = radeon_vm_map_gart(rdev, addr);
4712 value &= 0xFFFFFFFFFFFFF000ULL;
4713 } else if (flags & RADEON_VM_PAGE_VALID) {
4714 value = addr;
4715 } else {
4716 value = 0;
4717 }
4718 addr += incr;
4719 value |= r600_flags;
4720 ib->ptr[ib->length_dw++] = value;
4721 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4722 }
4723 }
4724 } else {
4725 /* DMA */
4726 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4727 }
4728 }
4729
4730 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4731 {
4732 struct radeon_ring *ring = &rdev->ring[ridx];
4733
4734 if (vm == NULL)
4735 return;
4736
4737 /* write new base address */
4738 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4739 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4740 WRITE_DATA_DST_SEL(0)));
4741
4742 if (vm->id < 8) {
4743 radeon_ring_write(ring,
4744 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4745 } else {
4746 radeon_ring_write(ring,
4747 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4748 }
4749 radeon_ring_write(ring, 0);
4750 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4751
4752 /* flush hdp cache */
4753 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4754 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4755 WRITE_DATA_DST_SEL(0)));
4756 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4757 radeon_ring_write(ring, 0);
4758 radeon_ring_write(ring, 0x1);
4759
4760 /* bits 0-15 are the VM contexts0-15 */
4761 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4762 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4763 WRITE_DATA_DST_SEL(0)));
4764 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4765 radeon_ring_write(ring, 0);
4766 radeon_ring_write(ring, 1 << vm->id);
4767
4768 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4769 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4770 radeon_ring_write(ring, 0x0);
4771 }
4772
4773 /*
4774 * Power and clock gating
4775 */
4776 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4777 {
4778 int i;
4779
4780 for (i = 0; i < rdev->usec_timeout; i++) {
4781 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4782 break;
4783 udelay(1);
4784 }
4785
4786 for (i = 0; i < rdev->usec_timeout; i++) {
4787 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4788 break;
4789 udelay(1);
4790 }
4791 }
4792
4793 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4794 bool enable)
4795 {
4796 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4797 u32 mask;
4798 int i;
4799
4800 if (enable)
4801 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4802 else
4803 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4804 WREG32(CP_INT_CNTL_RING0, tmp);
4805
4806 if (!enable) {
4807 /* read a gfx register */
4808 tmp = RREG32(DB_DEPTH_INFO);
4809
4810 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4811 for (i = 0; i < rdev->usec_timeout; i++) {
4812 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4813 break;
4814 udelay(1);
4815 }
4816 }
4817 }
4818
4819 static void si_set_uvd_dcm(struct radeon_device *rdev,
4820 bool sw_mode)
4821 {
4822 u32 tmp, tmp2;
4823
4824 tmp = RREG32(UVD_CGC_CTRL);
4825 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4826 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4827
4828 if (sw_mode) {
4829 tmp &= ~0x7ffff800;
4830 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4831 } else {
4832 tmp |= 0x7ffff800;
4833 tmp2 = 0;
4834 }
4835
4836 WREG32(UVD_CGC_CTRL, tmp);
4837 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4838 }
4839
4840 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4841 {
4842 bool hw_mode = true;
4843
4844 if (hw_mode) {
4845 si_set_uvd_dcm(rdev, false);
4846 } else {
4847 u32 tmp = RREG32(UVD_CGC_CTRL);
4848 tmp &= ~DCM;
4849 WREG32(UVD_CGC_CTRL, tmp);
4850 }
4851 }
4852
4853 static u32 si_halt_rlc(struct radeon_device *rdev)
4854 {
4855 u32 data, orig;
4856
4857 orig = data = RREG32(RLC_CNTL);
4858
4859 if (data & RLC_ENABLE) {
4860 data &= ~RLC_ENABLE;
4861 WREG32(RLC_CNTL, data);
4862
4863 si_wait_for_rlc_serdes(rdev);
4864 }
4865
4866 return orig;
4867 }
4868
4869 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4870 {
4871 u32 tmp;
4872
4873 tmp = RREG32(RLC_CNTL);
4874 if (tmp != rlc)
4875 WREG32(RLC_CNTL, rlc);
4876 }
4877
4878 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4879 {
4880 u32 data, orig;
4881
4882 orig = data = RREG32(DMA_PG);
4883 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4884 data |= PG_CNTL_ENABLE;
4885 else
4886 data &= ~PG_CNTL_ENABLE;
4887 if (orig != data)
4888 WREG32(DMA_PG, data);
4889 }
4890
4891 static void si_init_dma_pg(struct radeon_device *rdev)
4892 {
4893 u32 tmp;
4894
4895 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4896 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4897
4898 for (tmp = 0; tmp < 5; tmp++)
4899 WREG32(DMA_PGFSM_WRITE, 0);
4900 }
4901
4902 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4903 bool enable)
4904 {
4905 u32 tmp;
4906
4907 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4908 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4909 WREG32(RLC_TTOP_D, tmp);
4910
4911 tmp = RREG32(RLC_PG_CNTL);
4912 tmp |= GFX_PG_ENABLE;
4913 WREG32(RLC_PG_CNTL, tmp);
4914
4915 tmp = RREG32(RLC_AUTO_PG_CTRL);
4916 tmp |= AUTO_PG_EN;
4917 WREG32(RLC_AUTO_PG_CTRL, tmp);
4918 } else {
4919 tmp = RREG32(RLC_AUTO_PG_CTRL);
4920 tmp &= ~AUTO_PG_EN;
4921 WREG32(RLC_AUTO_PG_CTRL, tmp);
4922
4923 tmp = RREG32(DB_RENDER_CONTROL);
4924 }
4925 }
4926
4927 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4928 {
4929 u32 tmp;
4930
4931 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4932
4933 tmp = RREG32(RLC_PG_CNTL);
4934 tmp |= GFX_PG_SRC;
4935 WREG32(RLC_PG_CNTL, tmp);
4936
4937 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4938
4939 tmp = RREG32(RLC_AUTO_PG_CTRL);
4940
4941 tmp &= ~GRBM_REG_SGIT_MASK;
4942 tmp |= GRBM_REG_SGIT(0x700);
4943 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4944 WREG32(RLC_AUTO_PG_CTRL, tmp);
4945 }
4946
4947 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4948 {
4949 u32 mask = 0, tmp, tmp1;
4950 int i;
4951
4952 si_select_se_sh(rdev, se, sh);
4953 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4954 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4955 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4956
4957 tmp &= 0xffff0000;
4958
4959 tmp |= tmp1;
4960 tmp >>= 16;
4961
4962 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4963 mask <<= 1;
4964 mask |= 1;
4965 }
4966
4967 return (~tmp) & mask;
4968 }
4969
4970 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4971 {
4972 u32 i, j, k, active_cu_number = 0;
4973 u32 mask, counter, cu_bitmap;
4974 u32 tmp = 0;
4975
4976 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4977 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4978 mask = 1;
4979 cu_bitmap = 0;
4980 counter = 0;
4981 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4982 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4983 if (counter < 2)
4984 cu_bitmap |= mask;
4985 counter++;
4986 }
4987 mask <<= 1;
4988 }
4989
4990 active_cu_number += counter;
4991 tmp |= (cu_bitmap << (i * 16 + j * 8));
4992 }
4993 }
4994
4995 WREG32(RLC_PG_AO_CU_MASK, tmp);
4996
4997 tmp = RREG32(RLC_MAX_PG_CU);
4998 tmp &= ~MAX_PU_CU_MASK;
4999 tmp |= MAX_PU_CU(active_cu_number);
5000 WREG32(RLC_MAX_PG_CU, tmp);
5001 }
5002
5003 static void si_enable_cgcg(struct radeon_device *rdev,
5004 bool enable)
5005 {
5006 u32 data, orig, tmp;
5007
5008 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5009
5010 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5011 si_enable_gui_idle_interrupt(rdev, true);
5012
5013 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5014
5015 tmp = si_halt_rlc(rdev);
5016
5017 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5018 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5019 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5020
5021 si_wait_for_rlc_serdes(rdev);
5022
5023 si_update_rlc(rdev, tmp);
5024
5025 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5026
5027 data |= CGCG_EN | CGLS_EN;
5028 } else {
5029 si_enable_gui_idle_interrupt(rdev, false);
5030
5031 RREG32(CB_CGTT_SCLK_CTRL);
5032 RREG32(CB_CGTT_SCLK_CTRL);
5033 RREG32(CB_CGTT_SCLK_CTRL);
5034 RREG32(CB_CGTT_SCLK_CTRL);
5035
5036 data &= ~(CGCG_EN | CGLS_EN);
5037 }
5038
5039 if (orig != data)
5040 WREG32(RLC_CGCG_CGLS_CTRL, data);
5041 }
5042
5043 static void si_enable_mgcg(struct radeon_device *rdev,
5044 bool enable)
5045 {
5046 u32 data, orig, tmp = 0;
5047
5048 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5049 orig = data = RREG32(CGTS_SM_CTRL_REG);
5050 data = 0x96940200;
5051 if (orig != data)
5052 WREG32(CGTS_SM_CTRL_REG, data);
5053
5054 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5055 orig = data = RREG32(CP_MEM_SLP_CNTL);
5056 data |= CP_MEM_LS_EN;
5057 if (orig != data)
5058 WREG32(CP_MEM_SLP_CNTL, data);
5059 }
5060
5061 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5062 data &= 0xffffffc0;
5063 if (orig != data)
5064 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5065
5066 tmp = si_halt_rlc(rdev);
5067
5068 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5069 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5070 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5071
5072 si_update_rlc(rdev, tmp);
5073 } else {
5074 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5075 data |= 0x00000003;
5076 if (orig != data)
5077 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5078
5079 data = RREG32(CP_MEM_SLP_CNTL);
5080 if (data & CP_MEM_LS_EN) {
5081 data &= ~CP_MEM_LS_EN;
5082 WREG32(CP_MEM_SLP_CNTL, data);
5083 }
5084 orig = data = RREG32(CGTS_SM_CTRL_REG);
5085 data |= LS_OVERRIDE | OVERRIDE;
5086 if (orig != data)
5087 WREG32(CGTS_SM_CTRL_REG, data);
5088
5089 tmp = si_halt_rlc(rdev);
5090
5091 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5092 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5093 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5094
5095 si_update_rlc(rdev, tmp);
5096 }
5097 }
5098
5099 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5100 bool enable)
5101 {
5102 u32 orig, data, tmp;
5103
5104 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5105 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5106 tmp |= 0x3fff;
5107 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5108
5109 orig = data = RREG32(UVD_CGC_CTRL);
5110 data |= DCM;
5111 if (orig != data)
5112 WREG32(UVD_CGC_CTRL, data);
5113
5114 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5115 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5116 } else {
5117 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5118 tmp &= ~0x3fff;
5119 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5120
5121 orig = data = RREG32(UVD_CGC_CTRL);
5122 data &= ~DCM;
5123 if (orig != data)
5124 WREG32(UVD_CGC_CTRL, data);
5125
5126 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5127 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5128 }
5129 }
5130
5131 static const u32 mc_cg_registers[] =
5132 {
5133 MC_HUB_MISC_HUB_CG,
5134 MC_HUB_MISC_SIP_CG,
5135 MC_HUB_MISC_VM_CG,
5136 MC_XPB_CLK_GAT,
5137 ATC_MISC_CG,
5138 MC_CITF_MISC_WR_CG,
5139 MC_CITF_MISC_RD_CG,
5140 MC_CITF_MISC_VM_CG,
5141 VM_L2_CG,
5142 };
5143
5144 static void si_enable_mc_ls(struct radeon_device *rdev,
5145 bool enable)
5146 {
5147 int i;
5148 u32 orig, data;
5149
5150 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5151 orig = data = RREG32(mc_cg_registers[i]);
5152 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5153 data |= MC_LS_ENABLE;
5154 else
5155 data &= ~MC_LS_ENABLE;
5156 if (data != orig)
5157 WREG32(mc_cg_registers[i], data);
5158 }
5159 }
5160
5161 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5162 bool enable)
5163 {
5164 int i;
5165 u32 orig, data;
5166
5167 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5168 orig = data = RREG32(mc_cg_registers[i]);
5169 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5170 data |= MC_CG_ENABLE;
5171 else
5172 data &= ~MC_CG_ENABLE;
5173 if (data != orig)
5174 WREG32(mc_cg_registers[i], data);
5175 }
5176 }
5177
5178 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5179 bool enable)
5180 {
5181 u32 orig, data, offset;
5182 int i;
5183
5184 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5185 for (i = 0; i < 2; i++) {
5186 if (i == 0)
5187 offset = DMA0_REGISTER_OFFSET;
5188 else
5189 offset = DMA1_REGISTER_OFFSET;
5190 orig = data = RREG32(DMA_POWER_CNTL + offset);
5191 data &= ~MEM_POWER_OVERRIDE;
5192 if (data != orig)
5193 WREG32(DMA_POWER_CNTL + offset, data);
5194 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5195 }
5196 } else {
5197 for (i = 0; i < 2; i++) {
5198 if (i == 0)
5199 offset = DMA0_REGISTER_OFFSET;
5200 else
5201 offset = DMA1_REGISTER_OFFSET;
5202 orig = data = RREG32(DMA_POWER_CNTL + offset);
5203 data |= MEM_POWER_OVERRIDE;
5204 if (data != orig)
5205 WREG32(DMA_POWER_CNTL + offset, data);
5206
5207 orig = data = RREG32(DMA_CLK_CTRL + offset);
5208 data = 0xff000000;
5209 if (data != orig)
5210 WREG32(DMA_CLK_CTRL + offset, data);
5211 }
5212 }
5213 }
5214
5215 static void si_enable_bif_mgls(struct radeon_device *rdev,
5216 bool enable)
5217 {
5218 u32 orig, data;
5219
5220 orig = data = RREG32_PCIE(PCIE_CNTL2);
5221
5222 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5223 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5224 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5225 else
5226 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5227 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5228
5229 if (orig != data)
5230 WREG32_PCIE(PCIE_CNTL2, data);
5231 }
5232
5233 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5234 bool enable)
5235 {
5236 u32 orig, data;
5237
5238 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5239
5240 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5241 data &= ~CLOCK_GATING_DIS;
5242 else
5243 data |= CLOCK_GATING_DIS;
5244
5245 if (orig != data)
5246 WREG32(HDP_HOST_PATH_CNTL, data);
5247 }
5248
5249 static void si_enable_hdp_ls(struct radeon_device *rdev,
5250 bool enable)
5251 {
5252 u32 orig, data;
5253
5254 orig = data = RREG32(HDP_MEM_POWER_LS);
5255
5256 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5257 data |= HDP_LS_ENABLE;
5258 else
5259 data &= ~HDP_LS_ENABLE;
5260
5261 if (orig != data)
5262 WREG32(HDP_MEM_POWER_LS, data);
5263 }
5264
5265 void si_update_cg(struct radeon_device *rdev,
5266 u32 block, bool enable)
5267 {
5268 if (block & RADEON_CG_BLOCK_GFX) {
5269 si_enable_gui_idle_interrupt(rdev, false);
5270 /* order matters! */
5271 if (enable) {
5272 si_enable_mgcg(rdev, true);
5273 si_enable_cgcg(rdev, true);
5274 } else {
5275 si_enable_cgcg(rdev, false);
5276 si_enable_mgcg(rdev, false);
5277 }
5278 si_enable_gui_idle_interrupt(rdev, true);
5279 }
5280
5281 if (block & RADEON_CG_BLOCK_MC) {
5282 si_enable_mc_mgcg(rdev, enable);
5283 si_enable_mc_ls(rdev, enable);
5284 }
5285
5286 if (block & RADEON_CG_BLOCK_SDMA) {
5287 si_enable_dma_mgcg(rdev, enable);
5288 }
5289
5290 if (block & RADEON_CG_BLOCK_BIF) {
5291 si_enable_bif_mgls(rdev, enable);
5292 }
5293
5294 if (block & RADEON_CG_BLOCK_UVD) {
5295 if (rdev->has_uvd) {
5296 si_enable_uvd_mgcg(rdev, enable);
5297 }
5298 }
5299
5300 if (block & RADEON_CG_BLOCK_HDP) {
5301 si_enable_hdp_mgcg(rdev, enable);
5302 si_enable_hdp_ls(rdev, enable);
5303 }
5304 }
5305
5306 static void si_init_cg(struct radeon_device *rdev)
5307 {
5308 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5309 RADEON_CG_BLOCK_MC |
5310 RADEON_CG_BLOCK_SDMA |
5311 RADEON_CG_BLOCK_BIF |
5312 RADEON_CG_BLOCK_HDP), true);
5313 if (rdev->has_uvd) {
5314 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5315 si_init_uvd_internal_cg(rdev);
5316 }
5317 }
5318
5319 static void si_fini_cg(struct radeon_device *rdev)
5320 {
5321 if (rdev->has_uvd) {
5322 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5323 }
5324 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5325 RADEON_CG_BLOCK_MC |
5326 RADEON_CG_BLOCK_SDMA |
5327 RADEON_CG_BLOCK_BIF |
5328 RADEON_CG_BLOCK_HDP), false);
5329 }
5330
5331 u32 si_get_csb_size(struct radeon_device *rdev)
5332 {
5333 u32 count = 0;
5334 const struct cs_section_def *sect = NULL;
5335 const struct cs_extent_def *ext = NULL;
5336
5337 if (rdev->rlc.cs_data == NULL)
5338 return 0;
5339
5340 /* begin clear state */
5341 count += 2;
5342 /* context control state */
5343 count += 3;
5344
5345 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5346 for (ext = sect->section; ext->extent != NULL; ++ext) {
5347 if (sect->id == SECT_CONTEXT)
5348 count += 2 + ext->reg_count;
5349 else
5350 return 0;
5351 }
5352 }
5353 /* pa_sc_raster_config */
5354 count += 3;
5355 /* end clear state */
5356 count += 2;
5357 /* clear state */
5358 count += 2;
5359
5360 return count;
5361 }
5362
5363 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5364 {
5365 u32 count = 0, i;
5366 const struct cs_section_def *sect = NULL;
5367 const struct cs_extent_def *ext = NULL;
5368
5369 if (rdev->rlc.cs_data == NULL)
5370 return;
5371 if (buffer == NULL)
5372 return;
5373
5374 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5375 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5376
5377 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5378 buffer[count++] = 0x80000000;
5379 buffer[count++] = 0x80000000;
5380
5381 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5382 for (ext = sect->section; ext->extent != NULL; ++ext) {
5383 if (sect->id == SECT_CONTEXT) {
5384 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5385 buffer[count++] = ext->reg_index - 0xa000;
5386 for (i = 0; i < ext->reg_count; i++)
5387 buffer[count++] = ext->extent[i];
5388 } else {
5389 return;
5390 }
5391 }
5392 }
5393
5394 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5395 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5396 switch (rdev->family) {
5397 case CHIP_TAHITI:
5398 case CHIP_PITCAIRN:
5399 buffer[count++] = 0x2a00126a;
5400 break;
5401 case CHIP_VERDE:
5402 buffer[count++] = 0x0000124a;
5403 break;
5404 case CHIP_OLAND:
5405 buffer[count++] = 0x00000082;
5406 break;
5407 case CHIP_HAINAN:
5408 buffer[count++] = 0x00000000;
5409 break;
5410 default:
5411 buffer[count++] = 0x00000000;
5412 break;
5413 }
5414
5415 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5416 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5417
5418 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5419 buffer[count++] = 0;
5420 }
5421
5422 static void si_init_pg(struct radeon_device *rdev)
5423 {
5424 if (rdev->pg_flags) {
5425 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5426 si_init_dma_pg(rdev);
5427 }
5428 si_init_ao_cu_mask(rdev);
5429 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5430 si_init_gfx_cgpg(rdev);
5431 }
5432 si_enable_dma_pg(rdev, true);
5433 si_enable_gfx_cgpg(rdev, true);
5434 } else {
5435 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5436 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5437 }
5438 }
5439
5440 static void si_fini_pg(struct radeon_device *rdev)
5441 {
5442 if (rdev->pg_flags) {
5443 si_enable_dma_pg(rdev, false);
5444 si_enable_gfx_cgpg(rdev, false);
5445 }
5446 }
5447
5448 /*
5449 * RLC
5450 */
5451 void si_rlc_reset(struct radeon_device *rdev)
5452 {
5453 u32 tmp = RREG32(GRBM_SOFT_RESET);
5454
5455 tmp |= SOFT_RESET_RLC;
5456 WREG32(GRBM_SOFT_RESET, tmp);
5457 udelay(50);
5458 tmp &= ~SOFT_RESET_RLC;
5459 WREG32(GRBM_SOFT_RESET, tmp);
5460 udelay(50);
5461 }
5462
5463 static void si_rlc_stop(struct radeon_device *rdev)
5464 {
5465 WREG32(RLC_CNTL, 0);
5466
5467 si_enable_gui_idle_interrupt(rdev, false);
5468
5469 si_wait_for_rlc_serdes(rdev);
5470 }
5471
5472 static void si_rlc_start(struct radeon_device *rdev)
5473 {
5474 WREG32(RLC_CNTL, RLC_ENABLE);
5475
5476 si_enable_gui_idle_interrupt(rdev, true);
5477
5478 udelay(50);
5479 }
5480
5481 static bool si_lbpw_supported(struct radeon_device *rdev)
5482 {
5483 u32 tmp;
5484
5485 /* Enable LBPW only for DDR3 */
5486 tmp = RREG32(MC_SEQ_MISC0);
5487 if ((tmp & 0xF0000000) == 0xB0000000)
5488 return true;
5489 return false;
5490 }
5491
5492 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5493 {
5494 u32 tmp;
5495
5496 tmp = RREG32(RLC_LB_CNTL);
5497 if (enable)
5498 tmp |= LOAD_BALANCE_ENABLE;
5499 else
5500 tmp &= ~LOAD_BALANCE_ENABLE;
5501 WREG32(RLC_LB_CNTL, tmp);
5502
5503 if (!enable) {
5504 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5505 WREG32(SPI_LB_CU_MASK, 0x00ff);
5506 }
5507 }
5508
5509 static int si_rlc_resume(struct radeon_device *rdev)
5510 {
5511 u32 i;
5512 const __be32 *fw_data;
5513
5514 if (!rdev->rlc_fw)
5515 return -EINVAL;
5516
5517 si_rlc_stop(rdev);
5518
5519 si_rlc_reset(rdev);
5520
5521 si_init_pg(rdev);
5522
5523 si_init_cg(rdev);
5524
5525 WREG32(RLC_RL_BASE, 0);
5526 WREG32(RLC_RL_SIZE, 0);
5527 WREG32(RLC_LB_CNTL, 0);
5528 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5529 WREG32(RLC_LB_CNTR_INIT, 0);
5530 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5531
5532 WREG32(RLC_MC_CNTL, 0);
5533 WREG32(RLC_UCODE_CNTL, 0);
5534
5535 fw_data = (const __be32 *)rdev->rlc_fw->data;
5536 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5537 WREG32(RLC_UCODE_ADDR, i);
5538 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5539 }
5540 WREG32(RLC_UCODE_ADDR, 0);
5541
5542 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5543
5544 si_rlc_start(rdev);
5545
5546 return 0;
5547 }
5548
5549 static void si_enable_interrupts(struct radeon_device *rdev)
5550 {
5551 u32 ih_cntl = RREG32(IH_CNTL);
5552 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5553
5554 ih_cntl |= ENABLE_INTR;
5555 ih_rb_cntl |= IH_RB_ENABLE;
5556 WREG32(IH_CNTL, ih_cntl);
5557 WREG32(IH_RB_CNTL, ih_rb_cntl);
5558 rdev->ih.enabled = true;
5559 }
5560
5561 static void si_disable_interrupts(struct radeon_device *rdev)
5562 {
5563 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5564 u32 ih_cntl = RREG32(IH_CNTL);
5565
5566 ih_rb_cntl &= ~IH_RB_ENABLE;
5567 ih_cntl &= ~ENABLE_INTR;
5568 WREG32(IH_RB_CNTL, ih_rb_cntl);
5569 WREG32(IH_CNTL, ih_cntl);
5570 /* set rptr, wptr to 0 */
5571 WREG32(IH_RB_RPTR, 0);
5572 WREG32(IH_RB_WPTR, 0);
5573 rdev->ih.enabled = false;
5574 rdev->ih.rptr = 0;
5575 }
5576
5577 static void si_disable_interrupt_state(struct radeon_device *rdev)
5578 {
5579 u32 tmp;
5580
5581 tmp = RREG32(CP_INT_CNTL_RING0) &
5582 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5583 WREG32(CP_INT_CNTL_RING0, tmp);
5584 WREG32(CP_INT_CNTL_RING1, 0);
5585 WREG32(CP_INT_CNTL_RING2, 0);
5586 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5587 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5588 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5589 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5590 WREG32(GRBM_INT_CNTL, 0);
5591 if (rdev->num_crtc >= 2) {
5592 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5593 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5594 }
5595 if (rdev->num_crtc >= 4) {
5596 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5597 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5598 }
5599 if (rdev->num_crtc >= 6) {
5600 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5601 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5602 }
5603
5604 if (rdev->num_crtc >= 2) {
5605 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5606 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5607 }
5608 if (rdev->num_crtc >= 4) {
5609 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5610 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5611 }
5612 if (rdev->num_crtc >= 6) {
5613 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5614 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5615 }
5616
5617 if (!ASIC_IS_NODCE(rdev)) {
5618 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5619
5620 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5621 WREG32(DC_HPD1_INT_CONTROL, tmp);
5622 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5623 WREG32(DC_HPD2_INT_CONTROL, tmp);
5624 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5625 WREG32(DC_HPD3_INT_CONTROL, tmp);
5626 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5627 WREG32(DC_HPD4_INT_CONTROL, tmp);
5628 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5629 WREG32(DC_HPD5_INT_CONTROL, tmp);
5630 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5631 WREG32(DC_HPD6_INT_CONTROL, tmp);
5632 }
5633 }
5634
5635 static int si_irq_init(struct radeon_device *rdev)
5636 {
5637 int ret = 0;
5638 int rb_bufsz;
5639 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5640
5641 /* allocate ring */
5642 ret = r600_ih_ring_alloc(rdev);
5643 if (ret)
5644 return ret;
5645
5646 /* disable irqs */
5647 si_disable_interrupts(rdev);
5648
5649 /* init rlc */
5650 ret = si_rlc_resume(rdev);
5651 if (ret) {
5652 r600_ih_ring_fini(rdev);
5653 return ret;
5654 }
5655
5656 /* setup interrupt control */
5657 /* set dummy read address to ring address */
5658 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5659 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5660 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5661 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5662 */
5663 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5664 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5665 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5666 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5667
5668 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5669 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5670
5671 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5672 IH_WPTR_OVERFLOW_CLEAR |
5673 (rb_bufsz << 1));
5674
5675 if (rdev->wb.enabled)
5676 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5677
5678 /* set the writeback address whether it's enabled or not */
5679 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5680 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5681
5682 WREG32(IH_RB_CNTL, ih_rb_cntl);
5683
5684 /* set rptr, wptr to 0 */
5685 WREG32(IH_RB_RPTR, 0);
5686 WREG32(IH_RB_WPTR, 0);
5687
5688 /* Default settings for IH_CNTL (disabled at first) */
5689 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5690 /* RPTR_REARM only works if msi's are enabled */
5691 if (rdev->msi_enabled)
5692 ih_cntl |= RPTR_REARM;
5693 WREG32(IH_CNTL, ih_cntl);
5694
5695 /* force the active interrupt state to all disabled */
5696 si_disable_interrupt_state(rdev);
5697
5698 pci_set_master(rdev->pdev);
5699
5700 /* enable irqs */
5701 si_enable_interrupts(rdev);
5702
5703 return ret;
5704 }
5705
5706 int si_irq_set(struct radeon_device *rdev)
5707 {
5708 u32 cp_int_cntl;
5709 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5710 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5711 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5712 u32 grbm_int_cntl = 0;
5713 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5714 u32 dma_cntl, dma_cntl1;
5715 u32 thermal_int = 0;
5716
5717 if (!rdev->irq.installed) {
5718 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5719 return -EINVAL;
5720 }
5721 /* don't enable anything if the ih is disabled */
5722 if (!rdev->ih.enabled) {
5723 si_disable_interrupts(rdev);
5724 /* force the active interrupt state to all disabled */
5725 si_disable_interrupt_state(rdev);
5726 return 0;
5727 }
5728
5729 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5730 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5731
5732 if (!ASIC_IS_NODCE(rdev)) {
5733 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5734 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5735 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5736 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5737 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5738 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5739 }
5740
5741 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5742 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5743
5744 thermal_int = RREG32(CG_THERMAL_INT) &
5745 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5746
5747 /* enable CP interrupts on all rings */
5748 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5749 DRM_DEBUG("si_irq_set: sw int gfx\n");
5750 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5751 }
5752 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5753 DRM_DEBUG("si_irq_set: sw int cp1\n");
5754 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5755 }
5756 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5757 DRM_DEBUG("si_irq_set: sw int cp2\n");
5758 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5759 }
5760 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5761 DRM_DEBUG("si_irq_set: sw int dma\n");
5762 dma_cntl |= TRAP_ENABLE;
5763 }
5764
5765 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5766 DRM_DEBUG("si_irq_set: sw int dma1\n");
5767 dma_cntl1 |= TRAP_ENABLE;
5768 }
5769 if (rdev->irq.crtc_vblank_int[0] ||
5770 atomic_read(&rdev->irq.pflip[0])) {
5771 DRM_DEBUG("si_irq_set: vblank 0\n");
5772 crtc1 |= VBLANK_INT_MASK;
5773 }
5774 if (rdev->irq.crtc_vblank_int[1] ||
5775 atomic_read(&rdev->irq.pflip[1])) {
5776 DRM_DEBUG("si_irq_set: vblank 1\n");
5777 crtc2 |= VBLANK_INT_MASK;
5778 }
5779 if (rdev->irq.crtc_vblank_int[2] ||
5780 atomic_read(&rdev->irq.pflip[2])) {
5781 DRM_DEBUG("si_irq_set: vblank 2\n");
5782 crtc3 |= VBLANK_INT_MASK;
5783 }
5784 if (rdev->irq.crtc_vblank_int[3] ||
5785 atomic_read(&rdev->irq.pflip[3])) {
5786 DRM_DEBUG("si_irq_set: vblank 3\n");
5787 crtc4 |= VBLANK_INT_MASK;
5788 }
5789 if (rdev->irq.crtc_vblank_int[4] ||
5790 atomic_read(&rdev->irq.pflip[4])) {
5791 DRM_DEBUG("si_irq_set: vblank 4\n");
5792 crtc5 |= VBLANK_INT_MASK;
5793 }
5794 if (rdev->irq.crtc_vblank_int[5] ||
5795 atomic_read(&rdev->irq.pflip[5])) {
5796 DRM_DEBUG("si_irq_set: vblank 5\n");
5797 crtc6 |= VBLANK_INT_MASK;
5798 }
5799 if (rdev->irq.hpd[0]) {
5800 DRM_DEBUG("si_irq_set: hpd 1\n");
5801 hpd1 |= DC_HPDx_INT_EN;
5802 }
5803 if (rdev->irq.hpd[1]) {
5804 DRM_DEBUG("si_irq_set: hpd 2\n");
5805 hpd2 |= DC_HPDx_INT_EN;
5806 }
5807 if (rdev->irq.hpd[2]) {
5808 DRM_DEBUG("si_irq_set: hpd 3\n");
5809 hpd3 |= DC_HPDx_INT_EN;
5810 }
5811 if (rdev->irq.hpd[3]) {
5812 DRM_DEBUG("si_irq_set: hpd 4\n");
5813 hpd4 |= DC_HPDx_INT_EN;
5814 }
5815 if (rdev->irq.hpd[4]) {
5816 DRM_DEBUG("si_irq_set: hpd 5\n");
5817 hpd5 |= DC_HPDx_INT_EN;
5818 }
5819 if (rdev->irq.hpd[5]) {
5820 DRM_DEBUG("si_irq_set: hpd 6\n");
5821 hpd6 |= DC_HPDx_INT_EN;
5822 }
5823
5824 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5825 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5826 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5827
5828 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5829 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5830
5831 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5832
5833 if (rdev->irq.dpm_thermal) {
5834 DRM_DEBUG("dpm thermal\n");
5835 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5836 }
5837
5838 if (rdev->num_crtc >= 2) {
5839 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5840 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5841 }
5842 if (rdev->num_crtc >= 4) {
5843 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5844 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5845 }
5846 if (rdev->num_crtc >= 6) {
5847 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5848 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5849 }
5850
5851 if (rdev->num_crtc >= 2) {
5852 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5853 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5854 }
5855 if (rdev->num_crtc >= 4) {
5856 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5857 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5858 }
5859 if (rdev->num_crtc >= 6) {
5860 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5861 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5862 }
5863
5864 if (!ASIC_IS_NODCE(rdev)) {
5865 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5866 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5867 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5868 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5869 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5870 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5871 }
5872
5873 WREG32(CG_THERMAL_INT, thermal_int);
5874
5875 return 0;
5876 }
5877
5878 static inline void si_irq_ack(struct radeon_device *rdev)
5879 {
5880 u32 tmp;
5881
5882 if (ASIC_IS_NODCE(rdev))
5883 return;
5884
5885 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5886 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5887 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5888 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5889 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5890 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5891 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5892 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5893 if (rdev->num_crtc >= 4) {
5894 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5895 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5896 }
5897 if (rdev->num_crtc >= 6) {
5898 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5899 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5900 }
5901
5902 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5903 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5904 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5905 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5906 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5907 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5908 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5909 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5910 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5911 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5912 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5913 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5914
5915 if (rdev->num_crtc >= 4) {
5916 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5917 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5918 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5919 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5920 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5921 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5922 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5923 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5924 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5925 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5926 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5927 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5928 }
5929
5930 if (rdev->num_crtc >= 6) {
5931 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5932 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5933 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5934 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5935 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5936 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5937 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5938 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5939 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5940 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5941 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5942 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5943 }
5944
5945 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5946 tmp = RREG32(DC_HPD1_INT_CONTROL);
5947 tmp |= DC_HPDx_INT_ACK;
5948 WREG32(DC_HPD1_INT_CONTROL, tmp);
5949 }
5950 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5951 tmp = RREG32(DC_HPD2_INT_CONTROL);
5952 tmp |= DC_HPDx_INT_ACK;
5953 WREG32(DC_HPD2_INT_CONTROL, tmp);
5954 }
5955 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5956 tmp = RREG32(DC_HPD3_INT_CONTROL);
5957 tmp |= DC_HPDx_INT_ACK;
5958 WREG32(DC_HPD3_INT_CONTROL, tmp);
5959 }
5960 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5961 tmp = RREG32(DC_HPD4_INT_CONTROL);
5962 tmp |= DC_HPDx_INT_ACK;
5963 WREG32(DC_HPD4_INT_CONTROL, tmp);
5964 }
5965 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5966 tmp = RREG32(DC_HPD5_INT_CONTROL);
5967 tmp |= DC_HPDx_INT_ACK;
5968 WREG32(DC_HPD5_INT_CONTROL, tmp);
5969 }
5970 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5971 tmp = RREG32(DC_HPD5_INT_CONTROL);
5972 tmp |= DC_HPDx_INT_ACK;
5973 WREG32(DC_HPD6_INT_CONTROL, tmp);
5974 }
5975 }
5976
5977 static void si_irq_disable(struct radeon_device *rdev)
5978 {
5979 si_disable_interrupts(rdev);
5980 /* Wait and acknowledge irq */
5981 mdelay(1);
5982 si_irq_ack(rdev);
5983 si_disable_interrupt_state(rdev);
5984 }
5985
5986 static void si_irq_suspend(struct radeon_device *rdev)
5987 {
5988 si_irq_disable(rdev);
5989 si_rlc_stop(rdev);
5990 }
5991
5992 static void si_irq_fini(struct radeon_device *rdev)
5993 {
5994 si_irq_suspend(rdev);
5995 r600_ih_ring_fini(rdev);
5996 }
5997
5998 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5999 {
6000 u32 wptr, tmp;
6001
6002 if (rdev->wb.enabled)
6003 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6004 else
6005 wptr = RREG32(IH_RB_WPTR);
6006
6007 if (wptr & RB_OVERFLOW) {
6008 /* When a ring buffer overflow happen start parsing interrupt
6009 * from the last not overwritten vector (wptr + 16). Hopefully
6010 * this should allow us to catchup.
6011 */
6012 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6013 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6014 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6015 tmp = RREG32(IH_RB_CNTL);
6016 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6017 WREG32(IH_RB_CNTL, tmp);
6018 }
6019 return (wptr & rdev->ih.ptr_mask);
6020 }
6021
6022 /* SI IV Ring
6023 * Each IV ring entry is 128 bits:
6024 * [7:0] - interrupt source id
6025 * [31:8] - reserved
6026 * [59:32] - interrupt source data
6027 * [63:60] - reserved
6028 * [71:64] - RINGID
6029 * [79:72] - VMID
6030 * [127:80] - reserved
6031 */
6032 int si_irq_process(struct radeon_device *rdev)
6033 {
6034 u32 wptr;
6035 u32 rptr;
6036 u32 src_id, src_data, ring_id;
6037 u32 ring_index;
6038 bool queue_hotplug = false;
6039 bool queue_thermal = false;
6040 u32 status, addr;
6041
6042 if (!rdev->ih.enabled || rdev->shutdown)
6043 return IRQ_NONE;
6044
6045 wptr = si_get_ih_wptr(rdev);
6046
6047 restart_ih:
6048 /* is somebody else already processing irqs? */
6049 if (atomic_xchg(&rdev->ih.lock, 1))
6050 return IRQ_NONE;
6051
6052 rptr = rdev->ih.rptr;
6053 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6054
6055 /* Order reading of wptr vs. reading of IH ring data */
6056 rmb();
6057
6058 /* display interrupts */
6059 si_irq_ack(rdev);
6060
6061 while (rptr != wptr) {
6062 /* wptr/rptr are in bytes! */
6063 ring_index = rptr / 4;
6064 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6065 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6066 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6067
6068 switch (src_id) {
6069 case 1: /* D1 vblank/vline */
6070 switch (src_data) {
6071 case 0: /* D1 vblank */
6072 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6073 if (rdev->irq.crtc_vblank_int[0]) {
6074 drm_handle_vblank(rdev->ddev, 0);
6075 rdev->pm.vblank_sync = true;
6076 wake_up(&rdev->irq.vblank_queue);
6077 }
6078 if (atomic_read(&rdev->irq.pflip[0]))
6079 radeon_crtc_handle_flip(rdev, 0);
6080 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6081 DRM_DEBUG("IH: D1 vblank\n");
6082 }
6083 break;
6084 case 1: /* D1 vline */
6085 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6086 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6087 DRM_DEBUG("IH: D1 vline\n");
6088 }
6089 break;
6090 default:
6091 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6092 break;
6093 }
6094 break;
6095 case 2: /* D2 vblank/vline */
6096 switch (src_data) {
6097 case 0: /* D2 vblank */
6098 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6099 if (rdev->irq.crtc_vblank_int[1]) {
6100 drm_handle_vblank(rdev->ddev, 1);
6101 rdev->pm.vblank_sync = true;
6102 wake_up(&rdev->irq.vblank_queue);
6103 }
6104 if (atomic_read(&rdev->irq.pflip[1]))
6105 radeon_crtc_handle_flip(rdev, 1);
6106 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6107 DRM_DEBUG("IH: D2 vblank\n");
6108 }
6109 break;
6110 case 1: /* D2 vline */
6111 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6112 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6113 DRM_DEBUG("IH: D2 vline\n");
6114 }
6115 break;
6116 default:
6117 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6118 break;
6119 }
6120 break;
6121 case 3: /* D3 vblank/vline */
6122 switch (src_data) {
6123 case 0: /* D3 vblank */
6124 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6125 if (rdev->irq.crtc_vblank_int[2]) {
6126 drm_handle_vblank(rdev->ddev, 2);
6127 rdev->pm.vblank_sync = true;
6128 wake_up(&rdev->irq.vblank_queue);
6129 }
6130 if (atomic_read(&rdev->irq.pflip[2]))
6131 radeon_crtc_handle_flip(rdev, 2);
6132 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6133 DRM_DEBUG("IH: D3 vblank\n");
6134 }
6135 break;
6136 case 1: /* D3 vline */
6137 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6138 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6139 DRM_DEBUG("IH: D3 vline\n");
6140 }
6141 break;
6142 default:
6143 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6144 break;
6145 }
6146 break;
6147 case 4: /* D4 vblank/vline */
6148 switch (src_data) {
6149 case 0: /* D4 vblank */
6150 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6151 if (rdev->irq.crtc_vblank_int[3]) {
6152 drm_handle_vblank(rdev->ddev, 3);
6153 rdev->pm.vblank_sync = true;
6154 wake_up(&rdev->irq.vblank_queue);
6155 }
6156 if (atomic_read(&rdev->irq.pflip[3]))
6157 radeon_crtc_handle_flip(rdev, 3);
6158 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6159 DRM_DEBUG("IH: D4 vblank\n");
6160 }
6161 break;
6162 case 1: /* D4 vline */
6163 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6164 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6165 DRM_DEBUG("IH: D4 vline\n");
6166 }
6167 break;
6168 default:
6169 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6170 break;
6171 }
6172 break;
6173 case 5: /* D5 vblank/vline */
6174 switch (src_data) {
6175 case 0: /* D5 vblank */
6176 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6177 if (rdev->irq.crtc_vblank_int[4]) {
6178 drm_handle_vblank(rdev->ddev, 4);
6179 rdev->pm.vblank_sync = true;
6180 wake_up(&rdev->irq.vblank_queue);
6181 }
6182 if (atomic_read(&rdev->irq.pflip[4]))
6183 radeon_crtc_handle_flip(rdev, 4);
6184 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6185 DRM_DEBUG("IH: D5 vblank\n");
6186 }
6187 break;
6188 case 1: /* D5 vline */
6189 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6190 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6191 DRM_DEBUG("IH: D5 vline\n");
6192 }
6193 break;
6194 default:
6195 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6196 break;
6197 }
6198 break;
6199 case 6: /* D6 vblank/vline */
6200 switch (src_data) {
6201 case 0: /* D6 vblank */
6202 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6203 if (rdev->irq.crtc_vblank_int[5]) {
6204 drm_handle_vblank(rdev->ddev, 5);
6205 rdev->pm.vblank_sync = true;
6206 wake_up(&rdev->irq.vblank_queue);
6207 }
6208 if (atomic_read(&rdev->irq.pflip[5]))
6209 radeon_crtc_handle_flip(rdev, 5);
6210 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6211 DRM_DEBUG("IH: D6 vblank\n");
6212 }
6213 break;
6214 case 1: /* D6 vline */
6215 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6216 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6217 DRM_DEBUG("IH: D6 vline\n");
6218 }
6219 break;
6220 default:
6221 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6222 break;
6223 }
6224 break;
6225 case 42: /* HPD hotplug */
6226 switch (src_data) {
6227 case 0:
6228 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6229 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6230 queue_hotplug = true;
6231 DRM_DEBUG("IH: HPD1\n");
6232 }
6233 break;
6234 case 1:
6235 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6236 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6237 queue_hotplug = true;
6238 DRM_DEBUG("IH: HPD2\n");
6239 }
6240 break;
6241 case 2:
6242 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6243 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6244 queue_hotplug = true;
6245 DRM_DEBUG("IH: HPD3\n");
6246 }
6247 break;
6248 case 3:
6249 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6250 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6251 queue_hotplug = true;
6252 DRM_DEBUG("IH: HPD4\n");
6253 }
6254 break;
6255 case 4:
6256 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6257 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6258 queue_hotplug = true;
6259 DRM_DEBUG("IH: HPD5\n");
6260 }
6261 break;
6262 case 5:
6263 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6264 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6265 queue_hotplug = true;
6266 DRM_DEBUG("IH: HPD6\n");
6267 }
6268 break;
6269 default:
6270 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6271 break;
6272 }
6273 break;
6274 case 146:
6275 case 147:
6276 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6277 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6278 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6279 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6280 addr);
6281 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6282 status);
6283 si_vm_decode_fault(rdev, status, addr);
6284 /* reset addr and status */
6285 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6286 break;
6287 case 176: /* RINGID0 CP_INT */
6288 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6289 break;
6290 case 177: /* RINGID1 CP_INT */
6291 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6292 break;
6293 case 178: /* RINGID2 CP_INT */
6294 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6295 break;
6296 case 181: /* CP EOP event */
6297 DRM_DEBUG("IH: CP EOP\n");
6298 switch (ring_id) {
6299 case 0:
6300 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6301 break;
6302 case 1:
6303 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6304 break;
6305 case 2:
6306 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6307 break;
6308 }
6309 break;
6310 case 224: /* DMA trap event */
6311 DRM_DEBUG("IH: DMA trap\n");
6312 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6313 break;
6314 case 230: /* thermal low to high */
6315 DRM_DEBUG("IH: thermal low to high\n");
6316 rdev->pm.dpm.thermal.high_to_low = false;
6317 queue_thermal = true;
6318 break;
6319 case 231: /* thermal high to low */
6320 DRM_DEBUG("IH: thermal high to low\n");
6321 rdev->pm.dpm.thermal.high_to_low = true;
6322 queue_thermal = true;
6323 break;
6324 case 233: /* GUI IDLE */
6325 DRM_DEBUG("IH: GUI idle\n");
6326 break;
6327 case 244: /* DMA trap event */
6328 DRM_DEBUG("IH: DMA1 trap\n");
6329 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6330 break;
6331 default:
6332 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6333 break;
6334 }
6335
6336 /* wptr/rptr are in bytes! */
6337 rptr += 16;
6338 rptr &= rdev->ih.ptr_mask;
6339 }
6340 if (queue_hotplug)
6341 schedule_work(&rdev->hotplug_work);
6342 if (queue_thermal && rdev->pm.dpm_enabled)
6343 schedule_work(&rdev->pm.dpm.thermal.work);
6344 rdev->ih.rptr = rptr;
6345 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6346 atomic_set(&rdev->ih.lock, 0);
6347
6348 /* make sure wptr hasn't changed while processing */
6349 wptr = si_get_ih_wptr(rdev);
6350 if (wptr != rptr)
6351 goto restart_ih;
6352
6353 return IRQ_HANDLED;
6354 }
6355
6356 /*
6357 * startup/shutdown callbacks
6358 */
6359 static int si_startup(struct radeon_device *rdev)
6360 {
6361 struct radeon_ring *ring;
6362 int r;
6363
6364 /* enable pcie gen2/3 link */
6365 si_pcie_gen3_enable(rdev);
6366 /* enable aspm */
6367 si_program_aspm(rdev);
6368
6369 /* scratch needs to be initialized before MC */
6370 r = r600_vram_scratch_init(rdev);
6371 if (r)
6372 return r;
6373
6374 si_mc_program(rdev);
6375
6376 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6377 !rdev->rlc_fw || !rdev->mc_fw) {
6378 r = si_init_microcode(rdev);
6379 if (r) {
6380 DRM_ERROR("Failed to load firmware!\n");
6381 return r;
6382 }
6383 }
6384
6385 r = si_mc_load_microcode(rdev);
6386 if (r) {
6387 DRM_ERROR("Failed to load MC firmware!\n");
6388 return r;
6389 }
6390
6391 r = si_pcie_gart_enable(rdev);
6392 if (r)
6393 return r;
6394 si_gpu_init(rdev);
6395
6396 /* allocate rlc buffers */
6397 if (rdev->family == CHIP_VERDE) {
6398 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6399 rdev->rlc.reg_list_size =
6400 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6401 }
6402 rdev->rlc.cs_data = si_cs_data;
6403 r = sumo_rlc_init(rdev);
6404 if (r) {
6405 DRM_ERROR("Failed to init rlc BOs!\n");
6406 return r;
6407 }
6408
6409 /* allocate wb buffer */
6410 r = radeon_wb_init(rdev);
6411 if (r)
6412 return r;
6413
6414 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6415 if (r) {
6416 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6417 return r;
6418 }
6419
6420 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6421 if (r) {
6422 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6423 return r;
6424 }
6425
6426 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6427 if (r) {
6428 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6429 return r;
6430 }
6431
6432 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6433 if (r) {
6434 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6435 return r;
6436 }
6437
6438 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6439 if (r) {
6440 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6441 return r;
6442 }
6443
6444 if (rdev->has_uvd) {
6445 r = uvd_v2_2_resume(rdev);
6446 if (!r) {
6447 r = radeon_fence_driver_start_ring(rdev,
6448 R600_RING_TYPE_UVD_INDEX);
6449 if (r)
6450 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6451 }
6452 if (r)
6453 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6454 }
6455
6456 /* Enable IRQ */
6457 if (!rdev->irq.installed) {
6458 r = radeon_irq_kms_init(rdev);
6459 if (r)
6460 return r;
6461 }
6462
6463 r = si_irq_init(rdev);
6464 if (r) {
6465 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6466 radeon_irq_kms_fini(rdev);
6467 return r;
6468 }
6469 si_irq_set(rdev);
6470
6471 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6472 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6473 CP_RB0_RPTR, CP_RB0_WPTR,
6474 RADEON_CP_PACKET2);
6475 if (r)
6476 return r;
6477
6478 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6479 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6480 CP_RB1_RPTR, CP_RB1_WPTR,
6481 RADEON_CP_PACKET2);
6482 if (r)
6483 return r;
6484
6485 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6486 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6487 CP_RB2_RPTR, CP_RB2_WPTR,
6488 RADEON_CP_PACKET2);
6489 if (r)
6490 return r;
6491
6492 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6493 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6494 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6495 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6496 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6497 if (r)
6498 return r;
6499
6500 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6501 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6502 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6503 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6504 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6505 if (r)
6506 return r;
6507
6508 r = si_cp_load_microcode(rdev);
6509 if (r)
6510 return r;
6511 r = si_cp_resume(rdev);
6512 if (r)
6513 return r;
6514
6515 r = cayman_dma_resume(rdev);
6516 if (r)
6517 return r;
6518
6519 if (rdev->has_uvd) {
6520 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6521 if (ring->ring_size) {
6522 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6523 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6524 RADEON_CP_PACKET2);
6525 if (!r)
6526 r = uvd_v1_0_init(rdev);
6527 if (r)
6528 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6529 }
6530 }
6531
6532 r = radeon_ib_pool_init(rdev);
6533 if (r) {
6534 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6535 return r;
6536 }
6537
6538 r = radeon_vm_manager_init(rdev);
6539 if (r) {
6540 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6541 return r;
6542 }
6543
6544 r = dce6_audio_init(rdev);
6545 if (r)
6546 return r;
6547
6548 return 0;
6549 }
6550
6551 int si_resume(struct radeon_device *rdev)
6552 {
6553 int r;
6554
6555 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6556 * posting will perform necessary task to bring back GPU into good
6557 * shape.
6558 */
6559 /* post card */
6560 atom_asic_init(rdev->mode_info.atom_context);
6561
6562 /* init golden registers */
6563 si_init_golden_registers(rdev);
6564
6565 rdev->accel_working = true;
6566 r = si_startup(rdev);
6567 if (r) {
6568 DRM_ERROR("si startup failed on resume\n");
6569 rdev->accel_working = false;
6570 return r;
6571 }
6572
6573 return r;
6574
6575 }
6576
6577 int si_suspend(struct radeon_device *rdev)
6578 {
6579 dce6_audio_fini(rdev);
6580 radeon_vm_manager_fini(rdev);
6581 si_cp_enable(rdev, false);
6582 cayman_dma_stop(rdev);
6583 if (rdev->has_uvd) {
6584 uvd_v1_0_fini(rdev);
6585 radeon_uvd_suspend(rdev);
6586 }
6587 si_fini_pg(rdev);
6588 si_fini_cg(rdev);
6589 si_irq_suspend(rdev);
6590 radeon_wb_disable(rdev);
6591 si_pcie_gart_disable(rdev);
6592 return 0;
6593 }
6594
6595 /* Plan is to move initialization in that function and use
6596 * helper function so that radeon_device_init pretty much
6597 * do nothing more than calling asic specific function. This
6598 * should also allow to remove a bunch of callback function
6599 * like vram_info.
6600 */
6601 int si_init(struct radeon_device *rdev)
6602 {
6603 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6604 int r;
6605
6606 /* Read BIOS */
6607 if (!radeon_get_bios(rdev)) {
6608 if (ASIC_IS_AVIVO(rdev))
6609 return -EINVAL;
6610 }
6611 /* Must be an ATOMBIOS */
6612 if (!rdev->is_atom_bios) {
6613 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6614 return -EINVAL;
6615 }
6616 r = radeon_atombios_init(rdev);
6617 if (r)
6618 return r;
6619
6620 /* Post card if necessary */
6621 if (!radeon_card_posted(rdev)) {
6622 if (!rdev->bios) {
6623 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6624 return -EINVAL;
6625 }
6626 DRM_INFO("GPU not posted. posting now...\n");
6627 atom_asic_init(rdev->mode_info.atom_context);
6628 }
6629 /* init golden registers */
6630 si_init_golden_registers(rdev);
6631 /* Initialize scratch registers */
6632 si_scratch_init(rdev);
6633 /* Initialize surface registers */
6634 radeon_surface_init(rdev);
6635 /* Initialize clocks */
6636 radeon_get_clock_info(rdev->ddev);
6637
6638 /* Fence driver */
6639 r = radeon_fence_driver_init(rdev);
6640 if (r)
6641 return r;
6642
6643 /* initialize memory controller */
6644 r = si_mc_init(rdev);
6645 if (r)
6646 return r;
6647 /* Memory manager */
6648 r = radeon_bo_init(rdev);
6649 if (r)
6650 return r;
6651
6652 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6653 ring->ring_obj = NULL;
6654 r600_ring_init(rdev, ring, 1024 * 1024);
6655
6656 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6657 ring->ring_obj = NULL;
6658 r600_ring_init(rdev, ring, 1024 * 1024);
6659
6660 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6661 ring->ring_obj = NULL;
6662 r600_ring_init(rdev, ring, 1024 * 1024);
6663
6664 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6665 ring->ring_obj = NULL;
6666 r600_ring_init(rdev, ring, 64 * 1024);
6667
6668 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6669 ring->ring_obj = NULL;
6670 r600_ring_init(rdev, ring, 64 * 1024);
6671
6672 if (rdev->has_uvd) {
6673 r = radeon_uvd_init(rdev);
6674 if (!r) {
6675 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6676 ring->ring_obj = NULL;
6677 r600_ring_init(rdev, ring, 4096);
6678 }
6679 }
6680
6681 rdev->ih.ring_obj = NULL;
6682 r600_ih_ring_init(rdev, 64 * 1024);
6683
6684 r = r600_pcie_gart_init(rdev);
6685 if (r)
6686 return r;
6687
6688 rdev->accel_working = true;
6689 r = si_startup(rdev);
6690 if (r) {
6691 dev_err(rdev->dev, "disabling GPU acceleration\n");
6692 si_cp_fini(rdev);
6693 cayman_dma_fini(rdev);
6694 si_irq_fini(rdev);
6695 sumo_rlc_fini(rdev);
6696 radeon_wb_fini(rdev);
6697 radeon_ib_pool_fini(rdev);
6698 radeon_vm_manager_fini(rdev);
6699 radeon_irq_kms_fini(rdev);
6700 si_pcie_gart_fini(rdev);
6701 rdev->accel_working = false;
6702 }
6703
6704 /* Don't start up if the MC ucode is missing.
6705 * The default clocks and voltages before the MC ucode
6706 * is loaded are not suffient for advanced operations.
6707 */
6708 if (!rdev->mc_fw) {
6709 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6710 return -EINVAL;
6711 }
6712
6713 return 0;
6714 }
6715
6716 void si_fini(struct radeon_device *rdev)
6717 {
6718 si_cp_fini(rdev);
6719 cayman_dma_fini(rdev);
6720 si_fini_pg(rdev);
6721 si_fini_cg(rdev);
6722 si_irq_fini(rdev);
6723 sumo_rlc_fini(rdev);
6724 radeon_wb_fini(rdev);
6725 radeon_vm_manager_fini(rdev);
6726 radeon_ib_pool_fini(rdev);
6727 radeon_irq_kms_fini(rdev);
6728 if (rdev->has_uvd) {
6729 uvd_v1_0_fini(rdev);
6730 radeon_uvd_fini(rdev);
6731 }
6732 si_pcie_gart_fini(rdev);
6733 r600_vram_scratch_fini(rdev);
6734 radeon_gem_fini(rdev);
6735 radeon_fence_driver_fini(rdev);
6736 radeon_bo_fini(rdev);
6737 radeon_atombios_fini(rdev);
6738 kfree(rdev->bios);
6739 rdev->bios = NULL;
6740 }
6741
6742 /**
6743 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6744 *
6745 * @rdev: radeon_device pointer
6746 *
6747 * Fetches a GPU clock counter snapshot (SI).
6748 * Returns the 64 bit clock counter snapshot.
6749 */
6750 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6751 {
6752 uint64_t clock;
6753
6754 mutex_lock(&rdev->gpu_clock_mutex);
6755 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6756 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6757 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6758 mutex_unlock(&rdev->gpu_clock_mutex);
6759 return clock;
6760 }
6761
6762 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6763 {
6764 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6765 int r;
6766
6767 /* bypass vclk and dclk with bclk */
6768 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6769 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6770 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6771
6772 /* put PLL in bypass mode */
6773 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6774
6775 if (!vclk || !dclk) {
6776 /* keep the Bypass mode, put PLL to sleep */
6777 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6778 return 0;
6779 }
6780
6781 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6782 16384, 0x03FFFFFF, 0, 128, 5,
6783 &fb_div, &vclk_div, &dclk_div);
6784 if (r)
6785 return r;
6786
6787 /* set RESET_ANTI_MUX to 0 */
6788 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6789
6790 /* set VCO_MODE to 1 */
6791 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6792
6793 /* toggle UPLL_SLEEP to 1 then back to 0 */
6794 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6795 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6796
6797 /* deassert UPLL_RESET */
6798 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6799
6800 mdelay(1);
6801
6802 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6803 if (r)
6804 return r;
6805
6806 /* assert UPLL_RESET again */
6807 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6808
6809 /* disable spread spectrum. */
6810 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6811
6812 /* set feedback divider */
6813 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6814
6815 /* set ref divider to 0 */
6816 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6817
6818 if (fb_div < 307200)
6819 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6820 else
6821 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6822
6823 /* set PDIV_A and PDIV_B */
6824 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6825 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6826 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6827
6828 /* give the PLL some time to settle */
6829 mdelay(15);
6830
6831 /* deassert PLL_RESET */
6832 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6833
6834 mdelay(15);
6835
6836 /* switch from bypass mode to normal mode */
6837 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6838
6839 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6840 if (r)
6841 return r;
6842
6843 /* switch VCLK and DCLK selection */
6844 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6845 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6846 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6847
6848 mdelay(100);
6849
6850 return 0;
6851 }
6852
6853 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6854 {
6855 struct pci_dev *root = rdev->pdev->bus->self;
6856 int bridge_pos, gpu_pos;
6857 u32 speed_cntl, mask, current_data_rate;
6858 int ret, i;
6859 u16 tmp16;
6860
6861 if (radeon_pcie_gen2 == 0)
6862 return;
6863
6864 if (rdev->flags & RADEON_IS_IGP)
6865 return;
6866
6867 if (!(rdev->flags & RADEON_IS_PCIE))
6868 return;
6869
6870 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6871 if (ret != 0)
6872 return;
6873
6874 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6875 return;
6876
6877 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6878 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6879 LC_CURRENT_DATA_RATE_SHIFT;
6880 if (mask & DRM_PCIE_SPEED_80) {
6881 if (current_data_rate == 2) {
6882 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6883 return;
6884 }
6885 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6886 } else if (mask & DRM_PCIE_SPEED_50) {
6887 if (current_data_rate == 1) {
6888 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6889 return;
6890 }
6891 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6892 }
6893
6894 bridge_pos = pci_pcie_cap(root);
6895 if (!bridge_pos)
6896 return;
6897
6898 gpu_pos = pci_pcie_cap(rdev->pdev);
6899 if (!gpu_pos)
6900 return;
6901
6902 if (mask & DRM_PCIE_SPEED_80) {
6903 /* re-try equalization if gen3 is not already enabled */
6904 if (current_data_rate != 2) {
6905 u16 bridge_cfg, gpu_cfg;
6906 u16 bridge_cfg2, gpu_cfg2;
6907 u32 max_lw, current_lw, tmp;
6908
6909 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6910 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6911
6912 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6913 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6914
6915 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6916 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6917
6918 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6919 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6920 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6921
6922 if (current_lw < max_lw) {
6923 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6924 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6925 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6926 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6927 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6928 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6929 }
6930 }
6931
6932 for (i = 0; i < 10; i++) {
6933 /* check status */
6934 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6935 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6936 break;
6937
6938 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6939 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6940
6941 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6942 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6943
6944 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6945 tmp |= LC_SET_QUIESCE;
6946 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6947
6948 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6949 tmp |= LC_REDO_EQ;
6950 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6951
6952 mdelay(100);
6953
6954 /* linkctl */
6955 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6956 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6957 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6958 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6959
6960 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6961 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6962 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6963 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6964
6965 /* linkctl2 */
6966 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6967 tmp16 &= ~((1 << 4) | (7 << 9));
6968 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6969 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6970
6971 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6972 tmp16 &= ~((1 << 4) | (7 << 9));
6973 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6974 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6975
6976 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6977 tmp &= ~LC_SET_QUIESCE;
6978 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6979 }
6980 }
6981 }
6982
6983 /* set the link speed */
6984 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6985 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6986 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6987
6988 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6989 tmp16 &= ~0xf;
6990 if (mask & DRM_PCIE_SPEED_80)
6991 tmp16 |= 3; /* gen3 */
6992 else if (mask & DRM_PCIE_SPEED_50)
6993 tmp16 |= 2; /* gen2 */
6994 else
6995 tmp16 |= 1; /* gen1 */
6996 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6997
6998 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6999 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7000 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7001
7002 for (i = 0; i < rdev->usec_timeout; i++) {
7003 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7004 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7005 break;
7006 udelay(1);
7007 }
7008 }
7009
7010 static void si_program_aspm(struct radeon_device *rdev)
7011 {
7012 u32 data, orig;
7013 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7014 bool disable_clkreq = false;
7015
7016 if (radeon_aspm == 0)
7017 return;
7018
7019 if (!(rdev->flags & RADEON_IS_PCIE))
7020 return;
7021
7022 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7023 data &= ~LC_XMIT_N_FTS_MASK;
7024 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7025 if (orig != data)
7026 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7027
7028 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7029 data |= LC_GO_TO_RECOVERY;
7030 if (orig != data)
7031 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7032
7033 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7034 data |= P_IGNORE_EDB_ERR;
7035 if (orig != data)
7036 WREG32_PCIE(PCIE_P_CNTL, data);
7037
7038 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7039 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7040 data |= LC_PMI_TO_L1_DIS;
7041 if (!disable_l0s)
7042 data |= LC_L0S_INACTIVITY(7);
7043
7044 if (!disable_l1) {
7045 data |= LC_L1_INACTIVITY(7);
7046 data &= ~LC_PMI_TO_L1_DIS;
7047 if (orig != data)
7048 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7049
7050 if (!disable_plloff_in_l1) {
7051 bool clk_req_support;
7052
7053 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7054 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7055 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7056 if (orig != data)
7057 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7058
7059 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7060 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7061 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7062 if (orig != data)
7063 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7064
7065 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7066 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7067 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7068 if (orig != data)
7069 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7070
7071 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7072 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7073 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7074 if (orig != data)
7075 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7076
7077 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7078 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7079 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7080 if (orig != data)
7081 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7082
7083 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7084 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7085 if (orig != data)
7086 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7087
7088 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7089 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7090 if (orig != data)
7091 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7092
7093 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7094 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7095 if (orig != data)
7096 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7097
7098 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7099 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7100 if (orig != data)
7101 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7102
7103 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7104 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7105 if (orig != data)
7106 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7107
7108 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7109 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7110 if (orig != data)
7111 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7112
7113 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7114 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7115 if (orig != data)
7116 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7117 }
7118 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7119 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7120 data |= LC_DYN_LANES_PWR_STATE(3);
7121 if (orig != data)
7122 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7123
7124 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7125 data &= ~LS2_EXIT_TIME_MASK;
7126 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7127 data |= LS2_EXIT_TIME(5);
7128 if (orig != data)
7129 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7130
7131 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7132 data &= ~LS2_EXIT_TIME_MASK;
7133 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7134 data |= LS2_EXIT_TIME(5);
7135 if (orig != data)
7136 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7137
7138 if (!disable_clkreq) {
7139 struct pci_dev *root = rdev->pdev->bus->self;
7140 u32 lnkcap;
7141
7142 clk_req_support = false;
7143 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7144 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7145 clk_req_support = true;
7146 } else {
7147 clk_req_support = false;
7148 }
7149
7150 if (clk_req_support) {
7151 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7152 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7153 if (orig != data)
7154 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7155
7156 orig = data = RREG32(THM_CLK_CNTL);
7157 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7158 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7159 if (orig != data)
7160 WREG32(THM_CLK_CNTL, data);
7161
7162 orig = data = RREG32(MISC_CLK_CNTL);
7163 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7164 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7165 if (orig != data)
7166 WREG32(MISC_CLK_CNTL, data);
7167
7168 orig = data = RREG32(CG_CLKPIN_CNTL);
7169 data &= ~BCLK_AS_XCLK;
7170 if (orig != data)
7171 WREG32(CG_CLKPIN_CNTL, data);
7172
7173 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7174 data &= ~FORCE_BIF_REFCLK_EN;
7175 if (orig != data)
7176 WREG32(CG_CLKPIN_CNTL_2, data);
7177
7178 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7179 data &= ~MPLL_CLKOUT_SEL_MASK;
7180 data |= MPLL_CLKOUT_SEL(4);
7181 if (orig != data)
7182 WREG32(MPLL_BYPASSCLK_SEL, data);
7183
7184 orig = data = RREG32(SPLL_CNTL_MODE);
7185 data &= ~SPLL_REFCLK_SEL_MASK;
7186 if (orig != data)
7187 WREG32(SPLL_CNTL_MODE, data);
7188 }
7189 }
7190 } else {
7191 if (orig != data)
7192 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7193 }
7194
7195 orig = data = RREG32_PCIE(PCIE_CNTL2);
7196 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7197 if (orig != data)
7198 WREG32_PCIE(PCIE_CNTL2, data);
7199
7200 if (!disable_l0s) {
7201 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7202 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7203 data = RREG32_PCIE(PCIE_LC_STATUS1);
7204 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7205 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7206 data &= ~LC_L0S_INACTIVITY_MASK;
7207 if (orig != data)
7208 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7209 }
7210 }
7211 }
7212 }
Linux kernel - Deliverables
This page took 0.285578 seconds and 5 git commands to generate.