2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
30 #include "radeon_asic.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
36 #define SI_PFP_UCODE_SIZE 2144
37 #define SI_PM4_UCODE_SIZE 2144
38 #define SI_CE_UCODE_SIZE 2144
39 #define SI_RLC_UCODE_SIZE 2048
40 #define SI_MC_UCODE_SIZE 7769
41 #define OLAND_MC_UCODE_SIZE 7863
43 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54 MODULE_FIRMWARE("radeon/VERDE_me.bin");
55 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
69 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
70 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
71 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device
*rdev
);
72 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
73 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
74 extern u32
evergreen_get_number_of_dram_channels(struct radeon_device
*rdev
);
75 extern void evergreen_print_gpu_status_regs(struct radeon_device
*rdev
);
76 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
78 static const u32 tahiti_golden_rlc_registers
[] =
80 0xc424, 0xffffffff, 0x00601005,
81 0xc47c, 0xffffffff, 0x10104040,
82 0xc488, 0xffffffff, 0x0100000a,
83 0xc314, 0xffffffff, 0x00000800,
84 0xc30c, 0xffffffff, 0x800000f4,
85 0xf4a8, 0xffffffff, 0x00000000
88 static const u32 tahiti_golden_registers
[] =
90 0x9a10, 0x00010000, 0x00018208,
91 0x9830, 0xffffffff, 0x00000000,
92 0x9834, 0xf00fffff, 0x00000400,
93 0x9838, 0x0002021c, 0x00020200,
94 0xc78, 0x00000080, 0x00000000,
95 0xd030, 0x000300c0, 0x00800040,
96 0xd830, 0x000300c0, 0x00800040,
97 0x5bb0, 0x000000f0, 0x00000070,
98 0x5bc0, 0x00200000, 0x50100000,
99 0x7030, 0x31000311, 0x00000011,
100 0x277c, 0x00000003, 0x000007ff,
101 0x240c, 0x000007ff, 0x00000000,
102 0x8a14, 0xf000001f, 0x00000007,
103 0x8b24, 0xffffffff, 0x00ffffff,
104 0x8b10, 0x0000ff0f, 0x00000000,
105 0x28a4c, 0x07ffffff, 0x4e000000,
106 0x28350, 0x3f3f3fff, 0x2a00126a,
107 0x30, 0x000000ff, 0x0040,
108 0x34, 0x00000040, 0x00004040,
109 0x9100, 0x07ffffff, 0x03000000,
110 0x8e88, 0x01ff1f3f, 0x00000000,
111 0x8e84, 0x01ff1f3f, 0x00000000,
112 0x9060, 0x0000007f, 0x00000020,
113 0x9508, 0x00010000, 0x00010000,
114 0xac14, 0x00000200, 0x000002fb,
115 0xac10, 0xffffffff, 0x0000543b,
116 0xac0c, 0xffffffff, 0xa9210876,
117 0x88d0, 0xffffffff, 0x000fff40,
118 0x88d4, 0x0000001f, 0x00000010,
119 0x1410, 0x20000000, 0x20fffed8,
120 0x15c0, 0x000c0fc0, 0x000c0400
123 static const u32 tahiti_golden_registers2
[] =
125 0xc64, 0x00000001, 0x00000001
128 static const u32 pitcairn_golden_rlc_registers
[] =
130 0xc424, 0xffffffff, 0x00601004,
131 0xc47c, 0xffffffff, 0x10102020,
132 0xc488, 0xffffffff, 0x01000020,
133 0xc314, 0xffffffff, 0x00000800,
134 0xc30c, 0xffffffff, 0x800000a4
137 static const u32 pitcairn_golden_registers
[] =
139 0x9a10, 0x00010000, 0x00018208,
140 0x9830, 0xffffffff, 0x00000000,
141 0x9834, 0xf00fffff, 0x00000400,
142 0x9838, 0x0002021c, 0x00020200,
143 0xc78, 0x00000080, 0x00000000,
144 0xd030, 0x000300c0, 0x00800040,
145 0xd830, 0x000300c0, 0x00800040,
146 0x5bb0, 0x000000f0, 0x00000070,
147 0x5bc0, 0x00200000, 0x50100000,
148 0x7030, 0x31000311, 0x00000011,
149 0x2ae4, 0x00073ffe, 0x000022a2,
150 0x240c, 0x000007ff, 0x00000000,
151 0x8a14, 0xf000001f, 0x00000007,
152 0x8b24, 0xffffffff, 0x00ffffff,
153 0x8b10, 0x0000ff0f, 0x00000000,
154 0x28a4c, 0x07ffffff, 0x4e000000,
155 0x28350, 0x3f3f3fff, 0x2a00126a,
156 0x30, 0x000000ff, 0x0040,
157 0x34, 0x00000040, 0x00004040,
158 0x9100, 0x07ffffff, 0x03000000,
159 0x9060, 0x0000007f, 0x00000020,
160 0x9508, 0x00010000, 0x00010000,
161 0xac14, 0x000003ff, 0x000000f7,
162 0xac10, 0xffffffff, 0x00000000,
163 0xac0c, 0xffffffff, 0x32761054,
164 0x88d4, 0x0000001f, 0x00000010,
165 0x15c0, 0x000c0fc0, 0x000c0400
168 static const u32 verde_golden_rlc_registers
[] =
170 0xc424, 0xffffffff, 0x033f1005,
171 0xc47c, 0xffffffff, 0x10808020,
172 0xc488, 0xffffffff, 0x00800008,
173 0xc314, 0xffffffff, 0x00001000,
174 0xc30c, 0xffffffff, 0x80010014
177 static const u32 verde_golden_registers
[] =
179 0x9a10, 0x00010000, 0x00018208,
180 0x9830, 0xffffffff, 0x00000000,
181 0x9834, 0xf00fffff, 0x00000400,
182 0x9838, 0x0002021c, 0x00020200,
183 0xc78, 0x00000080, 0x00000000,
184 0xd030, 0x000300c0, 0x00800040,
185 0xd030, 0x000300c0, 0x00800040,
186 0xd830, 0x000300c0, 0x00800040,
187 0xd830, 0x000300c0, 0x00800040,
188 0x5bb0, 0x000000f0, 0x00000070,
189 0x5bc0, 0x00200000, 0x50100000,
190 0x7030, 0x31000311, 0x00000011,
191 0x2ae4, 0x00073ffe, 0x000022a2,
192 0x2ae4, 0x00073ffe, 0x000022a2,
193 0x2ae4, 0x00073ffe, 0x000022a2,
194 0x240c, 0x000007ff, 0x00000000,
195 0x240c, 0x000007ff, 0x00000000,
196 0x240c, 0x000007ff, 0x00000000,
197 0x8a14, 0xf000001f, 0x00000007,
198 0x8a14, 0xf000001f, 0x00000007,
199 0x8a14, 0xf000001f, 0x00000007,
200 0x8b24, 0xffffffff, 0x00ffffff,
201 0x8b10, 0x0000ff0f, 0x00000000,
202 0x28a4c, 0x07ffffff, 0x4e000000,
203 0x28350, 0x3f3f3fff, 0x0000124a,
204 0x28350, 0x3f3f3fff, 0x0000124a,
205 0x28350, 0x3f3f3fff, 0x0000124a,
206 0x30, 0x000000ff, 0x0040,
207 0x34, 0x00000040, 0x00004040,
208 0x9100, 0x07ffffff, 0x03000000,
209 0x9100, 0x07ffffff, 0x03000000,
210 0x8e88, 0x01ff1f3f, 0x00000000,
211 0x8e88, 0x01ff1f3f, 0x00000000,
212 0x8e88, 0x01ff1f3f, 0x00000000,
213 0x8e84, 0x01ff1f3f, 0x00000000,
214 0x8e84, 0x01ff1f3f, 0x00000000,
215 0x8e84, 0x01ff1f3f, 0x00000000,
216 0x9060, 0x0000007f, 0x00000020,
217 0x9508, 0x00010000, 0x00010000,
218 0xac14, 0x000003ff, 0x00000003,
219 0xac14, 0x000003ff, 0x00000003,
220 0xac14, 0x000003ff, 0x00000003,
221 0xac10, 0xffffffff, 0x00000000,
222 0xac10, 0xffffffff, 0x00000000,
223 0xac10, 0xffffffff, 0x00000000,
224 0xac0c, 0xffffffff, 0x00001032,
225 0xac0c, 0xffffffff, 0x00001032,
226 0xac0c, 0xffffffff, 0x00001032,
227 0x88d4, 0x0000001f, 0x00000010,
228 0x88d4, 0x0000001f, 0x00000010,
229 0x88d4, 0x0000001f, 0x00000010,
230 0x15c0, 0x000c0fc0, 0x000c0400
233 static const u32 oland_golden_rlc_registers
[] =
235 0xc424, 0xffffffff, 0x00601005,
236 0xc47c, 0xffffffff, 0x10104040,
237 0xc488, 0xffffffff, 0x0100000a,
238 0xc314, 0xffffffff, 0x00000800,
239 0xc30c, 0xffffffff, 0x800000f4
242 static const u32 oland_golden_registers
[] =
244 0x9a10, 0x00010000, 0x00018208,
245 0x9830, 0xffffffff, 0x00000000,
246 0x9834, 0xf00fffff, 0x00000400,
247 0x9838, 0x0002021c, 0x00020200,
248 0xc78, 0x00000080, 0x00000000,
249 0xd030, 0x000300c0, 0x00800040,
250 0xd830, 0x000300c0, 0x00800040,
251 0x5bb0, 0x000000f0, 0x00000070,
252 0x5bc0, 0x00200000, 0x50100000,
253 0x7030, 0x31000311, 0x00000011,
254 0x2ae4, 0x00073ffe, 0x000022a2,
255 0x240c, 0x000007ff, 0x00000000,
256 0x8a14, 0xf000001f, 0x00000007,
257 0x8b24, 0xffffffff, 0x00ffffff,
258 0x8b10, 0x0000ff0f, 0x00000000,
259 0x28a4c, 0x07ffffff, 0x4e000000,
260 0x28350, 0x3f3f3fff, 0x00000082,
261 0x30, 0x000000ff, 0x0040,
262 0x34, 0x00000040, 0x00004040,
263 0x9100, 0x07ffffff, 0x03000000,
264 0x9060, 0x0000007f, 0x00000020,
265 0x9508, 0x00010000, 0x00010000,
266 0xac14, 0x000003ff, 0x000000f3,
267 0xac10, 0xffffffff, 0x00000000,
268 0xac0c, 0xffffffff, 0x00003210,
269 0x88d4, 0x0000001f, 0x00000010,
270 0x15c0, 0x000c0fc0, 0x000c0400
273 static const u32 hainan_golden_registers
[] =
275 0x9a10, 0x00010000, 0x00018208,
276 0x9830, 0xffffffff, 0x00000000,
277 0x9834, 0xf00fffff, 0x00000400,
278 0x9838, 0x0002021c, 0x00020200,
279 0xd0c0, 0xff000fff, 0x00000100,
280 0xd030, 0x000300c0, 0x00800040,
281 0xd8c0, 0xff000fff, 0x00000100,
282 0xd830, 0x000300c0, 0x00800040,
283 0x2ae4, 0x00073ffe, 0x000022a2,
284 0x240c, 0x000007ff, 0x00000000,
285 0x8a14, 0xf000001f, 0x00000007,
286 0x8b24, 0xffffffff, 0x00ffffff,
287 0x8b10, 0x0000ff0f, 0x00000000,
288 0x28a4c, 0x07ffffff, 0x4e000000,
289 0x28350, 0x3f3f3fff, 0x00000000,
290 0x30, 0x000000ff, 0x0040,
291 0x34, 0x00000040, 0x00004040,
292 0x9100, 0x03e00000, 0x03600000,
293 0x9060, 0x0000007f, 0x00000020,
294 0x9508, 0x00010000, 0x00010000,
295 0xac14, 0x000003ff, 0x000000f1,
296 0xac10, 0xffffffff, 0x00000000,
297 0xac0c, 0xffffffff, 0x00003210,
298 0x88d4, 0x0000001f, 0x00000010,
299 0x15c0, 0x000c0fc0, 0x000c0400
302 static const u32 hainan_golden_registers2
[] =
304 0x98f8, 0xffffffff, 0x02010001
307 static const u32 tahiti_mgcg_cgcg_init
[] =
309 0xc400, 0xffffffff, 0xfffffffc,
310 0x802c, 0xffffffff, 0xe0000000,
311 0x9a60, 0xffffffff, 0x00000100,
312 0x92a4, 0xffffffff, 0x00000100,
313 0xc164, 0xffffffff, 0x00000100,
314 0x9774, 0xffffffff, 0x00000100,
315 0x8984, 0xffffffff, 0x06000100,
316 0x8a18, 0xffffffff, 0x00000100,
317 0x92a0, 0xffffffff, 0x00000100,
318 0xc380, 0xffffffff, 0x00000100,
319 0x8b28, 0xffffffff, 0x00000100,
320 0x9144, 0xffffffff, 0x00000100,
321 0x8d88, 0xffffffff, 0x00000100,
322 0x8d8c, 0xffffffff, 0x00000100,
323 0x9030, 0xffffffff, 0x00000100,
324 0x9034, 0xffffffff, 0x00000100,
325 0x9038, 0xffffffff, 0x00000100,
326 0x903c, 0xffffffff, 0x00000100,
327 0xad80, 0xffffffff, 0x00000100,
328 0xac54, 0xffffffff, 0x00000100,
329 0x897c, 0xffffffff, 0x06000100,
330 0x9868, 0xffffffff, 0x00000100,
331 0x9510, 0xffffffff, 0x00000100,
332 0xaf04, 0xffffffff, 0x00000100,
333 0xae04, 0xffffffff, 0x00000100,
334 0x949c, 0xffffffff, 0x00000100,
335 0x802c, 0xffffffff, 0xe0000000,
336 0x9160, 0xffffffff, 0x00010000,
337 0x9164, 0xffffffff, 0x00030002,
338 0x9168, 0xffffffff, 0x00040007,
339 0x916c, 0xffffffff, 0x00060005,
340 0x9170, 0xffffffff, 0x00090008,
341 0x9174, 0xffffffff, 0x00020001,
342 0x9178, 0xffffffff, 0x00040003,
343 0x917c, 0xffffffff, 0x00000007,
344 0x9180, 0xffffffff, 0x00060005,
345 0x9184, 0xffffffff, 0x00090008,
346 0x9188, 0xffffffff, 0x00030002,
347 0x918c, 0xffffffff, 0x00050004,
348 0x9190, 0xffffffff, 0x00000008,
349 0x9194, 0xffffffff, 0x00070006,
350 0x9198, 0xffffffff, 0x000a0009,
351 0x919c, 0xffffffff, 0x00040003,
352 0x91a0, 0xffffffff, 0x00060005,
353 0x91a4, 0xffffffff, 0x00000009,
354 0x91a8, 0xffffffff, 0x00080007,
355 0x91ac, 0xffffffff, 0x000b000a,
356 0x91b0, 0xffffffff, 0x00050004,
357 0x91b4, 0xffffffff, 0x00070006,
358 0x91b8, 0xffffffff, 0x0008000b,
359 0x91bc, 0xffffffff, 0x000a0009,
360 0x91c0, 0xffffffff, 0x000d000c,
361 0x91c4, 0xffffffff, 0x00060005,
362 0x91c8, 0xffffffff, 0x00080007,
363 0x91cc, 0xffffffff, 0x0000000b,
364 0x91d0, 0xffffffff, 0x000a0009,
365 0x91d4, 0xffffffff, 0x000d000c,
366 0x91d8, 0xffffffff, 0x00070006,
367 0x91dc, 0xffffffff, 0x00090008,
368 0x91e0, 0xffffffff, 0x0000000c,
369 0x91e4, 0xffffffff, 0x000b000a,
370 0x91e8, 0xffffffff, 0x000e000d,
371 0x91ec, 0xffffffff, 0x00080007,
372 0x91f0, 0xffffffff, 0x000a0009,
373 0x91f4, 0xffffffff, 0x0000000d,
374 0x91f8, 0xffffffff, 0x000c000b,
375 0x91fc, 0xffffffff, 0x000f000e,
376 0x9200, 0xffffffff, 0x00090008,
377 0x9204, 0xffffffff, 0x000b000a,
378 0x9208, 0xffffffff, 0x000c000f,
379 0x920c, 0xffffffff, 0x000e000d,
380 0x9210, 0xffffffff, 0x00110010,
381 0x9214, 0xffffffff, 0x000a0009,
382 0x9218, 0xffffffff, 0x000c000b,
383 0x921c, 0xffffffff, 0x0000000f,
384 0x9220, 0xffffffff, 0x000e000d,
385 0x9224, 0xffffffff, 0x00110010,
386 0x9228, 0xffffffff, 0x000b000a,
387 0x922c, 0xffffffff, 0x000d000c,
388 0x9230, 0xffffffff, 0x00000010,
389 0x9234, 0xffffffff, 0x000f000e,
390 0x9238, 0xffffffff, 0x00120011,
391 0x923c, 0xffffffff, 0x000c000b,
392 0x9240, 0xffffffff, 0x000e000d,
393 0x9244, 0xffffffff, 0x00000011,
394 0x9248, 0xffffffff, 0x0010000f,
395 0x924c, 0xffffffff, 0x00130012,
396 0x9250, 0xffffffff, 0x000d000c,
397 0x9254, 0xffffffff, 0x000f000e,
398 0x9258, 0xffffffff, 0x00100013,
399 0x925c, 0xffffffff, 0x00120011,
400 0x9260, 0xffffffff, 0x00150014,
401 0x9264, 0xffffffff, 0x000e000d,
402 0x9268, 0xffffffff, 0x0010000f,
403 0x926c, 0xffffffff, 0x00000013,
404 0x9270, 0xffffffff, 0x00120011,
405 0x9274, 0xffffffff, 0x00150014,
406 0x9278, 0xffffffff, 0x000f000e,
407 0x927c, 0xffffffff, 0x00110010,
408 0x9280, 0xffffffff, 0x00000014,
409 0x9284, 0xffffffff, 0x00130012,
410 0x9288, 0xffffffff, 0x00160015,
411 0x928c, 0xffffffff, 0x0010000f,
412 0x9290, 0xffffffff, 0x00120011,
413 0x9294, 0xffffffff, 0x00000015,
414 0x9298, 0xffffffff, 0x00140013,
415 0x929c, 0xffffffff, 0x00170016,
416 0x9150, 0xffffffff, 0x96940200,
417 0x8708, 0xffffffff, 0x00900100,
418 0xc478, 0xffffffff, 0x00000080,
419 0xc404, 0xffffffff, 0x0020003f,
420 0x30, 0xffffffff, 0x0000001c,
421 0x34, 0x000f0000, 0x000f0000,
422 0x160c, 0xffffffff, 0x00000100,
423 0x1024, 0xffffffff, 0x00000100,
424 0x102c, 0x00000101, 0x00000000,
425 0x20a8, 0xffffffff, 0x00000104,
426 0x264c, 0x000c0000, 0x000c0000,
427 0x2648, 0x000c0000, 0x000c0000,
428 0x55e4, 0xff000fff, 0x00000100,
429 0x55e8, 0x00000001, 0x00000001,
430 0x2f50, 0x00000001, 0x00000001,
431 0x30cc, 0xc0000fff, 0x00000104,
432 0xc1e4, 0x00000001, 0x00000001,
433 0xd0c0, 0xfffffff0, 0x00000100,
434 0xd8c0, 0xfffffff0, 0x00000100
437 static const u32 pitcairn_mgcg_cgcg_init
[] =
439 0xc400, 0xffffffff, 0xfffffffc,
440 0x802c, 0xffffffff, 0xe0000000,
441 0x9a60, 0xffffffff, 0x00000100,
442 0x92a4, 0xffffffff, 0x00000100,
443 0xc164, 0xffffffff, 0x00000100,
444 0x9774, 0xffffffff, 0x00000100,
445 0x8984, 0xffffffff, 0x06000100,
446 0x8a18, 0xffffffff, 0x00000100,
447 0x92a0, 0xffffffff, 0x00000100,
448 0xc380, 0xffffffff, 0x00000100,
449 0x8b28, 0xffffffff, 0x00000100,
450 0x9144, 0xffffffff, 0x00000100,
451 0x8d88, 0xffffffff, 0x00000100,
452 0x8d8c, 0xffffffff, 0x00000100,
453 0x9030, 0xffffffff, 0x00000100,
454 0x9034, 0xffffffff, 0x00000100,
455 0x9038, 0xffffffff, 0x00000100,
456 0x903c, 0xffffffff, 0x00000100,
457 0xad80, 0xffffffff, 0x00000100,
458 0xac54, 0xffffffff, 0x00000100,
459 0x897c, 0xffffffff, 0x06000100,
460 0x9868, 0xffffffff, 0x00000100,
461 0x9510, 0xffffffff, 0x00000100,
462 0xaf04, 0xffffffff, 0x00000100,
463 0xae04, 0xffffffff, 0x00000100,
464 0x949c, 0xffffffff, 0x00000100,
465 0x802c, 0xffffffff, 0xe0000000,
466 0x9160, 0xffffffff, 0x00010000,
467 0x9164, 0xffffffff, 0x00030002,
468 0x9168, 0xffffffff, 0x00040007,
469 0x916c, 0xffffffff, 0x00060005,
470 0x9170, 0xffffffff, 0x00090008,
471 0x9174, 0xffffffff, 0x00020001,
472 0x9178, 0xffffffff, 0x00040003,
473 0x917c, 0xffffffff, 0x00000007,
474 0x9180, 0xffffffff, 0x00060005,
475 0x9184, 0xffffffff, 0x00090008,
476 0x9188, 0xffffffff, 0x00030002,
477 0x918c, 0xffffffff, 0x00050004,
478 0x9190, 0xffffffff, 0x00000008,
479 0x9194, 0xffffffff, 0x00070006,
480 0x9198, 0xffffffff, 0x000a0009,
481 0x919c, 0xffffffff, 0x00040003,
482 0x91a0, 0xffffffff, 0x00060005,
483 0x91a4, 0xffffffff, 0x00000009,
484 0x91a8, 0xffffffff, 0x00080007,
485 0x91ac, 0xffffffff, 0x000b000a,
486 0x91b0, 0xffffffff, 0x00050004,
487 0x91b4, 0xffffffff, 0x00070006,
488 0x91b8, 0xffffffff, 0x0008000b,
489 0x91bc, 0xffffffff, 0x000a0009,
490 0x91c0, 0xffffffff, 0x000d000c,
491 0x9200, 0xffffffff, 0x00090008,
492 0x9204, 0xffffffff, 0x000b000a,
493 0x9208, 0xffffffff, 0x000c000f,
494 0x920c, 0xffffffff, 0x000e000d,
495 0x9210, 0xffffffff, 0x00110010,
496 0x9214, 0xffffffff, 0x000a0009,
497 0x9218, 0xffffffff, 0x000c000b,
498 0x921c, 0xffffffff, 0x0000000f,
499 0x9220, 0xffffffff, 0x000e000d,
500 0x9224, 0xffffffff, 0x00110010,
501 0x9228, 0xffffffff, 0x000b000a,
502 0x922c, 0xffffffff, 0x000d000c,
503 0x9230, 0xffffffff, 0x00000010,
504 0x9234, 0xffffffff, 0x000f000e,
505 0x9238, 0xffffffff, 0x00120011,
506 0x923c, 0xffffffff, 0x000c000b,
507 0x9240, 0xffffffff, 0x000e000d,
508 0x9244, 0xffffffff, 0x00000011,
509 0x9248, 0xffffffff, 0x0010000f,
510 0x924c, 0xffffffff, 0x00130012,
511 0x9250, 0xffffffff, 0x000d000c,
512 0x9254, 0xffffffff, 0x000f000e,
513 0x9258, 0xffffffff, 0x00100013,
514 0x925c, 0xffffffff, 0x00120011,
515 0x9260, 0xffffffff, 0x00150014,
516 0x9150, 0xffffffff, 0x96940200,
517 0x8708, 0xffffffff, 0x00900100,
518 0xc478, 0xffffffff, 0x00000080,
519 0xc404, 0xffffffff, 0x0020003f,
520 0x30, 0xffffffff, 0x0000001c,
521 0x34, 0x000f0000, 0x000f0000,
522 0x160c, 0xffffffff, 0x00000100,
523 0x1024, 0xffffffff, 0x00000100,
524 0x102c, 0x00000101, 0x00000000,
525 0x20a8, 0xffffffff, 0x00000104,
526 0x55e4, 0xff000fff, 0x00000100,
527 0x55e8, 0x00000001, 0x00000001,
528 0x2f50, 0x00000001, 0x00000001,
529 0x30cc, 0xc0000fff, 0x00000104,
530 0xc1e4, 0x00000001, 0x00000001,
531 0xd0c0, 0xfffffff0, 0x00000100,
532 0xd8c0, 0xfffffff0, 0x00000100
535 static const u32 verde_mgcg_cgcg_init
[] =
537 0xc400, 0xffffffff, 0xfffffffc,
538 0x802c, 0xffffffff, 0xe0000000,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x92a4, 0xffffffff, 0x00000100,
541 0xc164, 0xffffffff, 0x00000100,
542 0x9774, 0xffffffff, 0x00000100,
543 0x8984, 0xffffffff, 0x06000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x92a0, 0xffffffff, 0x00000100,
546 0xc380, 0xffffffff, 0x00000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x8d88, 0xffffffff, 0x00000100,
550 0x8d8c, 0xffffffff, 0x00000100,
551 0x9030, 0xffffffff, 0x00000100,
552 0x9034, 0xffffffff, 0x00000100,
553 0x9038, 0xffffffff, 0x00000100,
554 0x903c, 0xffffffff, 0x00000100,
555 0xad80, 0xffffffff, 0x00000100,
556 0xac54, 0xffffffff, 0x00000100,
557 0x897c, 0xffffffff, 0x06000100,
558 0x9868, 0xffffffff, 0x00000100,
559 0x9510, 0xffffffff, 0x00000100,
560 0xaf04, 0xffffffff, 0x00000100,
561 0xae04, 0xffffffff, 0x00000100,
562 0x949c, 0xffffffff, 0x00000100,
563 0x802c, 0xffffffff, 0xe0000000,
564 0x9160, 0xffffffff, 0x00010000,
565 0x9164, 0xffffffff, 0x00030002,
566 0x9168, 0xffffffff, 0x00040007,
567 0x916c, 0xffffffff, 0x00060005,
568 0x9170, 0xffffffff, 0x00090008,
569 0x9174, 0xffffffff, 0x00020001,
570 0x9178, 0xffffffff, 0x00040003,
571 0x917c, 0xffffffff, 0x00000007,
572 0x9180, 0xffffffff, 0x00060005,
573 0x9184, 0xffffffff, 0x00090008,
574 0x9188, 0xffffffff, 0x00030002,
575 0x918c, 0xffffffff, 0x00050004,
576 0x9190, 0xffffffff, 0x00000008,
577 0x9194, 0xffffffff, 0x00070006,
578 0x9198, 0xffffffff, 0x000a0009,
579 0x919c, 0xffffffff, 0x00040003,
580 0x91a0, 0xffffffff, 0x00060005,
581 0x91a4, 0xffffffff, 0x00000009,
582 0x91a8, 0xffffffff, 0x00080007,
583 0x91ac, 0xffffffff, 0x000b000a,
584 0x91b0, 0xffffffff, 0x00050004,
585 0x91b4, 0xffffffff, 0x00070006,
586 0x91b8, 0xffffffff, 0x0008000b,
587 0x91bc, 0xffffffff, 0x000a0009,
588 0x91c0, 0xffffffff, 0x000d000c,
589 0x9200, 0xffffffff, 0x00090008,
590 0x9204, 0xffffffff, 0x000b000a,
591 0x9208, 0xffffffff, 0x000c000f,
592 0x920c, 0xffffffff, 0x000e000d,
593 0x9210, 0xffffffff, 0x00110010,
594 0x9214, 0xffffffff, 0x000a0009,
595 0x9218, 0xffffffff, 0x000c000b,
596 0x921c, 0xffffffff, 0x0000000f,
597 0x9220, 0xffffffff, 0x000e000d,
598 0x9224, 0xffffffff, 0x00110010,
599 0x9228, 0xffffffff, 0x000b000a,
600 0x922c, 0xffffffff, 0x000d000c,
601 0x9230, 0xffffffff, 0x00000010,
602 0x9234, 0xffffffff, 0x000f000e,
603 0x9238, 0xffffffff, 0x00120011,
604 0x923c, 0xffffffff, 0x000c000b,
605 0x9240, 0xffffffff, 0x000e000d,
606 0x9244, 0xffffffff, 0x00000011,
607 0x9248, 0xffffffff, 0x0010000f,
608 0x924c, 0xffffffff, 0x00130012,
609 0x9250, 0xffffffff, 0x000d000c,
610 0x9254, 0xffffffff, 0x000f000e,
611 0x9258, 0xffffffff, 0x00100013,
612 0x925c, 0xffffffff, 0x00120011,
613 0x9260, 0xffffffff, 0x00150014,
614 0x9150, 0xffffffff, 0x96940200,
615 0x8708, 0xffffffff, 0x00900100,
616 0xc478, 0xffffffff, 0x00000080,
617 0xc404, 0xffffffff, 0x0020003f,
618 0x30, 0xffffffff, 0x0000001c,
619 0x34, 0x000f0000, 0x000f0000,
620 0x160c, 0xffffffff, 0x00000100,
621 0x1024, 0xffffffff, 0x00000100,
622 0x102c, 0x00000101, 0x00000000,
623 0x20a8, 0xffffffff, 0x00000104,
624 0x264c, 0x000c0000, 0x000c0000,
625 0x2648, 0x000c0000, 0x000c0000,
626 0x55e4, 0xff000fff, 0x00000100,
627 0x55e8, 0x00000001, 0x00000001,
628 0x2f50, 0x00000001, 0x00000001,
629 0x30cc, 0xc0000fff, 0x00000104,
630 0xc1e4, 0x00000001, 0x00000001,
631 0xd0c0, 0xfffffff0, 0x00000100,
632 0xd8c0, 0xfffffff0, 0x00000100
635 static const u32 oland_mgcg_cgcg_init
[] =
637 0xc400, 0xffffffff, 0xfffffffc,
638 0x802c, 0xffffffff, 0xe0000000,
639 0x9a60, 0xffffffff, 0x00000100,
640 0x92a4, 0xffffffff, 0x00000100,
641 0xc164, 0xffffffff, 0x00000100,
642 0x9774, 0xffffffff, 0x00000100,
643 0x8984, 0xffffffff, 0x06000100,
644 0x8a18, 0xffffffff, 0x00000100,
645 0x92a0, 0xffffffff, 0x00000100,
646 0xc380, 0xffffffff, 0x00000100,
647 0x8b28, 0xffffffff, 0x00000100,
648 0x9144, 0xffffffff, 0x00000100,
649 0x8d88, 0xffffffff, 0x00000100,
650 0x8d8c, 0xffffffff, 0x00000100,
651 0x9030, 0xffffffff, 0x00000100,
652 0x9034, 0xffffffff, 0x00000100,
653 0x9038, 0xffffffff, 0x00000100,
654 0x903c, 0xffffffff, 0x00000100,
655 0xad80, 0xffffffff, 0x00000100,
656 0xac54, 0xffffffff, 0x00000100,
657 0x897c, 0xffffffff, 0x06000100,
658 0x9868, 0xffffffff, 0x00000100,
659 0x9510, 0xffffffff, 0x00000100,
660 0xaf04, 0xffffffff, 0x00000100,
661 0xae04, 0xffffffff, 0x00000100,
662 0x949c, 0xffffffff, 0x00000100,
663 0x802c, 0xffffffff, 0xe0000000,
664 0x9160, 0xffffffff, 0x00010000,
665 0x9164, 0xffffffff, 0x00030002,
666 0x9168, 0xffffffff, 0x00040007,
667 0x916c, 0xffffffff, 0x00060005,
668 0x9170, 0xffffffff, 0x00090008,
669 0x9174, 0xffffffff, 0x00020001,
670 0x9178, 0xffffffff, 0x00040003,
671 0x917c, 0xffffffff, 0x00000007,
672 0x9180, 0xffffffff, 0x00060005,
673 0x9184, 0xffffffff, 0x00090008,
674 0x9188, 0xffffffff, 0x00030002,
675 0x918c, 0xffffffff, 0x00050004,
676 0x9190, 0xffffffff, 0x00000008,
677 0x9194, 0xffffffff, 0x00070006,
678 0x9198, 0xffffffff, 0x000a0009,
679 0x919c, 0xffffffff, 0x00040003,
680 0x91a0, 0xffffffff, 0x00060005,
681 0x91a4, 0xffffffff, 0x00000009,
682 0x91a8, 0xffffffff, 0x00080007,
683 0x91ac, 0xffffffff, 0x000b000a,
684 0x91b0, 0xffffffff, 0x00050004,
685 0x91b4, 0xffffffff, 0x00070006,
686 0x91b8, 0xffffffff, 0x0008000b,
687 0x91bc, 0xffffffff, 0x000a0009,
688 0x91c0, 0xffffffff, 0x000d000c,
689 0x91c4, 0xffffffff, 0x00060005,
690 0x91c8, 0xffffffff, 0x00080007,
691 0x91cc, 0xffffffff, 0x0000000b,
692 0x91d0, 0xffffffff, 0x000a0009,
693 0x91d4, 0xffffffff, 0x000d000c,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
715 static const u32 hainan_mgcg_cgcg_init
[] =
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x91c4, 0xffffffff, 0x00060005,
770 0x91c8, 0xffffffff, 0x00080007,
771 0x91cc, 0xffffffff, 0x0000000b,
772 0x91d0, 0xffffffff, 0x000a0009,
773 0x91d4, 0xffffffff, 0x000d000c,
774 0x9150, 0xffffffff, 0x96940200,
775 0x8708, 0xffffffff, 0x00900100,
776 0xc478, 0xffffffff, 0x00000080,
777 0xc404, 0xffffffff, 0x0020003f,
778 0x30, 0xffffffff, 0x0000001c,
779 0x34, 0x000f0000, 0x000f0000,
780 0x160c, 0xffffffff, 0x00000100,
781 0x1024, 0xffffffff, 0x00000100,
782 0x20a8, 0xffffffff, 0x00000104,
783 0x264c, 0x000c0000, 0x000c0000,
784 0x2648, 0x000c0000, 0x000c0000,
785 0x2f50, 0x00000001, 0x00000001,
786 0x30cc, 0xc0000fff, 0x00000104,
787 0xc1e4, 0x00000001, 0x00000001,
788 0xd0c0, 0xfffffff0, 0x00000100,
789 0xd8c0, 0xfffffff0, 0x00000100
792 static u32 verde_pg_init
[] =
794 0x353c, 0xffffffff, 0x40000,
795 0x3538, 0xffffffff, 0x200010ff,
796 0x353c, 0xffffffff, 0x0,
797 0x353c, 0xffffffff, 0x0,
798 0x353c, 0xffffffff, 0x0,
799 0x353c, 0xffffffff, 0x0,
800 0x353c, 0xffffffff, 0x0,
801 0x353c, 0xffffffff, 0x7007,
802 0x3538, 0xffffffff, 0x300010ff,
803 0x353c, 0xffffffff, 0x0,
804 0x353c, 0xffffffff, 0x0,
805 0x353c, 0xffffffff, 0x0,
806 0x353c, 0xffffffff, 0x0,
807 0x353c, 0xffffffff, 0x0,
808 0x353c, 0xffffffff, 0x400000,
809 0x3538, 0xffffffff, 0x100010ff,
810 0x353c, 0xffffffff, 0x0,
811 0x353c, 0xffffffff, 0x0,
812 0x353c, 0xffffffff, 0x0,
813 0x353c, 0xffffffff, 0x0,
814 0x353c, 0xffffffff, 0x0,
815 0x353c, 0xffffffff, 0x120200,
816 0x3538, 0xffffffff, 0x500010ff,
817 0x353c, 0xffffffff, 0x0,
818 0x353c, 0xffffffff, 0x0,
819 0x353c, 0xffffffff, 0x0,
820 0x353c, 0xffffffff, 0x0,
821 0x353c, 0xffffffff, 0x0,
822 0x353c, 0xffffffff, 0x1e1e16,
823 0x3538, 0xffffffff, 0x600010ff,
824 0x353c, 0xffffffff, 0x0,
825 0x353c, 0xffffffff, 0x0,
826 0x353c, 0xffffffff, 0x0,
827 0x353c, 0xffffffff, 0x0,
828 0x353c, 0xffffffff, 0x0,
829 0x353c, 0xffffffff, 0x171f1e,
830 0x3538, 0xffffffff, 0x700010ff,
831 0x353c, 0xffffffff, 0x0,
832 0x353c, 0xffffffff, 0x0,
833 0x353c, 0xffffffff, 0x0,
834 0x353c, 0xffffffff, 0x0,
835 0x353c, 0xffffffff, 0x0,
836 0x353c, 0xffffffff, 0x0,
837 0x3538, 0xffffffff, 0x9ff,
838 0x3500, 0xffffffff, 0x0,
839 0x3504, 0xffffffff, 0x10000800,
840 0x3504, 0xffffffff, 0xf,
841 0x3504, 0xffffffff, 0xf,
842 0x3500, 0xffffffff, 0x4,
843 0x3504, 0xffffffff, 0x1000051e,
844 0x3504, 0xffffffff, 0xffff,
845 0x3504, 0xffffffff, 0xffff,
846 0x3500, 0xffffffff, 0x8,
847 0x3504, 0xffffffff, 0x80500,
848 0x3500, 0xffffffff, 0x12,
849 0x3504, 0xffffffff, 0x9050c,
850 0x3500, 0xffffffff, 0x1d,
851 0x3504, 0xffffffff, 0xb052c,
852 0x3500, 0xffffffff, 0x2a,
853 0x3504, 0xffffffff, 0x1053e,
854 0x3500, 0xffffffff, 0x2d,
855 0x3504, 0xffffffff, 0x10546,
856 0x3500, 0xffffffff, 0x30,
857 0x3504, 0xffffffff, 0xa054e,
858 0x3500, 0xffffffff, 0x3c,
859 0x3504, 0xffffffff, 0x1055f,
860 0x3500, 0xffffffff, 0x3f,
861 0x3504, 0xffffffff, 0x10567,
862 0x3500, 0xffffffff, 0x42,
863 0x3504, 0xffffffff, 0x1056f,
864 0x3500, 0xffffffff, 0x45,
865 0x3504, 0xffffffff, 0x10572,
866 0x3500, 0xffffffff, 0x48,
867 0x3504, 0xffffffff, 0x20575,
868 0x3500, 0xffffffff, 0x4c,
869 0x3504, 0xffffffff, 0x190801,
870 0x3500, 0xffffffff, 0x67,
871 0x3504, 0xffffffff, 0x1082a,
872 0x3500, 0xffffffff, 0x6a,
873 0x3504, 0xffffffff, 0x1b082d,
874 0x3500, 0xffffffff, 0x87,
875 0x3504, 0xffffffff, 0x310851,
876 0x3500, 0xffffffff, 0xba,
877 0x3504, 0xffffffff, 0x891,
878 0x3500, 0xffffffff, 0xbc,
879 0x3504, 0xffffffff, 0x893,
880 0x3500, 0xffffffff, 0xbe,
881 0x3504, 0xffffffff, 0x20895,
882 0x3500, 0xffffffff, 0xc2,
883 0x3504, 0xffffffff, 0x20899,
884 0x3500, 0xffffffff, 0xc6,
885 0x3504, 0xffffffff, 0x2089d,
886 0x3500, 0xffffffff, 0xca,
887 0x3504, 0xffffffff, 0x8a1,
888 0x3500, 0xffffffff, 0xcc,
889 0x3504, 0xffffffff, 0x8a3,
890 0x3500, 0xffffffff, 0xce,
891 0x3504, 0xffffffff, 0x308a5,
892 0x3500, 0xffffffff, 0xd3,
893 0x3504, 0xffffffff, 0x6d08cd,
894 0x3500, 0xffffffff, 0x142,
895 0x3504, 0xffffffff, 0x2000095a,
896 0x3504, 0xffffffff, 0x1,
897 0x3500, 0xffffffff, 0x144,
898 0x3504, 0xffffffff, 0x301f095b,
899 0x3500, 0xffffffff, 0x165,
900 0x3504, 0xffffffff, 0xc094d,
901 0x3500, 0xffffffff, 0x173,
902 0x3504, 0xffffffff, 0xf096d,
903 0x3500, 0xffffffff, 0x184,
904 0x3504, 0xffffffff, 0x15097f,
905 0x3500, 0xffffffff, 0x19b,
906 0x3504, 0xffffffff, 0xc0998,
907 0x3500, 0xffffffff, 0x1a9,
908 0x3504, 0xffffffff, 0x409a7,
909 0x3500, 0xffffffff, 0x1af,
910 0x3504, 0xffffffff, 0xcdc,
911 0x3500, 0xffffffff, 0x1b1,
912 0x3504, 0xffffffff, 0x800,
913 0x3508, 0xffffffff, 0x6c9b2000,
914 0x3510, 0xfc00, 0x2000,
915 0x3544, 0xffffffff, 0xfc0,
916 0x28d4, 0x00000100, 0x100
919 static void si_init_golden_registers(struct radeon_device
*rdev
)
921 switch (rdev
->family
) {
923 radeon_program_register_sequence(rdev
,
924 tahiti_golden_registers
,
925 (const u32
)ARRAY_SIZE(tahiti_golden_registers
));
926 radeon_program_register_sequence(rdev
,
927 tahiti_golden_rlc_registers
,
928 (const u32
)ARRAY_SIZE(tahiti_golden_rlc_registers
));
929 radeon_program_register_sequence(rdev
,
930 tahiti_mgcg_cgcg_init
,
931 (const u32
)ARRAY_SIZE(tahiti_mgcg_cgcg_init
));
932 radeon_program_register_sequence(rdev
,
933 tahiti_golden_registers2
,
934 (const u32
)ARRAY_SIZE(tahiti_golden_registers2
));
937 radeon_program_register_sequence(rdev
,
938 pitcairn_golden_registers
,
939 (const u32
)ARRAY_SIZE(pitcairn_golden_registers
));
940 radeon_program_register_sequence(rdev
,
941 pitcairn_golden_rlc_registers
,
942 (const u32
)ARRAY_SIZE(pitcairn_golden_rlc_registers
));
943 radeon_program_register_sequence(rdev
,
944 pitcairn_mgcg_cgcg_init
,
945 (const u32
)ARRAY_SIZE(pitcairn_mgcg_cgcg_init
));
948 radeon_program_register_sequence(rdev
,
949 verde_golden_registers
,
950 (const u32
)ARRAY_SIZE(verde_golden_registers
));
951 radeon_program_register_sequence(rdev
,
952 verde_golden_rlc_registers
,
953 (const u32
)ARRAY_SIZE(verde_golden_rlc_registers
));
954 radeon_program_register_sequence(rdev
,
955 verde_mgcg_cgcg_init
,
956 (const u32
)ARRAY_SIZE(verde_mgcg_cgcg_init
));
957 radeon_program_register_sequence(rdev
,
959 (const u32
)ARRAY_SIZE(verde_pg_init
));
962 radeon_program_register_sequence(rdev
,
963 oland_golden_registers
,
964 (const u32
)ARRAY_SIZE(oland_golden_registers
));
965 radeon_program_register_sequence(rdev
,
966 oland_golden_rlc_registers
,
967 (const u32
)ARRAY_SIZE(oland_golden_rlc_registers
));
968 radeon_program_register_sequence(rdev
,
969 oland_mgcg_cgcg_init
,
970 (const u32
)ARRAY_SIZE(oland_mgcg_cgcg_init
));
973 radeon_program_register_sequence(rdev
,
974 hainan_golden_registers
,
975 (const u32
)ARRAY_SIZE(hainan_golden_registers
));
976 radeon_program_register_sequence(rdev
,
977 hainan_golden_registers2
,
978 (const u32
)ARRAY_SIZE(hainan_golden_registers2
));
979 radeon_program_register_sequence(rdev
,
980 hainan_mgcg_cgcg_init
,
981 (const u32
)ARRAY_SIZE(hainan_mgcg_cgcg_init
));
988 #define PCIE_BUS_CLK 10000
989 #define TCLK (PCIE_BUS_CLK / 10)
992 * si_get_xclk - get the xclk
994 * @rdev: radeon_device pointer
996 * Returns the reference clock used by the gfx engine
999 u32
si_get_xclk(struct radeon_device
*rdev
)
1001 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1004 tmp
= RREG32(CG_CLKPIN_CNTL_2
);
1005 if (tmp
& MUX_TCLK_TO_XCLK
)
1008 tmp
= RREG32(CG_CLKPIN_CNTL
);
1009 if (tmp
& XTALIN_DIVIDE
)
1010 return reference_clock
/ 4;
1012 return reference_clock
;
1015 /* get temperature in millidegrees */
1016 int si_get_temp(struct radeon_device
*rdev
)
1019 int actual_temp
= 0;
1021 temp
= (RREG32(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
1027 actual_temp
= temp
& 0x1ff;
1029 actual_temp
= (actual_temp
* 1000);
1034 #define TAHITI_IO_MC_REGS_SIZE 36
1036 static const u32 tahiti_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1037 {0x0000006f, 0x03044000},
1038 {0x00000070, 0x0480c018},
1039 {0x00000071, 0x00000040},
1040 {0x00000072, 0x01000000},
1041 {0x00000074, 0x000000ff},
1042 {0x00000075, 0x00143400},
1043 {0x00000076, 0x08ec0800},
1044 {0x00000077, 0x040000cc},
1045 {0x00000079, 0x00000000},
1046 {0x0000007a, 0x21000409},
1047 {0x0000007c, 0x00000000},
1048 {0x0000007d, 0xe8000000},
1049 {0x0000007e, 0x044408a8},
1050 {0x0000007f, 0x00000003},
1051 {0x00000080, 0x00000000},
1052 {0x00000081, 0x01000000},
1053 {0x00000082, 0x02000000},
1054 {0x00000083, 0x00000000},
1055 {0x00000084, 0xe3f3e4f4},
1056 {0x00000085, 0x00052024},
1057 {0x00000087, 0x00000000},
1058 {0x00000088, 0x66036603},
1059 {0x00000089, 0x01000000},
1060 {0x0000008b, 0x1c0a0000},
1061 {0x0000008c, 0xff010000},
1062 {0x0000008e, 0xffffefff},
1063 {0x0000008f, 0xfff3efff},
1064 {0x00000090, 0xfff3efbf},
1065 {0x00000094, 0x00101101},
1066 {0x00000095, 0x00000fff},
1067 {0x00000096, 0x00116fff},
1068 {0x00000097, 0x60010000},
1069 {0x00000098, 0x10010000},
1070 {0x00000099, 0x00006000},
1071 {0x0000009a, 0x00001000},
1072 {0x0000009f, 0x00a77400}
1075 static const u32 pitcairn_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1076 {0x0000006f, 0x03044000},
1077 {0x00000070, 0x0480c018},
1078 {0x00000071, 0x00000040},
1079 {0x00000072, 0x01000000},
1080 {0x00000074, 0x000000ff},
1081 {0x00000075, 0x00143400},
1082 {0x00000076, 0x08ec0800},
1083 {0x00000077, 0x040000cc},
1084 {0x00000079, 0x00000000},
1085 {0x0000007a, 0x21000409},
1086 {0x0000007c, 0x00000000},
1087 {0x0000007d, 0xe8000000},
1088 {0x0000007e, 0x044408a8},
1089 {0x0000007f, 0x00000003},
1090 {0x00000080, 0x00000000},
1091 {0x00000081, 0x01000000},
1092 {0x00000082, 0x02000000},
1093 {0x00000083, 0x00000000},
1094 {0x00000084, 0xe3f3e4f4},
1095 {0x00000085, 0x00052024},
1096 {0x00000087, 0x00000000},
1097 {0x00000088, 0x66036603},
1098 {0x00000089, 0x01000000},
1099 {0x0000008b, 0x1c0a0000},
1100 {0x0000008c, 0xff010000},
1101 {0x0000008e, 0xffffefff},
1102 {0x0000008f, 0xfff3efff},
1103 {0x00000090, 0xfff3efbf},
1104 {0x00000094, 0x00101101},
1105 {0x00000095, 0x00000fff},
1106 {0x00000096, 0x00116fff},
1107 {0x00000097, 0x60010000},
1108 {0x00000098, 0x10010000},
1109 {0x00000099, 0x00006000},
1110 {0x0000009a, 0x00001000},
1111 {0x0000009f, 0x00a47400}
1114 static const u32 verde_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1115 {0x0000006f, 0x03044000},
1116 {0x00000070, 0x0480c018},
1117 {0x00000071, 0x00000040},
1118 {0x00000072, 0x01000000},
1119 {0x00000074, 0x000000ff},
1120 {0x00000075, 0x00143400},
1121 {0x00000076, 0x08ec0800},
1122 {0x00000077, 0x040000cc},
1123 {0x00000079, 0x00000000},
1124 {0x0000007a, 0x21000409},
1125 {0x0000007c, 0x00000000},
1126 {0x0000007d, 0xe8000000},
1127 {0x0000007e, 0x044408a8},
1128 {0x0000007f, 0x00000003},
1129 {0x00000080, 0x00000000},
1130 {0x00000081, 0x01000000},
1131 {0x00000082, 0x02000000},
1132 {0x00000083, 0x00000000},
1133 {0x00000084, 0xe3f3e4f4},
1134 {0x00000085, 0x00052024},
1135 {0x00000087, 0x00000000},
1136 {0x00000088, 0x66036603},
1137 {0x00000089, 0x01000000},
1138 {0x0000008b, 0x1c0a0000},
1139 {0x0000008c, 0xff010000},
1140 {0x0000008e, 0xffffefff},
1141 {0x0000008f, 0xfff3efff},
1142 {0x00000090, 0xfff3efbf},
1143 {0x00000094, 0x00101101},
1144 {0x00000095, 0x00000fff},
1145 {0x00000096, 0x00116fff},
1146 {0x00000097, 0x60010000},
1147 {0x00000098, 0x10010000},
1148 {0x00000099, 0x00006000},
1149 {0x0000009a, 0x00001000},
1150 {0x0000009f, 0x00a37400}
1153 static const u32 oland_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1154 {0x0000006f, 0x03044000},
1155 {0x00000070, 0x0480c018},
1156 {0x00000071, 0x00000040},
1157 {0x00000072, 0x01000000},
1158 {0x00000074, 0x000000ff},
1159 {0x00000075, 0x00143400},
1160 {0x00000076, 0x08ec0800},
1161 {0x00000077, 0x040000cc},
1162 {0x00000079, 0x00000000},
1163 {0x0000007a, 0x21000409},
1164 {0x0000007c, 0x00000000},
1165 {0x0000007d, 0xe8000000},
1166 {0x0000007e, 0x044408a8},
1167 {0x0000007f, 0x00000003},
1168 {0x00000080, 0x00000000},
1169 {0x00000081, 0x01000000},
1170 {0x00000082, 0x02000000},
1171 {0x00000083, 0x00000000},
1172 {0x00000084, 0xe3f3e4f4},
1173 {0x00000085, 0x00052024},
1174 {0x00000087, 0x00000000},
1175 {0x00000088, 0x66036603},
1176 {0x00000089, 0x01000000},
1177 {0x0000008b, 0x1c0a0000},
1178 {0x0000008c, 0xff010000},
1179 {0x0000008e, 0xffffefff},
1180 {0x0000008f, 0xfff3efff},
1181 {0x00000090, 0xfff3efbf},
1182 {0x00000094, 0x00101101},
1183 {0x00000095, 0x00000fff},
1184 {0x00000096, 0x00116fff},
1185 {0x00000097, 0x60010000},
1186 {0x00000098, 0x10010000},
1187 {0x00000099, 0x00006000},
1188 {0x0000009a, 0x00001000},
1189 {0x0000009f, 0x00a17730}
1192 static const u32 hainan_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1193 {0x0000006f, 0x03044000},
1194 {0x00000070, 0x0480c018},
1195 {0x00000071, 0x00000040},
1196 {0x00000072, 0x01000000},
1197 {0x00000074, 0x000000ff},
1198 {0x00000075, 0x00143400},
1199 {0x00000076, 0x08ec0800},
1200 {0x00000077, 0x040000cc},
1201 {0x00000079, 0x00000000},
1202 {0x0000007a, 0x21000409},
1203 {0x0000007c, 0x00000000},
1204 {0x0000007d, 0xe8000000},
1205 {0x0000007e, 0x044408a8},
1206 {0x0000007f, 0x00000003},
1207 {0x00000080, 0x00000000},
1208 {0x00000081, 0x01000000},
1209 {0x00000082, 0x02000000},
1210 {0x00000083, 0x00000000},
1211 {0x00000084, 0xe3f3e4f4},
1212 {0x00000085, 0x00052024},
1213 {0x00000087, 0x00000000},
1214 {0x00000088, 0x66036603},
1215 {0x00000089, 0x01000000},
1216 {0x0000008b, 0x1c0a0000},
1217 {0x0000008c, 0xff010000},
1218 {0x0000008e, 0xffffefff},
1219 {0x0000008f, 0xfff3efff},
1220 {0x00000090, 0xfff3efbf},
1221 {0x00000094, 0x00101101},
1222 {0x00000095, 0x00000fff},
1223 {0x00000096, 0x00116fff},
1224 {0x00000097, 0x60010000},
1225 {0x00000098, 0x10010000},
1226 {0x00000099, 0x00006000},
1227 {0x0000009a, 0x00001000},
1228 {0x0000009f, 0x00a07730}
1232 static int si_mc_load_microcode(struct radeon_device
*rdev
)
1234 const __be32
*fw_data
;
1235 u32 running
, blackout
= 0;
1237 int i
, ucode_size
, regs_size
;
1242 switch (rdev
->family
) {
1244 io_mc_regs
= (u32
*)&tahiti_io_mc_regs
;
1245 ucode_size
= SI_MC_UCODE_SIZE
;
1246 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1249 io_mc_regs
= (u32
*)&pitcairn_io_mc_regs
;
1250 ucode_size
= SI_MC_UCODE_SIZE
;
1251 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1255 io_mc_regs
= (u32
*)&verde_io_mc_regs
;
1256 ucode_size
= SI_MC_UCODE_SIZE
;
1257 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1260 io_mc_regs
= (u32
*)&oland_io_mc_regs
;
1261 ucode_size
= OLAND_MC_UCODE_SIZE
;
1262 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1265 io_mc_regs
= (u32
*)&hainan_io_mc_regs
;
1266 ucode_size
= OLAND_MC_UCODE_SIZE
;
1267 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1271 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1275 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1276 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1279 /* reset the engine and set to writable */
1280 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1281 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1283 /* load mc io regs */
1284 for (i
= 0; i
< regs_size
; i
++) {
1285 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1286 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1288 /* load the MC ucode */
1289 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1290 for (i
= 0; i
< ucode_size
; i
++)
1291 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1293 /* put the engine back into the active state */
1294 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1295 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1296 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1298 /* wait for training to complete */
1299 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1300 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1304 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1305 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1311 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1317 static int si_init_microcode(struct radeon_device
*rdev
)
1319 struct platform_device
*pdev
;
1320 const char *chip_name
;
1321 const char *rlc_chip_name
;
1322 size_t pfp_req_size
, me_req_size
, ce_req_size
, rlc_req_size
, mc_req_size
;
1328 pdev
= platform_device_register_simple("radeon_cp", 0, NULL
, 0);
1331 printk(KERN_ERR
"radeon_cp: Failed to register firmware\n");
1335 switch (rdev
->family
) {
1337 chip_name
= "TAHITI";
1338 rlc_chip_name
= "TAHITI";
1339 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1340 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1341 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1342 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1343 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1346 chip_name
= "PITCAIRN";
1347 rlc_chip_name
= "PITCAIRN";
1348 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1349 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1350 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1351 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1352 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1355 chip_name
= "VERDE";
1356 rlc_chip_name
= "VERDE";
1357 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1358 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1359 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1360 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1361 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1364 chip_name
= "OLAND";
1365 rlc_chip_name
= "OLAND";
1366 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1367 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1368 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1369 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1370 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1373 chip_name
= "HAINAN";
1374 rlc_chip_name
= "HAINAN";
1375 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1376 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1377 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1378 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1379 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1384 DRM_INFO("Loading %s Microcode\n", chip_name
);
1386 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1387 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, &pdev
->dev
);
1390 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1392 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1393 rdev
->pfp_fw
->size
, fw_name
);
1398 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1399 err
= request_firmware(&rdev
->me_fw
, fw_name
, &pdev
->dev
);
1402 if (rdev
->me_fw
->size
!= me_req_size
) {
1404 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1405 rdev
->me_fw
->size
, fw_name
);
1409 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1410 err
= request_firmware(&rdev
->ce_fw
, fw_name
, &pdev
->dev
);
1413 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1415 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1416 rdev
->ce_fw
->size
, fw_name
);
1420 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", rlc_chip_name
);
1421 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, &pdev
->dev
);
1424 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1426 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1427 rdev
->rlc_fw
->size
, fw_name
);
1431 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1432 err
= request_firmware(&rdev
->mc_fw
, fw_name
, &pdev
->dev
);
1435 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1437 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1438 rdev
->mc_fw
->size
, fw_name
);
1443 platform_device_unregister(pdev
);
1448 "si_cp: Failed to load firmware \"%s\"\n",
1450 release_firmware(rdev
->pfp_fw
);
1451 rdev
->pfp_fw
= NULL
;
1452 release_firmware(rdev
->me_fw
);
1454 release_firmware(rdev
->ce_fw
);
1456 release_firmware(rdev
->rlc_fw
);
1457 rdev
->rlc_fw
= NULL
;
1458 release_firmware(rdev
->mc_fw
);
1464 /* watermark setup */
1465 static u32
dce6_line_buffer_adjust(struct radeon_device
*rdev
,
1466 struct radeon_crtc
*radeon_crtc
,
1467 struct drm_display_mode
*mode
,
1468 struct drm_display_mode
*other_mode
)
1473 * There are 3 line buffers, each one shared by 2 display controllers.
1474 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1475 * the display controllers. The paritioning is done via one of four
1476 * preset allocations specified in bits 21:20:
1478 * 2 - whole lb, other crtc must be disabled
1480 /* this can get tricky if we have two large displays on a paired group
1481 * of crtcs. Ideally for multiple large displays we'd assign them to
1482 * non-linked crtcs for maximum line buffer allocation.
1484 if (radeon_crtc
->base
.enabled
&& mode
) {
1488 tmp
= 2; /* whole */
1492 WREG32(DC_LB_MEMORY_SPLIT
+ radeon_crtc
->crtc_offset
,
1493 DC_LB_MEMORY_CONFIG(tmp
));
1495 if (radeon_crtc
->base
.enabled
&& mode
) {
1505 /* controller not enabled, so no lb used */
1509 static u32
si_get_number_of_dram_channels(struct radeon_device
*rdev
)
1511 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
1513 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
1536 struct dce6_wm_params
{
1537 u32 dram_channels
; /* number of dram channels */
1538 u32 yclk
; /* bandwidth per dram data pin in kHz */
1539 u32 sclk
; /* engine clock in kHz */
1540 u32 disp_clk
; /* display clock in kHz */
1541 u32 src_width
; /* viewport width */
1542 u32 active_time
; /* active display time in ns */
1543 u32 blank_time
; /* blank time in ns */
1544 bool interlaced
; /* mode is interlaced */
1545 fixed20_12 vsc
; /* vertical scale ratio */
1546 u32 num_heads
; /* number of active crtcs */
1547 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
1548 u32 lb_size
; /* line buffer allocated to pipe */
1549 u32 vtaps
; /* vertical scaler taps */
1552 static u32
dce6_dram_bandwidth(struct dce6_wm_params
*wm
)
1554 /* Calculate raw DRAM Bandwidth */
1555 fixed20_12 dram_efficiency
; /* 0.7 */
1556 fixed20_12 yclk
, dram_channels
, bandwidth
;
1559 a
.full
= dfixed_const(1000);
1560 yclk
.full
= dfixed_const(wm
->yclk
);
1561 yclk
.full
= dfixed_div(yclk
, a
);
1562 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1563 a
.full
= dfixed_const(10);
1564 dram_efficiency
.full
= dfixed_const(7);
1565 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
1566 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1567 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
1569 return dfixed_trunc(bandwidth
);
1572 static u32
dce6_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1574 /* Calculate DRAM Bandwidth and the part allocated to display. */
1575 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
1576 fixed20_12 yclk
, dram_channels
, bandwidth
;
1579 a
.full
= dfixed_const(1000);
1580 yclk
.full
= dfixed_const(wm
->yclk
);
1581 yclk
.full
= dfixed_div(yclk
, a
);
1582 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1583 a
.full
= dfixed_const(10);
1584 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
1585 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
1586 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1587 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
1589 return dfixed_trunc(bandwidth
);
1592 static u32
dce6_data_return_bandwidth(struct dce6_wm_params
*wm
)
1594 /* Calculate the display Data return Bandwidth */
1595 fixed20_12 return_efficiency
; /* 0.8 */
1596 fixed20_12 sclk
, bandwidth
;
1599 a
.full
= dfixed_const(1000);
1600 sclk
.full
= dfixed_const(wm
->sclk
);
1601 sclk
.full
= dfixed_div(sclk
, a
);
1602 a
.full
= dfixed_const(10);
1603 return_efficiency
.full
= dfixed_const(8);
1604 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
1605 a
.full
= dfixed_const(32);
1606 bandwidth
.full
= dfixed_mul(a
, sclk
);
1607 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
1609 return dfixed_trunc(bandwidth
);
1612 static u32
dce6_get_dmif_bytes_per_request(struct dce6_wm_params
*wm
)
1617 static u32
dce6_dmif_request_bandwidth(struct dce6_wm_params
*wm
)
1619 /* Calculate the DMIF Request Bandwidth */
1620 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
1621 fixed20_12 disp_clk
, sclk
, bandwidth
;
1622 fixed20_12 a
, b1
, b2
;
1625 a
.full
= dfixed_const(1000);
1626 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
1627 disp_clk
.full
= dfixed_div(disp_clk
, a
);
1628 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
) / 2);
1629 b1
.full
= dfixed_mul(a
, disp_clk
);
1631 a
.full
= dfixed_const(1000);
1632 sclk
.full
= dfixed_const(wm
->sclk
);
1633 sclk
.full
= dfixed_div(sclk
, a
);
1634 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
));
1635 b2
.full
= dfixed_mul(a
, sclk
);
1637 a
.full
= dfixed_const(10);
1638 disp_clk_request_efficiency
.full
= dfixed_const(8);
1639 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
1641 min_bandwidth
= min(dfixed_trunc(b1
), dfixed_trunc(b2
));
1643 a
.full
= dfixed_const(min_bandwidth
);
1644 bandwidth
.full
= dfixed_mul(a
, disp_clk_request_efficiency
);
1646 return dfixed_trunc(bandwidth
);
1649 static u32
dce6_available_bandwidth(struct dce6_wm_params
*wm
)
1651 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1652 u32 dram_bandwidth
= dce6_dram_bandwidth(wm
);
1653 u32 data_return_bandwidth
= dce6_data_return_bandwidth(wm
);
1654 u32 dmif_req_bandwidth
= dce6_dmif_request_bandwidth(wm
);
1656 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
1659 static u32
dce6_average_bandwidth(struct dce6_wm_params
*wm
)
1661 /* Calculate the display mode Average Bandwidth
1662 * DisplayMode should contain the source and destination dimensions,
1666 fixed20_12 line_time
;
1667 fixed20_12 src_width
;
1668 fixed20_12 bandwidth
;
1671 a
.full
= dfixed_const(1000);
1672 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
1673 line_time
.full
= dfixed_div(line_time
, a
);
1674 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
1675 src_width
.full
= dfixed_const(wm
->src_width
);
1676 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
1677 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
1678 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
1680 return dfixed_trunc(bandwidth
);
1683 static u32
dce6_latency_watermark(struct dce6_wm_params
*wm
)
1685 /* First calcualte the latency in ns */
1686 u32 mc_latency
= 2000; /* 2000 ns. */
1687 u32 available_bandwidth
= dce6_available_bandwidth(wm
);
1688 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
1689 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
1690 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
1691 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
1692 (wm
->num_heads
* cursor_line_pair_return_time
);
1693 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
1694 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
1695 u32 tmp
, dmif_size
= 12288;
1698 if (wm
->num_heads
== 0)
1701 a
.full
= dfixed_const(2);
1702 b
.full
= dfixed_const(1);
1703 if ((wm
->vsc
.full
> a
.full
) ||
1704 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
1706 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
1707 max_src_lines_per_dst_line
= 4;
1709 max_src_lines_per_dst_line
= 2;
1711 a
.full
= dfixed_const(available_bandwidth
);
1712 b
.full
= dfixed_const(wm
->num_heads
);
1713 a
.full
= dfixed_div(a
, b
);
1715 b
.full
= dfixed_const(mc_latency
+ 512);
1716 c
.full
= dfixed_const(wm
->disp_clk
);
1717 b
.full
= dfixed_div(b
, c
);
1719 c
.full
= dfixed_const(dmif_size
);
1720 b
.full
= dfixed_div(c
, b
);
1722 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
1724 b
.full
= dfixed_const(1000);
1725 c
.full
= dfixed_const(wm
->disp_clk
);
1726 b
.full
= dfixed_div(c
, b
);
1727 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
1728 b
.full
= dfixed_mul(b
, c
);
1730 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
1732 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
1733 b
.full
= dfixed_const(1000);
1734 c
.full
= dfixed_const(lb_fill_bw
);
1735 b
.full
= dfixed_div(c
, b
);
1736 a
.full
= dfixed_div(a
, b
);
1737 line_fill_time
= dfixed_trunc(a
);
1739 if (line_fill_time
< wm
->active_time
)
1742 return latency
+ (line_fill_time
- wm
->active_time
);
1746 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1748 if (dce6_average_bandwidth(wm
) <=
1749 (dce6_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
1755 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params
*wm
)
1757 if (dce6_average_bandwidth(wm
) <=
1758 (dce6_available_bandwidth(wm
) / wm
->num_heads
))
1764 static bool dce6_check_latency_hiding(struct dce6_wm_params
*wm
)
1766 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
1767 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
1768 u32 latency_tolerant_lines
;
1772 a
.full
= dfixed_const(1);
1773 if (wm
->vsc
.full
> a
.full
)
1774 latency_tolerant_lines
= 1;
1776 if (lb_partitions
<= (wm
->vtaps
+ 1))
1777 latency_tolerant_lines
= 1;
1779 latency_tolerant_lines
= 2;
1782 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
1784 if (dce6_latency_watermark(wm
) <= latency_hiding
)
1790 static void dce6_program_watermarks(struct radeon_device
*rdev
,
1791 struct radeon_crtc
*radeon_crtc
,
1792 u32 lb_size
, u32 num_heads
)
1794 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
1795 struct dce6_wm_params wm
;
1798 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
1799 u32 priority_a_mark
= 0, priority_b_mark
= 0;
1800 u32 priority_a_cnt
= PRIORITY_OFF
;
1801 u32 priority_b_cnt
= PRIORITY_OFF
;
1802 u32 tmp
, arb_control3
;
1805 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
1806 pixel_period
= 1000000 / (u32
)mode
->clock
;
1807 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
1811 wm
.yclk
= rdev
->pm
.current_mclk
* 10;
1812 wm
.sclk
= rdev
->pm
.current_sclk
* 10;
1813 wm
.disp_clk
= mode
->clock
;
1814 wm
.src_width
= mode
->crtc_hdisplay
;
1815 wm
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
1816 wm
.blank_time
= line_time
- wm
.active_time
;
1817 wm
.interlaced
= false;
1818 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
1819 wm
.interlaced
= true;
1820 wm
.vsc
= radeon_crtc
->vsc
;
1822 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
1824 wm
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
1825 wm
.lb_size
= lb_size
;
1826 if (rdev
->family
== CHIP_ARUBA
)
1827 wm
.dram_channels
= evergreen_get_number_of_dram_channels(rdev
);
1829 wm
.dram_channels
= si_get_number_of_dram_channels(rdev
);
1830 wm
.num_heads
= num_heads
;
1832 /* set for high clocks */
1833 latency_watermark_a
= min(dce6_latency_watermark(&wm
), (u32
)65535);
1834 /* set for low clocks */
1835 /* wm.yclk = low clk; wm.sclk = low clk */
1836 latency_watermark_b
= min(dce6_latency_watermark(&wm
), (u32
)65535);
1838 /* possibly force display priority to high */
1839 /* should really do this at mode validation time... */
1840 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm
) ||
1841 !dce6_average_bandwidth_vs_available_bandwidth(&wm
) ||
1842 !dce6_check_latency_hiding(&wm
) ||
1843 (rdev
->disp_priority
== 2)) {
1844 DRM_DEBUG_KMS("force priority to high\n");
1845 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
1846 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
1849 a
.full
= dfixed_const(1000);
1850 b
.full
= dfixed_const(mode
->clock
);
1851 b
.full
= dfixed_div(b
, a
);
1852 c
.full
= dfixed_const(latency_watermark_a
);
1853 c
.full
= dfixed_mul(c
, b
);
1854 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
1855 c
.full
= dfixed_div(c
, a
);
1856 a
.full
= dfixed_const(16);
1857 c
.full
= dfixed_div(c
, a
);
1858 priority_a_mark
= dfixed_trunc(c
);
1859 priority_a_cnt
|= priority_a_mark
& PRIORITY_MARK_MASK
;
1861 a
.full
= dfixed_const(1000);
1862 b
.full
= dfixed_const(mode
->clock
);
1863 b
.full
= dfixed_div(b
, a
);
1864 c
.full
= dfixed_const(latency_watermark_b
);
1865 c
.full
= dfixed_mul(c
, b
);
1866 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
1867 c
.full
= dfixed_div(c
, a
);
1868 a
.full
= dfixed_const(16);
1869 c
.full
= dfixed_div(c
, a
);
1870 priority_b_mark
= dfixed_trunc(c
);
1871 priority_b_cnt
|= priority_b_mark
& PRIORITY_MARK_MASK
;
1875 arb_control3
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
1877 tmp
&= ~LATENCY_WATERMARK_MASK(3);
1878 tmp
|= LATENCY_WATERMARK_MASK(1);
1879 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
1880 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
1881 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
1882 LATENCY_HIGH_WATERMARK(line_time
)));
1884 tmp
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
1885 tmp
&= ~LATENCY_WATERMARK_MASK(3);
1886 tmp
|= LATENCY_WATERMARK_MASK(2);
1887 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
1888 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
1889 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
1890 LATENCY_HIGH_WATERMARK(line_time
)));
1891 /* restore original selection */
1892 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, arb_control3
);
1894 /* write the priority marks */
1895 WREG32(PRIORITY_A_CNT
+ radeon_crtc
->crtc_offset
, priority_a_cnt
);
1896 WREG32(PRIORITY_B_CNT
+ radeon_crtc
->crtc_offset
, priority_b_cnt
);
1900 void dce6_bandwidth_update(struct radeon_device
*rdev
)
1902 struct drm_display_mode
*mode0
= NULL
;
1903 struct drm_display_mode
*mode1
= NULL
;
1904 u32 num_heads
= 0, lb_size
;
1907 radeon_update_display_priority(rdev
);
1909 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
1910 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
1913 for (i
= 0; i
< rdev
->num_crtc
; i
+= 2) {
1914 mode0
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
1915 mode1
= &rdev
->mode_info
.crtcs
[i
+1]->base
.mode
;
1916 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode0
, mode1
);
1917 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
1918 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
+1], mode1
, mode0
);
1919 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
+1], lb_size
, num_heads
);
1926 static void si_tiling_mode_table_init(struct radeon_device
*rdev
)
1928 const u32 num_tile_mode_states
= 32;
1929 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
1931 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
1933 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
1937 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
1940 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
1944 if ((rdev
->family
== CHIP_TAHITI
) ||
1945 (rdev
->family
== CHIP_PITCAIRN
)) {
1946 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
1947 switch (reg_offset
) {
1948 case 0: /* non-AA compressed depth or any compressed stencil */
1949 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1950 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
1951 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1952 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
1953 NUM_BANKS(ADDR_SURF_16_BANK
) |
1954 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
1958 case 1: /* 2xAA/4xAA compressed depth only */
1959 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1960 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
1961 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1962 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
1963 NUM_BANKS(ADDR_SURF_16_BANK
) |
1964 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
1968 case 2: /* 8xAA compressed depth only */
1969 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1970 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
1971 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1972 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
1973 NUM_BANKS(ADDR_SURF_16_BANK
) |
1974 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
1978 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1979 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
1980 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
1981 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1982 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
1983 NUM_BANKS(ADDR_SURF_16_BANK
) |
1984 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
1986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
1988 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1989 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
1990 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
1991 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
1992 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
1993 NUM_BANKS(ADDR_SURF_16_BANK
) |
1994 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
1995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
1996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
1998 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1999 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2000 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2001 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2002 TILE_SPLIT(split_equal_to_row_size
) |
2003 NUM_BANKS(ADDR_SURF_16_BANK
) |
2004 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2008 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2009 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2010 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2011 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2012 TILE_SPLIT(split_equal_to_row_size
) |
2013 NUM_BANKS(ADDR_SURF_16_BANK
) |
2014 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2015 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2016 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2018 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2019 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2020 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2021 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2022 TILE_SPLIT(split_equal_to_row_size
) |
2023 NUM_BANKS(ADDR_SURF_16_BANK
) |
2024 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2025 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2026 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2028 case 8: /* 1D and 1D Array Surfaces */
2029 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2030 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2031 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2032 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2033 NUM_BANKS(ADDR_SURF_16_BANK
) |
2034 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2035 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2036 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2038 case 9: /* Displayable maps. */
2039 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2040 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2041 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2042 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2043 NUM_BANKS(ADDR_SURF_16_BANK
) |
2044 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2045 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2046 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2048 case 10: /* Display 8bpp. */
2049 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2050 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2051 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2052 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2053 NUM_BANKS(ADDR_SURF_16_BANK
) |
2054 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2055 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2056 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2058 case 11: /* Display 16bpp. */
2059 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2060 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2061 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2062 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2063 NUM_BANKS(ADDR_SURF_16_BANK
) |
2064 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2065 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2066 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2068 case 12: /* Display 32bpp. */
2069 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2070 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2071 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2072 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2073 NUM_BANKS(ADDR_SURF_16_BANK
) |
2074 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2078 case 13: /* Thin. */
2079 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2080 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2081 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2082 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2083 NUM_BANKS(ADDR_SURF_16_BANK
) |
2084 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2085 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2086 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2088 case 14: /* Thin 8 bpp. */
2089 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2090 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2091 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2092 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2093 NUM_BANKS(ADDR_SURF_16_BANK
) |
2094 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2095 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2096 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2098 case 15: /* Thin 16 bpp. */
2099 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2100 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2101 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2102 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2103 NUM_BANKS(ADDR_SURF_16_BANK
) |
2104 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2105 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2106 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2108 case 16: /* Thin 32 bpp. */
2109 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2110 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2111 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2113 NUM_BANKS(ADDR_SURF_16_BANK
) |
2114 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2115 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2116 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2118 case 17: /* Thin 64 bpp. */
2119 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2120 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2121 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2122 TILE_SPLIT(split_equal_to_row_size
) |
2123 NUM_BANKS(ADDR_SURF_16_BANK
) |
2124 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2125 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2126 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2128 case 21: /* 8 bpp PRT. */
2129 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2130 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2131 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2133 NUM_BANKS(ADDR_SURF_16_BANK
) |
2134 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2135 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2136 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2138 case 22: /* 16 bpp PRT */
2139 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2140 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2141 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2142 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2143 NUM_BANKS(ADDR_SURF_16_BANK
) |
2144 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2148 case 23: /* 32 bpp PRT */
2149 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2150 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2151 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2152 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2153 NUM_BANKS(ADDR_SURF_16_BANK
) |
2154 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2155 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2156 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2158 case 24: /* 64 bpp PRT */
2159 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2160 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2161 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2162 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2163 NUM_BANKS(ADDR_SURF_16_BANK
) |
2164 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2168 case 25: /* 128 bpp PRT */
2169 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2170 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2171 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2172 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2173 NUM_BANKS(ADDR_SURF_8_BANK
) |
2174 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2182 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2183 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2185 } else if ((rdev
->family
== CHIP_VERDE
) ||
2186 (rdev
->family
== CHIP_OLAND
) ||
2187 (rdev
->family
== CHIP_HAINAN
)) {
2188 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2189 switch (reg_offset
) {
2190 case 0: /* non-AA compressed depth or any compressed stencil */
2191 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2192 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2193 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2194 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2195 NUM_BANKS(ADDR_SURF_16_BANK
) |
2196 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2200 case 1: /* 2xAA/4xAA compressed depth only */
2201 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2202 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2203 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2204 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2205 NUM_BANKS(ADDR_SURF_16_BANK
) |
2206 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2210 case 2: /* 8xAA compressed depth only */
2211 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2212 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2213 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2214 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2215 NUM_BANKS(ADDR_SURF_16_BANK
) |
2216 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2217 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2218 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2220 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2221 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2222 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2223 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2224 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2225 NUM_BANKS(ADDR_SURF_16_BANK
) |
2226 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2227 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2228 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2230 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2231 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2232 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2233 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2235 NUM_BANKS(ADDR_SURF_16_BANK
) |
2236 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2240 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2241 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2242 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2243 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2244 TILE_SPLIT(split_equal_to_row_size
) |
2245 NUM_BANKS(ADDR_SURF_16_BANK
) |
2246 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2250 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2251 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2252 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2253 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2254 TILE_SPLIT(split_equal_to_row_size
) |
2255 NUM_BANKS(ADDR_SURF_16_BANK
) |
2256 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2260 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2261 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2263 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2264 TILE_SPLIT(split_equal_to_row_size
) |
2265 NUM_BANKS(ADDR_SURF_16_BANK
) |
2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2270 case 8: /* 1D and 1D Array Surfaces */
2271 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2272 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2273 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2275 NUM_BANKS(ADDR_SURF_16_BANK
) |
2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2280 case 9: /* Displayable maps. */
2281 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2282 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2283 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2285 NUM_BANKS(ADDR_SURF_16_BANK
) |
2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2290 case 10: /* Display 8bpp. */
2291 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2292 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2293 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2295 NUM_BANKS(ADDR_SURF_16_BANK
) |
2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2300 case 11: /* Display 16bpp. */
2301 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2302 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2303 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2305 NUM_BANKS(ADDR_SURF_16_BANK
) |
2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2310 case 12: /* Display 32bpp. */
2311 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2312 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2313 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2314 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2315 NUM_BANKS(ADDR_SURF_16_BANK
) |
2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2320 case 13: /* Thin. */
2321 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2322 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2323 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2324 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2325 NUM_BANKS(ADDR_SURF_16_BANK
) |
2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2330 case 14: /* Thin 8 bpp. */
2331 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2332 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2333 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2334 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2335 NUM_BANKS(ADDR_SURF_16_BANK
) |
2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2340 case 15: /* Thin 16 bpp. */
2341 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2342 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2343 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2345 NUM_BANKS(ADDR_SURF_16_BANK
) |
2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2350 case 16: /* Thin 32 bpp. */
2351 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2352 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2353 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2355 NUM_BANKS(ADDR_SURF_16_BANK
) |
2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2360 case 17: /* Thin 64 bpp. */
2361 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2362 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2363 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2364 TILE_SPLIT(split_equal_to_row_size
) |
2365 NUM_BANKS(ADDR_SURF_16_BANK
) |
2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2370 case 21: /* 8 bpp PRT. */
2371 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2372 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2375 NUM_BANKS(ADDR_SURF_16_BANK
) |
2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2380 case 22: /* 16 bpp PRT */
2381 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2382 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2385 NUM_BANKS(ADDR_SURF_16_BANK
) |
2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2390 case 23: /* 32 bpp PRT */
2391 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2395 NUM_BANKS(ADDR_SURF_16_BANK
) |
2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2400 case 24: /* 64 bpp PRT */
2401 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2405 NUM_BANKS(ADDR_SURF_16_BANK
) |
2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2410 case 25: /* 128 bpp PRT */
2411 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2415 NUM_BANKS(ADDR_SURF_8_BANK
) |
2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2424 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2425 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2428 DRM_ERROR("unknown asic: 0x%x\n", rdev
->family
);
2431 static void si_select_se_sh(struct radeon_device
*rdev
,
2432 u32 se_num
, u32 sh_num
)
2434 u32 data
= INSTANCE_BROADCAST_WRITES
;
2436 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
2437 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
2438 else if (se_num
== 0xffffffff)
2439 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
2440 else if (sh_num
== 0xffffffff)
2441 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
2443 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
2444 WREG32(GRBM_GFX_INDEX
, data
);
2447 static u32
si_create_bitmask(u32 bit_width
)
2451 for (i
= 0; i
< bit_width
; i
++) {
2458 static u32
si_get_cu_enabled(struct radeon_device
*rdev
, u32 cu_per_sh
)
2462 data
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
2464 data
&= INACTIVE_CUS_MASK
;
2467 data
|= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
2469 data
>>= INACTIVE_CUS_SHIFT
;
2471 mask
= si_create_bitmask(cu_per_sh
);
2473 return ~data
& mask
;
2476 static void si_setup_spi(struct radeon_device
*rdev
,
2477 u32 se_num
, u32 sh_per_se
,
2481 u32 data
, mask
, active_cu
;
2483 for (i
= 0; i
< se_num
; i
++) {
2484 for (j
= 0; j
< sh_per_se
; j
++) {
2485 si_select_se_sh(rdev
, i
, j
);
2486 data
= RREG32(SPI_STATIC_THREAD_MGMT_3
);
2487 active_cu
= si_get_cu_enabled(rdev
, cu_per_sh
);
2490 for (k
= 0; k
< 16; k
++) {
2492 if (active_cu
& mask
) {
2494 WREG32(SPI_STATIC_THREAD_MGMT_3
, data
);
2500 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2503 static u32
si_get_rb_disabled(struct radeon_device
*rdev
,
2504 u32 max_rb_num
, u32 se_num
,
2509 data
= RREG32(CC_RB_BACKEND_DISABLE
);
2511 data
&= BACKEND_DISABLE_MASK
;
2514 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
2516 data
>>= BACKEND_DISABLE_SHIFT
;
2518 mask
= si_create_bitmask(max_rb_num
/ se_num
/ sh_per_se
);
2523 static void si_setup_rb(struct radeon_device
*rdev
,
2524 u32 se_num
, u32 sh_per_se
,
2529 u32 disabled_rbs
= 0;
2530 u32 enabled_rbs
= 0;
2532 for (i
= 0; i
< se_num
; i
++) {
2533 for (j
= 0; j
< sh_per_se
; j
++) {
2534 si_select_se_sh(rdev
, i
, j
);
2535 data
= si_get_rb_disabled(rdev
, max_rb_num
, se_num
, sh_per_se
);
2536 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * TAHITI_RB_BITMAP_WIDTH_PER_SH
);
2539 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2542 for (i
= 0; i
< max_rb_num
; i
++) {
2543 if (!(disabled_rbs
& mask
))
2544 enabled_rbs
|= mask
;
2548 for (i
= 0; i
< se_num
; i
++) {
2549 si_select_se_sh(rdev
, i
, 0xffffffff);
2551 for (j
= 0; j
< sh_per_se
; j
++) {
2552 switch (enabled_rbs
& 3) {
2554 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
2557 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
2561 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
2566 WREG32(PA_SC_RASTER_CONFIG
, data
);
2568 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2571 static void si_gpu_init(struct radeon_device
*rdev
)
2573 u32 gb_addr_config
= 0;
2574 u32 mc_shared_chmap
, mc_arb_ramcfg
;
2576 u32 hdp_host_path_cntl
;
2580 switch (rdev
->family
) {
2582 rdev
->config
.si
.max_shader_engines
= 2;
2583 rdev
->config
.si
.max_tile_pipes
= 12;
2584 rdev
->config
.si
.max_cu_per_sh
= 8;
2585 rdev
->config
.si
.max_sh_per_se
= 2;
2586 rdev
->config
.si
.max_backends_per_se
= 4;
2587 rdev
->config
.si
.max_texture_channel_caches
= 12;
2588 rdev
->config
.si
.max_gprs
= 256;
2589 rdev
->config
.si
.max_gs_threads
= 32;
2590 rdev
->config
.si
.max_hw_contexts
= 8;
2592 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2593 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2594 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2595 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2596 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2599 rdev
->config
.si
.max_shader_engines
= 2;
2600 rdev
->config
.si
.max_tile_pipes
= 8;
2601 rdev
->config
.si
.max_cu_per_sh
= 5;
2602 rdev
->config
.si
.max_sh_per_se
= 2;
2603 rdev
->config
.si
.max_backends_per_se
= 4;
2604 rdev
->config
.si
.max_texture_channel_caches
= 8;
2605 rdev
->config
.si
.max_gprs
= 256;
2606 rdev
->config
.si
.max_gs_threads
= 32;
2607 rdev
->config
.si
.max_hw_contexts
= 8;
2609 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2610 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2611 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2612 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2613 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2617 rdev
->config
.si
.max_shader_engines
= 1;
2618 rdev
->config
.si
.max_tile_pipes
= 4;
2619 rdev
->config
.si
.max_cu_per_sh
= 5;
2620 rdev
->config
.si
.max_sh_per_se
= 2;
2621 rdev
->config
.si
.max_backends_per_se
= 4;
2622 rdev
->config
.si
.max_texture_channel_caches
= 4;
2623 rdev
->config
.si
.max_gprs
= 256;
2624 rdev
->config
.si
.max_gs_threads
= 32;
2625 rdev
->config
.si
.max_hw_contexts
= 8;
2627 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2628 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2629 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2630 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2631 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2634 rdev
->config
.si
.max_shader_engines
= 1;
2635 rdev
->config
.si
.max_tile_pipes
= 4;
2636 rdev
->config
.si
.max_cu_per_sh
= 6;
2637 rdev
->config
.si
.max_sh_per_se
= 1;
2638 rdev
->config
.si
.max_backends_per_se
= 2;
2639 rdev
->config
.si
.max_texture_channel_caches
= 4;
2640 rdev
->config
.si
.max_gprs
= 256;
2641 rdev
->config
.si
.max_gs_threads
= 16;
2642 rdev
->config
.si
.max_hw_contexts
= 8;
2644 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2645 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2646 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2647 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2648 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2651 rdev
->config
.si
.max_shader_engines
= 1;
2652 rdev
->config
.si
.max_tile_pipes
= 4;
2653 rdev
->config
.si
.max_cu_per_sh
= 5;
2654 rdev
->config
.si
.max_sh_per_se
= 1;
2655 rdev
->config
.si
.max_backends_per_se
= 1;
2656 rdev
->config
.si
.max_texture_channel_caches
= 2;
2657 rdev
->config
.si
.max_gprs
= 256;
2658 rdev
->config
.si
.max_gs_threads
= 16;
2659 rdev
->config
.si
.max_hw_contexts
= 8;
2661 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2662 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2663 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2664 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2665 gb_addr_config
= HAINAN_GB_ADDR_CONFIG_GOLDEN
;
2669 /* Initialize HDP */
2670 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
2671 WREG32((0x2c14 + j
), 0x00000000);
2672 WREG32((0x2c18 + j
), 0x00000000);
2673 WREG32((0x2c1c + j
), 0x00000000);
2674 WREG32((0x2c20 + j
), 0x00000000);
2675 WREG32((0x2c24 + j
), 0x00000000);
2678 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
2680 evergreen_fix_pci_max_read_req_size(rdev
);
2682 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
2684 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
2685 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
2687 rdev
->config
.si
.num_tile_pipes
= rdev
->config
.si
.max_tile_pipes
;
2688 rdev
->config
.si
.mem_max_burst_length_bytes
= 256;
2689 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
2690 rdev
->config
.si
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
2691 if (rdev
->config
.si
.mem_row_size_in_kb
> 4)
2692 rdev
->config
.si
.mem_row_size_in_kb
= 4;
2693 /* XXX use MC settings? */
2694 rdev
->config
.si
.shader_engine_tile_size
= 32;
2695 rdev
->config
.si
.num_gpus
= 1;
2696 rdev
->config
.si
.multi_gpu_tile_size
= 64;
2698 /* fix up row size */
2699 gb_addr_config
&= ~ROW_SIZE_MASK
;
2700 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
2703 gb_addr_config
|= ROW_SIZE(0);
2706 gb_addr_config
|= ROW_SIZE(1);
2709 gb_addr_config
|= ROW_SIZE(2);
2713 /* setup tiling info dword. gb_addr_config is not adequate since it does
2714 * not have bank info, so create a custom tiling dword.
2715 * bits 3:0 num_pipes
2716 * bits 7:4 num_banks
2717 * bits 11:8 group_size
2718 * bits 15:12 row_size
2720 rdev
->config
.si
.tile_config
= 0;
2721 switch (rdev
->config
.si
.num_tile_pipes
) {
2723 rdev
->config
.si
.tile_config
|= (0 << 0);
2726 rdev
->config
.si
.tile_config
|= (1 << 0);
2729 rdev
->config
.si
.tile_config
|= (2 << 0);
2733 /* XXX what about 12? */
2734 rdev
->config
.si
.tile_config
|= (3 << 0);
2737 switch ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) {
2738 case 0: /* four banks */
2739 rdev
->config
.si
.tile_config
|= 0 << 4;
2741 case 1: /* eight banks */
2742 rdev
->config
.si
.tile_config
|= 1 << 4;
2744 case 2: /* sixteen banks */
2746 rdev
->config
.si
.tile_config
|= 2 << 4;
2749 rdev
->config
.si
.tile_config
|=
2750 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
2751 rdev
->config
.si
.tile_config
|=
2752 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
2754 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
2755 WREG32(DMIF_ADDR_CONFIG
, gb_addr_config
);
2756 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
2757 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
2758 WREG32(DMA_TILING_CONFIG
+ DMA0_REGISTER_OFFSET
, gb_addr_config
);
2759 WREG32(DMA_TILING_CONFIG
+ DMA1_REGISTER_OFFSET
, gb_addr_config
);
2760 if (rdev
->has_uvd
) {
2761 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
2762 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
2763 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
2766 si_tiling_mode_table_init(rdev
);
2768 si_setup_rb(rdev
, rdev
->config
.si
.max_shader_engines
,
2769 rdev
->config
.si
.max_sh_per_se
,
2770 rdev
->config
.si
.max_backends_per_se
);
2772 si_setup_spi(rdev
, rdev
->config
.si
.max_shader_engines
,
2773 rdev
->config
.si
.max_sh_per_se
,
2774 rdev
->config
.si
.max_cu_per_sh
);
2777 /* set HW defaults for 3D engine */
2778 WREG32(CP_QUEUE_THRESHOLDS
, (ROQ_IB1_START(0x16) |
2779 ROQ_IB2_START(0x2b)));
2780 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
2782 sx_debug_1
= RREG32(SX_DEBUG_1
);
2783 WREG32(SX_DEBUG_1
, sx_debug_1
);
2785 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
2787 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_frontend
) |
2788 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_backend
) |
2789 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_hiz_tile_fifo_size
) |
2790 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_earlyz_tile_fifo_size
)));
2792 WREG32(VGT_NUM_INSTANCES
, 1);
2794 WREG32(CP_PERFMON_CNTL
, 0);
2796 WREG32(SQ_CONFIG
, 0);
2798 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
2799 FORCE_EOV_MAX_REZ_CNT(255)));
2801 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
2802 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
2804 WREG32(VGT_GS_VERTEX_REUSE
, 16);
2805 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
2807 WREG32(CB_PERFCOUNTER0_SELECT0
, 0);
2808 WREG32(CB_PERFCOUNTER0_SELECT1
, 0);
2809 WREG32(CB_PERFCOUNTER1_SELECT0
, 0);
2810 WREG32(CB_PERFCOUNTER1_SELECT1
, 0);
2811 WREG32(CB_PERFCOUNTER2_SELECT0
, 0);
2812 WREG32(CB_PERFCOUNTER2_SELECT1
, 0);
2813 WREG32(CB_PERFCOUNTER3_SELECT0
, 0);
2814 WREG32(CB_PERFCOUNTER3_SELECT1
, 0);
2816 tmp
= RREG32(HDP_MISC_CNTL
);
2817 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
2818 WREG32(HDP_MISC_CNTL
, tmp
);
2820 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
2821 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
2823 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
2829 * GPU scratch registers helpers function.
2831 static void si_scratch_init(struct radeon_device
*rdev
)
2835 rdev
->scratch
.num_reg
= 7;
2836 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
2837 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
2838 rdev
->scratch
.free
[i
] = true;
2839 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
2843 void si_fence_ring_emit(struct radeon_device
*rdev
,
2844 struct radeon_fence
*fence
)
2846 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
2847 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
2849 /* flush read cache over gart */
2850 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
2851 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
2852 radeon_ring_write(ring
, 0);
2853 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
2854 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
2855 PACKET3_TC_ACTION_ENA
|
2856 PACKET3_SH_KCACHE_ACTION_ENA
|
2857 PACKET3_SH_ICACHE_ACTION_ENA
);
2858 radeon_ring_write(ring
, 0xFFFFFFFF);
2859 radeon_ring_write(ring
, 0);
2860 radeon_ring_write(ring
, 10); /* poll interval */
2861 /* EVENT_WRITE_EOP - flush caches, send int */
2862 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
2863 radeon_ring_write(ring
, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) | EVENT_INDEX(5));
2864 radeon_ring_write(ring
, addr
& 0xffffffff);
2865 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2866 radeon_ring_write(ring
, fence
->seq
);
2867 radeon_ring_write(ring
, 0);
2873 void si_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
2875 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
2878 if (ib
->is_const_ib
) {
2879 /* set switch buffer packet before const IB */
2880 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
2881 radeon_ring_write(ring
, 0);
2883 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
2886 if (ring
->rptr_save_reg
) {
2887 next_rptr
= ring
->wptr
+ 3 + 4 + 8;
2888 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
2889 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
2890 PACKET3_SET_CONFIG_REG_START
) >> 2));
2891 radeon_ring_write(ring
, next_rptr
);
2892 } else if (rdev
->wb
.enabled
) {
2893 next_rptr
= ring
->wptr
+ 5 + 4 + 8;
2894 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
2895 radeon_ring_write(ring
, (1 << 8));
2896 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
2897 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
2898 radeon_ring_write(ring
, next_rptr
);
2901 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
2904 radeon_ring_write(ring
, header
);
2905 radeon_ring_write(ring
,
2909 (ib
->gpu_addr
& 0xFFFFFFFC));
2910 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
2911 radeon_ring_write(ring
, ib
->length_dw
|
2912 (ib
->vm
? (ib
->vm
->id
<< 24) : 0));
2914 if (!ib
->is_const_ib
) {
2915 /* flush read cache over gart for this vmid */
2916 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
2917 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
2918 radeon_ring_write(ring
, ib
->vm
? ib
->vm
->id
: 0);
2919 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
2920 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
2921 PACKET3_TC_ACTION_ENA
|
2922 PACKET3_SH_KCACHE_ACTION_ENA
|
2923 PACKET3_SH_ICACHE_ACTION_ENA
);
2924 radeon_ring_write(ring
, 0xFFFFFFFF);
2925 radeon_ring_write(ring
, 0);
2926 radeon_ring_write(ring
, 10); /* poll interval */
2933 static void si_cp_enable(struct radeon_device
*rdev
, bool enable
)
2936 WREG32(CP_ME_CNTL
, 0);
2938 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
2939 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
2940 WREG32(SCRATCH_UMSK
, 0);
2941 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
2942 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
2943 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
2948 static int si_cp_load_microcode(struct radeon_device
*rdev
)
2950 const __be32
*fw_data
;
2953 if (!rdev
->me_fw
|| !rdev
->pfp_fw
)
2956 si_cp_enable(rdev
, false);
2959 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
2960 WREG32(CP_PFP_UCODE_ADDR
, 0);
2961 for (i
= 0; i
< SI_PFP_UCODE_SIZE
; i
++)
2962 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
2963 WREG32(CP_PFP_UCODE_ADDR
, 0);
2966 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
2967 WREG32(CP_CE_UCODE_ADDR
, 0);
2968 for (i
= 0; i
< SI_CE_UCODE_SIZE
; i
++)
2969 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
2970 WREG32(CP_CE_UCODE_ADDR
, 0);
2973 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
2974 WREG32(CP_ME_RAM_WADDR
, 0);
2975 for (i
= 0; i
< SI_PM4_UCODE_SIZE
; i
++)
2976 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
2977 WREG32(CP_ME_RAM_WADDR
, 0);
2979 WREG32(CP_PFP_UCODE_ADDR
, 0);
2980 WREG32(CP_CE_UCODE_ADDR
, 0);
2981 WREG32(CP_ME_RAM_WADDR
, 0);
2982 WREG32(CP_ME_RAM_RADDR
, 0);
2986 static int si_cp_start(struct radeon_device
*rdev
)
2988 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
2991 r
= radeon_ring_lock(rdev
, ring
, 7 + 4);
2993 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
2997 radeon_ring_write(ring
, PACKET3(PACKET3_ME_INITIALIZE
, 5));
2998 radeon_ring_write(ring
, 0x1);
2999 radeon_ring_write(ring
, 0x0);
3000 radeon_ring_write(ring
, rdev
->config
.si
.max_hw_contexts
- 1);
3001 radeon_ring_write(ring
, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3002 radeon_ring_write(ring
, 0);
3003 radeon_ring_write(ring
, 0);
3005 /* init the CE partitions */
3006 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3007 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3008 radeon_ring_write(ring
, 0xc000);
3009 radeon_ring_write(ring
, 0xe000);
3010 radeon_ring_unlock_commit(rdev
, ring
);
3012 si_cp_enable(rdev
, true);
3014 r
= radeon_ring_lock(rdev
, ring
, si_default_size
+ 10);
3016 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3020 /* setup clear context state */
3021 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3022 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3024 for (i
= 0; i
< si_default_size
; i
++)
3025 radeon_ring_write(ring
, si_default_state
[i
]);
3027 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3028 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3030 /* set clear context state */
3031 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3032 radeon_ring_write(ring
, 0);
3034 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3035 radeon_ring_write(ring
, 0x00000316);
3036 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3037 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3039 radeon_ring_unlock_commit(rdev
, ring
);
3041 for (i
= RADEON_RING_TYPE_GFX_INDEX
; i
<= CAYMAN_RING_TYPE_CP2_INDEX
; ++i
) {
3042 ring
= &rdev
->ring
[i
];
3043 r
= radeon_ring_lock(rdev
, ring
, 2);
3045 /* clear the compute context state */
3046 radeon_ring_write(ring
, PACKET3_COMPUTE(PACKET3_CLEAR_STATE
, 0));
3047 radeon_ring_write(ring
, 0);
3049 radeon_ring_unlock_commit(rdev
, ring
);
3055 static void si_cp_fini(struct radeon_device
*rdev
)
3057 struct radeon_ring
*ring
;
3058 si_cp_enable(rdev
, false);
3060 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3061 radeon_ring_fini(rdev
, ring
);
3062 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3064 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3065 radeon_ring_fini(rdev
, ring
);
3066 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3068 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3069 radeon_ring_fini(rdev
, ring
);
3070 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3073 static int si_cp_resume(struct radeon_device
*rdev
)
3075 struct radeon_ring
*ring
;
3080 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3081 WREG32(GRBM_SOFT_RESET
, (SOFT_RESET_CP
|
3086 RREG32(GRBM_SOFT_RESET
);
3088 WREG32(GRBM_SOFT_RESET
, 0);
3089 RREG32(GRBM_SOFT_RESET
);
3091 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
3092 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
3094 /* Set the write pointer delay */
3095 WREG32(CP_RB_WPTR_DELAY
, 0);
3097 WREG32(CP_DEBUG
, 0);
3098 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
3100 /* ring 0 - compute and gfx */
3101 /* Set ring buffer size */
3102 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3103 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3104 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3106 tmp
|= BUF_SWAP_32BIT
;
3108 WREG32(CP_RB0_CNTL
, tmp
);
3110 /* Initialize the ring buffer's read and write pointers */
3111 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3113 WREG32(CP_RB0_WPTR
, ring
->wptr
);
3115 /* set the wb address whether it's enabled or not */
3116 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
3117 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
3119 if (rdev
->wb
.enabled
)
3120 WREG32(SCRATCH_UMSK
, 0xff);
3122 tmp
|= RB_NO_UPDATE
;
3123 WREG32(SCRATCH_UMSK
, 0);
3127 WREG32(CP_RB0_CNTL
, tmp
);
3129 WREG32(CP_RB0_BASE
, ring
->gpu_addr
>> 8);
3131 ring
->rptr
= RREG32(CP_RB0_RPTR
);
3133 /* ring1 - compute only */
3134 /* Set ring buffer size */
3135 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3136 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3137 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3139 tmp
|= BUF_SWAP_32BIT
;
3141 WREG32(CP_RB1_CNTL
, tmp
);
3143 /* Initialize the ring buffer's read and write pointers */
3144 WREG32(CP_RB1_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3146 WREG32(CP_RB1_WPTR
, ring
->wptr
);
3148 /* set the wb address whether it's enabled or not */
3149 WREG32(CP_RB1_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFFFFFFFC);
3150 WREG32(CP_RB1_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFF);
3153 WREG32(CP_RB1_CNTL
, tmp
);
3155 WREG32(CP_RB1_BASE
, ring
->gpu_addr
>> 8);
3157 ring
->rptr
= RREG32(CP_RB1_RPTR
);
3159 /* ring2 - compute only */
3160 /* Set ring buffer size */
3161 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3162 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3163 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3165 tmp
|= BUF_SWAP_32BIT
;
3167 WREG32(CP_RB2_CNTL
, tmp
);
3169 /* Initialize the ring buffer's read and write pointers */
3170 WREG32(CP_RB2_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3172 WREG32(CP_RB2_WPTR
, ring
->wptr
);
3174 /* set the wb address whether it's enabled or not */
3175 WREG32(CP_RB2_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFFFFFFFC);
3176 WREG32(CP_RB2_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFF);
3179 WREG32(CP_RB2_CNTL
, tmp
);
3181 WREG32(CP_RB2_BASE
, ring
->gpu_addr
>> 8);
3183 ring
->rptr
= RREG32(CP_RB2_RPTR
);
3185 /* start the rings */
3187 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
3188 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= true;
3189 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= true;
3190 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3192 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3193 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3194 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3197 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
]);
3199 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3201 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
]);
3203 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3209 static u32
si_gpu_check_soft_reset(struct radeon_device
*rdev
)
3215 tmp
= RREG32(GRBM_STATUS
);
3216 if (tmp
& (PA_BUSY
| SC_BUSY
|
3217 BCI_BUSY
| SX_BUSY
|
3218 TA_BUSY
| VGT_BUSY
|
3220 GDS_BUSY
| SPI_BUSY
|
3221 IA_BUSY
| IA_BUSY_NO_DMA
))
3222 reset_mask
|= RADEON_RESET_GFX
;
3224 if (tmp
& (CF_RQ_PENDING
| PF_RQ_PENDING
|
3225 CP_BUSY
| CP_COHERENCY_BUSY
))
3226 reset_mask
|= RADEON_RESET_CP
;
3228 if (tmp
& GRBM_EE_BUSY
)
3229 reset_mask
|= RADEON_RESET_GRBM
| RADEON_RESET_GFX
| RADEON_RESET_CP
;
3232 tmp
= RREG32(GRBM_STATUS2
);
3233 if (tmp
& (RLC_RQ_PENDING
| RLC_BUSY
))
3234 reset_mask
|= RADEON_RESET_RLC
;
3236 /* DMA_STATUS_REG 0 */
3237 tmp
= RREG32(DMA_STATUS_REG
+ DMA0_REGISTER_OFFSET
);
3238 if (!(tmp
& DMA_IDLE
))
3239 reset_mask
|= RADEON_RESET_DMA
;
3241 /* DMA_STATUS_REG 1 */
3242 tmp
= RREG32(DMA_STATUS_REG
+ DMA1_REGISTER_OFFSET
);
3243 if (!(tmp
& DMA_IDLE
))
3244 reset_mask
|= RADEON_RESET_DMA1
;
3247 tmp
= RREG32(SRBM_STATUS2
);
3249 reset_mask
|= RADEON_RESET_DMA
;
3251 if (tmp
& DMA1_BUSY
)
3252 reset_mask
|= RADEON_RESET_DMA1
;
3255 tmp
= RREG32(SRBM_STATUS
);
3258 reset_mask
|= RADEON_RESET_IH
;
3261 reset_mask
|= RADEON_RESET_SEM
;
3263 if (tmp
& GRBM_RQ_PENDING
)
3264 reset_mask
|= RADEON_RESET_GRBM
;
3267 reset_mask
|= RADEON_RESET_VMC
;
3269 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
3270 MCC_BUSY
| MCD_BUSY
))
3271 reset_mask
|= RADEON_RESET_MC
;
3273 if (evergreen_is_display_hung(rdev
))
3274 reset_mask
|= RADEON_RESET_DISPLAY
;
3277 tmp
= RREG32(VM_L2_STATUS
);
3279 reset_mask
|= RADEON_RESET_VMC
;
3281 /* Skip MC reset as it's mostly likely not hung, just busy */
3282 if (reset_mask
& RADEON_RESET_MC
) {
3283 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
3284 reset_mask
&= ~RADEON_RESET_MC
;
3290 static void si_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
3292 struct evergreen_mc_save save
;
3293 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
3296 if (reset_mask
== 0)
3299 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
3301 evergreen_print_gpu_status_regs(rdev
);
3302 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3303 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
3304 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3305 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
3307 /* Disable CP parsing/prefetching */
3308 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
3310 if (reset_mask
& RADEON_RESET_DMA
) {
3312 tmp
= RREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
);
3313 tmp
&= ~DMA_RB_ENABLE
;
3314 WREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
3316 if (reset_mask
& RADEON_RESET_DMA1
) {
3318 tmp
= RREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
);
3319 tmp
&= ~DMA_RB_ENABLE
;
3320 WREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
3325 evergreen_mc_stop(rdev
, &save
);
3326 if (evergreen_mc_wait_for_idle(rdev
)) {
3327 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3330 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
)) {
3331 grbm_soft_reset
= SOFT_RESET_CB
|
3345 if (reset_mask
& RADEON_RESET_CP
) {
3346 grbm_soft_reset
|= SOFT_RESET_CP
| SOFT_RESET_VGT
;
3348 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3351 if (reset_mask
& RADEON_RESET_DMA
)
3352 srbm_soft_reset
|= SOFT_RESET_DMA
;
3354 if (reset_mask
& RADEON_RESET_DMA1
)
3355 srbm_soft_reset
|= SOFT_RESET_DMA1
;
3357 if (reset_mask
& RADEON_RESET_DISPLAY
)
3358 srbm_soft_reset
|= SOFT_RESET_DC
;
3360 if (reset_mask
& RADEON_RESET_RLC
)
3361 grbm_soft_reset
|= SOFT_RESET_RLC
;
3363 if (reset_mask
& RADEON_RESET_SEM
)
3364 srbm_soft_reset
|= SOFT_RESET_SEM
;
3366 if (reset_mask
& RADEON_RESET_IH
)
3367 srbm_soft_reset
|= SOFT_RESET_IH
;
3369 if (reset_mask
& RADEON_RESET_GRBM
)
3370 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3372 if (reset_mask
& RADEON_RESET_VMC
)
3373 srbm_soft_reset
|= SOFT_RESET_VMC
;
3375 if (reset_mask
& RADEON_RESET_MC
)
3376 srbm_soft_reset
|= SOFT_RESET_MC
;
3378 if (grbm_soft_reset
) {
3379 tmp
= RREG32(GRBM_SOFT_RESET
);
3380 tmp
|= grbm_soft_reset
;
3381 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
3382 WREG32(GRBM_SOFT_RESET
, tmp
);
3383 tmp
= RREG32(GRBM_SOFT_RESET
);
3387 tmp
&= ~grbm_soft_reset
;
3388 WREG32(GRBM_SOFT_RESET
, tmp
);
3389 tmp
= RREG32(GRBM_SOFT_RESET
);
3392 if (srbm_soft_reset
) {
3393 tmp
= RREG32(SRBM_SOFT_RESET
);
3394 tmp
|= srbm_soft_reset
;
3395 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
3396 WREG32(SRBM_SOFT_RESET
, tmp
);
3397 tmp
= RREG32(SRBM_SOFT_RESET
);
3401 tmp
&= ~srbm_soft_reset
;
3402 WREG32(SRBM_SOFT_RESET
, tmp
);
3403 tmp
= RREG32(SRBM_SOFT_RESET
);
3406 /* Wait a little for things to settle down */
3409 evergreen_mc_resume(rdev
, &save
);
3412 evergreen_print_gpu_status_regs(rdev
);
3415 int si_asic_reset(struct radeon_device
*rdev
)
3419 reset_mask
= si_gpu_check_soft_reset(rdev
);
3422 r600_set_bios_scratch_engine_hung(rdev
, true);
3424 si_gpu_soft_reset(rdev
, reset_mask
);
3426 reset_mask
= si_gpu_check_soft_reset(rdev
);
3429 r600_set_bios_scratch_engine_hung(rdev
, false);
3435 * si_gfx_is_lockup - Check if the GFX engine is locked up
3437 * @rdev: radeon_device pointer
3438 * @ring: radeon_ring structure holding ring information
3440 * Check if the GFX engine is locked up.
3441 * Returns true if the engine appears to be locked up, false if not.
3443 bool si_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3445 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3447 if (!(reset_mask
& (RADEON_RESET_GFX
|
3448 RADEON_RESET_COMPUTE
|
3449 RADEON_RESET_CP
))) {
3450 radeon_ring_lockup_update(ring
);
3453 /* force CP activities */
3454 radeon_ring_force_activity(rdev
, ring
);
3455 return radeon_ring_test_lockup(rdev
, ring
);
3459 * si_dma_is_lockup - Check if the DMA engine is locked up
3461 * @rdev: radeon_device pointer
3462 * @ring: radeon_ring structure holding ring information
3464 * Check if the async DMA engine is locked up.
3465 * Returns true if the engine appears to be locked up, false if not.
3467 bool si_dma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3469 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3472 if (ring
->idx
== R600_RING_TYPE_DMA_INDEX
)
3473 mask
= RADEON_RESET_DMA
;
3475 mask
= RADEON_RESET_DMA1
;
3477 if (!(reset_mask
& mask
)) {
3478 radeon_ring_lockup_update(ring
);
3481 /* force ring activities */
3482 radeon_ring_force_activity(rdev
, ring
);
3483 return radeon_ring_test_lockup(rdev
, ring
);
3487 static void si_mc_program(struct radeon_device
*rdev
)
3489 struct evergreen_mc_save save
;
3493 /* Initialize HDP */
3494 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
3495 WREG32((0x2c14 + j
), 0x00000000);
3496 WREG32((0x2c18 + j
), 0x00000000);
3497 WREG32((0x2c1c + j
), 0x00000000);
3498 WREG32((0x2c20 + j
), 0x00000000);
3499 WREG32((0x2c24 + j
), 0x00000000);
3501 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
3503 evergreen_mc_stop(rdev
, &save
);
3504 if (radeon_mc_wait_for_idle(rdev
)) {
3505 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3507 if (!ASIC_IS_NODCE(rdev
))
3508 /* Lockout access through VGA aperture*/
3509 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
3510 /* Update configuration */
3511 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
3512 rdev
->mc
.vram_start
>> 12);
3513 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
3514 rdev
->mc
.vram_end
>> 12);
3515 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
3516 rdev
->vram_scratch
.gpu_addr
>> 12);
3517 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
3518 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
3519 WREG32(MC_VM_FB_LOCATION
, tmp
);
3520 /* XXX double check these! */
3521 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
3522 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
3523 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
3524 WREG32(MC_VM_AGP_BASE
, 0);
3525 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
3526 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
3527 if (radeon_mc_wait_for_idle(rdev
)) {
3528 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3530 evergreen_mc_resume(rdev
, &save
);
3531 if (!ASIC_IS_NODCE(rdev
)) {
3532 /* we need to own VRAM, so turn off the VGA renderer here
3533 * to stop it overwriting our objects */
3534 rv515_vga_render_disable(rdev
);
3538 static void si_vram_gtt_location(struct radeon_device
*rdev
,
3539 struct radeon_mc
*mc
)
3541 if (mc
->mc_vram_size
> 0xFFC0000000ULL
) {
3542 /* leave room for at least 1024M GTT */
3543 dev_warn(rdev
->dev
, "limiting VRAM\n");
3544 mc
->real_vram_size
= 0xFFC0000000ULL
;
3545 mc
->mc_vram_size
= 0xFFC0000000ULL
;
3547 radeon_vram_location(rdev
, &rdev
->mc
, 0);
3548 rdev
->mc
.gtt_base_align
= 0;
3549 radeon_gtt_location(rdev
, mc
);
3552 static int si_mc_init(struct radeon_device
*rdev
)
3555 int chansize
, numchan
;
3557 /* Get VRAM informations */
3558 rdev
->mc
.vram_is_ddr
= true;
3559 tmp
= RREG32(MC_ARB_RAMCFG
);
3560 if (tmp
& CHANSIZE_OVERRIDE
) {
3562 } else if (tmp
& CHANSIZE_MASK
) {
3567 tmp
= RREG32(MC_SHARED_CHMAP
);
3568 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
3598 rdev
->mc
.vram_width
= numchan
* chansize
;
3599 /* Could aper size report 0 ? */
3600 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
3601 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
3602 /* size in MB on si */
3603 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3604 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3605 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
3606 si_vram_gtt_location(rdev
, &rdev
->mc
);
3607 radeon_update_bandwidth_info(rdev
);
3615 void si_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
3617 /* flush hdp cache */
3618 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0x1);
3620 /* bits 0-15 are the VM contexts0-15 */
3621 WREG32(VM_INVALIDATE_REQUEST
, 1);
3624 static int si_pcie_gart_enable(struct radeon_device
*rdev
)
3628 if (rdev
->gart
.robj
== NULL
) {
3629 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
3632 r
= radeon_gart_table_vram_pin(rdev
);
3635 radeon_gart_restore(rdev
);
3636 /* Setup TLB control */
3637 WREG32(MC_VM_MX_L1_TLB_CNTL
,
3640 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
3641 ENABLE_ADVANCED_DRIVER_MODEL
|
3642 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
3643 /* Setup L2 cache */
3644 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
3645 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
3646 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
3647 EFFECTIVE_L2_QUEUE_SIZE(7) |
3648 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3649 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
3650 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
3651 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3652 /* setup context0 */
3653 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
3654 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
3655 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
3656 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
3657 (u32
)(rdev
->dummy_page
.addr
>> 12));
3658 WREG32(VM_CONTEXT0_CNTL2
, 0);
3659 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
3660 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
3666 /* empty context1-15 */
3667 /* set vm size, must be a multiple of 4 */
3668 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
3669 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
3670 /* Assign the pt base to something valid for now; the pts used for
3671 * the VMs are determined by the application and setup and assigned
3672 * on the fly in the vm part of radeon_gart.c
3674 for (i
= 1; i
< 16; i
++) {
3676 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
3677 rdev
->gart
.table_addr
>> 12);
3679 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
3680 rdev
->gart
.table_addr
>> 12);
3683 /* enable context1-15 */
3684 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
3685 (u32
)(rdev
->dummy_page
.addr
>> 12));
3686 WREG32(VM_CONTEXT1_CNTL2
, 4);
3687 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
3688 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3689 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3690 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3691 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3692 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3693 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
3694 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3695 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
3696 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3697 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
3698 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3699 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
3701 si_pcie_gart_tlb_flush(rdev
);
3702 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3703 (unsigned)(rdev
->mc
.gtt_size
>> 20),
3704 (unsigned long long)rdev
->gart
.table_addr
);
3705 rdev
->gart
.ready
= true;
3709 static void si_pcie_gart_disable(struct radeon_device
*rdev
)
3711 /* Disable all tables */
3712 WREG32(VM_CONTEXT0_CNTL
, 0);
3713 WREG32(VM_CONTEXT1_CNTL
, 0);
3714 /* Setup TLB control */
3715 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
3716 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
3717 /* Setup L2 cache */
3718 WREG32(VM_L2_CNTL
, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
3719 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
3720 EFFECTIVE_L2_QUEUE_SIZE(7) |
3721 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3722 WREG32(VM_L2_CNTL2
, 0);
3723 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
3724 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3725 radeon_gart_table_vram_unpin(rdev
);
3728 static void si_pcie_gart_fini(struct radeon_device
*rdev
)
3730 si_pcie_gart_disable(rdev
);
3731 radeon_gart_table_vram_free(rdev
);
3732 radeon_gart_fini(rdev
);
3736 static bool si_vm_reg_valid(u32 reg
)
3738 /* context regs are fine */
3742 /* check config regs */
3744 case GRBM_GFX_INDEX
:
3745 case CP_STRMOUT_CNTL
:
3746 case VGT_VTX_VECT_EJECT_REG
:
3747 case VGT_CACHE_INVALIDATION
:
3748 case VGT_ESGS_RING_SIZE
:
3749 case VGT_GSVS_RING_SIZE
:
3750 case VGT_GS_VERTEX_REUSE
:
3751 case VGT_PRIMITIVE_TYPE
:
3752 case VGT_INDEX_TYPE
:
3753 case VGT_NUM_INDICES
:
3754 case VGT_NUM_INSTANCES
:
3755 case VGT_TF_RING_SIZE
:
3756 case VGT_HS_OFFCHIP_PARAM
:
3757 case VGT_TF_MEMORY_BASE
:
3759 case PA_SU_LINE_STIPPLE_VALUE
:
3760 case PA_SC_LINE_STIPPLE_STATE
:
3763 case SPI_STATIC_THREAD_MGMT_1
:
3764 case SPI_STATIC_THREAD_MGMT_2
:
3765 case SPI_STATIC_THREAD_MGMT_3
:
3766 case SPI_PS_MAX_WAVE_ID
:
3767 case SPI_CONFIG_CNTL
:
3768 case SPI_CONFIG_CNTL_1
:
3772 DRM_ERROR("Invalid register 0x%x in CS\n", reg
);
3777 static int si_vm_packet3_ce_check(struct radeon_device
*rdev
,
3778 u32
*ib
, struct radeon_cs_packet
*pkt
)
3780 switch (pkt
->opcode
) {
3782 case PACKET3_SET_BASE
:
3783 case PACKET3_SET_CE_DE_COUNTERS
:
3784 case PACKET3_LOAD_CONST_RAM
:
3785 case PACKET3_WRITE_CONST_RAM
:
3786 case PACKET3_WRITE_CONST_RAM_OFFSET
:
3787 case PACKET3_DUMP_CONST_RAM
:
3788 case PACKET3_INCREMENT_CE_COUNTER
:
3789 case PACKET3_WAIT_ON_DE_COUNTER
:
3790 case PACKET3_CE_WRITE
:
3793 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt
->opcode
);
3799 static int si_vm_packet3_gfx_check(struct radeon_device
*rdev
,
3800 u32
*ib
, struct radeon_cs_packet
*pkt
)
3802 u32 idx
= pkt
->idx
+ 1;
3803 u32 idx_value
= ib
[idx
];
3804 u32 start_reg
, end_reg
, reg
, i
;
3807 switch (pkt
->opcode
) {
3809 case PACKET3_SET_BASE
:
3810 case PACKET3_CLEAR_STATE
:
3811 case PACKET3_INDEX_BUFFER_SIZE
:
3812 case PACKET3_DISPATCH_DIRECT
:
3813 case PACKET3_DISPATCH_INDIRECT
:
3814 case PACKET3_ALLOC_GDS
:
3815 case PACKET3_WRITE_GDS_RAM
:
3816 case PACKET3_ATOMIC_GDS
:
3817 case PACKET3_ATOMIC
:
3818 case PACKET3_OCCLUSION_QUERY
:
3819 case PACKET3_SET_PREDICATION
:
3820 case PACKET3_COND_EXEC
:
3821 case PACKET3_PRED_EXEC
:
3822 case PACKET3_DRAW_INDIRECT
:
3823 case PACKET3_DRAW_INDEX_INDIRECT
:
3824 case PACKET3_INDEX_BASE
:
3825 case PACKET3_DRAW_INDEX_2
:
3826 case PACKET3_CONTEXT_CONTROL
:
3827 case PACKET3_INDEX_TYPE
:
3828 case PACKET3_DRAW_INDIRECT_MULTI
:
3829 case PACKET3_DRAW_INDEX_AUTO
:
3830 case PACKET3_DRAW_INDEX_IMMD
:
3831 case PACKET3_NUM_INSTANCES
:
3832 case PACKET3_DRAW_INDEX_MULTI_AUTO
:
3833 case PACKET3_STRMOUT_BUFFER_UPDATE
:
3834 case PACKET3_DRAW_INDEX_OFFSET_2
:
3835 case PACKET3_DRAW_INDEX_MULTI_ELEMENT
:
3836 case PACKET3_DRAW_INDEX_INDIRECT_MULTI
:
3837 case PACKET3_MPEG_INDEX
:
3838 case PACKET3_WAIT_REG_MEM
:
3839 case PACKET3_MEM_WRITE
:
3840 case PACKET3_PFP_SYNC_ME
:
3841 case PACKET3_SURFACE_SYNC
:
3842 case PACKET3_EVENT_WRITE
:
3843 case PACKET3_EVENT_WRITE_EOP
:
3844 case PACKET3_EVENT_WRITE_EOS
:
3845 case PACKET3_SET_CONTEXT_REG
:
3846 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
3847 case PACKET3_SET_SH_REG
:
3848 case PACKET3_SET_SH_REG_OFFSET
:
3849 case PACKET3_INCREMENT_DE_COUNTER
:
3850 case PACKET3_WAIT_ON_CE_COUNTER
:
3851 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
3852 case PACKET3_ME_WRITE
:
3854 case PACKET3_COPY_DATA
:
3855 if ((idx_value
& 0xf00) == 0) {
3856 reg
= ib
[idx
+ 3] * 4;
3857 if (!si_vm_reg_valid(reg
))
3861 case PACKET3_WRITE_DATA
:
3862 if ((idx_value
& 0xf00) == 0) {
3863 start_reg
= ib
[idx
+ 1] * 4;
3864 if (idx_value
& 0x10000) {
3865 if (!si_vm_reg_valid(start_reg
))
3868 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
3869 reg
= start_reg
+ (4 * i
);
3870 if (!si_vm_reg_valid(reg
))
3876 case PACKET3_COND_WRITE
:
3877 if (idx_value
& 0x100) {
3878 reg
= ib
[idx
+ 5] * 4;
3879 if (!si_vm_reg_valid(reg
))
3883 case PACKET3_COPY_DW
:
3884 if (idx_value
& 0x2) {
3885 reg
= ib
[idx
+ 3] * 4;
3886 if (!si_vm_reg_valid(reg
))
3890 case PACKET3_SET_CONFIG_REG
:
3891 start_reg
= (idx_value
<< 2) + PACKET3_SET_CONFIG_REG_START
;
3892 end_reg
= 4 * pkt
->count
+ start_reg
- 4;
3893 if ((start_reg
< PACKET3_SET_CONFIG_REG_START
) ||
3894 (start_reg
>= PACKET3_SET_CONFIG_REG_END
) ||
3895 (end_reg
>= PACKET3_SET_CONFIG_REG_END
)) {
3896 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3899 for (i
= 0; i
< pkt
->count
; i
++) {
3900 reg
= start_reg
+ (4 * i
);
3901 if (!si_vm_reg_valid(reg
))
3905 case PACKET3_CP_DMA
:
3906 command
= ib
[idx
+ 4];
3908 if (command
& PACKET3_CP_DMA_CMD_SAS
) {
3909 /* src address space is register */
3910 if (((info
& 0x60000000) >> 29) == 0) {
3911 start_reg
= idx_value
<< 2;
3912 if (command
& PACKET3_CP_DMA_CMD_SAIC
) {
3914 if (!si_vm_reg_valid(reg
)) {
3915 DRM_ERROR("CP DMA Bad SRC register\n");
3919 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
3920 reg
= start_reg
+ (4 * i
);
3921 if (!si_vm_reg_valid(reg
)) {
3922 DRM_ERROR("CP DMA Bad SRC register\n");
3929 if (command
& PACKET3_CP_DMA_CMD_DAS
) {
3930 /* dst address space is register */
3931 if (((info
& 0x00300000) >> 20) == 0) {
3932 start_reg
= ib
[idx
+ 2];
3933 if (command
& PACKET3_CP_DMA_CMD_DAIC
) {
3935 if (!si_vm_reg_valid(reg
)) {
3936 DRM_ERROR("CP DMA Bad DST register\n");
3940 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
3941 reg
= start_reg
+ (4 * i
);
3942 if (!si_vm_reg_valid(reg
)) {
3943 DRM_ERROR("CP DMA Bad DST register\n");
3952 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt
->opcode
);
3958 static int si_vm_packet3_compute_check(struct radeon_device
*rdev
,
3959 u32
*ib
, struct radeon_cs_packet
*pkt
)
3961 u32 idx
= pkt
->idx
+ 1;
3962 u32 idx_value
= ib
[idx
];
3963 u32 start_reg
, reg
, i
;
3965 switch (pkt
->opcode
) {
3967 case PACKET3_SET_BASE
:
3968 case PACKET3_CLEAR_STATE
:
3969 case PACKET3_DISPATCH_DIRECT
:
3970 case PACKET3_DISPATCH_INDIRECT
:
3971 case PACKET3_ALLOC_GDS
:
3972 case PACKET3_WRITE_GDS_RAM
:
3973 case PACKET3_ATOMIC_GDS
:
3974 case PACKET3_ATOMIC
:
3975 case PACKET3_OCCLUSION_QUERY
:
3976 case PACKET3_SET_PREDICATION
:
3977 case PACKET3_COND_EXEC
:
3978 case PACKET3_PRED_EXEC
:
3979 case PACKET3_CONTEXT_CONTROL
:
3980 case PACKET3_STRMOUT_BUFFER_UPDATE
:
3981 case PACKET3_WAIT_REG_MEM
:
3982 case PACKET3_MEM_WRITE
:
3983 case PACKET3_PFP_SYNC_ME
:
3984 case PACKET3_SURFACE_SYNC
:
3985 case PACKET3_EVENT_WRITE
:
3986 case PACKET3_EVENT_WRITE_EOP
:
3987 case PACKET3_EVENT_WRITE_EOS
:
3988 case PACKET3_SET_CONTEXT_REG
:
3989 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
3990 case PACKET3_SET_SH_REG
:
3991 case PACKET3_SET_SH_REG_OFFSET
:
3992 case PACKET3_INCREMENT_DE_COUNTER
:
3993 case PACKET3_WAIT_ON_CE_COUNTER
:
3994 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
3995 case PACKET3_ME_WRITE
:
3997 case PACKET3_COPY_DATA
:
3998 if ((idx_value
& 0xf00) == 0) {
3999 reg
= ib
[idx
+ 3] * 4;
4000 if (!si_vm_reg_valid(reg
))
4004 case PACKET3_WRITE_DATA
:
4005 if ((idx_value
& 0xf00) == 0) {
4006 start_reg
= ib
[idx
+ 1] * 4;
4007 if (idx_value
& 0x10000) {
4008 if (!si_vm_reg_valid(start_reg
))
4011 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
4012 reg
= start_reg
+ (4 * i
);
4013 if (!si_vm_reg_valid(reg
))
4019 case PACKET3_COND_WRITE
:
4020 if (idx_value
& 0x100) {
4021 reg
= ib
[idx
+ 5] * 4;
4022 if (!si_vm_reg_valid(reg
))
4026 case PACKET3_COPY_DW
:
4027 if (idx_value
& 0x2) {
4028 reg
= ib
[idx
+ 3] * 4;
4029 if (!si_vm_reg_valid(reg
))
4034 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt
->opcode
);
4040 int si_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
4044 struct radeon_cs_packet pkt
;
4048 pkt
.type
= RADEON_CP_PACKET_GET_TYPE(ib
->ptr
[idx
]);
4049 pkt
.count
= RADEON_CP_PACKET_GET_COUNT(ib
->ptr
[idx
]);
4052 case RADEON_PACKET_TYPE0
:
4053 dev_err(rdev
->dev
, "Packet0 not allowed!\n");
4056 case RADEON_PACKET_TYPE2
:
4059 case RADEON_PACKET_TYPE3
:
4060 pkt
.opcode
= RADEON_CP_PACKET3_GET_OPCODE(ib
->ptr
[idx
]);
4061 if (ib
->is_const_ib
)
4062 ret
= si_vm_packet3_ce_check(rdev
, ib
->ptr
, &pkt
);
4065 case RADEON_RING_TYPE_GFX_INDEX
:
4066 ret
= si_vm_packet3_gfx_check(rdev
, ib
->ptr
, &pkt
);
4068 case CAYMAN_RING_TYPE_CP1_INDEX
:
4069 case CAYMAN_RING_TYPE_CP2_INDEX
:
4070 ret
= si_vm_packet3_compute_check(rdev
, ib
->ptr
, &pkt
);
4073 dev_err(rdev
->dev
, "Non-PM4 ring %d !\n", ib
->ring
);
4078 idx
+= pkt
.count
+ 2;
4081 dev_err(rdev
->dev
, "Unknown packet type %d !\n", pkt
.type
);
4087 } while (idx
< ib
->length_dw
);
4095 int si_vm_init(struct radeon_device
*rdev
)
4098 rdev
->vm_manager
.nvm
= 16;
4099 /* base offset of vram pages */
4100 rdev
->vm_manager
.vram_base_offset
= 0;
4105 void si_vm_fini(struct radeon_device
*rdev
)
4110 * si_vm_set_page - update the page tables using the CP
4112 * @rdev: radeon_device pointer
4113 * @ib: indirect buffer to fill with commands
4114 * @pe: addr of the page entry
4115 * @addr: dst addr to write into pe
4116 * @count: number of page entries to update
4117 * @incr: increase next addr by incr bytes
4118 * @flags: access flags
4120 * Update the page tables using the CP (SI).
4122 void si_vm_set_page(struct radeon_device
*rdev
,
4123 struct radeon_ib
*ib
,
4125 uint64_t addr
, unsigned count
,
4126 uint32_t incr
, uint32_t flags
)
4128 uint32_t r600_flags
= cayman_vm_page_flags(rdev
, flags
);
4132 if (rdev
->asic
->vm
.pt_ring_index
== RADEON_RING_TYPE_GFX_INDEX
) {
4134 ndw
= 2 + count
* 2;
4138 ib
->ptr
[ib
->length_dw
++] = PACKET3(PACKET3_WRITE_DATA
, ndw
);
4139 ib
->ptr
[ib
->length_dw
++] = (WRITE_DATA_ENGINE_SEL(0) |
4140 WRITE_DATA_DST_SEL(1));
4141 ib
->ptr
[ib
->length_dw
++] = pe
;
4142 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
4143 for (; ndw
> 2; ndw
-= 2, --count
, pe
+= 8) {
4144 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4145 value
= radeon_vm_map_gart(rdev
, addr
);
4146 value
&= 0xFFFFFFFFFFFFF000ULL
;
4147 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4153 value
|= r600_flags
;
4154 ib
->ptr
[ib
->length_dw
++] = value
;
4155 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4160 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4166 /* for non-physically contiguous pages (system) */
4167 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_WRITE
, 0, 0, 0, ndw
);
4168 ib
->ptr
[ib
->length_dw
++] = pe
;
4169 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4170 for (; ndw
> 0; ndw
-= 2, --count
, pe
+= 8) {
4171 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4172 value
= radeon_vm_map_gart(rdev
, addr
);
4173 value
&= 0xFFFFFFFFFFFFF000ULL
;
4174 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4180 value
|= r600_flags
;
4181 ib
->ptr
[ib
->length_dw
++] = value
;
4182 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4191 if (flags
& RADEON_VM_PAGE_VALID
)
4195 /* for physically contiguous pages (vram) */
4196 ib
->ptr
[ib
->length_dw
++] = DMA_PTE_PDE_PACKET(ndw
);
4197 ib
->ptr
[ib
->length_dw
++] = pe
; /* dst addr */
4198 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4199 ib
->ptr
[ib
->length_dw
++] = r600_flags
; /* mask */
4200 ib
->ptr
[ib
->length_dw
++] = 0;
4201 ib
->ptr
[ib
->length_dw
++] = value
; /* value */
4202 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4203 ib
->ptr
[ib
->length_dw
++] = incr
; /* increment size */
4204 ib
->ptr
[ib
->length_dw
++] = 0;
4206 addr
+= (ndw
/ 2) * incr
;
4210 while (ib
->length_dw
& 0x7)
4211 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0);
4215 void si_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4217 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4222 /* write new base address */
4223 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4224 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4225 WRITE_DATA_DST_SEL(0)));
4228 radeon_ring_write(ring
,
4229 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
4231 radeon_ring_write(ring
,
4232 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
4234 radeon_ring_write(ring
, 0);
4235 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4237 /* flush hdp cache */
4238 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4239 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4240 WRITE_DATA_DST_SEL(0)));
4241 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
4242 radeon_ring_write(ring
, 0);
4243 radeon_ring_write(ring
, 0x1);
4245 /* bits 0-15 are the VM contexts0-15 */
4246 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4247 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4248 WRITE_DATA_DST_SEL(0)));
4249 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
4250 radeon_ring_write(ring
, 0);
4251 radeon_ring_write(ring
, 1 << vm
->id
);
4253 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4254 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
4255 radeon_ring_write(ring
, 0x0);
4258 void si_dma_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4260 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4265 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4267 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2));
4269 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2));
4271 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4273 /* flush hdp cache */
4274 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4275 radeon_ring_write(ring
, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2));
4276 radeon_ring_write(ring
, 1);
4278 /* bits 0-7 are the VM contexts0-7 */
4279 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4280 radeon_ring_write(ring
, (0xf << 16) | (VM_INVALIDATE_REQUEST
>> 2));
4281 radeon_ring_write(ring
, 1 << vm
->id
);
4287 void si_rlc_fini(struct radeon_device
*rdev
)
4291 /* save restore block */
4292 if (rdev
->rlc
.save_restore_obj
) {
4293 r
= radeon_bo_reserve(rdev
->rlc
.save_restore_obj
, false);
4294 if (unlikely(r
!= 0))
4295 dev_warn(rdev
->dev
, "(%d) reserve RLC sr bo failed\n", r
);
4296 radeon_bo_unpin(rdev
->rlc
.save_restore_obj
);
4297 radeon_bo_unreserve(rdev
->rlc
.save_restore_obj
);
4299 radeon_bo_unref(&rdev
->rlc
.save_restore_obj
);
4300 rdev
->rlc
.save_restore_obj
= NULL
;
4303 /* clear state block */
4304 if (rdev
->rlc
.clear_state_obj
) {
4305 r
= radeon_bo_reserve(rdev
->rlc
.clear_state_obj
, false);
4306 if (unlikely(r
!= 0))
4307 dev_warn(rdev
->dev
, "(%d) reserve RLC c bo failed\n", r
);
4308 radeon_bo_unpin(rdev
->rlc
.clear_state_obj
);
4309 radeon_bo_unreserve(rdev
->rlc
.clear_state_obj
);
4311 radeon_bo_unref(&rdev
->rlc
.clear_state_obj
);
4312 rdev
->rlc
.clear_state_obj
= NULL
;
4316 int si_rlc_init(struct radeon_device
*rdev
)
4320 /* save restore block */
4321 if (rdev
->rlc
.save_restore_obj
== NULL
) {
4322 r
= radeon_bo_create(rdev
, RADEON_GPU_PAGE_SIZE
, PAGE_SIZE
, true,
4323 RADEON_GEM_DOMAIN_VRAM
, NULL
,
4324 &rdev
->rlc
.save_restore_obj
);
4326 dev_warn(rdev
->dev
, "(%d) create RLC sr bo failed\n", r
);
4331 r
= radeon_bo_reserve(rdev
->rlc
.save_restore_obj
, false);
4332 if (unlikely(r
!= 0)) {
4336 r
= radeon_bo_pin(rdev
->rlc
.save_restore_obj
, RADEON_GEM_DOMAIN_VRAM
,
4337 &rdev
->rlc
.save_restore_gpu_addr
);
4338 radeon_bo_unreserve(rdev
->rlc
.save_restore_obj
);
4340 dev_warn(rdev
->dev
, "(%d) pin RLC sr bo failed\n", r
);
4345 /* clear state block */
4346 if (rdev
->rlc
.clear_state_obj
== NULL
) {
4347 r
= radeon_bo_create(rdev
, RADEON_GPU_PAGE_SIZE
, PAGE_SIZE
, true,
4348 RADEON_GEM_DOMAIN_VRAM
, NULL
,
4349 &rdev
->rlc
.clear_state_obj
);
4351 dev_warn(rdev
->dev
, "(%d) create RLC c bo failed\n", r
);
4356 r
= radeon_bo_reserve(rdev
->rlc
.clear_state_obj
, false);
4357 if (unlikely(r
!= 0)) {
4361 r
= radeon_bo_pin(rdev
->rlc
.clear_state_obj
, RADEON_GEM_DOMAIN_VRAM
,
4362 &rdev
->rlc
.clear_state_gpu_addr
);
4363 radeon_bo_unreserve(rdev
->rlc
.clear_state_obj
);
4365 dev_warn(rdev
->dev
, "(%d) pin RLC c bo failed\n", r
);
4373 static void si_rlc_stop(struct radeon_device
*rdev
)
4375 WREG32(RLC_CNTL
, 0);
4378 static void si_rlc_start(struct radeon_device
*rdev
)
4380 WREG32(RLC_CNTL
, RLC_ENABLE
);
4383 static int si_rlc_resume(struct radeon_device
*rdev
)
4386 const __be32
*fw_data
;
4393 WREG32(RLC_RL_BASE
, 0);
4394 WREG32(RLC_RL_SIZE
, 0);
4395 WREG32(RLC_LB_CNTL
, 0);
4396 WREG32(RLC_LB_CNTR_MAX
, 0xffffffff);
4397 WREG32(RLC_LB_CNTR_INIT
, 0);
4399 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
4400 WREG32(RLC_CLEAR_STATE_RESTORE_BASE
, rdev
->rlc
.clear_state_gpu_addr
>> 8);
4402 WREG32(RLC_MC_CNTL
, 0);
4403 WREG32(RLC_UCODE_CNTL
, 0);
4405 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
4406 for (i
= 0; i
< SI_RLC_UCODE_SIZE
; i
++) {
4407 WREG32(RLC_UCODE_ADDR
, i
);
4408 WREG32(RLC_UCODE_DATA
, be32_to_cpup(fw_data
++));
4410 WREG32(RLC_UCODE_ADDR
, 0);
4417 static void si_enable_interrupts(struct radeon_device
*rdev
)
4419 u32 ih_cntl
= RREG32(IH_CNTL
);
4420 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
4422 ih_cntl
|= ENABLE_INTR
;
4423 ih_rb_cntl
|= IH_RB_ENABLE
;
4424 WREG32(IH_CNTL
, ih_cntl
);
4425 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
4426 rdev
->ih
.enabled
= true;
4429 static void si_disable_interrupts(struct radeon_device
*rdev
)
4431 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
4432 u32 ih_cntl
= RREG32(IH_CNTL
);
4434 ih_rb_cntl
&= ~IH_RB_ENABLE
;
4435 ih_cntl
&= ~ENABLE_INTR
;
4436 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
4437 WREG32(IH_CNTL
, ih_cntl
);
4438 /* set rptr, wptr to 0 */
4439 WREG32(IH_RB_RPTR
, 0);
4440 WREG32(IH_RB_WPTR
, 0);
4441 rdev
->ih
.enabled
= false;
4445 static void si_disable_interrupt_state(struct radeon_device
*rdev
)
4449 WREG32(CP_INT_CNTL_RING0
, CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
4450 WREG32(CP_INT_CNTL_RING1
, 0);
4451 WREG32(CP_INT_CNTL_RING2
, 0);
4452 tmp
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
4453 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
4454 tmp
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
4455 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
4456 WREG32(GRBM_INT_CNTL
, 0);
4457 if (rdev
->num_crtc
>= 2) {
4458 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
4459 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
4461 if (rdev
->num_crtc
>= 4) {
4462 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
4463 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
4465 if (rdev
->num_crtc
>= 6) {
4466 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
4467 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
4470 if (rdev
->num_crtc
>= 2) {
4471 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
4472 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
4474 if (rdev
->num_crtc
>= 4) {
4475 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
4476 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
4478 if (rdev
->num_crtc
>= 6) {
4479 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
4480 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
4483 if (!ASIC_IS_NODCE(rdev
)) {
4484 WREG32(DACA_AUTODETECT_INT_CONTROL
, 0);
4486 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4487 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
4488 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4489 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
4490 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4491 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
4492 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4493 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
4494 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4495 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
4496 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
4497 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
4501 static int si_irq_init(struct radeon_device
*rdev
)
4505 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
4508 ret
= r600_ih_ring_alloc(rdev
);
4513 si_disable_interrupts(rdev
);
4516 ret
= si_rlc_resume(rdev
);
4518 r600_ih_ring_fini(rdev
);
4522 /* setup interrupt control */
4523 /* set dummy read address to ring address */
4524 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
4525 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
4526 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4527 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4529 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
4530 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4531 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
4532 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
4534 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
4535 rb_bufsz
= drm_order(rdev
->ih
.ring_size
/ 4);
4537 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
4538 IH_WPTR_OVERFLOW_CLEAR
|
4541 if (rdev
->wb
.enabled
)
4542 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
4544 /* set the writeback address whether it's enabled or not */
4545 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
4546 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
4548 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
4550 /* set rptr, wptr to 0 */
4551 WREG32(IH_RB_RPTR
, 0);
4552 WREG32(IH_RB_WPTR
, 0);
4554 /* Default settings for IH_CNTL (disabled at first) */
4555 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4556 /* RPTR_REARM only works if msi's are enabled */
4557 if (rdev
->msi_enabled
)
4558 ih_cntl
|= RPTR_REARM
;
4559 WREG32(IH_CNTL
, ih_cntl
);
4561 /* force the active interrupt state to all disabled */
4562 si_disable_interrupt_state(rdev
);
4564 pci_set_master(rdev
->pdev
);
4567 si_enable_interrupts(rdev
);
4572 int si_irq_set(struct radeon_device
*rdev
)
4574 u32 cp_int_cntl
= CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
;
4575 u32 cp_int_cntl1
= 0, cp_int_cntl2
= 0;
4576 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
4577 u32 hpd1
= 0, hpd2
= 0, hpd3
= 0, hpd4
= 0, hpd5
= 0, hpd6
= 0;
4578 u32 grbm_int_cntl
= 0;
4579 u32 grph1
= 0, grph2
= 0, grph3
= 0, grph4
= 0, grph5
= 0, grph6
= 0;
4580 u32 dma_cntl
, dma_cntl1
;
4582 if (!rdev
->irq
.installed
) {
4583 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4586 /* don't enable anything if the ih is disabled */
4587 if (!rdev
->ih
.enabled
) {
4588 si_disable_interrupts(rdev
);
4589 /* force the active interrupt state to all disabled */
4590 si_disable_interrupt_state(rdev
);
4594 if (!ASIC_IS_NODCE(rdev
)) {
4595 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4596 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4597 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4598 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4599 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4600 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
4603 dma_cntl
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
4604 dma_cntl1
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
4606 /* enable CP interrupts on all rings */
4607 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
4608 DRM_DEBUG("si_irq_set: sw int gfx\n");
4609 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
4611 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
4612 DRM_DEBUG("si_irq_set: sw int cp1\n");
4613 cp_int_cntl1
|= TIME_STAMP_INT_ENABLE
;
4615 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
4616 DRM_DEBUG("si_irq_set: sw int cp2\n");
4617 cp_int_cntl2
|= TIME_STAMP_INT_ENABLE
;
4619 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
4620 DRM_DEBUG("si_irq_set: sw int dma\n");
4621 dma_cntl
|= TRAP_ENABLE
;
4624 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
4625 DRM_DEBUG("si_irq_set: sw int dma1\n");
4626 dma_cntl1
|= TRAP_ENABLE
;
4628 if (rdev
->irq
.crtc_vblank_int
[0] ||
4629 atomic_read(&rdev
->irq
.pflip
[0])) {
4630 DRM_DEBUG("si_irq_set: vblank 0\n");
4631 crtc1
|= VBLANK_INT_MASK
;
4633 if (rdev
->irq
.crtc_vblank_int
[1] ||
4634 atomic_read(&rdev
->irq
.pflip
[1])) {
4635 DRM_DEBUG("si_irq_set: vblank 1\n");
4636 crtc2
|= VBLANK_INT_MASK
;
4638 if (rdev
->irq
.crtc_vblank_int
[2] ||
4639 atomic_read(&rdev
->irq
.pflip
[2])) {
4640 DRM_DEBUG("si_irq_set: vblank 2\n");
4641 crtc3
|= VBLANK_INT_MASK
;
4643 if (rdev
->irq
.crtc_vblank_int
[3] ||
4644 atomic_read(&rdev
->irq
.pflip
[3])) {
4645 DRM_DEBUG("si_irq_set: vblank 3\n");
4646 crtc4
|= VBLANK_INT_MASK
;
4648 if (rdev
->irq
.crtc_vblank_int
[4] ||
4649 atomic_read(&rdev
->irq
.pflip
[4])) {
4650 DRM_DEBUG("si_irq_set: vblank 4\n");
4651 crtc5
|= VBLANK_INT_MASK
;
4653 if (rdev
->irq
.crtc_vblank_int
[5] ||
4654 atomic_read(&rdev
->irq
.pflip
[5])) {
4655 DRM_DEBUG("si_irq_set: vblank 5\n");
4656 crtc6
|= VBLANK_INT_MASK
;
4658 if (rdev
->irq
.hpd
[0]) {
4659 DRM_DEBUG("si_irq_set: hpd 1\n");
4660 hpd1
|= DC_HPDx_INT_EN
;
4662 if (rdev
->irq
.hpd
[1]) {
4663 DRM_DEBUG("si_irq_set: hpd 2\n");
4664 hpd2
|= DC_HPDx_INT_EN
;
4666 if (rdev
->irq
.hpd
[2]) {
4667 DRM_DEBUG("si_irq_set: hpd 3\n");
4668 hpd3
|= DC_HPDx_INT_EN
;
4670 if (rdev
->irq
.hpd
[3]) {
4671 DRM_DEBUG("si_irq_set: hpd 4\n");
4672 hpd4
|= DC_HPDx_INT_EN
;
4674 if (rdev
->irq
.hpd
[4]) {
4675 DRM_DEBUG("si_irq_set: hpd 5\n");
4676 hpd5
|= DC_HPDx_INT_EN
;
4678 if (rdev
->irq
.hpd
[5]) {
4679 DRM_DEBUG("si_irq_set: hpd 6\n");
4680 hpd6
|= DC_HPDx_INT_EN
;
4683 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
4684 WREG32(CP_INT_CNTL_RING1
, cp_int_cntl1
);
4685 WREG32(CP_INT_CNTL_RING2
, cp_int_cntl2
);
4687 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, dma_cntl
);
4688 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, dma_cntl1
);
4690 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
4692 if (rdev
->num_crtc
>= 2) {
4693 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
4694 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
4696 if (rdev
->num_crtc
>= 4) {
4697 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
4698 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
4700 if (rdev
->num_crtc
>= 6) {
4701 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
4702 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
4705 if (rdev
->num_crtc
>= 2) {
4706 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, grph1
);
4707 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, grph2
);
4709 if (rdev
->num_crtc
>= 4) {
4710 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, grph3
);
4711 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, grph4
);
4713 if (rdev
->num_crtc
>= 6) {
4714 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, grph5
);
4715 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, grph6
);
4718 if (!ASIC_IS_NODCE(rdev
)) {
4719 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
4720 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
4721 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
4722 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
4723 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
4724 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
4730 static inline void si_irq_ack(struct radeon_device
*rdev
)
4734 if (ASIC_IS_NODCE(rdev
))
4737 rdev
->irq
.stat_regs
.evergreen
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
4738 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
4739 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
4740 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
4741 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
4742 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
4743 rdev
->irq
.stat_regs
.evergreen
.d1grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
4744 rdev
->irq
.stat_regs
.evergreen
.d2grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
4745 if (rdev
->num_crtc
>= 4) {
4746 rdev
->irq
.stat_regs
.evergreen
.d3grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
4747 rdev
->irq
.stat_regs
.evergreen
.d4grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
4749 if (rdev
->num_crtc
>= 6) {
4750 rdev
->irq
.stat_regs
.evergreen
.d5grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
4751 rdev
->irq
.stat_regs
.evergreen
.d6grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
4754 if (rdev
->irq
.stat_regs
.evergreen
.d1grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4755 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4756 if (rdev
->irq
.stat_regs
.evergreen
.d2grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4757 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4758 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
4759 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
4760 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
)
4761 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
4762 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
4763 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
4764 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
4765 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
4767 if (rdev
->num_crtc
>= 4) {
4768 if (rdev
->irq
.stat_regs
.evergreen
.d3grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4769 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4770 if (rdev
->irq
.stat_regs
.evergreen
.d4grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4771 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4772 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
4773 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
4774 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
4775 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
4776 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
4777 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
4778 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
4779 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
4782 if (rdev
->num_crtc
>= 6) {
4783 if (rdev
->irq
.stat_regs
.evergreen
.d5grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4784 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4785 if (rdev
->irq
.stat_regs
.evergreen
.d6grph_int
& GRPH_PFLIP_INT_OCCURRED
)
4786 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
4787 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
4788 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
4789 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
4790 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
4791 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
4792 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
4793 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
4794 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
4797 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
4798 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
4799 tmp
|= DC_HPDx_INT_ACK
;
4800 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
4802 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
4803 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
4804 tmp
|= DC_HPDx_INT_ACK
;
4805 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
4807 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
4808 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
4809 tmp
|= DC_HPDx_INT_ACK
;
4810 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
4812 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
4813 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
4814 tmp
|= DC_HPDx_INT_ACK
;
4815 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
4817 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
4818 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
4819 tmp
|= DC_HPDx_INT_ACK
;
4820 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
4822 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
4823 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
4824 tmp
|= DC_HPDx_INT_ACK
;
4825 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
4829 static void si_irq_disable(struct radeon_device
*rdev
)
4831 si_disable_interrupts(rdev
);
4832 /* Wait and acknowledge irq */
4835 si_disable_interrupt_state(rdev
);
4838 static void si_irq_suspend(struct radeon_device
*rdev
)
4840 si_irq_disable(rdev
);
4844 static void si_irq_fini(struct radeon_device
*rdev
)
4846 si_irq_suspend(rdev
);
4847 r600_ih_ring_fini(rdev
);
4850 static inline u32
si_get_ih_wptr(struct radeon_device
*rdev
)
4854 if (rdev
->wb
.enabled
)
4855 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
4857 wptr
= RREG32(IH_RB_WPTR
);
4859 if (wptr
& RB_OVERFLOW
) {
4860 /* When a ring buffer overflow happen start parsing interrupt
4861 * from the last not overwritten vector (wptr + 16). Hopefully
4862 * this should allow us to catchup.
4864 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4865 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
4866 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
4867 tmp
= RREG32(IH_RB_CNTL
);
4868 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
4869 WREG32(IH_RB_CNTL
, tmp
);
4871 return (wptr
& rdev
->ih
.ptr_mask
);
4875 * Each IV ring entry is 128 bits:
4876 * [7:0] - interrupt source id
4878 * [59:32] - interrupt source data
4879 * [63:60] - reserved
4882 * [127:80] - reserved
4884 int si_irq_process(struct radeon_device
*rdev
)
4888 u32 src_id
, src_data
, ring_id
;
4890 bool queue_hotplug
= false;
4892 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
4895 wptr
= si_get_ih_wptr(rdev
);
4898 /* is somebody else already processing irqs? */
4899 if (atomic_xchg(&rdev
->ih
.lock
, 1))
4902 rptr
= rdev
->ih
.rptr
;
4903 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
4905 /* Order reading of wptr vs. reading of IH ring data */
4908 /* display interrupts */
4911 while (rptr
!= wptr
) {
4912 /* wptr/rptr are in bytes! */
4913 ring_index
= rptr
/ 4;
4914 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
4915 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
4916 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
4919 case 1: /* D1 vblank/vline */
4921 case 0: /* D1 vblank */
4922 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
4923 if (rdev
->irq
.crtc_vblank_int
[0]) {
4924 drm_handle_vblank(rdev
->ddev
, 0);
4925 rdev
->pm
.vblank_sync
= true;
4926 wake_up(&rdev
->irq
.vblank_queue
);
4928 if (atomic_read(&rdev
->irq
.pflip
[0]))
4929 radeon_crtc_handle_flip(rdev
, 0);
4930 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
4931 DRM_DEBUG("IH: D1 vblank\n");
4934 case 1: /* D1 vline */
4935 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
4936 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
4937 DRM_DEBUG("IH: D1 vline\n");
4941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
4945 case 2: /* D2 vblank/vline */
4947 case 0: /* D2 vblank */
4948 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
4949 if (rdev
->irq
.crtc_vblank_int
[1]) {
4950 drm_handle_vblank(rdev
->ddev
, 1);
4951 rdev
->pm
.vblank_sync
= true;
4952 wake_up(&rdev
->irq
.vblank_queue
);
4954 if (atomic_read(&rdev
->irq
.pflip
[1]))
4955 radeon_crtc_handle_flip(rdev
, 1);
4956 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
4957 DRM_DEBUG("IH: D2 vblank\n");
4960 case 1: /* D2 vline */
4961 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
4962 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
4963 DRM_DEBUG("IH: D2 vline\n");
4967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
4971 case 3: /* D3 vblank/vline */
4973 case 0: /* D3 vblank */
4974 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
4975 if (rdev
->irq
.crtc_vblank_int
[2]) {
4976 drm_handle_vblank(rdev
->ddev
, 2);
4977 rdev
->pm
.vblank_sync
= true;
4978 wake_up(&rdev
->irq
.vblank_queue
);
4980 if (atomic_read(&rdev
->irq
.pflip
[2]))
4981 radeon_crtc_handle_flip(rdev
, 2);
4982 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
4983 DRM_DEBUG("IH: D3 vblank\n");
4986 case 1: /* D3 vline */
4987 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
4988 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
4989 DRM_DEBUG("IH: D3 vline\n");
4993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
4997 case 4: /* D4 vblank/vline */
4999 case 0: /* D4 vblank */
5000 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
5001 if (rdev
->irq
.crtc_vblank_int
[3]) {
5002 drm_handle_vblank(rdev
->ddev
, 3);
5003 rdev
->pm
.vblank_sync
= true;
5004 wake_up(&rdev
->irq
.vblank_queue
);
5006 if (atomic_read(&rdev
->irq
.pflip
[3]))
5007 radeon_crtc_handle_flip(rdev
, 3);
5008 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
5009 DRM_DEBUG("IH: D4 vblank\n");
5012 case 1: /* D4 vline */
5013 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
5014 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
5015 DRM_DEBUG("IH: D4 vline\n");
5019 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5023 case 5: /* D5 vblank/vline */
5025 case 0: /* D5 vblank */
5026 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
5027 if (rdev
->irq
.crtc_vblank_int
[4]) {
5028 drm_handle_vblank(rdev
->ddev
, 4);
5029 rdev
->pm
.vblank_sync
= true;
5030 wake_up(&rdev
->irq
.vblank_queue
);
5032 if (atomic_read(&rdev
->irq
.pflip
[4]))
5033 radeon_crtc_handle_flip(rdev
, 4);
5034 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
5035 DRM_DEBUG("IH: D5 vblank\n");
5038 case 1: /* D5 vline */
5039 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
5040 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
5041 DRM_DEBUG("IH: D5 vline\n");
5045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5049 case 6: /* D6 vblank/vline */
5051 case 0: /* D6 vblank */
5052 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
5053 if (rdev
->irq
.crtc_vblank_int
[5]) {
5054 drm_handle_vblank(rdev
->ddev
, 5);
5055 rdev
->pm
.vblank_sync
= true;
5056 wake_up(&rdev
->irq
.vblank_queue
);
5058 if (atomic_read(&rdev
->irq
.pflip
[5]))
5059 radeon_crtc_handle_flip(rdev
, 5);
5060 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
5061 DRM_DEBUG("IH: D6 vblank\n");
5064 case 1: /* D6 vline */
5065 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
5066 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
5067 DRM_DEBUG("IH: D6 vline\n");
5071 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5075 case 42: /* HPD hotplug */
5078 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
5079 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~DC_HPD1_INTERRUPT
;
5080 queue_hotplug
= true;
5081 DRM_DEBUG("IH: HPD1\n");
5085 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
5086 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
5087 queue_hotplug
= true;
5088 DRM_DEBUG("IH: HPD2\n");
5092 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
5093 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
5094 queue_hotplug
= true;
5095 DRM_DEBUG("IH: HPD3\n");
5099 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
5100 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
5101 queue_hotplug
= true;
5102 DRM_DEBUG("IH: HPD4\n");
5106 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
5107 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
5108 queue_hotplug
= true;
5109 DRM_DEBUG("IH: HPD5\n");
5113 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
5114 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
5115 queue_hotplug
= true;
5116 DRM_DEBUG("IH: HPD6\n");
5120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5126 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
5127 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5128 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
5129 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5130 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
5131 /* reset addr and status */
5132 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
5134 case 176: /* RINGID0 CP_INT */
5135 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
5137 case 177: /* RINGID1 CP_INT */
5138 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
5140 case 178: /* RINGID2 CP_INT */
5141 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
5143 case 181: /* CP EOP event */
5144 DRM_DEBUG("IH: CP EOP\n");
5147 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
5150 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
5153 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
5157 case 224: /* DMA trap event */
5158 DRM_DEBUG("IH: DMA trap\n");
5159 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
5161 case 233: /* GUI IDLE */
5162 DRM_DEBUG("IH: GUI idle\n");
5164 case 244: /* DMA trap event */
5165 DRM_DEBUG("IH: DMA1 trap\n");
5166 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
5169 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5173 /* wptr/rptr are in bytes! */
5175 rptr
&= rdev
->ih
.ptr_mask
;
5178 schedule_work(&rdev
->hotplug_work
);
5179 rdev
->ih
.rptr
= rptr
;
5180 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
5181 atomic_set(&rdev
->ih
.lock
, 0);
5183 /* make sure wptr hasn't changed while processing */
5184 wptr
= si_get_ih_wptr(rdev
);
5192 * si_copy_dma - copy pages using the DMA engine
5194 * @rdev: radeon_device pointer
5195 * @src_offset: src GPU address
5196 * @dst_offset: dst GPU address
5197 * @num_gpu_pages: number of GPU pages to xfer
5198 * @fence: radeon fence object
5200 * Copy GPU paging using the DMA engine (SI).
5201 * Used by the radeon ttm implementation to move pages if
5202 * registered as the asic copy callback.
5204 int si_copy_dma(struct radeon_device
*rdev
,
5205 uint64_t src_offset
, uint64_t dst_offset
,
5206 unsigned num_gpu_pages
,
5207 struct radeon_fence
**fence
)
5209 struct radeon_semaphore
*sem
= NULL
;
5210 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
5211 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
5212 u32 size_in_bytes
, cur_size_in_bytes
;
5216 r
= radeon_semaphore_create(rdev
, &sem
);
5218 DRM_ERROR("radeon: moving bo (%d).\n", r
);
5222 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
5223 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0xfffff);
5224 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 5 + 11);
5226 DRM_ERROR("radeon: moving bo (%d).\n", r
);
5227 radeon_semaphore_free(rdev
, &sem
, NULL
);
5231 if (radeon_fence_need_sync(*fence
, ring
->idx
)) {
5232 radeon_semaphore_sync_rings(rdev
, sem
, (*fence
)->ring
,
5234 radeon_fence_note_sync(*fence
, ring
->idx
);
5236 radeon_semaphore_free(rdev
, &sem
, NULL
);
5239 for (i
= 0; i
< num_loops
; i
++) {
5240 cur_size_in_bytes
= size_in_bytes
;
5241 if (cur_size_in_bytes
> 0xFFFFF)
5242 cur_size_in_bytes
= 0xFFFFF;
5243 size_in_bytes
-= cur_size_in_bytes
;
5244 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_COPY
, 1, 0, 0, cur_size_in_bytes
));
5245 radeon_ring_write(ring
, dst_offset
& 0xffffffff);
5246 radeon_ring_write(ring
, src_offset
& 0xffffffff);
5247 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xff);
5248 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xff);
5249 src_offset
+= cur_size_in_bytes
;
5250 dst_offset
+= cur_size_in_bytes
;
5253 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
5255 radeon_ring_unlock_undo(rdev
, ring
);
5259 radeon_ring_unlock_commit(rdev
, ring
);
5260 radeon_semaphore_free(rdev
, &sem
, *fence
);
5266 * startup/shutdown callbacks
5268 static int si_startup(struct radeon_device
*rdev
)
5270 struct radeon_ring
*ring
;
5273 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
5274 !rdev
->rlc_fw
|| !rdev
->mc_fw
) {
5275 r
= si_init_microcode(rdev
);
5277 DRM_ERROR("Failed to load firmware!\n");
5282 r
= si_mc_load_microcode(rdev
);
5284 DRM_ERROR("Failed to load MC firmware!\n");
5288 r
= r600_vram_scratch_init(rdev
);
5292 si_mc_program(rdev
);
5293 r
= si_pcie_gart_enable(rdev
);
5298 /* allocate rlc buffers */
5299 r
= si_rlc_init(rdev
);
5301 DRM_ERROR("Failed to init rlc BOs!\n");
5305 /* allocate wb buffer */
5306 r
= radeon_wb_init(rdev
);
5310 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
5312 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
5316 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
5318 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
5322 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
5324 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
5328 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
5330 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
5334 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
5336 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
5340 if (rdev
->has_uvd
) {
5341 r
= rv770_uvd_resume(rdev
);
5343 r
= radeon_fence_driver_start_ring(rdev
,
5344 R600_RING_TYPE_UVD_INDEX
);
5346 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
5349 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
5353 r
= si_irq_init(rdev
);
5355 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
5356 radeon_irq_kms_fini(rdev
);
5361 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
5362 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
5363 CP_RB0_RPTR
, CP_RB0_WPTR
,
5364 0, 0xfffff, RADEON_CP_PACKET2
);
5368 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
5369 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
5370 CP_RB1_RPTR
, CP_RB1_WPTR
,
5371 0, 0xfffff, RADEON_CP_PACKET2
);
5375 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
5376 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
5377 CP_RB2_RPTR
, CP_RB2_WPTR
,
5378 0, 0xfffff, RADEON_CP_PACKET2
);
5382 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
5383 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
5384 DMA_RB_RPTR
+ DMA0_REGISTER_OFFSET
,
5385 DMA_RB_WPTR
+ DMA0_REGISTER_OFFSET
,
5386 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
5390 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
5391 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
5392 DMA_RB_RPTR
+ DMA1_REGISTER_OFFSET
,
5393 DMA_RB_WPTR
+ DMA1_REGISTER_OFFSET
,
5394 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
5398 r
= si_cp_load_microcode(rdev
);
5401 r
= si_cp_resume(rdev
);
5405 r
= cayman_dma_resume(rdev
);
5409 if (rdev
->has_uvd
) {
5410 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
5411 if (ring
->ring_size
) {
5412 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
,
5413 R600_WB_UVD_RPTR_OFFSET
,
5414 UVD_RBC_RB_RPTR
, UVD_RBC_RB_WPTR
,
5415 0, 0xfffff, RADEON_CP_PACKET2
);
5417 r
= r600_uvd_init(rdev
);
5419 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
5423 r
= radeon_ib_pool_init(rdev
);
5425 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
5429 r
= radeon_vm_manager_init(rdev
);
5431 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
5438 int si_resume(struct radeon_device
*rdev
)
5442 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5443 * posting will perform necessary task to bring back GPU into good
5447 atom_asic_init(rdev
->mode_info
.atom_context
);
5449 /* init golden registers */
5450 si_init_golden_registers(rdev
);
5452 rdev
->accel_working
= true;
5453 r
= si_startup(rdev
);
5455 DRM_ERROR("si startup failed on resume\n");
5456 rdev
->accel_working
= false;
5464 int si_suspend(struct radeon_device
*rdev
)
5466 radeon_vm_manager_fini(rdev
);
5467 si_cp_enable(rdev
, false);
5468 cayman_dma_stop(rdev
);
5469 if (rdev
->has_uvd
) {
5470 r600_uvd_rbc_stop(rdev
);
5471 radeon_uvd_suspend(rdev
);
5473 si_irq_suspend(rdev
);
5474 radeon_wb_disable(rdev
);
5475 si_pcie_gart_disable(rdev
);
5479 /* Plan is to move initialization in that function and use
5480 * helper function so that radeon_device_init pretty much
5481 * do nothing more than calling asic specific function. This
5482 * should also allow to remove a bunch of callback function
5485 int si_init(struct radeon_device
*rdev
)
5487 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
5491 if (!radeon_get_bios(rdev
)) {
5492 if (ASIC_IS_AVIVO(rdev
))
5495 /* Must be an ATOMBIOS */
5496 if (!rdev
->is_atom_bios
) {
5497 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
5500 r
= radeon_atombios_init(rdev
);
5504 /* Post card if necessary */
5505 if (!radeon_card_posted(rdev
)) {
5507 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
5510 DRM_INFO("GPU not posted. posting now...\n");
5511 atom_asic_init(rdev
->mode_info
.atom_context
);
5513 /* init golden registers */
5514 si_init_golden_registers(rdev
);
5515 /* Initialize scratch registers */
5516 si_scratch_init(rdev
);
5517 /* Initialize surface registers */
5518 radeon_surface_init(rdev
);
5519 /* Initialize clocks */
5520 radeon_get_clock_info(rdev
->ddev
);
5523 r
= radeon_fence_driver_init(rdev
);
5527 /* initialize memory controller */
5528 r
= si_mc_init(rdev
);
5531 /* Memory manager */
5532 r
= radeon_bo_init(rdev
);
5536 r
= radeon_irq_kms_init(rdev
);
5540 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
5541 ring
->ring_obj
= NULL
;
5542 r600_ring_init(rdev
, ring
, 1024 * 1024);
5544 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
5545 ring
->ring_obj
= NULL
;
5546 r600_ring_init(rdev
, ring
, 1024 * 1024);
5548 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
5549 ring
->ring_obj
= NULL
;
5550 r600_ring_init(rdev
, ring
, 1024 * 1024);
5552 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
5553 ring
->ring_obj
= NULL
;
5554 r600_ring_init(rdev
, ring
, 64 * 1024);
5556 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
5557 ring
->ring_obj
= NULL
;
5558 r600_ring_init(rdev
, ring
, 64 * 1024);
5560 if (rdev
->has_uvd
) {
5561 r
= radeon_uvd_init(rdev
);
5563 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
5564 ring
->ring_obj
= NULL
;
5565 r600_ring_init(rdev
, ring
, 4096);
5569 rdev
->ih
.ring_obj
= NULL
;
5570 r600_ih_ring_init(rdev
, 64 * 1024);
5572 r
= r600_pcie_gart_init(rdev
);
5576 rdev
->accel_working
= true;
5577 r
= si_startup(rdev
);
5579 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
5581 cayman_dma_fini(rdev
);
5584 radeon_wb_fini(rdev
);
5585 radeon_ib_pool_fini(rdev
);
5586 radeon_vm_manager_fini(rdev
);
5587 radeon_irq_kms_fini(rdev
);
5588 si_pcie_gart_fini(rdev
);
5589 rdev
->accel_working
= false;
5592 /* Don't start up if the MC ucode is missing.
5593 * The default clocks and voltages before the MC ucode
5594 * is loaded are not suffient for advanced operations.
5597 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5604 void si_fini(struct radeon_device
*rdev
)
5607 cayman_dma_fini(rdev
);
5610 radeon_wb_fini(rdev
);
5611 radeon_vm_manager_fini(rdev
);
5612 radeon_ib_pool_fini(rdev
);
5613 radeon_irq_kms_fini(rdev
);
5615 radeon_uvd_fini(rdev
);
5616 si_pcie_gart_fini(rdev
);
5617 r600_vram_scratch_fini(rdev
);
5618 radeon_gem_fini(rdev
);
5619 radeon_fence_driver_fini(rdev
);
5620 radeon_bo_fini(rdev
);
5621 radeon_atombios_fini(rdev
);
5627 * si_get_gpu_clock_counter - return GPU clock counter snapshot
5629 * @rdev: radeon_device pointer
5631 * Fetches a GPU clock counter snapshot (SI).
5632 * Returns the 64 bit clock counter snapshot.
5634 uint64_t si_get_gpu_clock_counter(struct radeon_device
*rdev
)
5638 mutex_lock(&rdev
->gpu_clock_mutex
);
5639 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
5640 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
5641 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
5642 mutex_unlock(&rdev
->gpu_clock_mutex
);
5646 int si_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
5648 unsigned fb_div
= 0, vclk_div
= 0, dclk_div
= 0;
5651 /* bypass vclk and dclk with bclk */
5652 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
5653 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5654 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));
5656 /* put PLL in bypass mode */
5657 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_BYPASS_EN_MASK
, ~UPLL_BYPASS_EN_MASK
);
5659 if (!vclk
|| !dclk
) {
5660 /* keep the Bypass mode, put PLL to sleep */
5661 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
5665 r
= radeon_uvd_calc_upll_dividers(rdev
, vclk
, dclk
, 125000, 250000,
5666 16384, 0x03FFFFFF, 0, 128, 5,
5667 &fb_div
, &vclk_div
, &dclk_div
);
5671 /* set RESET_ANTI_MUX to 0 */
5672 WREG32_P(CG_UPLL_FUNC_CNTL_5
, 0, ~RESET_ANTI_MUX_MASK
);
5674 /* set VCO_MODE to 1 */
5675 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_VCO_MODE_MASK
, ~UPLL_VCO_MODE_MASK
);
5677 /* toggle UPLL_SLEEP to 1 then back to 0 */
5678 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
5679 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_SLEEP_MASK
);
5681 /* deassert UPLL_RESET */
5682 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
5686 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
5690 /* assert UPLL_RESET again */
5691 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_RESET_MASK
, ~UPLL_RESET_MASK
);
5693 /* disable spread spectrum. */
5694 WREG32_P(CG_UPLL_SPREAD_SPECTRUM
, 0, ~SSEN_MASK
);
5696 /* set feedback divider */
5697 WREG32_P(CG_UPLL_FUNC_CNTL_3
, UPLL_FB_DIV(fb_div
), ~UPLL_FB_DIV_MASK
);
5699 /* set ref divider to 0 */
5700 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_REF_DIV_MASK
);
5702 if (fb_div
< 307200)
5703 WREG32_P(CG_UPLL_FUNC_CNTL_4
, 0, ~UPLL_SPARE_ISPARE9
);
5705 WREG32_P(CG_UPLL_FUNC_CNTL_4
, UPLL_SPARE_ISPARE9
, ~UPLL_SPARE_ISPARE9
);
5707 /* set PDIV_A and PDIV_B */
5708 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
5709 UPLL_PDIV_A(vclk_div
) | UPLL_PDIV_B(dclk_div
),
5710 ~(UPLL_PDIV_A_MASK
| UPLL_PDIV_B_MASK
));
5712 /* give the PLL some time to settle */
5715 /* deassert PLL_RESET */
5716 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
5720 /* switch from bypass mode to normal mode */
5721 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_BYPASS_EN_MASK
);
5723 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
5727 /* switch VCLK and DCLK selection */
5728 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
5729 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5730 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));