projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'fbdev-fixes-3.11-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git...
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "cikd.h"
31 #include "atom.h"
32 #include "cik_blit_shaders.h"
33
34 /* GFX */
35 #define CIK_PFP_UCODE_SIZE 2144
36 #define CIK_ME_UCODE_SIZE 2144
37 #define CIK_CE_UCODE_SIZE 2144
38 /* compute */
39 #define CIK_MEC_UCODE_SIZE 4192
40 /* interrupts */
41 #define BONAIRE_RLC_UCODE_SIZE 2048
42 #define KB_RLC_UCODE_SIZE 2560
43 #define KV_RLC_UCODE_SIZE 2560
44 /* gddr controller */
45 #define CIK_MC_UCODE_SIZE 7866
46 /* sdma */
47 #define CIK_SDMA_UCODE_SIZE 1050
48 #define CIK_SDMA_UCODE_VERSION 64
49
50 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
51 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
52 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
53 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
54 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
55 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
56 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
60 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
61 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
62 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
63 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
64 MODULE_FIRMWARE("radeon/KABINI_me.bin");
65 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
66 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
67 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
68 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
69
70 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
71 extern void r600_ih_ring_fini(struct radeon_device *rdev);
72 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
73 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
74 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
75 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
76 extern void si_rlc_fini(struct radeon_device *rdev);
77 extern int si_rlc_init(struct radeon_device *rdev);
78 static void cik_rlc_stop(struct radeon_device *rdev);
79
80 /*
81 * Indirect registers accessor
82 */
83 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
84 {
85 u32 r;
86
87 WREG32(PCIE_INDEX, reg);
88 (void)RREG32(PCIE_INDEX);
89 r = RREG32(PCIE_DATA);
90 return r;
91 }
92
93 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
94 {
95 WREG32(PCIE_INDEX, reg);
96 (void)RREG32(PCIE_INDEX);
97 WREG32(PCIE_DATA, v);
98 (void)RREG32(PCIE_DATA);
99 }
100
101 static const u32 bonaire_golden_spm_registers[] =
102 {
103 0x30800, 0xe0ffffff, 0xe0000000
104 };
105
106 static const u32 bonaire_golden_common_registers[] =
107 {
108 0xc770, 0xffffffff, 0x00000800,
109 0xc774, 0xffffffff, 0x00000800,
110 0xc798, 0xffffffff, 0x00007fbf,
111 0xc79c, 0xffffffff, 0x00007faf
112 };
113
114 static const u32 bonaire_golden_registers[] =
115 {
116 0x3354, 0x00000333, 0x00000333,
117 0x3350, 0x000c0fc0, 0x00040200,
118 0x9a10, 0x00010000, 0x00058208,
119 0x3c000, 0xffff1fff, 0x00140000,
120 0x3c200, 0xfdfc0fff, 0x00000100,
121 0x3c234, 0x40000000, 0x40000200,
122 0x9830, 0xffffffff, 0x00000000,
123 0x9834, 0xf00fffff, 0x00000400,
124 0x9838, 0x0002021c, 0x00020200,
125 0xc78, 0x00000080, 0x00000000,
126 0x5bb0, 0x000000f0, 0x00000070,
127 0x5bc0, 0xf0311fff, 0x80300000,
128 0x98f8, 0x73773777, 0x12010001,
129 0x350c, 0x00810000, 0x408af000,
130 0x7030, 0x31000111, 0x00000011,
131 0x2f48, 0x73773777, 0x12010001,
132 0x220c, 0x00007fb6, 0x0021a1b1,
133 0x2210, 0x00007fb6, 0x002021b1,
134 0x2180, 0x00007fb6, 0x00002191,
135 0x2218, 0x00007fb6, 0x002121b1,
136 0x221c, 0x00007fb6, 0x002021b1,
137 0x21dc, 0x00007fb6, 0x00002191,
138 0x21e0, 0x00007fb6, 0x00002191,
139 0x3628, 0x0000003f, 0x0000000a,
140 0x362c, 0x0000003f, 0x0000000a,
141 0x2ae4, 0x00073ffe, 0x000022a2,
142 0x240c, 0x000007ff, 0x00000000,
143 0x8a14, 0xf000003f, 0x00000007,
144 0x8bf0, 0x00002001, 0x00000001,
145 0x8b24, 0xffffffff, 0x00ffffff,
146 0x30a04, 0x0000ff0f, 0x00000000,
147 0x28a4c, 0x07ffffff, 0x06000000,
148 0x4d8, 0x00000fff, 0x00000100,
149 0x3e78, 0x00000001, 0x00000002,
150 0x9100, 0x03000000, 0x0362c688,
151 0x8c00, 0x000000ff, 0x00000001,
152 0xe40, 0x00001fff, 0x00001fff,
153 0x9060, 0x0000007f, 0x00000020,
154 0x9508, 0x00010000, 0x00010000,
155 0xac14, 0x000003ff, 0x000000f3,
156 0xac0c, 0xffffffff, 0x00001032
157 };
158
159 static const u32 bonaire_mgcg_cgcg_init[] =
160 {
161 0xc420, 0xffffffff, 0xfffffffc,
162 0x30800, 0xffffffff, 0xe0000000,
163 0x3c2a0, 0xffffffff, 0x00000100,
164 0x3c208, 0xffffffff, 0x00000100,
165 0x3c2c0, 0xffffffff, 0xc0000100,
166 0x3c2c8, 0xffffffff, 0xc0000100,
167 0x3c2c4, 0xffffffff, 0xc0000100,
168 0x55e4, 0xffffffff, 0x00600100,
169 0x3c280, 0xffffffff, 0x00000100,
170 0x3c214, 0xffffffff, 0x06000100,
171 0x3c220, 0xffffffff, 0x00000100,
172 0x3c218, 0xffffffff, 0x06000100,
173 0x3c204, 0xffffffff, 0x00000100,
174 0x3c2e0, 0xffffffff, 0x00000100,
175 0x3c224, 0xffffffff, 0x00000100,
176 0x3c200, 0xffffffff, 0x00000100,
177 0x3c230, 0xffffffff, 0x00000100,
178 0x3c234, 0xffffffff, 0x00000100,
179 0x3c250, 0xffffffff, 0x00000100,
180 0x3c254, 0xffffffff, 0x00000100,
181 0x3c258, 0xffffffff, 0x00000100,
182 0x3c25c, 0xffffffff, 0x00000100,
183 0x3c260, 0xffffffff, 0x00000100,
184 0x3c27c, 0xffffffff, 0x00000100,
185 0x3c278, 0xffffffff, 0x00000100,
186 0x3c210, 0xffffffff, 0x06000100,
187 0x3c290, 0xffffffff, 0x00000100,
188 0x3c274, 0xffffffff, 0x00000100,
189 0x3c2b4, 0xffffffff, 0x00000100,
190 0x3c2b0, 0xffffffff, 0x00000100,
191 0x3c270, 0xffffffff, 0x00000100,
192 0x30800, 0xffffffff, 0xe0000000,
193 0x3c020, 0xffffffff, 0x00010000,
194 0x3c024, 0xffffffff, 0x00030002,
195 0x3c028, 0xffffffff, 0x00040007,
196 0x3c02c, 0xffffffff, 0x00060005,
197 0x3c030, 0xffffffff, 0x00090008,
198 0x3c034, 0xffffffff, 0x00010000,
199 0x3c038, 0xffffffff, 0x00030002,
200 0x3c03c, 0xffffffff, 0x00040007,
201 0x3c040, 0xffffffff, 0x00060005,
202 0x3c044, 0xffffffff, 0x00090008,
203 0x3c048, 0xffffffff, 0x00010000,
204 0x3c04c, 0xffffffff, 0x00030002,
205 0x3c050, 0xffffffff, 0x00040007,
206 0x3c054, 0xffffffff, 0x00060005,
207 0x3c058, 0xffffffff, 0x00090008,
208 0x3c05c, 0xffffffff, 0x00010000,
209 0x3c060, 0xffffffff, 0x00030002,
210 0x3c064, 0xffffffff, 0x00040007,
211 0x3c068, 0xffffffff, 0x00060005,
212 0x3c06c, 0xffffffff, 0x00090008,
213 0x3c070, 0xffffffff, 0x00010000,
214 0x3c074, 0xffffffff, 0x00030002,
215 0x3c078, 0xffffffff, 0x00040007,
216 0x3c07c, 0xffffffff, 0x00060005,
217 0x3c080, 0xffffffff, 0x00090008,
218 0x3c084, 0xffffffff, 0x00010000,
219 0x3c088, 0xffffffff, 0x00030002,
220 0x3c08c, 0xffffffff, 0x00040007,
221 0x3c090, 0xffffffff, 0x00060005,
222 0x3c094, 0xffffffff, 0x00090008,
223 0x3c098, 0xffffffff, 0x00010000,
224 0x3c09c, 0xffffffff, 0x00030002,
225 0x3c0a0, 0xffffffff, 0x00040007,
226 0x3c0a4, 0xffffffff, 0x00060005,
227 0x3c0a8, 0xffffffff, 0x00090008,
228 0x3c000, 0xffffffff, 0x96e00200,
229 0x8708, 0xffffffff, 0x00900100,
230 0xc424, 0xffffffff, 0x0020003f,
231 0x38, 0xffffffff, 0x0140001c,
232 0x3c, 0x000f0000, 0x000f0000,
233 0x220, 0xffffffff, 0xC060000C,
234 0x224, 0xc0000fff, 0x00000100,
235 0xf90, 0xffffffff, 0x00000100,
236 0xf98, 0x00000101, 0x00000000,
237 0x20a8, 0xffffffff, 0x00000104,
238 0x55e4, 0xff000fff, 0x00000100,
239 0x30cc, 0xc0000fff, 0x00000104,
240 0xc1e4, 0x00000001, 0x00000001,
241 0xd00c, 0xff000ff0, 0x00000100,
242 0xd80c, 0xff000ff0, 0x00000100
243 };
244
245 static const u32 spectre_golden_spm_registers[] =
246 {
247 0x30800, 0xe0ffffff, 0xe0000000
248 };
249
250 static const u32 spectre_golden_common_registers[] =
251 {
252 0xc770, 0xffffffff, 0x00000800,
253 0xc774, 0xffffffff, 0x00000800,
254 0xc798, 0xffffffff, 0x00007fbf,
255 0xc79c, 0xffffffff, 0x00007faf
256 };
257
258 static const u32 spectre_golden_registers[] =
259 {
260 0x3c000, 0xffff1fff, 0x96940200,
261 0x3c00c, 0xffff0001, 0xff000000,
262 0x3c200, 0xfffc0fff, 0x00000100,
263 0x6ed8, 0x00010101, 0x00010000,
264 0x9834, 0xf00fffff, 0x00000400,
265 0x9838, 0xfffffffc, 0x00020200,
266 0x5bb0, 0x000000f0, 0x00000070,
267 0x5bc0, 0xf0311fff, 0x80300000,
268 0x98f8, 0x73773777, 0x12010001,
269 0x9b7c, 0x00ff0000, 0x00fc0000,
270 0x2f48, 0x73773777, 0x12010001,
271 0x8a14, 0xf000003f, 0x00000007,
272 0x8b24, 0xffffffff, 0x00ffffff,
273 0x28350, 0x3f3f3fff, 0x00000082,
274 0x28355, 0x0000003f, 0x00000000,
275 0x3e78, 0x00000001, 0x00000002,
276 0x913c, 0xffff03df, 0x00000004,
277 0xc768, 0x00000008, 0x00000008,
278 0x8c00, 0x000008ff, 0x00000800,
279 0x9508, 0x00010000, 0x00010000,
280 0xac0c, 0xffffffff, 0x54763210,
281 0x214f8, 0x01ff01ff, 0x00000002,
282 0x21498, 0x007ff800, 0x00200000,
283 0x2015c, 0xffffffff, 0x00000f40,
284 0x30934, 0xffffffff, 0x00000001
285 };
286
287 static const u32 spectre_mgcg_cgcg_init[] =
288 {
289 0xc420, 0xffffffff, 0xfffffffc,
290 0x30800, 0xffffffff, 0xe0000000,
291 0x3c2a0, 0xffffffff, 0x00000100,
292 0x3c208, 0xffffffff, 0x00000100,
293 0x3c2c0, 0xffffffff, 0x00000100,
294 0x3c2c8, 0xffffffff, 0x00000100,
295 0x3c2c4, 0xffffffff, 0x00000100,
296 0x55e4, 0xffffffff, 0x00600100,
297 0x3c280, 0xffffffff, 0x00000100,
298 0x3c214, 0xffffffff, 0x06000100,
299 0x3c220, 0xffffffff, 0x00000100,
300 0x3c218, 0xffffffff, 0x06000100,
301 0x3c204, 0xffffffff, 0x00000100,
302 0x3c2e0, 0xffffffff, 0x00000100,
303 0x3c224, 0xffffffff, 0x00000100,
304 0x3c200, 0xffffffff, 0x00000100,
305 0x3c230, 0xffffffff, 0x00000100,
306 0x3c234, 0xffffffff, 0x00000100,
307 0x3c250, 0xffffffff, 0x00000100,
308 0x3c254, 0xffffffff, 0x00000100,
309 0x3c258, 0xffffffff, 0x00000100,
310 0x3c25c, 0xffffffff, 0x00000100,
311 0x3c260, 0xffffffff, 0x00000100,
312 0x3c27c, 0xffffffff, 0x00000100,
313 0x3c278, 0xffffffff, 0x00000100,
314 0x3c210, 0xffffffff, 0x06000100,
315 0x3c290, 0xffffffff, 0x00000100,
316 0x3c274, 0xffffffff, 0x00000100,
317 0x3c2b4, 0xffffffff, 0x00000100,
318 0x3c2b0, 0xffffffff, 0x00000100,
319 0x3c270, 0xffffffff, 0x00000100,
320 0x30800, 0xffffffff, 0xe0000000,
321 0x3c020, 0xffffffff, 0x00010000,
322 0x3c024, 0xffffffff, 0x00030002,
323 0x3c028, 0xffffffff, 0x00040007,
324 0x3c02c, 0xffffffff, 0x00060005,
325 0x3c030, 0xffffffff, 0x00090008,
326 0x3c034, 0xffffffff, 0x00010000,
327 0x3c038, 0xffffffff, 0x00030002,
328 0x3c03c, 0xffffffff, 0x00040007,
329 0x3c040, 0xffffffff, 0x00060005,
330 0x3c044, 0xffffffff, 0x00090008,
331 0x3c048, 0xffffffff, 0x00010000,
332 0x3c04c, 0xffffffff, 0x00030002,
333 0x3c050, 0xffffffff, 0x00040007,
334 0x3c054, 0xffffffff, 0x00060005,
335 0x3c058, 0xffffffff, 0x00090008,
336 0x3c05c, 0xffffffff, 0x00010000,
337 0x3c060, 0xffffffff, 0x00030002,
338 0x3c064, 0xffffffff, 0x00040007,
339 0x3c068, 0xffffffff, 0x00060005,
340 0x3c06c, 0xffffffff, 0x00090008,
341 0x3c070, 0xffffffff, 0x00010000,
342 0x3c074, 0xffffffff, 0x00030002,
343 0x3c078, 0xffffffff, 0x00040007,
344 0x3c07c, 0xffffffff, 0x00060005,
345 0x3c080, 0xffffffff, 0x00090008,
346 0x3c084, 0xffffffff, 0x00010000,
347 0x3c088, 0xffffffff, 0x00030002,
348 0x3c08c, 0xffffffff, 0x00040007,
349 0x3c090, 0xffffffff, 0x00060005,
350 0x3c094, 0xffffffff, 0x00090008,
351 0x3c098, 0xffffffff, 0x00010000,
352 0x3c09c, 0xffffffff, 0x00030002,
353 0x3c0a0, 0xffffffff, 0x00040007,
354 0x3c0a4, 0xffffffff, 0x00060005,
355 0x3c0a8, 0xffffffff, 0x00090008,
356 0x3c0ac, 0xffffffff, 0x00010000,
357 0x3c0b0, 0xffffffff, 0x00030002,
358 0x3c0b4, 0xffffffff, 0x00040007,
359 0x3c0b8, 0xffffffff, 0x00060005,
360 0x3c0bc, 0xffffffff, 0x00090008,
361 0x3c000, 0xffffffff, 0x96e00200,
362 0x8708, 0xffffffff, 0x00900100,
363 0xc424, 0xffffffff, 0x0020003f,
364 0x38, 0xffffffff, 0x0140001c,
365 0x3c, 0x000f0000, 0x000f0000,
366 0x220, 0xffffffff, 0xC060000C,
367 0x224, 0xc0000fff, 0x00000100,
368 0xf90, 0xffffffff, 0x00000100,
369 0xf98, 0x00000101, 0x00000000,
370 0x20a8, 0xffffffff, 0x00000104,
371 0x55e4, 0xff000fff, 0x00000100,
372 0x30cc, 0xc0000fff, 0x00000104,
373 0xc1e4, 0x00000001, 0x00000001,
374 0xd00c, 0xff000ff0, 0x00000100,
375 0xd80c, 0xff000ff0, 0x00000100
376 };
377
378 static const u32 kalindi_golden_spm_registers[] =
379 {
380 0x30800, 0xe0ffffff, 0xe0000000
381 };
382
383 static const u32 kalindi_golden_common_registers[] =
384 {
385 0xc770, 0xffffffff, 0x00000800,
386 0xc774, 0xffffffff, 0x00000800,
387 0xc798, 0xffffffff, 0x00007fbf,
388 0xc79c, 0xffffffff, 0x00007faf
389 };
390
391 static const u32 kalindi_golden_registers[] =
392 {
393 0x3c000, 0xffffdfff, 0x6e944040,
394 0x55e4, 0xff607fff, 0xfc000100,
395 0x3c220, 0xff000fff, 0x00000100,
396 0x3c224, 0xff000fff, 0x00000100,
397 0x3c200, 0xfffc0fff, 0x00000100,
398 0x6ed8, 0x00010101, 0x00010000,
399 0x9830, 0xffffffff, 0x00000000,
400 0x9834, 0xf00fffff, 0x00000400,
401 0x5bb0, 0x000000f0, 0x00000070,
402 0x5bc0, 0xf0311fff, 0x80300000,
403 0x98f8, 0x73773777, 0x12010001,
404 0x98fc, 0xffffffff, 0x00000010,
405 0x9b7c, 0x00ff0000, 0x00fc0000,
406 0x8030, 0x00001f0f, 0x0000100a,
407 0x2f48, 0x73773777, 0x12010001,
408 0x2408, 0x000fffff, 0x000c007f,
409 0x8a14, 0xf000003f, 0x00000007,
410 0x8b24, 0x3fff3fff, 0x00ffcfff,
411 0x30a04, 0x0000ff0f, 0x00000000,
412 0x28a4c, 0x07ffffff, 0x06000000,
413 0x4d8, 0x00000fff, 0x00000100,
414 0x3e78, 0x00000001, 0x00000002,
415 0xc768, 0x00000008, 0x00000008,
416 0x8c00, 0x000000ff, 0x00000003,
417 0x214f8, 0x01ff01ff, 0x00000002,
418 0x21498, 0x007ff800, 0x00200000,
419 0x2015c, 0xffffffff, 0x00000f40,
420 0x88c4, 0x001f3ae3, 0x00000082,
421 0x88d4, 0x0000001f, 0x00000010,
422 0x30934, 0xffffffff, 0x00000000
423 };
424
425 static const u32 kalindi_mgcg_cgcg_init[] =
426 {
427 0xc420, 0xffffffff, 0xfffffffc,
428 0x30800, 0xffffffff, 0xe0000000,
429 0x3c2a0, 0xffffffff, 0x00000100,
430 0x3c208, 0xffffffff, 0x00000100,
431 0x3c2c0, 0xffffffff, 0x00000100,
432 0x3c2c8, 0xffffffff, 0x00000100,
433 0x3c2c4, 0xffffffff, 0x00000100,
434 0x55e4, 0xffffffff, 0x00600100,
435 0x3c280, 0xffffffff, 0x00000100,
436 0x3c214, 0xffffffff, 0x06000100,
437 0x3c220, 0xffffffff, 0x00000100,
438 0x3c218, 0xffffffff, 0x06000100,
439 0x3c204, 0xffffffff, 0x00000100,
440 0x3c2e0, 0xffffffff, 0x00000100,
441 0x3c224, 0xffffffff, 0x00000100,
442 0x3c200, 0xffffffff, 0x00000100,
443 0x3c230, 0xffffffff, 0x00000100,
444 0x3c234, 0xffffffff, 0x00000100,
445 0x3c250, 0xffffffff, 0x00000100,
446 0x3c254, 0xffffffff, 0x00000100,
447 0x3c258, 0xffffffff, 0x00000100,
448 0x3c25c, 0xffffffff, 0x00000100,
449 0x3c260, 0xffffffff, 0x00000100,
450 0x3c27c, 0xffffffff, 0x00000100,
451 0x3c278, 0xffffffff, 0x00000100,
452 0x3c210, 0xffffffff, 0x06000100,
453 0x3c290, 0xffffffff, 0x00000100,
454 0x3c274, 0xffffffff, 0x00000100,
455 0x3c2b4, 0xffffffff, 0x00000100,
456 0x3c2b0, 0xffffffff, 0x00000100,
457 0x3c270, 0xffffffff, 0x00000100,
458 0x30800, 0xffffffff, 0xe0000000,
459 0x3c020, 0xffffffff, 0x00010000,
460 0x3c024, 0xffffffff, 0x00030002,
461 0x3c028, 0xffffffff, 0x00040007,
462 0x3c02c, 0xffffffff, 0x00060005,
463 0x3c030, 0xffffffff, 0x00090008,
464 0x3c034, 0xffffffff, 0x00010000,
465 0x3c038, 0xffffffff, 0x00030002,
466 0x3c03c, 0xffffffff, 0x00040007,
467 0x3c040, 0xffffffff, 0x00060005,
468 0x3c044, 0xffffffff, 0x00090008,
469 0x3c000, 0xffffffff, 0x96e00200,
470 0x8708, 0xffffffff, 0x00900100,
471 0xc424, 0xffffffff, 0x0020003f,
472 0x38, 0xffffffff, 0x0140001c,
473 0x3c, 0x000f0000, 0x000f0000,
474 0x220, 0xffffffff, 0xC060000C,
475 0x224, 0xc0000fff, 0x00000100,
476 0x20a8, 0xffffffff, 0x00000104,
477 0x55e4, 0xff000fff, 0x00000100,
478 0x30cc, 0xc0000fff, 0x00000104,
479 0xc1e4, 0x00000001, 0x00000001,
480 0xd00c, 0xff000ff0, 0x00000100,
481 0xd80c, 0xff000ff0, 0x00000100
482 };
483
484 static void cik_init_golden_registers(struct radeon_device *rdev)
485 {
486 switch (rdev->family) {
487 case CHIP_BONAIRE:
488 radeon_program_register_sequence(rdev,
489 bonaire_mgcg_cgcg_init,
490 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
491 radeon_program_register_sequence(rdev,
492 bonaire_golden_registers,
493 (const u32)ARRAY_SIZE(bonaire_golden_registers));
494 radeon_program_register_sequence(rdev,
495 bonaire_golden_common_registers,
496 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
497 radeon_program_register_sequence(rdev,
498 bonaire_golden_spm_registers,
499 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
500 break;
501 case CHIP_KABINI:
502 radeon_program_register_sequence(rdev,
503 kalindi_mgcg_cgcg_init,
504 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
505 radeon_program_register_sequence(rdev,
506 kalindi_golden_registers,
507 (const u32)ARRAY_SIZE(kalindi_golden_registers));
508 radeon_program_register_sequence(rdev,
509 kalindi_golden_common_registers,
510 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
511 radeon_program_register_sequence(rdev,
512 kalindi_golden_spm_registers,
513 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
514 break;
515 case CHIP_KAVERI:
516 radeon_program_register_sequence(rdev,
517 spectre_mgcg_cgcg_init,
518 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
519 radeon_program_register_sequence(rdev,
520 spectre_golden_registers,
521 (const u32)ARRAY_SIZE(spectre_golden_registers));
522 radeon_program_register_sequence(rdev,
523 spectre_golden_common_registers,
524 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
525 radeon_program_register_sequence(rdev,
526 spectre_golden_spm_registers,
527 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
528 break;
529 default:
530 break;
531 }
532 }
533
534 /**
535 * cik_get_xclk - get the xclk
536 *
537 * @rdev: radeon_device pointer
538 *
539 * Returns the reference clock used by the gfx engine
540 * (CIK).
541 */
542 u32 cik_get_xclk(struct radeon_device *rdev)
543 {
544 u32 reference_clock = rdev->clock.spll.reference_freq;
545
546 if (rdev->flags & RADEON_IS_IGP) {
547 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
548 return reference_clock / 2;
549 } else {
550 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
551 return reference_clock / 4;
552 }
553 return reference_clock;
554 }
555
556 /**
557 * cik_mm_rdoorbell - read a doorbell dword
558 *
559 * @rdev: radeon_device pointer
560 * @offset: byte offset into the aperture
561 *
562 * Returns the value in the doorbell aperture at the
563 * requested offset (CIK).
564 */
565 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
566 {
567 if (offset < rdev->doorbell.size) {
568 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
569 } else {
570 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
571 return 0;
572 }
573 }
574
575 /**
576 * cik_mm_wdoorbell - write a doorbell dword
577 *
578 * @rdev: radeon_device pointer
579 * @offset: byte offset into the aperture
580 * @v: value to write
581 *
582 * Writes @v to the doorbell aperture at the
583 * requested offset (CIK).
584 */
585 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
586 {
587 if (offset < rdev->doorbell.size) {
588 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
589 } else {
590 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
591 }
592 }
593
594 #define BONAIRE_IO_MC_REGS_SIZE 36
595
596 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
597 {
598 {0x00000070, 0x04400000},
599 {0x00000071, 0x80c01803},
600 {0x00000072, 0x00004004},
601 {0x00000073, 0x00000100},
602 {0x00000074, 0x00ff0000},
603 {0x00000075, 0x34000000},
604 {0x00000076, 0x08000014},
605 {0x00000077, 0x00cc08ec},
606 {0x00000078, 0x00000400},
607 {0x00000079, 0x00000000},
608 {0x0000007a, 0x04090000},
609 {0x0000007c, 0x00000000},
610 {0x0000007e, 0x4408a8e8},
611 {0x0000007f, 0x00000304},
612 {0x00000080, 0x00000000},
613 {0x00000082, 0x00000001},
614 {0x00000083, 0x00000002},
615 {0x00000084, 0xf3e4f400},
616 {0x00000085, 0x052024e3},
617 {0x00000087, 0x00000000},
618 {0x00000088, 0x01000000},
619 {0x0000008a, 0x1c0a0000},
620 {0x0000008b, 0xff010000},
621 {0x0000008d, 0xffffefff},
622 {0x0000008e, 0xfff3efff},
623 {0x0000008f, 0xfff3efbf},
624 {0x00000092, 0xf7ffffff},
625 {0x00000093, 0xffffff7f},
626 {0x00000095, 0x00101101},
627 {0x00000096, 0x00000fff},
628 {0x00000097, 0x00116fff},
629 {0x00000098, 0x60010000},
630 {0x00000099, 0x10010000},
631 {0x0000009a, 0x00006000},
632 {0x0000009b, 0x00001000},
633 {0x0000009f, 0x00b48000}
634 };
635
636 /**
637 * cik_srbm_select - select specific register instances
638 *
639 * @rdev: radeon_device pointer
640 * @me: selected ME (micro engine)
641 * @pipe: pipe
642 * @queue: queue
643 * @vmid: VMID
644 *
645 * Switches the currently active registers instances. Some
646 * registers are instanced per VMID, others are instanced per
647 * me/pipe/queue combination.
648 */
649 static void cik_srbm_select(struct radeon_device *rdev,
650 u32 me, u32 pipe, u32 queue, u32 vmid)
651 {
652 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
653 MEID(me & 0x3) |
654 VMID(vmid & 0xf) |
655 QUEUEID(queue & 0x7));
656 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
657 }
658
659 /* ucode loading */
660 /**
661 * ci_mc_load_microcode - load MC ucode into the hw
662 *
663 * @rdev: radeon_device pointer
664 *
665 * Load the GDDR MC ucode into the hw (CIK).
666 * Returns 0 on success, error on failure.
667 */
668 static int ci_mc_load_microcode(struct radeon_device *rdev)
669 {
670 const __be32 *fw_data;
671 u32 running, blackout = 0;
672 u32 *io_mc_regs;
673 int i, ucode_size, regs_size;
674
675 if (!rdev->mc_fw)
676 return -EINVAL;
677
678 switch (rdev->family) {
679 case CHIP_BONAIRE:
680 default:
681 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
682 ucode_size = CIK_MC_UCODE_SIZE;
683 regs_size = BONAIRE_IO_MC_REGS_SIZE;
684 break;
685 }
686
687 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
688
689 if (running == 0) {
690 if (running) {
691 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
692 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
693 }
694
695 /* reset the engine and set to writable */
696 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
697 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
698
699 /* load mc io regs */
700 for (i = 0; i < regs_size; i++) {
701 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
702 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
703 }
704 /* load the MC ucode */
705 fw_data = (const __be32 *)rdev->mc_fw->data;
706 for (i = 0; i < ucode_size; i++)
707 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
708
709 /* put the engine back into the active state */
710 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
711 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
712 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
713
714 /* wait for training to complete */
715 for (i = 0; i < rdev->usec_timeout; i++) {
716 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
717 break;
718 udelay(1);
719 }
720 for (i = 0; i < rdev->usec_timeout; i++) {
721 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
722 break;
723 udelay(1);
724 }
725
726 if (running)
727 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
728 }
729
730 return 0;
731 }
732
733 /**
734 * cik_init_microcode - load ucode images from disk
735 *
736 * @rdev: radeon_device pointer
737 *
738 * Use the firmware interface to load the ucode images into
739 * the driver (not loaded into hw).
740 * Returns 0 on success, error on failure.
741 */
742 static int cik_init_microcode(struct radeon_device *rdev)
743 {
744 const char *chip_name;
745 size_t pfp_req_size, me_req_size, ce_req_size,
746 mec_req_size, rlc_req_size, mc_req_size,
747 sdma_req_size;
748 char fw_name[30];
749 int err;
750
751 DRM_DEBUG("\n");
752
753 switch (rdev->family) {
754 case CHIP_BONAIRE:
755 chip_name = "BONAIRE";
756 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
757 me_req_size = CIK_ME_UCODE_SIZE * 4;
758 ce_req_size = CIK_CE_UCODE_SIZE * 4;
759 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
760 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
761 mc_req_size = CIK_MC_UCODE_SIZE * 4;
762 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
763 break;
764 case CHIP_KAVERI:
765 chip_name = "KAVERI";
766 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
767 me_req_size = CIK_ME_UCODE_SIZE * 4;
768 ce_req_size = CIK_CE_UCODE_SIZE * 4;
769 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
770 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
771 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
772 break;
773 case CHIP_KABINI:
774 chip_name = "KABINI";
775 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
776 me_req_size = CIK_ME_UCODE_SIZE * 4;
777 ce_req_size = CIK_CE_UCODE_SIZE * 4;
778 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
779 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
780 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
781 break;
782 default: BUG();
783 }
784
785 DRM_INFO("Loading %s Microcode\n", chip_name);
786
787 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
788 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
789 if (err)
790 goto out;
791 if (rdev->pfp_fw->size != pfp_req_size) {
792 printk(KERN_ERR
793 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
794 rdev->pfp_fw->size, fw_name);
795 err = -EINVAL;
796 goto out;
797 }
798
799 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
800 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
801 if (err)
802 goto out;
803 if (rdev->me_fw->size != me_req_size) {
804 printk(KERN_ERR
805 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
806 rdev->me_fw->size, fw_name);
807 err = -EINVAL;
808 }
809
810 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
811 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
812 if (err)
813 goto out;
814 if (rdev->ce_fw->size != ce_req_size) {
815 printk(KERN_ERR
816 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
817 rdev->ce_fw->size, fw_name);
818 err = -EINVAL;
819 }
820
821 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
822 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
823 if (err)
824 goto out;
825 if (rdev->mec_fw->size != mec_req_size) {
826 printk(KERN_ERR
827 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
828 rdev->mec_fw->size, fw_name);
829 err = -EINVAL;
830 }
831
832 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
833 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
834 if (err)
835 goto out;
836 if (rdev->rlc_fw->size != rlc_req_size) {
837 printk(KERN_ERR
838 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
839 rdev->rlc_fw->size, fw_name);
840 err = -EINVAL;
841 }
842
843 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
844 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
845 if (err)
846 goto out;
847 if (rdev->sdma_fw->size != sdma_req_size) {
848 printk(KERN_ERR
849 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
850 rdev->sdma_fw->size, fw_name);
851 err = -EINVAL;
852 }
853
854 /* No MC ucode on APUs */
855 if (!(rdev->flags & RADEON_IS_IGP)) {
856 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
857 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
858 if (err)
859 goto out;
860 if (rdev->mc_fw->size != mc_req_size) {
861 printk(KERN_ERR
862 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
863 rdev->mc_fw->size, fw_name);
864 err = -EINVAL;
865 }
866 }
867
868 out:
869 if (err) {
870 if (err != -EINVAL)
871 printk(KERN_ERR
872 "cik_cp: Failed to load firmware \"%s\"\n",
873 fw_name);
874 release_firmware(rdev->pfp_fw);
875 rdev->pfp_fw = NULL;
876 release_firmware(rdev->me_fw);
877 rdev->me_fw = NULL;
878 release_firmware(rdev->ce_fw);
879 rdev->ce_fw = NULL;
880 release_firmware(rdev->rlc_fw);
881 rdev->rlc_fw = NULL;
882 release_firmware(rdev->mc_fw);
883 rdev->mc_fw = NULL;
884 }
885 return err;
886 }
887
888 /*
889 * Core functions
890 */
891 /**
892 * cik_tiling_mode_table_init - init the hw tiling table
893 *
894 * @rdev: radeon_device pointer
895 *
896 * Starting with SI, the tiling setup is done globally in a
897 * set of 32 tiling modes. Rather than selecting each set of
898 * parameters per surface as on older asics, we just select
899 * which index in the tiling table we want to use, and the
900 * surface uses those parameters (CIK).
901 */
902 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
903 {
904 const u32 num_tile_mode_states = 32;
905 const u32 num_secondary_tile_mode_states = 16;
906 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
907 u32 num_pipe_configs;
908 u32 num_rbs = rdev->config.cik.max_backends_per_se *
909 rdev->config.cik.max_shader_engines;
910
911 switch (rdev->config.cik.mem_row_size_in_kb) {
912 case 1:
913 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
914 break;
915 case 2:
916 default:
917 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
918 break;
919 case 4:
920 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
921 break;
922 }
923
924 num_pipe_configs = rdev->config.cik.max_tile_pipes;
925 if (num_pipe_configs > 8)
926 num_pipe_configs = 8; /* ??? */
927
928 if (num_pipe_configs == 8) {
929 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
930 switch (reg_offset) {
931 case 0:
932 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
933 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
934 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
935 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
936 break;
937 case 1:
938 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
939 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
940 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
941 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
942 break;
943 case 2:
944 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
945 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
946 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
947 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
948 break;
949 case 3:
950 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
951 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
952 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
953 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
954 break;
955 case 4:
956 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
957 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
958 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
959 TILE_SPLIT(split_equal_to_row_size));
960 break;
961 case 5:
962 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
963 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
964 break;
965 case 6:
966 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
967 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
968 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
969 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
970 break;
971 case 7:
972 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
973 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
974 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
975 TILE_SPLIT(split_equal_to_row_size));
976 break;
977 case 8:
978 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
979 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
980 break;
981 case 9:
982 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
983 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
984 break;
985 case 10:
986 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
987 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
988 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
989 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
990 break;
991 case 11:
992 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
993 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
994 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
995 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
996 break;
997 case 12:
998 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
999 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1000 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1001 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1002 break;
1003 case 13:
1004 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1005 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1006 break;
1007 case 14:
1008 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1009 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1010 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1011 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1012 break;
1013 case 16:
1014 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1015 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1016 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1017 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1018 break;
1019 case 17:
1020 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1021 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1022 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1023 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1024 break;
1025 case 27:
1026 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1027 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1028 break;
1029 case 28:
1030 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1031 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1032 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1033 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1034 break;
1035 case 29:
1036 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1037 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1038 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1039 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1040 break;
1041 case 30:
1042 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1043 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1044 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1045 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1046 break;
1047 default:
1048 gb_tile_moden = 0;
1049 break;
1050 }
1051 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1052 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1053 }
1054 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1055 switch (reg_offset) {
1056 case 0:
1057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1060 NUM_BANKS(ADDR_SURF_16_BANK));
1061 break;
1062 case 1:
1063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1066 NUM_BANKS(ADDR_SURF_16_BANK));
1067 break;
1068 case 2:
1069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1072 NUM_BANKS(ADDR_SURF_16_BANK));
1073 break;
1074 case 3:
1075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1078 NUM_BANKS(ADDR_SURF_16_BANK));
1079 break;
1080 case 4:
1081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1084 NUM_BANKS(ADDR_SURF_8_BANK));
1085 break;
1086 case 5:
1087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1090 NUM_BANKS(ADDR_SURF_4_BANK));
1091 break;
1092 case 6:
1093 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1094 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1095 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1096 NUM_BANKS(ADDR_SURF_2_BANK));
1097 break;
1098 case 8:
1099 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1100 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1101 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1102 NUM_BANKS(ADDR_SURF_16_BANK));
1103 break;
1104 case 9:
1105 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1106 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1107 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1108 NUM_BANKS(ADDR_SURF_16_BANK));
1109 break;
1110 case 10:
1111 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1112 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1113 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1114 NUM_BANKS(ADDR_SURF_16_BANK));
1115 break;
1116 case 11:
1117 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1118 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1119 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1120 NUM_BANKS(ADDR_SURF_16_BANK));
1121 break;
1122 case 12:
1123 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1124 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1125 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1126 NUM_BANKS(ADDR_SURF_8_BANK));
1127 break;
1128 case 13:
1129 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1130 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1131 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1132 NUM_BANKS(ADDR_SURF_4_BANK));
1133 break;
1134 case 14:
1135 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1136 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1137 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1138 NUM_BANKS(ADDR_SURF_2_BANK));
1139 break;
1140 default:
1141 gb_tile_moden = 0;
1142 break;
1143 }
1144 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1145 }
1146 } else if (num_pipe_configs == 4) {
1147 if (num_rbs == 4) {
1148 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1149 switch (reg_offset) {
1150 case 0:
1151 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1152 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1153 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1154 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1155 break;
1156 case 1:
1157 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1158 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1159 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1160 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1161 break;
1162 case 2:
1163 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1164 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1165 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1166 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1167 break;
1168 case 3:
1169 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1170 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1171 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1172 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1173 break;
1174 case 4:
1175 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1176 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1177 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1178 TILE_SPLIT(split_equal_to_row_size));
1179 break;
1180 case 5:
1181 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1182 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1183 break;
1184 case 6:
1185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1186 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1187 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1188 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1189 break;
1190 case 7:
1191 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1192 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1193 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1194 TILE_SPLIT(split_equal_to_row_size));
1195 break;
1196 case 8:
1197 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1198 PIPE_CONFIG(ADDR_SURF_P4_16x16));
1199 break;
1200 case 9:
1201 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1202 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1203 break;
1204 case 10:
1205 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1206 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1207 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1209 break;
1210 case 11:
1211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1212 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1213 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1215 break;
1216 case 12:
1217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1218 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1219 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1221 break;
1222 case 13:
1223 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1224 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1225 break;
1226 case 14:
1227 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1228 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1229 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1230 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1231 break;
1232 case 16:
1233 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1234 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1235 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1236 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1237 break;
1238 case 17:
1239 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1240 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1241 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1242 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1243 break;
1244 case 27:
1245 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1246 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1247 break;
1248 case 28:
1249 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1250 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1251 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1252 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1253 break;
1254 case 29:
1255 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1256 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1257 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1258 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1259 break;
1260 case 30:
1261 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1262 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1263 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1264 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1265 break;
1266 default:
1267 gb_tile_moden = 0;
1268 break;
1269 }
1270 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1271 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1272 }
1273 } else if (num_rbs < 4) {
1274 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1275 switch (reg_offset) {
1276 case 0:
1277 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1278 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1279 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1280 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1281 break;
1282 case 1:
1283 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1284 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1285 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1286 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1287 break;
1288 case 2:
1289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1290 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1291 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1292 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1293 break;
1294 case 3:
1295 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1296 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1297 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1298 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1299 break;
1300 case 4:
1301 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1302 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1303 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1304 TILE_SPLIT(split_equal_to_row_size));
1305 break;
1306 case 5:
1307 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1308 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1309 break;
1310 case 6:
1311 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1312 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1313 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1314 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1315 break;
1316 case 7:
1317 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1318 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1319 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1320 TILE_SPLIT(split_equal_to_row_size));
1321 break;
1322 case 8:
1323 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1324 PIPE_CONFIG(ADDR_SURF_P4_8x16));
1325 break;
1326 case 9:
1327 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1328 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1329 break;
1330 case 10:
1331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1332 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1333 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1335 break;
1336 case 11:
1337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1338 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1339 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1340 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1341 break;
1342 case 12:
1343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1344 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1345 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1346 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1347 break;
1348 case 13:
1349 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1350 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1351 break;
1352 case 14:
1353 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1354 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1355 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1356 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1357 break;
1358 case 16:
1359 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1360 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1361 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1362 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1363 break;
1364 case 17:
1365 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1366 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1367 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1368 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1369 break;
1370 case 27:
1371 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1372 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1373 break;
1374 case 28:
1375 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1376 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1377 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1378 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1379 break;
1380 case 29:
1381 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1382 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1383 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1385 break;
1386 case 30:
1387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1388 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1389 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1391 break;
1392 default:
1393 gb_tile_moden = 0;
1394 break;
1395 }
1396 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1397 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1398 }
1399 }
1400 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1401 switch (reg_offset) {
1402 case 0:
1403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1406 NUM_BANKS(ADDR_SURF_16_BANK));
1407 break;
1408 case 1:
1409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1412 NUM_BANKS(ADDR_SURF_16_BANK));
1413 break;
1414 case 2:
1415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1418 NUM_BANKS(ADDR_SURF_16_BANK));
1419 break;
1420 case 3:
1421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1424 NUM_BANKS(ADDR_SURF_16_BANK));
1425 break;
1426 case 4:
1427 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1428 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1429 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1430 NUM_BANKS(ADDR_SURF_16_BANK));
1431 break;
1432 case 5:
1433 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1436 NUM_BANKS(ADDR_SURF_8_BANK));
1437 break;
1438 case 6:
1439 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1440 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1441 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1442 NUM_BANKS(ADDR_SURF_4_BANK));
1443 break;
1444 case 8:
1445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1448 NUM_BANKS(ADDR_SURF_16_BANK));
1449 break;
1450 case 9:
1451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1454 NUM_BANKS(ADDR_SURF_16_BANK));
1455 break;
1456 case 10:
1457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1460 NUM_BANKS(ADDR_SURF_16_BANK));
1461 break;
1462 case 11:
1463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1466 NUM_BANKS(ADDR_SURF_16_BANK));
1467 break;
1468 case 12:
1469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1472 NUM_BANKS(ADDR_SURF_16_BANK));
1473 break;
1474 case 13:
1475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1478 NUM_BANKS(ADDR_SURF_8_BANK));
1479 break;
1480 case 14:
1481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1484 NUM_BANKS(ADDR_SURF_4_BANK));
1485 break;
1486 default:
1487 gb_tile_moden = 0;
1488 break;
1489 }
1490 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1491 }
1492 } else if (num_pipe_configs == 2) {
1493 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1494 switch (reg_offset) {
1495 case 0:
1496 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1497 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1498 PIPE_CONFIG(ADDR_SURF_P2) |
1499 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1500 break;
1501 case 1:
1502 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1503 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1504 PIPE_CONFIG(ADDR_SURF_P2) |
1505 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1506 break;
1507 case 2:
1508 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1509 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1510 PIPE_CONFIG(ADDR_SURF_P2) |
1511 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1512 break;
1513 case 3:
1514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1515 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1516 PIPE_CONFIG(ADDR_SURF_P2) |
1517 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1518 break;
1519 case 4:
1520 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1521 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1522 PIPE_CONFIG(ADDR_SURF_P2) |
1523 TILE_SPLIT(split_equal_to_row_size));
1524 break;
1525 case 5:
1526 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1527 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1528 break;
1529 case 6:
1530 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1531 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1532 PIPE_CONFIG(ADDR_SURF_P2) |
1533 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1534 break;
1535 case 7:
1536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1538 PIPE_CONFIG(ADDR_SURF_P2) |
1539 TILE_SPLIT(split_equal_to_row_size));
1540 break;
1541 case 8:
1542 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
1543 break;
1544 case 9:
1545 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1546 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1547 break;
1548 case 10:
1549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1550 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1551 PIPE_CONFIG(ADDR_SURF_P2) |
1552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1553 break;
1554 case 11:
1555 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1556 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1557 PIPE_CONFIG(ADDR_SURF_P2) |
1558 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1559 break;
1560 case 12:
1561 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1562 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1563 PIPE_CONFIG(ADDR_SURF_P2) |
1564 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1565 break;
1566 case 13:
1567 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1568 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1569 break;
1570 case 14:
1571 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1572 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1573 PIPE_CONFIG(ADDR_SURF_P2) |
1574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1575 break;
1576 case 16:
1577 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1578 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1579 PIPE_CONFIG(ADDR_SURF_P2) |
1580 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1581 break;
1582 case 17:
1583 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1584 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1585 PIPE_CONFIG(ADDR_SURF_P2) |
1586 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1587 break;
1588 case 27:
1589 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1590 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1591 break;
1592 case 28:
1593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1594 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1595 PIPE_CONFIG(ADDR_SURF_P2) |
1596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1597 break;
1598 case 29:
1599 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1600 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1601 PIPE_CONFIG(ADDR_SURF_P2) |
1602 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1603 break;
1604 case 30:
1605 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1606 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1607 PIPE_CONFIG(ADDR_SURF_P2) |
1608 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1609 break;
1610 default:
1611 gb_tile_moden = 0;
1612 break;
1613 }
1614 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1615 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1616 }
1617 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1618 switch (reg_offset) {
1619 case 0:
1620 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1623 NUM_BANKS(ADDR_SURF_16_BANK));
1624 break;
1625 case 1:
1626 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1627 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1628 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1629 NUM_BANKS(ADDR_SURF_16_BANK));
1630 break;
1631 case 2:
1632 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1635 NUM_BANKS(ADDR_SURF_16_BANK));
1636 break;
1637 case 3:
1638 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1641 NUM_BANKS(ADDR_SURF_16_BANK));
1642 break;
1643 case 4:
1644 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1645 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1646 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1647 NUM_BANKS(ADDR_SURF_16_BANK));
1648 break;
1649 case 5:
1650 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1651 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1652 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1653 NUM_BANKS(ADDR_SURF_16_BANK));
1654 break;
1655 case 6:
1656 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1657 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1658 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1659 NUM_BANKS(ADDR_SURF_8_BANK));
1660 break;
1661 case 8:
1662 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1663 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1664 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1665 NUM_BANKS(ADDR_SURF_16_BANK));
1666 break;
1667 case 9:
1668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1671 NUM_BANKS(ADDR_SURF_16_BANK));
1672 break;
1673 case 10:
1674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1677 NUM_BANKS(ADDR_SURF_16_BANK));
1678 break;
1679 case 11:
1680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1683 NUM_BANKS(ADDR_SURF_16_BANK));
1684 break;
1685 case 12:
1686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1689 NUM_BANKS(ADDR_SURF_16_BANK));
1690 break;
1691 case 13:
1692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1695 NUM_BANKS(ADDR_SURF_16_BANK));
1696 break;
1697 case 14:
1698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1701 NUM_BANKS(ADDR_SURF_8_BANK));
1702 break;
1703 default:
1704 gb_tile_moden = 0;
1705 break;
1706 }
1707 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1708 }
1709 } else
1710 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
1711 }
1712
1713 /**
1714 * cik_select_se_sh - select which SE, SH to address
1715 *
1716 * @rdev: radeon_device pointer
1717 * @se_num: shader engine to address
1718 * @sh_num: sh block to address
1719 *
1720 * Select which SE, SH combinations to address. Certain
1721 * registers are instanced per SE or SH. 0xffffffff means
1722 * broadcast to all SEs or SHs (CIK).
1723 */
1724 static void cik_select_se_sh(struct radeon_device *rdev,
1725 u32 se_num, u32 sh_num)
1726 {
1727 u32 data = INSTANCE_BROADCAST_WRITES;
1728
1729 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1730 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1731 else if (se_num == 0xffffffff)
1732 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
1733 else if (sh_num == 0xffffffff)
1734 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
1735 else
1736 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
1737 WREG32(GRBM_GFX_INDEX, data);
1738 }
1739
1740 /**
1741 * cik_create_bitmask - create a bitmask
1742 *
1743 * @bit_width: length of the mask
1744 *
1745 * create a variable length bit mask (CIK).
1746 * Returns the bitmask.
1747 */
1748 static u32 cik_create_bitmask(u32 bit_width)
1749 {
1750 u32 i, mask = 0;
1751
1752 for (i = 0; i < bit_width; i++) {
1753 mask <<= 1;
1754 mask |= 1;
1755 }
1756 return mask;
1757 }
1758
1759 /**
1760 * cik_select_se_sh - select which SE, SH to address
1761 *
1762 * @rdev: radeon_device pointer
1763 * @max_rb_num: max RBs (render backends) for the asic
1764 * @se_num: number of SEs (shader engines) for the asic
1765 * @sh_per_se: number of SH blocks per SE for the asic
1766 *
1767 * Calculates the bitmask of disabled RBs (CIK).
1768 * Returns the disabled RB bitmask.
1769 */
1770 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
1771 u32 max_rb_num, u32 se_num,
1772 u32 sh_per_se)
1773 {
1774 u32 data, mask;
1775
1776 data = RREG32(CC_RB_BACKEND_DISABLE);
1777 if (data & 1)
1778 data &= BACKEND_DISABLE_MASK;
1779 else
1780 data = 0;
1781 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
1782
1783 data >>= BACKEND_DISABLE_SHIFT;
1784
1785 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
1786
1787 return data & mask;
1788 }
1789
1790 /**
1791 * cik_setup_rb - setup the RBs on the asic
1792 *
1793 * @rdev: radeon_device pointer
1794 * @se_num: number of SEs (shader engines) for the asic
1795 * @sh_per_se: number of SH blocks per SE for the asic
1796 * @max_rb_num: max RBs (render backends) for the asic
1797 *
1798 * Configures per-SE/SH RB registers (CIK).
1799 */
1800 static void cik_setup_rb(struct radeon_device *rdev,
1801 u32 se_num, u32 sh_per_se,
1802 u32 max_rb_num)
1803 {
1804 int i, j;
1805 u32 data, mask;
1806 u32 disabled_rbs = 0;
1807 u32 enabled_rbs = 0;
1808
1809 for (i = 0; i < se_num; i++) {
1810 for (j = 0; j < sh_per_se; j++) {
1811 cik_select_se_sh(rdev, i, j);
1812 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
1813 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
1814 }
1815 }
1816 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1817
1818 mask = 1;
1819 for (i = 0; i < max_rb_num; i++) {
1820 if (!(disabled_rbs & mask))
1821 enabled_rbs |= mask;
1822 mask <<= 1;
1823 }
1824
1825 for (i = 0; i < se_num; i++) {
1826 cik_select_se_sh(rdev, i, 0xffffffff);
1827 data = 0;
1828 for (j = 0; j < sh_per_se; j++) {
1829 switch (enabled_rbs & 3) {
1830 case 1:
1831 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
1832 break;
1833 case 2:
1834 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
1835 break;
1836 case 3:
1837 default:
1838 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
1839 break;
1840 }
1841 enabled_rbs >>= 2;
1842 }
1843 WREG32(PA_SC_RASTER_CONFIG, data);
1844 }
1845 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1846 }
1847
1848 /**
1849 * cik_gpu_init - setup the 3D engine
1850 *
1851 * @rdev: radeon_device pointer
1852 *
1853 * Configures the 3D engine and tiling configuration
1854 * registers so that the 3D engine is usable.
1855 */
1856 static void cik_gpu_init(struct radeon_device *rdev)
1857 {
1858 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
1859 u32 mc_shared_chmap, mc_arb_ramcfg;
1860 u32 hdp_host_path_cntl;
1861 u32 tmp;
1862 int i, j;
1863
1864 switch (rdev->family) {
1865 case CHIP_BONAIRE:
1866 rdev->config.cik.max_shader_engines = 2;
1867 rdev->config.cik.max_tile_pipes = 4;
1868 rdev->config.cik.max_cu_per_sh = 7;
1869 rdev->config.cik.max_sh_per_se = 1;
1870 rdev->config.cik.max_backends_per_se = 2;
1871 rdev->config.cik.max_texture_channel_caches = 4;
1872 rdev->config.cik.max_gprs = 256;
1873 rdev->config.cik.max_gs_threads = 32;
1874 rdev->config.cik.max_hw_contexts = 8;
1875
1876 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
1877 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
1878 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
1879 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
1880 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
1881 break;
1882 case CHIP_KAVERI:
1883 /* TODO */
1884 break;
1885 case CHIP_KABINI:
1886 default:
1887 rdev->config.cik.max_shader_engines = 1;
1888 rdev->config.cik.max_tile_pipes = 2;
1889 rdev->config.cik.max_cu_per_sh = 2;
1890 rdev->config.cik.max_sh_per_se = 1;
1891 rdev->config.cik.max_backends_per_se = 1;
1892 rdev->config.cik.max_texture_channel_caches = 2;
1893 rdev->config.cik.max_gprs = 256;
1894 rdev->config.cik.max_gs_threads = 16;
1895 rdev->config.cik.max_hw_contexts = 8;
1896
1897 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
1898 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
1899 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
1900 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
1901 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
1902 break;
1903 }
1904
1905 /* Initialize HDP */
1906 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1907 WREG32((0x2c14 + j), 0x00000000);
1908 WREG32((0x2c18 + j), 0x00000000);
1909 WREG32((0x2c1c + j), 0x00000000);
1910 WREG32((0x2c20 + j), 0x00000000);
1911 WREG32((0x2c24 + j), 0x00000000);
1912 }
1913
1914 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1915
1916 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1917
1918 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1919 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1920
1921 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
1922 rdev->config.cik.mem_max_burst_length_bytes = 256;
1923 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1924 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1925 if (rdev->config.cik.mem_row_size_in_kb > 4)
1926 rdev->config.cik.mem_row_size_in_kb = 4;
1927 /* XXX use MC settings? */
1928 rdev->config.cik.shader_engine_tile_size = 32;
1929 rdev->config.cik.num_gpus = 1;
1930 rdev->config.cik.multi_gpu_tile_size = 64;
1931
1932 /* fix up row size */
1933 gb_addr_config &= ~ROW_SIZE_MASK;
1934 switch (rdev->config.cik.mem_row_size_in_kb) {
1935 case 1:
1936 default:
1937 gb_addr_config |= ROW_SIZE(0);
1938 break;
1939 case 2:
1940 gb_addr_config |= ROW_SIZE(1);
1941 break;
1942 case 4:
1943 gb_addr_config |= ROW_SIZE(2);
1944 break;
1945 }
1946
1947 /* setup tiling info dword. gb_addr_config is not adequate since it does
1948 * not have bank info, so create a custom tiling dword.
1949 * bits 3:0 num_pipes
1950 * bits 7:4 num_banks
1951 * bits 11:8 group_size
1952 * bits 15:12 row_size
1953 */
1954 rdev->config.cik.tile_config = 0;
1955 switch (rdev->config.cik.num_tile_pipes) {
1956 case 1:
1957 rdev->config.cik.tile_config |= (0 << 0);
1958 break;
1959 case 2:
1960 rdev->config.cik.tile_config |= (1 << 0);
1961 break;
1962 case 4:
1963 rdev->config.cik.tile_config |= (2 << 0);
1964 break;
1965 case 8:
1966 default:
1967 /* XXX what about 12? */
1968 rdev->config.cik.tile_config |= (3 << 0);
1969 break;
1970 }
1971 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1972 rdev->config.cik.tile_config |= 1 << 4;
1973 else
1974 rdev->config.cik.tile_config |= 0 << 4;
1975 rdev->config.cik.tile_config |=
1976 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1977 rdev->config.cik.tile_config |=
1978 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1979
1980 WREG32(GB_ADDR_CONFIG, gb_addr_config);
1981 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1982 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1983 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
1984 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
1985 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1986 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1987 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1988
1989 cik_tiling_mode_table_init(rdev);
1990
1991 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
1992 rdev->config.cik.max_sh_per_se,
1993 rdev->config.cik.max_backends_per_se);
1994
1995 /* set HW defaults for 3D engine */
1996 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1997
1998 WREG32(SX_DEBUG_1, 0x20);
1999
2000 WREG32(TA_CNTL_AUX, 0x00010000);
2001
2002 tmp = RREG32(SPI_CONFIG_CNTL);
2003 tmp |= 0x03000000;
2004 WREG32(SPI_CONFIG_CNTL, tmp);
2005
2006 WREG32(SQ_CONFIG, 1);
2007
2008 WREG32(DB_DEBUG, 0);
2009
2010 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2011 tmp |= 0x00000400;
2012 WREG32(DB_DEBUG2, tmp);
2013
2014 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2015 tmp |= 0x00020200;
2016 WREG32(DB_DEBUG3, tmp);
2017
2018 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2019 tmp |= 0x00018208;
2020 WREG32(CB_HW_CONTROL, tmp);
2021
2022 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2023
2024 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2025 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2026 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2027 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2028
2029 WREG32(VGT_NUM_INSTANCES, 1);
2030
2031 WREG32(CP_PERFMON_CNTL, 0);
2032
2033 WREG32(SQ_CONFIG, 0);
2034
2035 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2036 FORCE_EOV_MAX_REZ_CNT(255)));
2037
2038 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2039 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2040
2041 WREG32(VGT_GS_VERTEX_REUSE, 16);
2042 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2043
2044 tmp = RREG32(HDP_MISC_CNTL);
2045 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2046 WREG32(HDP_MISC_CNTL, tmp);
2047
2048 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2049 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2050
2051 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2052 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2053
2054 udelay(50);
2055 }
2056
2057 /*
2058 * GPU scratch registers helpers function.
2059 */
2060 /**
2061 * cik_scratch_init - setup driver info for CP scratch regs
2062 *
2063 * @rdev: radeon_device pointer
2064 *
2065 * Set up the number and offset of the CP scratch registers.
2066 * NOTE: use of CP scratch registers is a legacy inferface and
2067 * is not used by default on newer asics (r6xx+). On newer asics,
2068 * memory buffers are used for fences rather than scratch regs.
2069 */
2070 static void cik_scratch_init(struct radeon_device *rdev)
2071 {
2072 int i;
2073
2074 rdev->scratch.num_reg = 7;
2075 rdev->scratch.reg_base = SCRATCH_REG0;
2076 for (i = 0; i < rdev->scratch.num_reg; i++) {
2077 rdev->scratch.free[i] = true;
2078 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2079 }
2080 }
2081
2082 /**
2083 * cik_ring_test - basic gfx ring test
2084 *
2085 * @rdev: radeon_device pointer
2086 * @ring: radeon_ring structure holding ring information
2087 *
2088 * Allocate a scratch register and write to it using the gfx ring (CIK).
2089 * Provides a basic gfx ring test to verify that the ring is working.
2090 * Used by cik_cp_gfx_resume();
2091 * Returns 0 on success, error on failure.
2092 */
2093 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2094 {
2095 uint32_t scratch;
2096 uint32_t tmp = 0;
2097 unsigned i;
2098 int r;
2099
2100 r = radeon_scratch_get(rdev, &scratch);
2101 if (r) {
2102 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2103 return r;
2104 }
2105 WREG32(scratch, 0xCAFEDEAD);
2106 r = radeon_ring_lock(rdev, ring, 3);
2107 if (r) {
2108 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2109 radeon_scratch_free(rdev, scratch);
2110 return r;
2111 }
2112 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2113 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2114 radeon_ring_write(ring, 0xDEADBEEF);
2115 radeon_ring_unlock_commit(rdev, ring);
2116
2117 for (i = 0; i < rdev->usec_timeout; i++) {
2118 tmp = RREG32(scratch);
2119 if (tmp == 0xDEADBEEF)
2120 break;
2121 DRM_UDELAY(1);
2122 }
2123 if (i < rdev->usec_timeout) {
2124 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2125 } else {
2126 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2127 ring->idx, scratch, tmp);
2128 r = -EINVAL;
2129 }
2130 radeon_scratch_free(rdev, scratch);
2131 return r;
2132 }
2133
2134 /**
2135 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2136 *
2137 * @rdev: radeon_device pointer
2138 * @fence: radeon fence object
2139 *
2140 * Emits a fence sequnce number on the gfx ring and flushes
2141 * GPU caches.
2142 */
2143 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
2144 struct radeon_fence *fence)
2145 {
2146 struct radeon_ring *ring = &rdev->ring[fence->ring];
2147 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2148
2149 /* EVENT_WRITE_EOP - flush caches, send int */
2150 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2151 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
2152 EOP_TC_ACTION_EN |
2153 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2154 EVENT_INDEX(5)));
2155 radeon_ring_write(ring, addr & 0xfffffffc);
2156 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
2157 radeon_ring_write(ring, fence->seq);
2158 radeon_ring_write(ring, 0);
2159 /* HDP flush */
2160 /* We should be using the new WAIT_REG_MEM special op packet here
2161 * but it causes the CP to hang
2162 */
2163 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2164 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
2165 WRITE_DATA_DST_SEL(0)));
2166 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
2167 radeon_ring_write(ring, 0);
2168 radeon_ring_write(ring, 0);
2169 }
2170
2171 /**
2172 * cik_fence_compute_ring_emit - emit a fence on the compute ring
2173 *
2174 * @rdev: radeon_device pointer
2175 * @fence: radeon fence object
2176 *
2177 * Emits a fence sequnce number on the compute ring and flushes
2178 * GPU caches.
2179 */
2180 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
2181 struct radeon_fence *fence)
2182 {
2183 struct radeon_ring *ring = &rdev->ring[fence->ring];
2184 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2185
2186 /* RELEASE_MEM - flush caches, send int */
2187 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
2188 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
2189 EOP_TC_ACTION_EN |
2190 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2191 EVENT_INDEX(5)));
2192 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
2193 radeon_ring_write(ring, addr & 0xfffffffc);
2194 radeon_ring_write(ring, upper_32_bits(addr));
2195 radeon_ring_write(ring, fence->seq);
2196 radeon_ring_write(ring, 0);
2197 /* HDP flush */
2198 /* We should be using the new WAIT_REG_MEM special op packet here
2199 * but it causes the CP to hang
2200 */
2201 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2202 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
2203 WRITE_DATA_DST_SEL(0)));
2204 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
2205 radeon_ring_write(ring, 0);
2206 radeon_ring_write(ring, 0);
2207 }
2208
2209 void cik_semaphore_ring_emit(struct radeon_device *rdev,
2210 struct radeon_ring *ring,
2211 struct radeon_semaphore *semaphore,
2212 bool emit_wait)
2213 {
2214 uint64_t addr = semaphore->gpu_addr;
2215 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2216
2217 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2218 radeon_ring_write(ring, addr & 0xffffffff);
2219 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
2220 }
2221
2222 /*
2223 * IB stuff
2224 */
2225 /**
2226 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
2227 *
2228 * @rdev: radeon_device pointer
2229 * @ib: radeon indirect buffer object
2230 *
2231 * Emits an DE (drawing engine) or CE (constant engine) IB
2232 * on the gfx ring. IBs are usually generated by userspace
2233 * acceleration drivers and submitted to the kernel for
2234 * sheduling on the ring. This function schedules the IB
2235 * on the gfx ring for execution by the GPU.
2236 */
2237 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2238 {
2239 struct radeon_ring *ring = &rdev->ring[ib->ring];
2240 u32 header, control = INDIRECT_BUFFER_VALID;
2241
2242 if (ib->is_const_ib) {
2243 /* set switch buffer packet before const IB */
2244 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2245 radeon_ring_write(ring, 0);
2246
2247 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2248 } else {
2249 u32 next_rptr;
2250 if (ring->rptr_save_reg) {
2251 next_rptr = ring->wptr + 3 + 4;
2252 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2253 radeon_ring_write(ring, ((ring->rptr_save_reg -
2254 PACKET3_SET_UCONFIG_REG_START) >> 2));
2255 radeon_ring_write(ring, next_rptr);
2256 } else if (rdev->wb.enabled) {
2257 next_rptr = ring->wptr + 5 + 4;
2258 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2259 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
2260 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2261 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2262 radeon_ring_write(ring, next_rptr);
2263 }
2264
2265 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2266 }
2267
2268 control |= ib->length_dw |
2269 (ib->vm ? (ib->vm->id << 24) : 0);
2270
2271 radeon_ring_write(ring, header);
2272 radeon_ring_write(ring,
2273 #ifdef __BIG_ENDIAN
2274 (2 << 0) |
2275 #endif
2276 (ib->gpu_addr & 0xFFFFFFFC));
2277 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2278 radeon_ring_write(ring, control);
2279 }
2280
2281 /**
2282 * cik_ib_test - basic gfx ring IB test
2283 *
2284 * @rdev: radeon_device pointer
2285 * @ring: radeon_ring structure holding ring information
2286 *
2287 * Allocate an IB and execute it on the gfx ring (CIK).
2288 * Provides a basic gfx ring test to verify that IBs are working.
2289 * Returns 0 on success, error on failure.
2290 */
2291 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
2292 {
2293 struct radeon_ib ib;
2294 uint32_t scratch;
2295 uint32_t tmp = 0;
2296 unsigned i;
2297 int r;
2298
2299 r = radeon_scratch_get(rdev, &scratch);
2300 if (r) {
2301 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2302 return r;
2303 }
2304 WREG32(scratch, 0xCAFEDEAD);
2305 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2306 if (r) {
2307 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2308 return r;
2309 }
2310 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2311 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
2312 ib.ptr[2] = 0xDEADBEEF;
2313 ib.length_dw = 3;
2314 r = radeon_ib_schedule(rdev, &ib, NULL);
2315 if (r) {
2316 radeon_scratch_free(rdev, scratch);
2317 radeon_ib_free(rdev, &ib);
2318 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2319 return r;
2320 }
2321 r = radeon_fence_wait(ib.fence, false);
2322 if (r) {
2323 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2324 return r;
2325 }
2326 for (i = 0; i < rdev->usec_timeout; i++) {
2327 tmp = RREG32(scratch);
2328 if (tmp == 0xDEADBEEF)
2329 break;
2330 DRM_UDELAY(1);
2331 }
2332 if (i < rdev->usec_timeout) {
2333 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2334 } else {
2335 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
2336 scratch, tmp);
2337 r = -EINVAL;
2338 }
2339 radeon_scratch_free(rdev, scratch);
2340 radeon_ib_free(rdev, &ib);
2341 return r;
2342 }
2343
2344 /*
2345 * CP.
2346 * On CIK, gfx and compute now have independant command processors.
2347 *
2348 * GFX
2349 * Gfx consists of a single ring and can process both gfx jobs and
2350 * compute jobs. The gfx CP consists of three microengines (ME):
2351 * PFP - Pre-Fetch Parser
2352 * ME - Micro Engine
2353 * CE - Constant Engine
2354 * The PFP and ME make up what is considered the Drawing Engine (DE).
2355 * The CE is an asynchronous engine used for updating buffer desciptors
2356 * used by the DE so that they can be loaded into cache in parallel
2357 * while the DE is processing state update packets.
2358 *
2359 * Compute
2360 * The compute CP consists of two microengines (ME):
2361 * MEC1 - Compute MicroEngine 1
2362 * MEC2 - Compute MicroEngine 2
2363 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
2364 * The queues are exposed to userspace and are programmed directly
2365 * by the compute runtime.
2366 */
2367 /**
2368 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
2369 *
2370 * @rdev: radeon_device pointer
2371 * @enable: enable or disable the MEs
2372 *
2373 * Halts or unhalts the gfx MEs.
2374 */
2375 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
2376 {
2377 if (enable)
2378 WREG32(CP_ME_CNTL, 0);
2379 else {
2380 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2381 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2382 }
2383 udelay(50);
2384 }
2385
2386 /**
2387 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
2388 *
2389 * @rdev: radeon_device pointer
2390 *
2391 * Loads the gfx PFP, ME, and CE ucode.
2392 * Returns 0 for success, -EINVAL if the ucode is not available.
2393 */
2394 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
2395 {
2396 const __be32 *fw_data;
2397 int i;
2398
2399 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
2400 return -EINVAL;
2401
2402 cik_cp_gfx_enable(rdev, false);
2403
2404 /* PFP */
2405 fw_data = (const __be32 *)rdev->pfp_fw->data;
2406 WREG32(CP_PFP_UCODE_ADDR, 0);
2407 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
2408 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2409 WREG32(CP_PFP_UCODE_ADDR, 0);
2410
2411 /* CE */
2412 fw_data = (const __be32 *)rdev->ce_fw->data;
2413 WREG32(CP_CE_UCODE_ADDR, 0);
2414 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
2415 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2416 WREG32(CP_CE_UCODE_ADDR, 0);
2417
2418 /* ME */
2419 fw_data = (const __be32 *)rdev->me_fw->data;
2420 WREG32(CP_ME_RAM_WADDR, 0);
2421 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
2422 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2423 WREG32(CP_ME_RAM_WADDR, 0);
2424
2425 WREG32(CP_PFP_UCODE_ADDR, 0);
2426 WREG32(CP_CE_UCODE_ADDR, 0);
2427 WREG32(CP_ME_RAM_WADDR, 0);
2428 WREG32(CP_ME_RAM_RADDR, 0);
2429 return 0;
2430 }
2431
2432 /**
2433 * cik_cp_gfx_start - start the gfx ring
2434 *
2435 * @rdev: radeon_device pointer
2436 *
2437 * Enables the ring and loads the clear state context and other
2438 * packets required to init the ring.
2439 * Returns 0 for success, error for failure.
2440 */
2441 static int cik_cp_gfx_start(struct radeon_device *rdev)
2442 {
2443 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2444 int r, i;
2445
2446 /* init the CP */
2447 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
2448 WREG32(CP_ENDIAN_SWAP, 0);
2449 WREG32(CP_DEVICE_ID, 1);
2450
2451 cik_cp_gfx_enable(rdev, true);
2452
2453 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
2454 if (r) {
2455 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2456 return r;
2457 }
2458
2459 /* init the CE partitions. CE only used for gfx on CIK */
2460 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2461 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2462 radeon_ring_write(ring, 0xc000);
2463 radeon_ring_write(ring, 0xc000);
2464
2465 /* setup clear context state */
2466 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2467 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2468
2469 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2470 radeon_ring_write(ring, 0x80000000);
2471 radeon_ring_write(ring, 0x80000000);
2472
2473 for (i = 0; i < cik_default_size; i++)
2474 radeon_ring_write(ring, cik_default_state[i]);
2475
2476 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2477 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2478
2479 /* set clear context state */
2480 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2481 radeon_ring_write(ring, 0);
2482
2483 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2484 radeon_ring_write(ring, 0x00000316);
2485 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2486 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2487
2488 radeon_ring_unlock_commit(rdev, ring);
2489
2490 return 0;
2491 }
2492
2493 /**
2494 * cik_cp_gfx_fini - stop the gfx ring
2495 *
2496 * @rdev: radeon_device pointer
2497 *
2498 * Stop the gfx ring and tear down the driver ring
2499 * info.
2500 */
2501 static void cik_cp_gfx_fini(struct radeon_device *rdev)
2502 {
2503 cik_cp_gfx_enable(rdev, false);
2504 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2505 }
2506
2507 /**
2508 * cik_cp_gfx_resume - setup the gfx ring buffer registers
2509 *
2510 * @rdev: radeon_device pointer
2511 *
2512 * Program the location and size of the gfx ring buffer
2513 * and test it to make sure it's working.
2514 * Returns 0 for success, error for failure.
2515 */
2516 static int cik_cp_gfx_resume(struct radeon_device *rdev)
2517 {
2518 struct radeon_ring *ring;
2519 u32 tmp;
2520 u32 rb_bufsz;
2521 u64 rb_addr;
2522 int r;
2523
2524 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2525 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2526
2527 /* Set the write pointer delay */
2528 WREG32(CP_RB_WPTR_DELAY, 0);
2529
2530 /* set the RB to use vmid 0 */
2531 WREG32(CP_RB_VMID, 0);
2532
2533 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2534
2535 /* ring 0 - compute and gfx */
2536 /* Set ring buffer size */
2537 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2538 rb_bufsz = drm_order(ring->ring_size / 8);
2539 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2540 #ifdef __BIG_ENDIAN
2541 tmp |= BUF_SWAP_32BIT;
2542 #endif
2543 WREG32(CP_RB0_CNTL, tmp);
2544
2545 /* Initialize the ring buffer's read and write pointers */
2546 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2547 ring->wptr = 0;
2548 WREG32(CP_RB0_WPTR, ring->wptr);
2549
2550 /* set the wb address wether it's enabled or not */
2551 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2552 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2553
2554 /* scratch register shadowing is no longer supported */
2555 WREG32(SCRATCH_UMSK, 0);
2556
2557 if (!rdev->wb.enabled)
2558 tmp |= RB_NO_UPDATE;
2559
2560 mdelay(1);
2561 WREG32(CP_RB0_CNTL, tmp);
2562
2563 rb_addr = ring->gpu_addr >> 8;
2564 WREG32(CP_RB0_BASE, rb_addr);
2565 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
2566
2567 ring->rptr = RREG32(CP_RB0_RPTR);
2568
2569 /* start the ring */
2570 cik_cp_gfx_start(rdev);
2571 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2572 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2573 if (r) {
2574 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2575 return r;
2576 }
2577 return 0;
2578 }
2579
2580 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
2581 struct radeon_ring *ring)
2582 {
2583 u32 rptr;
2584
2585
2586
2587 if (rdev->wb.enabled) {
2588 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
2589 } else {
2590 mutex_lock(&rdev->srbm_mutex);
2591 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
2592 rptr = RREG32(CP_HQD_PQ_RPTR);
2593 cik_srbm_select(rdev, 0, 0, 0, 0);
2594 mutex_unlock(&rdev->srbm_mutex);
2595 }
2596 rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
2597
2598 return rptr;
2599 }
2600
2601 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
2602 struct radeon_ring *ring)
2603 {
2604 u32 wptr;
2605
2606 if (rdev->wb.enabled) {
2607 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
2608 } else {
2609 mutex_lock(&rdev->srbm_mutex);
2610 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
2611 wptr = RREG32(CP_HQD_PQ_WPTR);
2612 cik_srbm_select(rdev, 0, 0, 0, 0);
2613 mutex_unlock(&rdev->srbm_mutex);
2614 }
2615 wptr = (wptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
2616
2617 return wptr;
2618 }
2619
2620 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
2621 struct radeon_ring *ring)
2622 {
2623 u32 wptr = (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask;
2624
2625 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(wptr);
2626 WDOORBELL32(ring->doorbell_offset, wptr);
2627 }
2628
2629 /**
2630 * cik_cp_compute_enable - enable/disable the compute CP MEs
2631 *
2632 * @rdev: radeon_device pointer
2633 * @enable: enable or disable the MEs
2634 *
2635 * Halts or unhalts the compute MEs.
2636 */
2637 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
2638 {
2639 if (enable)
2640 WREG32(CP_MEC_CNTL, 0);
2641 else
2642 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
2643 udelay(50);
2644 }
2645
2646 /**
2647 * cik_cp_compute_load_microcode - load the compute CP ME ucode
2648 *
2649 * @rdev: radeon_device pointer
2650 *
2651 * Loads the compute MEC1&2 ucode.
2652 * Returns 0 for success, -EINVAL if the ucode is not available.
2653 */
2654 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
2655 {
2656 const __be32 *fw_data;
2657 int i;
2658
2659 if (!rdev->mec_fw)
2660 return -EINVAL;
2661
2662 cik_cp_compute_enable(rdev, false);
2663
2664 /* MEC1 */
2665 fw_data = (const __be32 *)rdev->mec_fw->data;
2666 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
2667 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
2668 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
2669 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
2670
2671 if (rdev->family == CHIP_KAVERI) {
2672 /* MEC2 */
2673 fw_data = (const __be32 *)rdev->mec_fw->data;
2674 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
2675 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
2676 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
2677 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
2678 }
2679
2680 return 0;
2681 }
2682
2683 /**
2684 * cik_cp_compute_start - start the compute queues
2685 *
2686 * @rdev: radeon_device pointer
2687 *
2688 * Enable the compute queues.
2689 * Returns 0 for success, error for failure.
2690 */
2691 static int cik_cp_compute_start(struct radeon_device *rdev)
2692 {
2693 cik_cp_compute_enable(rdev, true);
2694
2695 return 0;
2696 }
2697
2698 /**
2699 * cik_cp_compute_fini - stop the compute queues
2700 *
2701 * @rdev: radeon_device pointer
2702 *
2703 * Stop the compute queues and tear down the driver queue
2704 * info.
2705 */
2706 static void cik_cp_compute_fini(struct radeon_device *rdev)
2707 {
2708 int i, idx, r;
2709
2710 cik_cp_compute_enable(rdev, false);
2711
2712 for (i = 0; i < 2; i++) {
2713 if (i == 0)
2714 idx = CAYMAN_RING_TYPE_CP1_INDEX;
2715 else
2716 idx = CAYMAN_RING_TYPE_CP2_INDEX;
2717
2718 if (rdev->ring[idx].mqd_obj) {
2719 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
2720 if (unlikely(r != 0))
2721 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
2722
2723 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
2724 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
2725
2726 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
2727 rdev->ring[idx].mqd_obj = NULL;
2728 }
2729 }
2730 }
2731
2732 static void cik_mec_fini(struct radeon_device *rdev)
2733 {
2734 int r;
2735
2736 if (rdev->mec.hpd_eop_obj) {
2737 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
2738 if (unlikely(r != 0))
2739 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
2740 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
2741 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
2742
2743 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
2744 rdev->mec.hpd_eop_obj = NULL;
2745 }
2746 }
2747
2748 #define MEC_HPD_SIZE 2048
2749
2750 static int cik_mec_init(struct radeon_device *rdev)
2751 {
2752 int r;
2753 u32 *hpd;
2754
2755 /*
2756 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
2757 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
2758 */
2759 if (rdev->family == CHIP_KAVERI)
2760 rdev->mec.num_mec = 2;
2761 else
2762 rdev->mec.num_mec = 1;
2763 rdev->mec.num_pipe = 4;
2764 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
2765
2766 if (rdev->mec.hpd_eop_obj == NULL) {
2767 r = radeon_bo_create(rdev,
2768 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
2769 PAGE_SIZE, true,
2770 RADEON_GEM_DOMAIN_GTT, NULL,
2771 &rdev->mec.hpd_eop_obj);
2772 if (r) {
2773 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
2774 return r;
2775 }
2776 }
2777
2778 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
2779 if (unlikely(r != 0)) {
2780 cik_mec_fini(rdev);
2781 return r;
2782 }
2783 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
2784 &rdev->mec.hpd_eop_gpu_addr);
2785 if (r) {
2786 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
2787 cik_mec_fini(rdev);
2788 return r;
2789 }
2790 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
2791 if (r) {
2792 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
2793 cik_mec_fini(rdev);
2794 return r;
2795 }
2796
2797 /* clear memory. Not sure if this is required or not */
2798 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
2799
2800 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
2801 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
2802
2803 return 0;
2804 }
2805
2806 struct hqd_registers
2807 {
2808 u32 cp_mqd_base_addr;
2809 u32 cp_mqd_base_addr_hi;
2810 u32 cp_hqd_active;
2811 u32 cp_hqd_vmid;
2812 u32 cp_hqd_persistent_state;
2813 u32 cp_hqd_pipe_priority;
2814 u32 cp_hqd_queue_priority;
2815 u32 cp_hqd_quantum;
2816 u32 cp_hqd_pq_base;
2817 u32 cp_hqd_pq_base_hi;
2818 u32 cp_hqd_pq_rptr;
2819 u32 cp_hqd_pq_rptr_report_addr;
2820 u32 cp_hqd_pq_rptr_report_addr_hi;
2821 u32 cp_hqd_pq_wptr_poll_addr;
2822 u32 cp_hqd_pq_wptr_poll_addr_hi;
2823 u32 cp_hqd_pq_doorbell_control;
2824 u32 cp_hqd_pq_wptr;
2825 u32 cp_hqd_pq_control;
2826 u32 cp_hqd_ib_base_addr;
2827 u32 cp_hqd_ib_base_addr_hi;
2828 u32 cp_hqd_ib_rptr;
2829 u32 cp_hqd_ib_control;
2830 u32 cp_hqd_iq_timer;
2831 u32 cp_hqd_iq_rptr;
2832 u32 cp_hqd_dequeue_request;
2833 u32 cp_hqd_dma_offload;
2834 u32 cp_hqd_sema_cmd;
2835 u32 cp_hqd_msg_type;
2836 u32 cp_hqd_atomic0_preop_lo;
2837 u32 cp_hqd_atomic0_preop_hi;
2838 u32 cp_hqd_atomic1_preop_lo;
2839 u32 cp_hqd_atomic1_preop_hi;
2840 u32 cp_hqd_hq_scheduler0;
2841 u32 cp_hqd_hq_scheduler1;
2842 u32 cp_mqd_control;
2843 };
2844
2845 struct bonaire_mqd
2846 {
2847 u32 header;
2848 u32 dispatch_initiator;
2849 u32 dimensions[3];
2850 u32 start_idx[3];
2851 u32 num_threads[3];
2852 u32 pipeline_stat_enable;
2853 u32 perf_counter_enable;
2854 u32 pgm[2];
2855 u32 tba[2];
2856 u32 tma[2];
2857 u32 pgm_rsrc[2];
2858 u32 vmid;
2859 u32 resource_limits;
2860 u32 static_thread_mgmt01[2];
2861 u32 tmp_ring_size;
2862 u32 static_thread_mgmt23[2];
2863 u32 restart[3];
2864 u32 thread_trace_enable;
2865 u32 reserved1;
2866 u32 user_data[16];
2867 u32 vgtcs_invoke_count[2];
2868 struct hqd_registers queue_state;
2869 u32 dequeue_cntr;
2870 u32 interrupt_queue[64];
2871 };
2872
2873 /**
2874 * cik_cp_compute_resume - setup the compute queue registers
2875 *
2876 * @rdev: radeon_device pointer
2877 *
2878 * Program the compute queues and test them to make sure they
2879 * are working.
2880 * Returns 0 for success, error for failure.
2881 */
2882 static int cik_cp_compute_resume(struct radeon_device *rdev)
2883 {
2884 int r, i, idx;
2885 u32 tmp;
2886 bool use_doorbell = true;
2887 u64 hqd_gpu_addr;
2888 u64 mqd_gpu_addr;
2889 u64 eop_gpu_addr;
2890 u64 wb_gpu_addr;
2891 u32 *buf;
2892 struct bonaire_mqd *mqd;
2893
2894 r = cik_cp_compute_start(rdev);
2895 if (r)
2896 return r;
2897
2898 /* fix up chicken bits */
2899 tmp = RREG32(CP_CPF_DEBUG);
2900 tmp |= (1 << 23);
2901 WREG32(CP_CPF_DEBUG, tmp);
2902
2903 /* init the pipes */
2904 mutex_lock(&rdev->srbm_mutex);
2905 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
2906 int me = (i < 4) ? 1 : 2;
2907 int pipe = (i < 4) ? i : (i - 4);
2908
2909 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
2910
2911 cik_srbm_select(rdev, me, pipe, 0, 0);
2912
2913 /* write the EOP addr */
2914 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
2915 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
2916
2917 /* set the VMID assigned */
2918 WREG32(CP_HPD_EOP_VMID, 0);
2919
2920 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2921 tmp = RREG32(CP_HPD_EOP_CONTROL);
2922 tmp &= ~EOP_SIZE_MASK;
2923 tmp |= drm_order(MEC_HPD_SIZE / 8);
2924 WREG32(CP_HPD_EOP_CONTROL, tmp);
2925 }
2926 cik_srbm_select(rdev, 0, 0, 0, 0);
2927 mutex_unlock(&rdev->srbm_mutex);
2928
2929 /* init the queues. Just two for now. */
2930 for (i = 0; i < 2; i++) {
2931 if (i == 0)
2932 idx = CAYMAN_RING_TYPE_CP1_INDEX;
2933 else
2934 idx = CAYMAN_RING_TYPE_CP2_INDEX;
2935
2936 if (rdev->ring[idx].mqd_obj == NULL) {
2937 r = radeon_bo_create(rdev,
2938 sizeof(struct bonaire_mqd),
2939 PAGE_SIZE, true,
2940 RADEON_GEM_DOMAIN_GTT, NULL,
2941 &rdev->ring[idx].mqd_obj);
2942 if (r) {
2943 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
2944 return r;
2945 }
2946 }
2947
2948 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
2949 if (unlikely(r != 0)) {
2950 cik_cp_compute_fini(rdev);
2951 return r;
2952 }
2953 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
2954 &mqd_gpu_addr);
2955 if (r) {
2956 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
2957 cik_cp_compute_fini(rdev);
2958 return r;
2959 }
2960 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
2961 if (r) {
2962 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
2963 cik_cp_compute_fini(rdev);
2964 return r;
2965 }
2966
2967 /* doorbell offset */
2968 rdev->ring[idx].doorbell_offset =
2969 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
2970
2971 /* init the mqd struct */
2972 memset(buf, 0, sizeof(struct bonaire_mqd));
2973
2974 mqd = (struct bonaire_mqd *)buf;
2975 mqd->header = 0xC0310800;
2976 mqd->static_thread_mgmt01[0] = 0xffffffff;
2977 mqd->static_thread_mgmt01[1] = 0xffffffff;
2978 mqd->static_thread_mgmt23[0] = 0xffffffff;
2979 mqd->static_thread_mgmt23[1] = 0xffffffff;
2980
2981 mutex_lock(&rdev->srbm_mutex);
2982 cik_srbm_select(rdev, rdev->ring[idx].me,
2983 rdev->ring[idx].pipe,
2984 rdev->ring[idx].queue, 0);
2985
2986 /* disable wptr polling */
2987 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
2988 tmp &= ~WPTR_POLL_EN;
2989 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
2990
2991 /* enable doorbell? */
2992 mqd->queue_state.cp_hqd_pq_doorbell_control =
2993 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
2994 if (use_doorbell)
2995 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
2996 else
2997 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
2998 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
2999 mqd->queue_state.cp_hqd_pq_doorbell_control);
3000
3001 /* disable the queue if it's active */
3002 mqd->queue_state.cp_hqd_dequeue_request = 0;
3003 mqd->queue_state.cp_hqd_pq_rptr = 0;
3004 mqd->queue_state.cp_hqd_pq_wptr= 0;
3005 if (RREG32(CP_HQD_ACTIVE) & 1) {
3006 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3007 for (i = 0; i < rdev->usec_timeout; i++) {
3008 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3009 break;
3010 udelay(1);
3011 }
3012 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3013 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3014 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3015 }
3016
3017 /* set the pointer to the MQD */
3018 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3019 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3020 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3021 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3022 /* set MQD vmid to 0 */
3023 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3024 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3025 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3026
3027 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3028 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3029 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3030 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3031 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3032 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3033
3034 /* set up the HQD, this is similar to CP_RB0_CNTL */
3035 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3036 mqd->queue_state.cp_hqd_pq_control &=
3037 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3038
3039 mqd->queue_state.cp_hqd_pq_control |=
3040 drm_order(rdev->ring[idx].ring_size / 8);
3041 mqd->queue_state.cp_hqd_pq_control |=
3042 (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8);
3043 #ifdef __BIG_ENDIAN
3044 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3045 #endif
3046 mqd->queue_state.cp_hqd_pq_control &=
3047 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3048 mqd->queue_state.cp_hqd_pq_control |=
3049 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3050 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3051
3052 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3053 if (i == 0)
3054 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3055 else
3056 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3057 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3058 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3059 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3060 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3061 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3062
3063 /* set the wb address wether it's enabled or not */
3064 if (i == 0)
3065 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3066 else
3067 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3068 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3069 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3070 upper_32_bits(wb_gpu_addr) & 0xffff;
3071 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3072 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3073 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3074 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3075
3076 /* enable the doorbell if requested */
3077 if (use_doorbell) {
3078 mqd->queue_state.cp_hqd_pq_doorbell_control =
3079 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3080 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3081 mqd->queue_state.cp_hqd_pq_doorbell_control |=
3082 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3083 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3084 mqd->queue_state.cp_hqd_pq_doorbell_control &=
3085 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3086
3087 } else {
3088 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3089 }
3090 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3091 mqd->queue_state.cp_hqd_pq_doorbell_control);
3092
3093 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3094 rdev->ring[idx].wptr = 0;
3095 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3096 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3097 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3098 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3099
3100 /* set the vmid for the queue */
3101 mqd->queue_state.cp_hqd_vmid = 0;
3102 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3103
3104 /* activate the queue */
3105 mqd->queue_state.cp_hqd_active = 1;
3106 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3107
3108 cik_srbm_select(rdev, 0, 0, 0, 0);
3109 mutex_unlock(&rdev->srbm_mutex);
3110
3111 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3112 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3113
3114 rdev->ring[idx].ready = true;
3115 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3116 if (r)
3117 rdev->ring[idx].ready = false;
3118 }
3119
3120 return 0;
3121 }
3122
3123 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3124 {
3125 cik_cp_gfx_enable(rdev, enable);
3126 cik_cp_compute_enable(rdev, enable);
3127 }
3128
3129 static int cik_cp_load_microcode(struct radeon_device *rdev)
3130 {
3131 int r;
3132
3133 r = cik_cp_gfx_load_microcode(rdev);
3134 if (r)
3135 return r;
3136 r = cik_cp_compute_load_microcode(rdev);
3137 if (r)
3138 return r;
3139
3140 return 0;
3141 }
3142
3143 static void cik_cp_fini(struct radeon_device *rdev)
3144 {
3145 cik_cp_gfx_fini(rdev);
3146 cik_cp_compute_fini(rdev);
3147 }
3148
3149 static int cik_cp_resume(struct radeon_device *rdev)
3150 {
3151 int r;
3152
3153 /* Reset all cp blocks */
3154 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
3155 RREG32(GRBM_SOFT_RESET);
3156 mdelay(15);
3157 WREG32(GRBM_SOFT_RESET, 0);
3158 RREG32(GRBM_SOFT_RESET);
3159
3160 r = cik_cp_load_microcode(rdev);
3161 if (r)
3162 return r;
3163
3164 r = cik_cp_gfx_resume(rdev);
3165 if (r)
3166 return r;
3167 r = cik_cp_compute_resume(rdev);
3168 if (r)
3169 return r;
3170
3171 return 0;
3172 }
3173
3174 /*
3175 * sDMA - System DMA
3176 * Starting with CIK, the GPU has new asynchronous
3177 * DMA engines. These engines are used for compute
3178 * and gfx. There are two DMA engines (SDMA0, SDMA1)
3179 * and each one supports 1 ring buffer used for gfx
3180 * and 2 queues used for compute.
3181 *
3182 * The programming model is very similar to the CP
3183 * (ring buffer, IBs, etc.), but sDMA has it's own
3184 * packet format that is different from the PM4 format
3185 * used by the CP. sDMA supports copying data, writing
3186 * embedded data, solid fills, and a number of other
3187 * things. It also has support for tiling/detiling of
3188 * buffers.
3189 */
3190 /**
3191 * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
3192 *
3193 * @rdev: radeon_device pointer
3194 * @ib: IB object to schedule
3195 *
3196 * Schedule an IB in the DMA ring (CIK).
3197 */
3198 void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
3199 struct radeon_ib *ib)
3200 {
3201 struct radeon_ring *ring = &rdev->ring[ib->ring];
3202 u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
3203
3204 if (rdev->wb.enabled) {
3205 u32 next_rptr = ring->wptr + 5;
3206 while ((next_rptr & 7) != 4)
3207 next_rptr++;
3208 next_rptr += 4;
3209 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3212 radeon_ring_write(ring, 1); /* number of DWs to follow */
3213 radeon_ring_write(ring, next_rptr);
3214 }
3215
3216 /* IB packet must end on a 8 DW boundary */
3217 while ((ring->wptr & 7) != 4)
3218 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
3219 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
3220 radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
3221 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
3222 radeon_ring_write(ring, ib->length_dw);
3223
3224 }
3225
3226 /**
3227 * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
3228 *
3229 * @rdev: radeon_device pointer
3230 * @fence: radeon fence object
3231 *
3232 * Add a DMA fence packet to the ring to write
3233 * the fence seq number and DMA trap packet to generate
3234 * an interrupt if needed (CIK).
3235 */
3236 void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
3237 struct radeon_fence *fence)
3238 {
3239 struct radeon_ring *ring = &rdev->ring[fence->ring];
3240 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3241 u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
3242 SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
3243 u32 ref_and_mask;
3244
3245 if (fence->ring == R600_RING_TYPE_DMA_INDEX)
3246 ref_and_mask = SDMA0;
3247 else
3248 ref_and_mask = SDMA1;
3249
3250 /* write the fence */
3251 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
3252 radeon_ring_write(ring, addr & 0xffffffff);
3253 radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3254 radeon_ring_write(ring, fence->seq);
3255 /* generate an interrupt */
3256 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
3257 /* flush HDP */
3258 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
3259 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
3260 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
3261 radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
3262 radeon_ring_write(ring, ref_and_mask); /* MASK */
3263 radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
3264 }
3265
3266 /**
3267 * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
3268 *
3269 * @rdev: radeon_device pointer
3270 * @ring: radeon_ring structure holding ring information
3271 * @semaphore: radeon semaphore object
3272 * @emit_wait: wait or signal semaphore
3273 *
3274 * Add a DMA semaphore packet to the ring wait on or signal
3275 * other rings (CIK).
3276 */
3277 void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
3278 struct radeon_ring *ring,
3279 struct radeon_semaphore *semaphore,
3280 bool emit_wait)
3281 {
3282 u64 addr = semaphore->gpu_addr;
3283 u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
3284
3285 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
3286 radeon_ring_write(ring, addr & 0xfffffff8);
3287 radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3288 }
3289
3290 /**
3291 * cik_sdma_gfx_stop - stop the gfx async dma engines
3292 *
3293 * @rdev: radeon_device pointer
3294 *
3295 * Stop the gfx async dma ring buffers (CIK).
3296 */
3297 static void cik_sdma_gfx_stop(struct radeon_device *rdev)
3298 {
3299 u32 rb_cntl, reg_offset;
3300 int i;
3301
3302 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3303
3304 for (i = 0; i < 2; i++) {
3305 if (i == 0)
3306 reg_offset = SDMA0_REGISTER_OFFSET;
3307 else
3308 reg_offset = SDMA1_REGISTER_OFFSET;
3309 rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
3310 rb_cntl &= ~SDMA_RB_ENABLE;
3311 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
3312 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
3313 }
3314 }
3315
3316 /**
3317 * cik_sdma_rlc_stop - stop the compute async dma engines
3318 *
3319 * @rdev: radeon_device pointer
3320 *
3321 * Stop the compute async dma queues (CIK).
3322 */
3323 static void cik_sdma_rlc_stop(struct radeon_device *rdev)
3324 {
3325 /* XXX todo */
3326 }
3327
3328 /**
3329 * cik_sdma_enable - stop the async dma engines
3330 *
3331 * @rdev: radeon_device pointer
3332 * @enable: enable/disable the DMA MEs.
3333 *
3334 * Halt or unhalt the async dma engines (CIK).
3335 */
3336 static void cik_sdma_enable(struct radeon_device *rdev, bool enable)
3337 {
3338 u32 me_cntl, reg_offset;
3339 int i;
3340
3341 for (i = 0; i < 2; i++) {
3342 if (i == 0)
3343 reg_offset = SDMA0_REGISTER_OFFSET;
3344 else
3345 reg_offset = SDMA1_REGISTER_OFFSET;
3346 me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
3347 if (enable)
3348 me_cntl &= ~SDMA_HALT;
3349 else
3350 me_cntl |= SDMA_HALT;
3351 WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
3352 }
3353 }
3354
3355 /**
3356 * cik_sdma_gfx_resume - setup and start the async dma engines
3357 *
3358 * @rdev: radeon_device pointer
3359 *
3360 * Set up the gfx DMA ring buffers and enable them (CIK).
3361 * Returns 0 for success, error for failure.
3362 */
3363 static int cik_sdma_gfx_resume(struct radeon_device *rdev)
3364 {
3365 struct radeon_ring *ring;
3366 u32 rb_cntl, ib_cntl;
3367 u32 rb_bufsz;
3368 u32 reg_offset, wb_offset;
3369 int i, r;
3370
3371 for (i = 0; i < 2; i++) {
3372 if (i == 0) {
3373 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3374 reg_offset = SDMA0_REGISTER_OFFSET;
3375 wb_offset = R600_WB_DMA_RPTR_OFFSET;
3376 } else {
3377 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
3378 reg_offset = SDMA1_REGISTER_OFFSET;
3379 wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
3380 }
3381
3382 WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
3383 WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
3384
3385 /* Set ring buffer size in dwords */
3386 rb_bufsz = drm_order(ring->ring_size / 4);
3387 rb_cntl = rb_bufsz << 1;
3388 #ifdef __BIG_ENDIAN
3389 rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
3390 #endif
3391 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
3392
3393 /* Initialize the ring buffer's read and write pointers */
3394 WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
3395 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
3396
3397 /* set the wb address whether it's enabled or not */
3398 WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
3399 upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
3400 WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
3401 ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
3402
3403 if (rdev->wb.enabled)
3404 rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
3405
3406 WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
3407 WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
3408
3409 ring->wptr = 0;
3410 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
3411
3412 ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
3413
3414 /* enable DMA RB */
3415 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
3416
3417 ib_cntl = SDMA_IB_ENABLE;
3418 #ifdef __BIG_ENDIAN
3419 ib_cntl |= SDMA_IB_SWAP_ENABLE;
3420 #endif
3421 /* enable DMA IBs */
3422 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
3423
3424 ring->ready = true;
3425
3426 r = radeon_ring_test(rdev, ring->idx, ring);
3427 if (r) {
3428 ring->ready = false;
3429 return r;
3430 }
3431 }
3432
3433 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3434
3435 return 0;
3436 }
3437
3438 /**
3439 * cik_sdma_rlc_resume - setup and start the async dma engines
3440 *
3441 * @rdev: radeon_device pointer
3442 *
3443 * Set up the compute DMA queues and enable them (CIK).
3444 * Returns 0 for success, error for failure.
3445 */
3446 static int cik_sdma_rlc_resume(struct radeon_device *rdev)
3447 {
3448 /* XXX todo */
3449 return 0;
3450 }
3451
3452 /**
3453 * cik_sdma_load_microcode - load the sDMA ME ucode
3454 *
3455 * @rdev: radeon_device pointer
3456 *
3457 * Loads the sDMA0/1 ucode.
3458 * Returns 0 for success, -EINVAL if the ucode is not available.
3459 */
3460 static int cik_sdma_load_microcode(struct radeon_device *rdev)
3461 {
3462 const __be32 *fw_data;
3463 int i;
3464
3465 if (!rdev->sdma_fw)
3466 return -EINVAL;
3467
3468 /* stop the gfx rings and rlc compute queues */
3469 cik_sdma_gfx_stop(rdev);
3470 cik_sdma_rlc_stop(rdev);
3471
3472 /* halt the MEs */
3473 cik_sdma_enable(rdev, false);
3474
3475 /* sdma0 */
3476 fw_data = (const __be32 *)rdev->sdma_fw->data;
3477 WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
3478 for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
3479 WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
3480 WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
3481
3482 /* sdma1 */
3483 fw_data = (const __be32 *)rdev->sdma_fw->data;
3484 WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
3485 for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
3486 WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
3487 WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
3488
3489 WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
3490 WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
3491 return 0;
3492 }
3493
3494 /**
3495 * cik_sdma_resume - setup and start the async dma engines
3496 *
3497 * @rdev: radeon_device pointer
3498 *
3499 * Set up the DMA engines and enable them (CIK).
3500 * Returns 0 for success, error for failure.
3501 */
3502 static int cik_sdma_resume(struct radeon_device *rdev)
3503 {
3504 int r;
3505
3506 /* Reset dma */
3507 WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
3508 RREG32(SRBM_SOFT_RESET);
3509 udelay(50);
3510 WREG32(SRBM_SOFT_RESET, 0);
3511 RREG32(SRBM_SOFT_RESET);
3512
3513 r = cik_sdma_load_microcode(rdev);
3514 if (r)
3515 return r;
3516
3517 /* unhalt the MEs */
3518 cik_sdma_enable(rdev, true);
3519
3520 /* start the gfx rings and rlc compute queues */
3521 r = cik_sdma_gfx_resume(rdev);
3522 if (r)
3523 return r;
3524 r = cik_sdma_rlc_resume(rdev);
3525 if (r)
3526 return r;
3527
3528 return 0;
3529 }
3530
3531 /**
3532 * cik_sdma_fini - tear down the async dma engines
3533 *
3534 * @rdev: radeon_device pointer
3535 *
3536 * Stop the async dma engines and free the rings (CIK).
3537 */
3538 static void cik_sdma_fini(struct radeon_device *rdev)
3539 {
3540 /* stop the gfx rings and rlc compute queues */
3541 cik_sdma_gfx_stop(rdev);
3542 cik_sdma_rlc_stop(rdev);
3543 /* halt the MEs */
3544 cik_sdma_enable(rdev, false);
3545 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
3546 radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
3547 /* XXX - compute dma queue tear down */
3548 }
3549
3550 /**
3551 * cik_copy_dma - copy pages using the DMA engine
3552 *
3553 * @rdev: radeon_device pointer
3554 * @src_offset: src GPU address
3555 * @dst_offset: dst GPU address
3556 * @num_gpu_pages: number of GPU pages to xfer
3557 * @fence: radeon fence object
3558 *
3559 * Copy GPU paging using the DMA engine (CIK).
3560 * Used by the radeon ttm implementation to move pages if
3561 * registered as the asic copy callback.
3562 */
3563 int cik_copy_dma(struct radeon_device *rdev,
3564 uint64_t src_offset, uint64_t dst_offset,
3565 unsigned num_gpu_pages,
3566 struct radeon_fence **fence)
3567 {
3568 struct radeon_semaphore *sem = NULL;
3569 int ring_index = rdev->asic->copy.dma_ring_index;
3570 struct radeon_ring *ring = &rdev->ring[ring_index];
3571 u32 size_in_bytes, cur_size_in_bytes;
3572 int i, num_loops;
3573 int r = 0;
3574
3575 r = radeon_semaphore_create(rdev, &sem);
3576 if (r) {
3577 DRM_ERROR("radeon: moving bo (%d).\n", r);
3578 return r;
3579 }
3580
3581 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3582 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3583 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
3584 if (r) {
3585 DRM_ERROR("radeon: moving bo (%d).\n", r);
3586 radeon_semaphore_free(rdev, &sem, NULL);
3587 return r;
3588 }
3589
3590 if (radeon_fence_need_sync(*fence, ring->idx)) {
3591 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3592 ring->idx);
3593 radeon_fence_note_sync(*fence, ring->idx);
3594 } else {
3595 radeon_semaphore_free(rdev, &sem, NULL);
3596 }
3597
3598 for (i = 0; i < num_loops; i++) {
3599 cur_size_in_bytes = size_in_bytes;
3600 if (cur_size_in_bytes > 0x1fffff)
3601 cur_size_in_bytes = 0x1fffff;
3602 size_in_bytes -= cur_size_in_bytes;
3603 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
3604 radeon_ring_write(ring, cur_size_in_bytes);
3605 radeon_ring_write(ring, 0); /* src/dst endian swap */
3606 radeon_ring_write(ring, src_offset & 0xffffffff);
3607 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
3608 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3609 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
3610 src_offset += cur_size_in_bytes;
3611 dst_offset += cur_size_in_bytes;
3612 }
3613
3614 r = radeon_fence_emit(rdev, fence, ring->idx);
3615 if (r) {
3616 radeon_ring_unlock_undo(rdev, ring);
3617 return r;
3618 }
3619
3620 radeon_ring_unlock_commit(rdev, ring);
3621 radeon_semaphore_free(rdev, &sem, *fence);
3622
3623 return r;
3624 }
3625
3626 /**
3627 * cik_sdma_ring_test - simple async dma engine test
3628 *
3629 * @rdev: radeon_device pointer
3630 * @ring: radeon_ring structure holding ring information
3631 *
3632 * Test the DMA engine by writing using it to write an
3633 * value to memory. (CIK).
3634 * Returns 0 for success, error for failure.
3635 */
3636 int cik_sdma_ring_test(struct radeon_device *rdev,
3637 struct radeon_ring *ring)
3638 {
3639 unsigned i;
3640 int r;
3641 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3642 u32 tmp;
3643
3644 if (!ptr) {
3645 DRM_ERROR("invalid vram scratch pointer\n");
3646 return -EINVAL;
3647 }
3648
3649 tmp = 0xCAFEDEAD;
3650 writel(tmp, ptr);
3651
3652 r = radeon_ring_lock(rdev, ring, 4);
3653 if (r) {
3654 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
3655 return r;
3656 }
3657 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
3658 radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
3659 radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
3660 radeon_ring_write(ring, 1); /* number of DWs to follow */
3661 radeon_ring_write(ring, 0xDEADBEEF);
3662 radeon_ring_unlock_commit(rdev, ring);
3663
3664 for (i = 0; i < rdev->usec_timeout; i++) {
3665 tmp = readl(ptr);
3666 if (tmp == 0xDEADBEEF)
3667 break;
3668 DRM_UDELAY(1);
3669 }
3670
3671 if (i < rdev->usec_timeout) {
3672 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3673 } else {
3674 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
3675 ring->idx, tmp);
3676 r = -EINVAL;
3677 }
3678 return r;
3679 }
3680
3681 /**
3682 * cik_sdma_ib_test - test an IB on the DMA engine
3683 *
3684 * @rdev: radeon_device pointer
3685 * @ring: radeon_ring structure holding ring information
3686 *
3687 * Test a simple IB in the DMA ring (CIK).
3688 * Returns 0 on success, error on failure.
3689 */
3690 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3691 {
3692 struct radeon_ib ib;
3693 unsigned i;
3694 int r;
3695 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3696 u32 tmp = 0;
3697
3698 if (!ptr) {
3699 DRM_ERROR("invalid vram scratch pointer\n");
3700 return -EINVAL;
3701 }
3702
3703 tmp = 0xCAFEDEAD;
3704 writel(tmp, ptr);
3705
3706 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3707 if (r) {
3708 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3709 return r;
3710 }
3711
3712 ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
3713 ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3714 ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
3715 ib.ptr[3] = 1;
3716 ib.ptr[4] = 0xDEADBEEF;
3717 ib.length_dw = 5;
3718
3719 r = radeon_ib_schedule(rdev, &ib, NULL);
3720 if (r) {
3721 radeon_ib_free(rdev, &ib);
3722 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3723 return r;
3724 }
3725 r = radeon_fence_wait(ib.fence, false);
3726 if (r) {
3727 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3728 return r;
3729 }
3730 for (i = 0; i < rdev->usec_timeout; i++) {
3731 tmp = readl(ptr);
3732 if (tmp == 0xDEADBEEF)
3733 break;
3734 DRM_UDELAY(1);
3735 }
3736 if (i < rdev->usec_timeout) {
3737 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3738 } else {
3739 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3740 r = -EINVAL;
3741 }
3742 radeon_ib_free(rdev, &ib);
3743 return r;
3744 }
3745
3746
3747 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
3748 {
3749 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
3750 RREG32(GRBM_STATUS));
3751 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
3752 RREG32(GRBM_STATUS2));
3753 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
3754 RREG32(GRBM_STATUS_SE0));
3755 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
3756 RREG32(GRBM_STATUS_SE1));
3757 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
3758 RREG32(GRBM_STATUS_SE2));
3759 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
3760 RREG32(GRBM_STATUS_SE3));
3761 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
3762 RREG32(SRBM_STATUS));
3763 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
3764 RREG32(SRBM_STATUS2));
3765 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
3766 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
3767 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
3768 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
3769 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
3770 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
3771 RREG32(CP_STALLED_STAT1));
3772 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
3773 RREG32(CP_STALLED_STAT2));
3774 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
3775 RREG32(CP_STALLED_STAT3));
3776 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
3777 RREG32(CP_CPF_BUSY_STAT));
3778 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
3779 RREG32(CP_CPF_STALLED_STAT1));
3780 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
3781 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
3782 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
3783 RREG32(CP_CPC_STALLED_STAT1));
3784 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
3785 }
3786
3787 /**
3788 * cik_gpu_check_soft_reset - check which blocks are busy
3789 *
3790 * @rdev: radeon_device pointer
3791 *
3792 * Check which blocks are busy and return the relevant reset
3793 * mask to be used by cik_gpu_soft_reset().
3794 * Returns a mask of the blocks to be reset.
3795 */
3796 static u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
3797 {
3798 u32 reset_mask = 0;
3799 u32 tmp;
3800
3801 /* GRBM_STATUS */
3802 tmp = RREG32(GRBM_STATUS);
3803 if (tmp & (PA_BUSY | SC_BUSY |
3804 BCI_BUSY | SX_BUSY |
3805 TA_BUSY | VGT_BUSY |
3806 DB_BUSY | CB_BUSY |
3807 GDS_BUSY | SPI_BUSY |
3808 IA_BUSY | IA_BUSY_NO_DMA))
3809 reset_mask |= RADEON_RESET_GFX;
3810
3811 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
3812 reset_mask |= RADEON_RESET_CP;
3813
3814 /* GRBM_STATUS2 */
3815 tmp = RREG32(GRBM_STATUS2);
3816 if (tmp & RLC_BUSY)
3817 reset_mask |= RADEON_RESET_RLC;
3818
3819 /* SDMA0_STATUS_REG */
3820 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
3821 if (!(tmp & SDMA_IDLE))
3822 reset_mask |= RADEON_RESET_DMA;
3823
3824 /* SDMA1_STATUS_REG */
3825 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
3826 if (!(tmp & SDMA_IDLE))
3827 reset_mask |= RADEON_RESET_DMA1;
3828
3829 /* SRBM_STATUS2 */
3830 tmp = RREG32(SRBM_STATUS2);
3831 if (tmp & SDMA_BUSY)
3832 reset_mask |= RADEON_RESET_DMA;
3833
3834 if (tmp & SDMA1_BUSY)
3835 reset_mask |= RADEON_RESET_DMA1;
3836
3837 /* SRBM_STATUS */
3838 tmp = RREG32(SRBM_STATUS);
3839
3840 if (tmp & IH_BUSY)
3841 reset_mask |= RADEON_RESET_IH;
3842
3843 if (tmp & SEM_BUSY)
3844 reset_mask |= RADEON_RESET_SEM;
3845
3846 if (tmp & GRBM_RQ_PENDING)
3847 reset_mask |= RADEON_RESET_GRBM;
3848
3849 if (tmp & VMC_BUSY)
3850 reset_mask |= RADEON_RESET_VMC;
3851
3852 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3853 MCC_BUSY | MCD_BUSY))
3854 reset_mask |= RADEON_RESET_MC;
3855
3856 if (evergreen_is_display_hung(rdev))
3857 reset_mask |= RADEON_RESET_DISPLAY;
3858
3859 /* Skip MC reset as it's mostly likely not hung, just busy */
3860 if (reset_mask & RADEON_RESET_MC) {
3861 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3862 reset_mask &= ~RADEON_RESET_MC;
3863 }
3864
3865 return reset_mask;
3866 }
3867
3868 /**
3869 * cik_gpu_soft_reset - soft reset GPU
3870 *
3871 * @rdev: radeon_device pointer
3872 * @reset_mask: mask of which blocks to reset
3873 *
3874 * Soft reset the blocks specified in @reset_mask.
3875 */
3876 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3877 {
3878 struct evergreen_mc_save save;
3879 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3880 u32 tmp;
3881
3882 if (reset_mask == 0)
3883 return;
3884
3885 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3886
3887 cik_print_gpu_status_regs(rdev);
3888 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3889 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3890 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3891 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3892
3893 /* stop the rlc */
3894 cik_rlc_stop(rdev);
3895
3896 /* Disable GFX parsing/prefetching */
3897 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3898
3899 /* Disable MEC parsing/prefetching */
3900 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
3901
3902 if (reset_mask & RADEON_RESET_DMA) {
3903 /* sdma0 */
3904 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
3905 tmp |= SDMA_HALT;
3906 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
3907 }
3908 if (reset_mask & RADEON_RESET_DMA1) {
3909 /* sdma1 */
3910 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
3911 tmp |= SDMA_HALT;
3912 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
3913 }
3914
3915 evergreen_mc_stop(rdev, &save);
3916 if (evergreen_mc_wait_for_idle(rdev)) {
3917 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3918 }
3919
3920 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
3921 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
3922
3923 if (reset_mask & RADEON_RESET_CP) {
3924 grbm_soft_reset |= SOFT_RESET_CP;
3925
3926 srbm_soft_reset |= SOFT_RESET_GRBM;
3927 }
3928
3929 if (reset_mask & RADEON_RESET_DMA)
3930 srbm_soft_reset |= SOFT_RESET_SDMA;
3931
3932 if (reset_mask & RADEON_RESET_DMA1)
3933 srbm_soft_reset |= SOFT_RESET_SDMA1;
3934
3935 if (reset_mask & RADEON_RESET_DISPLAY)
3936 srbm_soft_reset |= SOFT_RESET_DC;
3937
3938 if (reset_mask & RADEON_RESET_RLC)
3939 grbm_soft_reset |= SOFT_RESET_RLC;
3940
3941 if (reset_mask & RADEON_RESET_SEM)
3942 srbm_soft_reset |= SOFT_RESET_SEM;
3943
3944 if (reset_mask & RADEON_RESET_IH)
3945 srbm_soft_reset |= SOFT_RESET_IH;
3946
3947 if (reset_mask & RADEON_RESET_GRBM)
3948 srbm_soft_reset |= SOFT_RESET_GRBM;
3949
3950 if (reset_mask & RADEON_RESET_VMC)
3951 srbm_soft_reset |= SOFT_RESET_VMC;
3952
3953 if (!(rdev->flags & RADEON_IS_IGP)) {
3954 if (reset_mask & RADEON_RESET_MC)
3955 srbm_soft_reset |= SOFT_RESET_MC;
3956 }
3957
3958 if (grbm_soft_reset) {
3959 tmp = RREG32(GRBM_SOFT_RESET);
3960 tmp |= grbm_soft_reset;
3961 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3962 WREG32(GRBM_SOFT_RESET, tmp);
3963 tmp = RREG32(GRBM_SOFT_RESET);
3964
3965 udelay(50);
3966
3967 tmp &= ~grbm_soft_reset;
3968 WREG32(GRBM_SOFT_RESET, tmp);
3969 tmp = RREG32(GRBM_SOFT_RESET);
3970 }
3971
3972 if (srbm_soft_reset) {
3973 tmp = RREG32(SRBM_SOFT_RESET);
3974 tmp |= srbm_soft_reset;
3975 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3976 WREG32(SRBM_SOFT_RESET, tmp);
3977 tmp = RREG32(SRBM_SOFT_RESET);
3978
3979 udelay(50);
3980
3981 tmp &= ~srbm_soft_reset;
3982 WREG32(SRBM_SOFT_RESET, tmp);
3983 tmp = RREG32(SRBM_SOFT_RESET);
3984 }
3985
3986 /* Wait a little for things to settle down */
3987 udelay(50);
3988
3989 evergreen_mc_resume(rdev, &save);
3990 udelay(50);
3991
3992 cik_print_gpu_status_regs(rdev);
3993 }
3994
3995 /**
3996 * cik_asic_reset - soft reset GPU
3997 *
3998 * @rdev: radeon_device pointer
3999 *
4000 * Look up which blocks are hung and attempt
4001 * to reset them.
4002 * Returns 0 for success.
4003 */
4004 int cik_asic_reset(struct radeon_device *rdev)
4005 {
4006 u32 reset_mask;
4007
4008 reset_mask = cik_gpu_check_soft_reset(rdev);
4009
4010 if (reset_mask)
4011 r600_set_bios_scratch_engine_hung(rdev, true);
4012
4013 cik_gpu_soft_reset(rdev, reset_mask);
4014
4015 reset_mask = cik_gpu_check_soft_reset(rdev);
4016
4017 if (!reset_mask)
4018 r600_set_bios_scratch_engine_hung(rdev, false);
4019
4020 return 0;
4021 }
4022
4023 /**
4024 * cik_gfx_is_lockup - check if the 3D engine is locked up
4025 *
4026 * @rdev: radeon_device pointer
4027 * @ring: radeon_ring structure holding ring information
4028 *
4029 * Check if the 3D engine is locked up (CIK).
4030 * Returns true if the engine is locked, false if not.
4031 */
4032 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4033 {
4034 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4035
4036 if (!(reset_mask & (RADEON_RESET_GFX |
4037 RADEON_RESET_COMPUTE |
4038 RADEON_RESET_CP))) {
4039 radeon_ring_lockup_update(ring);
4040 return false;
4041 }
4042 /* force CP activities */
4043 radeon_ring_force_activity(rdev, ring);
4044 return radeon_ring_test_lockup(rdev, ring);
4045 }
4046
4047 /**
4048 * cik_sdma_is_lockup - Check if the DMA engine is locked up
4049 *
4050 * @rdev: radeon_device pointer
4051 * @ring: radeon_ring structure holding ring information
4052 *
4053 * Check if the async DMA engine is locked up (CIK).
4054 * Returns true if the engine appears to be locked up, false if not.
4055 */
4056 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4057 {
4058 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4059 u32 mask;
4060
4061 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
4062 mask = RADEON_RESET_DMA;
4063 else
4064 mask = RADEON_RESET_DMA1;
4065
4066 if (!(reset_mask & mask)) {
4067 radeon_ring_lockup_update(ring);
4068 return false;
4069 }
4070 /* force ring activities */
4071 radeon_ring_force_activity(rdev, ring);
4072 return radeon_ring_test_lockup(rdev, ring);
4073 }
4074
4075 /* MC */
4076 /**
4077 * cik_mc_program - program the GPU memory controller
4078 *
4079 * @rdev: radeon_device pointer
4080 *
4081 * Set the location of vram, gart, and AGP in the GPU's
4082 * physical address space (CIK).
4083 */
4084 static void cik_mc_program(struct radeon_device *rdev)
4085 {
4086 struct evergreen_mc_save save;
4087 u32 tmp;
4088 int i, j;
4089
4090 /* Initialize HDP */
4091 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4092 WREG32((0x2c14 + j), 0x00000000);
4093 WREG32((0x2c18 + j), 0x00000000);
4094 WREG32((0x2c1c + j), 0x00000000);
4095 WREG32((0x2c20 + j), 0x00000000);
4096 WREG32((0x2c24 + j), 0x00000000);
4097 }
4098 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4099
4100 evergreen_mc_stop(rdev, &save);
4101 if (radeon_mc_wait_for_idle(rdev)) {
4102 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4103 }
4104 /* Lockout access through VGA aperture*/
4105 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4106 /* Update configuration */
4107 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4108 rdev->mc.vram_start >> 12);
4109 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4110 rdev->mc.vram_end >> 12);
4111 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4112 rdev->vram_scratch.gpu_addr >> 12);
4113 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4114 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4115 WREG32(MC_VM_FB_LOCATION, tmp);
4116 /* XXX double check these! */
4117 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4118 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4119 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4120 WREG32(MC_VM_AGP_BASE, 0);
4121 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4122 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4123 if (radeon_mc_wait_for_idle(rdev)) {
4124 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4125 }
4126 evergreen_mc_resume(rdev, &save);
4127 /* we need to own VRAM, so turn off the VGA renderer here
4128 * to stop it overwriting our objects */
4129 rv515_vga_render_disable(rdev);
4130 }
4131
4132 /**
4133 * cik_mc_init - initialize the memory controller driver params
4134 *
4135 * @rdev: radeon_device pointer
4136 *
4137 * Look up the amount of vram, vram width, and decide how to place
4138 * vram and gart within the GPU's physical address space (CIK).
4139 * Returns 0 for success.
4140 */
4141 static int cik_mc_init(struct radeon_device *rdev)
4142 {
4143 u32 tmp;
4144 int chansize, numchan;
4145
4146 /* Get VRAM informations */
4147 rdev->mc.vram_is_ddr = true;
4148 tmp = RREG32(MC_ARB_RAMCFG);
4149 if (tmp & CHANSIZE_MASK) {
4150 chansize = 64;
4151 } else {
4152 chansize = 32;
4153 }
4154 tmp = RREG32(MC_SHARED_CHMAP);
4155 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4156 case 0:
4157 default:
4158 numchan = 1;
4159 break;
4160 case 1:
4161 numchan = 2;
4162 break;
4163 case 2:
4164 numchan = 4;
4165 break;
4166 case 3:
4167 numchan = 8;
4168 break;
4169 case 4:
4170 numchan = 3;
4171 break;
4172 case 5:
4173 numchan = 6;
4174 break;
4175 case 6:
4176 numchan = 10;
4177 break;
4178 case 7:
4179 numchan = 12;
4180 break;
4181 case 8:
4182 numchan = 16;
4183 break;
4184 }
4185 rdev->mc.vram_width = numchan * chansize;
4186 /* Could aper size report 0 ? */
4187 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4188 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4189 /* size in MB on si */
4190 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4191 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4192 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4193 si_vram_gtt_location(rdev, &rdev->mc);
4194 radeon_update_bandwidth_info(rdev);
4195
4196 return 0;
4197 }
4198
4199 /*
4200 * GART
4201 * VMID 0 is the physical GPU addresses as used by the kernel.
4202 * VMIDs 1-15 are used for userspace clients and are handled
4203 * by the radeon vm/hsa code.
4204 */
4205 /**
4206 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4207 *
4208 * @rdev: radeon_device pointer
4209 *
4210 * Flush the TLB for the VMID 0 page table (CIK).
4211 */
4212 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4213 {
4214 /* flush hdp cache */
4215 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4216
4217 /* bits 0-15 are the VM contexts0-15 */
4218 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4219 }
4220
4221 /**
4222 * cik_pcie_gart_enable - gart enable
4223 *
4224 * @rdev: radeon_device pointer
4225 *
4226 * This sets up the TLBs, programs the page tables for VMID0,
4227 * sets up the hw for VMIDs 1-15 which are allocated on
4228 * demand, and sets up the global locations for the LDS, GDS,
4229 * and GPUVM for FSA64 clients (CIK).
4230 * Returns 0 for success, errors for failure.
4231 */
4232 static int cik_pcie_gart_enable(struct radeon_device *rdev)
4233 {
4234 int r, i;
4235
4236 if (rdev->gart.robj == NULL) {
4237 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4238 return -EINVAL;
4239 }
4240 r = radeon_gart_table_vram_pin(rdev);
4241 if (r)
4242 return r;
4243 radeon_gart_restore(rdev);
4244 /* Setup TLB control */
4245 WREG32(MC_VM_MX_L1_TLB_CNTL,
4246 (0xA << 7) |
4247 ENABLE_L1_TLB |
4248 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4249 ENABLE_ADVANCED_DRIVER_MODEL |
4250 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4251 /* Setup L2 cache */
4252 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4253 ENABLE_L2_FRAGMENT_PROCESSING |
4254 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4255 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4256 EFFECTIVE_L2_QUEUE_SIZE(7) |
4257 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4258 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4259 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4260 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4261 /* setup context0 */
4262 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4263 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4264 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4265 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4266 (u32)(rdev->dummy_page.addr >> 12));
4267 WREG32(VM_CONTEXT0_CNTL2, 0);
4268 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4269 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4270
4271 WREG32(0x15D4, 0);
4272 WREG32(0x15D8, 0);
4273 WREG32(0x15DC, 0);
4274
4275 /* empty context1-15 */
4276 /* FIXME start with 4G, once using 2 level pt switch to full
4277 * vm size space
4278 */
4279 /* set vm size, must be a multiple of 4 */
4280 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4281 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4282 for (i = 1; i < 16; i++) {
4283 if (i < 8)
4284 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4285 rdev->gart.table_addr >> 12);
4286 else
4287 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4288 rdev->gart.table_addr >> 12);
4289 }
4290
4291 /* enable context1-15 */
4292 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4293 (u32)(rdev->dummy_page.addr >> 12));
4294 WREG32(VM_CONTEXT1_CNTL2, 4);
4295 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4296 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4297 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4298 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4299 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4300 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4301 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4302 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4303 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4304 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4305 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4306 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4307 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4308
4309 /* TC cache setup ??? */
4310 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4311 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4312 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4313
4314 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4315 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4316 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4317 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4318 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4319
4320 WREG32(TC_CFG_L1_VOLATILE, 0);
4321 WREG32(TC_CFG_L2_VOLATILE, 0);
4322
4323 if (rdev->family == CHIP_KAVERI) {
4324 u32 tmp = RREG32(CHUB_CONTROL);
4325 tmp &= ~BYPASS_VM;
4326 WREG32(CHUB_CONTROL, tmp);
4327 }
4328
4329 /* XXX SH_MEM regs */
4330 /* where to put LDS, scratch, GPUVM in FSA64 space */
4331 mutex_lock(&rdev->srbm_mutex);
4332 for (i = 0; i < 16; i++) {
4333 cik_srbm_select(rdev, 0, 0, 0, i);
4334 /* CP and shaders */
4335 WREG32(SH_MEM_CONFIG, 0);
4336 WREG32(SH_MEM_APE1_BASE, 1);
4337 WREG32(SH_MEM_APE1_LIMIT, 0);
4338 WREG32(SH_MEM_BASES, 0);
4339 /* SDMA GFX */
4340 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4341 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4342 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4343 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4344 /* XXX SDMA RLC - todo */
4345 }
4346 cik_srbm_select(rdev, 0, 0, 0, 0);
4347 mutex_unlock(&rdev->srbm_mutex);
4348
4349 cik_pcie_gart_tlb_flush(rdev);
4350 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4351 (unsigned)(rdev->mc.gtt_size >> 20),
4352 (unsigned long long)rdev->gart.table_addr);
4353 rdev->gart.ready = true;
4354 return 0;
4355 }
4356
4357 /**
4358 * cik_pcie_gart_disable - gart disable
4359 *
4360 * @rdev: radeon_device pointer
4361 *
4362 * This disables all VM page table (CIK).
4363 */
4364 static void cik_pcie_gart_disable(struct radeon_device *rdev)
4365 {
4366 /* Disable all tables */
4367 WREG32(VM_CONTEXT0_CNTL, 0);
4368 WREG32(VM_CONTEXT1_CNTL, 0);
4369 /* Setup TLB control */
4370 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4371 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4372 /* Setup L2 cache */
4373 WREG32(VM_L2_CNTL,
4374 ENABLE_L2_FRAGMENT_PROCESSING |
4375 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4376 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4377 EFFECTIVE_L2_QUEUE_SIZE(7) |
4378 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4379 WREG32(VM_L2_CNTL2, 0);
4380 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4381 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4382 radeon_gart_table_vram_unpin(rdev);
4383 }
4384
4385 /**
4386 * cik_pcie_gart_fini - vm fini callback
4387 *
4388 * @rdev: radeon_device pointer
4389 *
4390 * Tears down the driver GART/VM setup (CIK).
4391 */
4392 static void cik_pcie_gart_fini(struct radeon_device *rdev)
4393 {
4394 cik_pcie_gart_disable(rdev);
4395 radeon_gart_table_vram_free(rdev);
4396 radeon_gart_fini(rdev);
4397 }
4398
4399 /* vm parser */
4400 /**
4401 * cik_ib_parse - vm ib_parse callback
4402 *
4403 * @rdev: radeon_device pointer
4404 * @ib: indirect buffer pointer
4405 *
4406 * CIK uses hw IB checking so this is a nop (CIK).
4407 */
4408 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4409 {
4410 return 0;
4411 }
4412
4413 /*
4414 * vm
4415 * VMID 0 is the physical GPU addresses as used by the kernel.
4416 * VMIDs 1-15 are used for userspace clients and are handled
4417 * by the radeon vm/hsa code.
4418 */
4419 /**
4420 * cik_vm_init - cik vm init callback
4421 *
4422 * @rdev: radeon_device pointer
4423 *
4424 * Inits cik specific vm parameters (number of VMs, base of vram for
4425 * VMIDs 1-15) (CIK).
4426 * Returns 0 for success.
4427 */
4428 int cik_vm_init(struct radeon_device *rdev)
4429 {
4430 /* number of VMs */
4431 rdev->vm_manager.nvm = 16;
4432 /* base offset of vram pages */
4433 if (rdev->flags & RADEON_IS_IGP) {
4434 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4435 tmp <<= 22;
4436 rdev->vm_manager.vram_base_offset = tmp;
4437 } else
4438 rdev->vm_manager.vram_base_offset = 0;
4439
4440 return 0;
4441 }
4442
4443 /**
4444 * cik_vm_fini - cik vm fini callback
4445 *
4446 * @rdev: radeon_device pointer
4447 *
4448 * Tear down any asic specific VM setup (CIK).
4449 */
4450 void cik_vm_fini(struct radeon_device *rdev)
4451 {
4452 }
4453
4454 /**
4455 * cik_vm_decode_fault - print human readable fault info
4456 *
4457 * @rdev: radeon_device pointer
4458 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4459 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4460 *
4461 * Print human readable fault information (CIK).
4462 */
4463 static void cik_vm_decode_fault(struct radeon_device *rdev,
4464 u32 status, u32 addr, u32 mc_client)
4465 {
4466 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4467 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4468 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4469 char *block = (char *)&mc_client;
4470
4471 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4472 protections, vmid, addr,
4473 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4474 block, mc_id);
4475 }
4476
4477 /**
4478 * cik_vm_flush - cik vm flush using the CP
4479 *
4480 * @rdev: radeon_device pointer
4481 *
4482 * Update the page table base and flush the VM TLB
4483 * using the CP (CIK).
4484 */
4485 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4486 {
4487 struct radeon_ring *ring = &rdev->ring[ridx];
4488
4489 if (vm == NULL)
4490 return;
4491
4492 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4493 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4494 WRITE_DATA_DST_SEL(0)));
4495 if (vm->id < 8) {
4496 radeon_ring_write(ring,
4497 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4498 } else {
4499 radeon_ring_write(ring,
4500 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4501 }
4502 radeon_ring_write(ring, 0);
4503 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4504
4505 /* update SH_MEM_* regs */
4506 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4507 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4508 WRITE_DATA_DST_SEL(0)));
4509 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4510 radeon_ring_write(ring, 0);
4511 radeon_ring_write(ring, VMID(vm->id));
4512
4513 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4514 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4515 WRITE_DATA_DST_SEL(0)));
4516 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4517 radeon_ring_write(ring, 0);
4518
4519 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4520 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4521 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4522 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4523
4524 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4525 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4526 WRITE_DATA_DST_SEL(0)));
4527 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4528 radeon_ring_write(ring, 0);
4529 radeon_ring_write(ring, VMID(0));
4530
4531 /* HDP flush */
4532 /* We should be using the WAIT_REG_MEM packet here like in
4533 * cik_fence_ring_emit(), but it causes the CP to hang in this
4534 * context...
4535 */
4536 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4537 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4538 WRITE_DATA_DST_SEL(0)));
4539 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4540 radeon_ring_write(ring, 0);
4541 radeon_ring_write(ring, 0);
4542
4543 /* bits 0-15 are the VM contexts0-15 */
4544 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4545 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4546 WRITE_DATA_DST_SEL(0)));
4547 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4548 radeon_ring_write(ring, 0);
4549 radeon_ring_write(ring, 1 << vm->id);
4550
4551 /* compute doesn't have PFP */
4552 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4553 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4554 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4555 radeon_ring_write(ring, 0x0);
4556 }
4557 }
4558
4559 /**
4560 * cik_vm_set_page - update the page tables using sDMA
4561 *
4562 * @rdev: radeon_device pointer
4563 * @ib: indirect buffer to fill with commands
4564 * @pe: addr of the page entry
4565 * @addr: dst addr to write into pe
4566 * @count: number of page entries to update
4567 * @incr: increase next addr by incr bytes
4568 * @flags: access flags
4569 *
4570 * Update the page tables using CP or sDMA (CIK).
4571 */
4572 void cik_vm_set_page(struct radeon_device *rdev,
4573 struct radeon_ib *ib,
4574 uint64_t pe,
4575 uint64_t addr, unsigned count,
4576 uint32_t incr, uint32_t flags)
4577 {
4578 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4579 uint64_t value;
4580 unsigned ndw;
4581
4582 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4583 /* CP */
4584 while (count) {
4585 ndw = 2 + count * 2;
4586 if (ndw > 0x3FFE)
4587 ndw = 0x3FFE;
4588
4589 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4590 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4591 WRITE_DATA_DST_SEL(1));
4592 ib->ptr[ib->length_dw++] = pe;
4593 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4594 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4595 if (flags & RADEON_VM_PAGE_SYSTEM) {
4596 value = radeon_vm_map_gart(rdev, addr);
4597 value &= 0xFFFFFFFFFFFFF000ULL;
4598 } else if (flags & RADEON_VM_PAGE_VALID) {
4599 value = addr;
4600 } else {
4601 value = 0;
4602 }
4603 addr += incr;
4604 value |= r600_flags;
4605 ib->ptr[ib->length_dw++] = value;
4606 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4607 }
4608 }
4609 } else {
4610 /* DMA */
4611 if (flags & RADEON_VM_PAGE_SYSTEM) {
4612 while (count) {
4613 ndw = count * 2;
4614 if (ndw > 0xFFFFE)
4615 ndw = 0xFFFFE;
4616
4617 /* for non-physically contiguous pages (system) */
4618 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
4619 ib->ptr[ib->length_dw++] = pe;
4620 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4621 ib->ptr[ib->length_dw++] = ndw;
4622 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4623 if (flags & RADEON_VM_PAGE_SYSTEM) {
4624 value = radeon_vm_map_gart(rdev, addr);
4625 value &= 0xFFFFFFFFFFFFF000ULL;
4626 } else if (flags & RADEON_VM_PAGE_VALID) {
4627 value = addr;
4628 } else {
4629 value = 0;
4630 }
4631 addr += incr;
4632 value |= r600_flags;
4633 ib->ptr[ib->length_dw++] = value;
4634 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4635 }
4636 }
4637 } else {
4638 while (count) {
4639 ndw = count;
4640 if (ndw > 0x7FFFF)
4641 ndw = 0x7FFFF;
4642
4643 if (flags & RADEON_VM_PAGE_VALID)
4644 value = addr;
4645 else
4646 value = 0;
4647 /* for physically contiguous pages (vram) */
4648 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
4649 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4650 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4651 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4652 ib->ptr[ib->length_dw++] = 0;
4653 ib->ptr[ib->length_dw++] = value; /* value */
4654 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4655 ib->ptr[ib->length_dw++] = incr; /* increment size */
4656 ib->ptr[ib->length_dw++] = 0;
4657 ib->ptr[ib->length_dw++] = ndw; /* number of entries */
4658 pe += ndw * 8;
4659 addr += ndw * incr;
4660 count -= ndw;
4661 }
4662 }
4663 while (ib->length_dw & 0x7)
4664 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
4665 }
4666 }
4667
4668 /**
4669 * cik_dma_vm_flush - cik vm flush using sDMA
4670 *
4671 * @rdev: radeon_device pointer
4672 *
4673 * Update the page table base and flush the VM TLB
4674 * using sDMA (CIK).
4675 */
4676 void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4677 {
4678 struct radeon_ring *ring = &rdev->ring[ridx];
4679 u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
4680 SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
4681 u32 ref_and_mask;
4682
4683 if (vm == NULL)
4684 return;
4685
4686 if (ridx == R600_RING_TYPE_DMA_INDEX)
4687 ref_and_mask = SDMA0;
4688 else
4689 ref_and_mask = SDMA1;
4690
4691 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4692 if (vm->id < 8) {
4693 radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4694 } else {
4695 radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4696 }
4697 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4698
4699 /* update SH_MEM_* regs */
4700 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4701 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4702 radeon_ring_write(ring, VMID(vm->id));
4703
4704 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4705 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4706 radeon_ring_write(ring, 0);
4707
4708 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4709 radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
4710 radeon_ring_write(ring, 0);
4711
4712 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4713 radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
4714 radeon_ring_write(ring, 1);
4715
4716 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4717 radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
4718 radeon_ring_write(ring, 0);
4719
4720 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4721 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4722 radeon_ring_write(ring, VMID(0));
4723
4724 /* flush HDP */
4725 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
4726 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
4727 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
4728 radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
4729 radeon_ring_write(ring, ref_and_mask); /* MASK */
4730 radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
4731
4732 /* flush TLB */
4733 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4734 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4735 radeon_ring_write(ring, 1 << vm->id);
4736 }
4737
4738 /*
4739 * RLC
4740 * The RLC is a multi-purpose microengine that handles a
4741 * variety of functions, the most important of which is
4742 * the interrupt controller.
4743 */
4744 /**
4745 * cik_rlc_stop - stop the RLC ME
4746 *
4747 * @rdev: radeon_device pointer
4748 *
4749 * Halt the RLC ME (MicroEngine) (CIK).
4750 */
4751 static void cik_rlc_stop(struct radeon_device *rdev)
4752 {
4753 int i, j, k;
4754 u32 mask, tmp;
4755
4756 tmp = RREG32(CP_INT_CNTL_RING0);
4757 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4758 WREG32(CP_INT_CNTL_RING0, tmp);
4759
4760 RREG32(CB_CGTT_SCLK_CTRL);
4761 RREG32(CB_CGTT_SCLK_CTRL);
4762 RREG32(CB_CGTT_SCLK_CTRL);
4763 RREG32(CB_CGTT_SCLK_CTRL);
4764
4765 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
4766 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
4767
4768 WREG32(RLC_CNTL, 0);
4769
4770 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4771 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4772 cik_select_se_sh(rdev, i, j);
4773 for (k = 0; k < rdev->usec_timeout; k++) {
4774 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4775 break;
4776 udelay(1);
4777 }
4778 }
4779 }
4780 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4781
4782 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4783 for (k = 0; k < rdev->usec_timeout; k++) {
4784 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4785 break;
4786 udelay(1);
4787 }
4788 }
4789
4790 /**
4791 * cik_rlc_start - start the RLC ME
4792 *
4793 * @rdev: radeon_device pointer
4794 *
4795 * Unhalt the RLC ME (MicroEngine) (CIK).
4796 */
4797 static void cik_rlc_start(struct radeon_device *rdev)
4798 {
4799 u32 tmp;
4800
4801 WREG32(RLC_CNTL, RLC_ENABLE);
4802
4803 tmp = RREG32(CP_INT_CNTL_RING0);
4804 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4805 WREG32(CP_INT_CNTL_RING0, tmp);
4806
4807 udelay(50);
4808 }
4809
4810 /**
4811 * cik_rlc_resume - setup the RLC hw
4812 *
4813 * @rdev: radeon_device pointer
4814 *
4815 * Initialize the RLC registers, load the ucode,
4816 * and start the RLC (CIK).
4817 * Returns 0 for success, -EINVAL if the ucode is not available.
4818 */
4819 static int cik_rlc_resume(struct radeon_device *rdev)
4820 {
4821 u32 i, size;
4822 u32 clear_state_info[3];
4823 const __be32 *fw_data;
4824
4825 if (!rdev->rlc_fw)
4826 return -EINVAL;
4827
4828 switch (rdev->family) {
4829 case CHIP_BONAIRE:
4830 default:
4831 size = BONAIRE_RLC_UCODE_SIZE;
4832 break;
4833 case CHIP_KAVERI:
4834 size = KV_RLC_UCODE_SIZE;
4835 break;
4836 case CHIP_KABINI:
4837 size = KB_RLC_UCODE_SIZE;
4838 break;
4839 }
4840
4841 cik_rlc_stop(rdev);
4842
4843 WREG32(GRBM_SOFT_RESET, SOFT_RESET_RLC);
4844 RREG32(GRBM_SOFT_RESET);
4845 udelay(50);
4846 WREG32(GRBM_SOFT_RESET, 0);
4847 RREG32(GRBM_SOFT_RESET);
4848 udelay(50);
4849
4850 WREG32(RLC_LB_CNTR_INIT, 0);
4851 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
4852
4853 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4854 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
4855 WREG32(RLC_LB_PARAMS, 0x00600408);
4856 WREG32(RLC_LB_CNTL, 0x80000004);
4857
4858 WREG32(RLC_MC_CNTL, 0);
4859 WREG32(RLC_UCODE_CNTL, 0);
4860
4861 fw_data = (const __be32 *)rdev->rlc_fw->data;
4862 WREG32(RLC_GPM_UCODE_ADDR, 0);
4863 for (i = 0; i < size; i++)
4864 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
4865 WREG32(RLC_GPM_UCODE_ADDR, 0);
4866
4867 /* XXX */
4868 clear_state_info[0] = 0;//upper_32_bits(rdev->rlc.save_restore_gpu_addr);
4869 clear_state_info[1] = 0;//rdev->rlc.save_restore_gpu_addr;
4870 clear_state_info[2] = 0;//cik_default_size;
4871 WREG32(RLC_GPM_SCRATCH_ADDR, 0x3d);
4872 for (i = 0; i < 3; i++)
4873 WREG32(RLC_GPM_SCRATCH_DATA, clear_state_info[i]);
4874 WREG32(RLC_DRIVER_DMA_STATUS, 0);
4875
4876 cik_rlc_start(rdev);
4877
4878 return 0;
4879 }
4880
4881 /*
4882 * Interrupts
4883 * Starting with r6xx, interrupts are handled via a ring buffer.
4884 * Ring buffers are areas of GPU accessible memory that the GPU
4885 * writes interrupt vectors into and the host reads vectors out of.
4886 * There is a rptr (read pointer) that determines where the
4887 * host is currently reading, and a wptr (write pointer)
4888 * which determines where the GPU has written. When the
4889 * pointers are equal, the ring is idle. When the GPU
4890 * writes vectors to the ring buffer, it increments the
4891 * wptr. When there is an interrupt, the host then starts
4892 * fetching commands and processing them until the pointers are
4893 * equal again at which point it updates the rptr.
4894 */
4895
4896 /**
4897 * cik_enable_interrupts - Enable the interrupt ring buffer
4898 *
4899 * @rdev: radeon_device pointer
4900 *
4901 * Enable the interrupt ring buffer (CIK).
4902 */
4903 static void cik_enable_interrupts(struct radeon_device *rdev)
4904 {
4905 u32 ih_cntl = RREG32(IH_CNTL);
4906 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4907
4908 ih_cntl |= ENABLE_INTR;
4909 ih_rb_cntl |= IH_RB_ENABLE;
4910 WREG32(IH_CNTL, ih_cntl);
4911 WREG32(IH_RB_CNTL, ih_rb_cntl);
4912 rdev->ih.enabled = true;
4913 }
4914
4915 /**
4916 * cik_disable_interrupts - Disable the interrupt ring buffer
4917 *
4918 * @rdev: radeon_device pointer
4919 *
4920 * Disable the interrupt ring buffer (CIK).
4921 */
4922 static void cik_disable_interrupts(struct radeon_device *rdev)
4923 {
4924 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4925 u32 ih_cntl = RREG32(IH_CNTL);
4926
4927 ih_rb_cntl &= ~IH_RB_ENABLE;
4928 ih_cntl &= ~ENABLE_INTR;
4929 WREG32(IH_RB_CNTL, ih_rb_cntl);
4930 WREG32(IH_CNTL, ih_cntl);
4931 /* set rptr, wptr to 0 */
4932 WREG32(IH_RB_RPTR, 0);
4933 WREG32(IH_RB_WPTR, 0);
4934 rdev->ih.enabled = false;
4935 rdev->ih.rptr = 0;
4936 }
4937
4938 /**
4939 * cik_disable_interrupt_state - Disable all interrupt sources
4940 *
4941 * @rdev: radeon_device pointer
4942 *
4943 * Clear all interrupt enable bits used by the driver (CIK).
4944 */
4945 static void cik_disable_interrupt_state(struct radeon_device *rdev)
4946 {
4947 u32 tmp;
4948
4949 /* gfx ring */
4950 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4951 /* sdma */
4952 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4953 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4954 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4955 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4956 /* compute queues */
4957 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
4958 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
4959 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
4960 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
4961 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
4962 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
4963 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
4964 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
4965 /* grbm */
4966 WREG32(GRBM_INT_CNTL, 0);
4967 /* vline/vblank, etc. */
4968 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4969 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4970 if (rdev->num_crtc >= 4) {
4971 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4972 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4973 }
4974 if (rdev->num_crtc >= 6) {
4975 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4976 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4977 }
4978
4979 /* dac hotplug */
4980 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
4981
4982 /* digital hotplug */
4983 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4984 WREG32(DC_HPD1_INT_CONTROL, tmp);
4985 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4986 WREG32(DC_HPD2_INT_CONTROL, tmp);
4987 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4988 WREG32(DC_HPD3_INT_CONTROL, tmp);
4989 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4990 WREG32(DC_HPD4_INT_CONTROL, tmp);
4991 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4992 WREG32(DC_HPD5_INT_CONTROL, tmp);
4993 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4994 WREG32(DC_HPD6_INT_CONTROL, tmp);
4995
4996 }
4997
4998 /**
4999 * cik_irq_init - init and enable the interrupt ring
5000 *
5001 * @rdev: radeon_device pointer
5002 *
5003 * Allocate a ring buffer for the interrupt controller,
5004 * enable the RLC, disable interrupts, enable the IH
5005 * ring buffer and enable it (CIK).
5006 * Called at device load and reume.
5007 * Returns 0 for success, errors for failure.
5008 */
5009 static int cik_irq_init(struct radeon_device *rdev)
5010 {
5011 int ret = 0;
5012 int rb_bufsz;
5013 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5014
5015 /* allocate ring */
5016 ret = r600_ih_ring_alloc(rdev);
5017 if (ret)
5018 return ret;
5019
5020 /* disable irqs */
5021 cik_disable_interrupts(rdev);
5022
5023 /* init rlc */
5024 ret = cik_rlc_resume(rdev);
5025 if (ret) {
5026 r600_ih_ring_fini(rdev);
5027 return ret;
5028 }
5029
5030 /* setup interrupt control */
5031 /* XXX this should actually be a bus address, not an MC address. same on older asics */
5032 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5033 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5034 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5035 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5036 */
5037 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5038 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5039 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5040 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5041
5042 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5043 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5044
5045 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5046 IH_WPTR_OVERFLOW_CLEAR |
5047 (rb_bufsz << 1));
5048
5049 if (rdev->wb.enabled)
5050 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5051
5052 /* set the writeback address whether it's enabled or not */
5053 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5054 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5055
5056 WREG32(IH_RB_CNTL, ih_rb_cntl);
5057
5058 /* set rptr, wptr to 0 */
5059 WREG32(IH_RB_RPTR, 0);
5060 WREG32(IH_RB_WPTR, 0);
5061
5062 /* Default settings for IH_CNTL (disabled at first) */
5063 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5064 /* RPTR_REARM only works if msi's are enabled */
5065 if (rdev->msi_enabled)
5066 ih_cntl |= RPTR_REARM;
5067 WREG32(IH_CNTL, ih_cntl);
5068
5069 /* force the active interrupt state to all disabled */
5070 cik_disable_interrupt_state(rdev);
5071
5072 pci_set_master(rdev->pdev);
5073
5074 /* enable irqs */
5075 cik_enable_interrupts(rdev);
5076
5077 return ret;
5078 }
5079
5080 /**
5081 * cik_irq_set - enable/disable interrupt sources
5082 *
5083 * @rdev: radeon_device pointer
5084 *
5085 * Enable interrupt sources on the GPU (vblanks, hpd,
5086 * etc.) (CIK).
5087 * Returns 0 for success, errors for failure.
5088 */
5089 int cik_irq_set(struct radeon_device *rdev)
5090 {
5091 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
5092 PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
5093 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
5094 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
5095 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5096 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
5097 u32 grbm_int_cntl = 0;
5098 u32 dma_cntl, dma_cntl1;
5099
5100 if (!rdev->irq.installed) {
5101 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5102 return -EINVAL;
5103 }
5104 /* don't enable anything if the ih is disabled */
5105 if (!rdev->ih.enabled) {
5106 cik_disable_interrupts(rdev);
5107 /* force the active interrupt state to all disabled */
5108 cik_disable_interrupt_state(rdev);
5109 return 0;
5110 }
5111
5112 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5113 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5114 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5115 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5116 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5117 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5118
5119 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5120 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5121
5122 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5123 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5124 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5125 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5126 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5127 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5128 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5129 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5130
5131 /* enable CP interrupts on all rings */
5132 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5133 DRM_DEBUG("cik_irq_set: sw int gfx\n");
5134 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5135 }
5136 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5137 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5138 DRM_DEBUG("si_irq_set: sw int cp1\n");
5139 if (ring->me == 1) {
5140 switch (ring->pipe) {
5141 case 0:
5142 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
5143 break;
5144 case 1:
5145 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
5146 break;
5147 case 2:
5148 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5149 break;
5150 case 3:
5151 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5152 break;
5153 default:
5154 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
5155 break;
5156 }
5157 } else if (ring->me == 2) {
5158 switch (ring->pipe) {
5159 case 0:
5160 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
5161 break;
5162 case 1:
5163 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
5164 break;
5165 case 2:
5166 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5167 break;
5168 case 3:
5169 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5170 break;
5171 default:
5172 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
5173 break;
5174 }
5175 } else {
5176 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
5177 }
5178 }
5179 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5180 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5181 DRM_DEBUG("si_irq_set: sw int cp2\n");
5182 if (ring->me == 1) {
5183 switch (ring->pipe) {
5184 case 0:
5185 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
5186 break;
5187 case 1:
5188 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
5189 break;
5190 case 2:
5191 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5192 break;
5193 case 3:
5194 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5195 break;
5196 default:
5197 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
5198 break;
5199 }
5200 } else if (ring->me == 2) {
5201 switch (ring->pipe) {
5202 case 0:
5203 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
5204 break;
5205 case 1:
5206 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
5207 break;
5208 case 2:
5209 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5210 break;
5211 case 3:
5212 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5213 break;
5214 default:
5215 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
5216 break;
5217 }
5218 } else {
5219 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
5220 }
5221 }
5222
5223 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5224 DRM_DEBUG("cik_irq_set: sw int dma\n");
5225 dma_cntl |= TRAP_ENABLE;
5226 }
5227
5228 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5229 DRM_DEBUG("cik_irq_set: sw int dma1\n");
5230 dma_cntl1 |= TRAP_ENABLE;
5231 }
5232
5233 if (rdev->irq.crtc_vblank_int[0] ||
5234 atomic_read(&rdev->irq.pflip[0])) {
5235 DRM_DEBUG("cik_irq_set: vblank 0\n");
5236 crtc1 |= VBLANK_INTERRUPT_MASK;
5237 }
5238 if (rdev->irq.crtc_vblank_int[1] ||
5239 atomic_read(&rdev->irq.pflip[1])) {
5240 DRM_DEBUG("cik_irq_set: vblank 1\n");
5241 crtc2 |= VBLANK_INTERRUPT_MASK;
5242 }
5243 if (rdev->irq.crtc_vblank_int[2] ||
5244 atomic_read(&rdev->irq.pflip[2])) {
5245 DRM_DEBUG("cik_irq_set: vblank 2\n");
5246 crtc3 |= VBLANK_INTERRUPT_MASK;
5247 }
5248 if (rdev->irq.crtc_vblank_int[3] ||
5249 atomic_read(&rdev->irq.pflip[3])) {
5250 DRM_DEBUG("cik_irq_set: vblank 3\n");
5251 crtc4 |= VBLANK_INTERRUPT_MASK;
5252 }
5253 if (rdev->irq.crtc_vblank_int[4] ||
5254 atomic_read(&rdev->irq.pflip[4])) {
5255 DRM_DEBUG("cik_irq_set: vblank 4\n");
5256 crtc5 |= VBLANK_INTERRUPT_MASK;
5257 }
5258 if (rdev->irq.crtc_vblank_int[5] ||
5259 atomic_read(&rdev->irq.pflip[5])) {
5260 DRM_DEBUG("cik_irq_set: vblank 5\n");
5261 crtc6 |= VBLANK_INTERRUPT_MASK;
5262 }
5263 if (rdev->irq.hpd[0]) {
5264 DRM_DEBUG("cik_irq_set: hpd 1\n");
5265 hpd1 |= DC_HPDx_INT_EN;
5266 }
5267 if (rdev->irq.hpd[1]) {
5268 DRM_DEBUG("cik_irq_set: hpd 2\n");
5269 hpd2 |= DC_HPDx_INT_EN;
5270 }
5271 if (rdev->irq.hpd[2]) {
5272 DRM_DEBUG("cik_irq_set: hpd 3\n");
5273 hpd3 |= DC_HPDx_INT_EN;
5274 }
5275 if (rdev->irq.hpd[3]) {
5276 DRM_DEBUG("cik_irq_set: hpd 4\n");
5277 hpd4 |= DC_HPDx_INT_EN;
5278 }
5279 if (rdev->irq.hpd[4]) {
5280 DRM_DEBUG("cik_irq_set: hpd 5\n");
5281 hpd5 |= DC_HPDx_INT_EN;
5282 }
5283 if (rdev->irq.hpd[5]) {
5284 DRM_DEBUG("cik_irq_set: hpd 6\n");
5285 hpd6 |= DC_HPDx_INT_EN;
5286 }
5287
5288 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5289
5290 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
5291 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
5292
5293 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
5294 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
5295 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
5296 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
5297 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
5298 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
5299 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
5300 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
5301
5302 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5303
5304 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5305 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5306 if (rdev->num_crtc >= 4) {
5307 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5308 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5309 }
5310 if (rdev->num_crtc >= 6) {
5311 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5312 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5313 }
5314
5315 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5316 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5317 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5318 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5319 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5320 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5321
5322 return 0;
5323 }
5324
5325 /**
5326 * cik_irq_ack - ack interrupt sources
5327 *
5328 * @rdev: radeon_device pointer
5329 *
5330 * Ack interrupt sources on the GPU (vblanks, hpd,
5331 * etc.) (CIK). Certain interrupts sources are sw
5332 * generated and do not require an explicit ack.
5333 */
5334 static inline void cik_irq_ack(struct radeon_device *rdev)
5335 {
5336 u32 tmp;
5337
5338 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5339 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5340 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5341 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5342 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5343 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5344 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
5345
5346 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
5347 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5348 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
5349 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5350 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5351 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5352 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5353 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5354
5355 if (rdev->num_crtc >= 4) {
5356 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5357 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5358 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5359 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5360 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5361 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5362 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5363 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5364 }
5365
5366 if (rdev->num_crtc >= 6) {
5367 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5368 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5369 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5370 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5371 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5372 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5373 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5374 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5375 }
5376
5377 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
5378 tmp = RREG32(DC_HPD1_INT_CONTROL);
5379 tmp |= DC_HPDx_INT_ACK;
5380 WREG32(DC_HPD1_INT_CONTROL, tmp);
5381 }
5382 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
5383 tmp = RREG32(DC_HPD2_INT_CONTROL);
5384 tmp |= DC_HPDx_INT_ACK;
5385 WREG32(DC_HPD2_INT_CONTROL, tmp);
5386 }
5387 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5388 tmp = RREG32(DC_HPD3_INT_CONTROL);
5389 tmp |= DC_HPDx_INT_ACK;
5390 WREG32(DC_HPD3_INT_CONTROL, tmp);
5391 }
5392 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5393 tmp = RREG32(DC_HPD4_INT_CONTROL);
5394 tmp |= DC_HPDx_INT_ACK;
5395 WREG32(DC_HPD4_INT_CONTROL, tmp);
5396 }
5397 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5398 tmp = RREG32(DC_HPD5_INT_CONTROL);
5399 tmp |= DC_HPDx_INT_ACK;
5400 WREG32(DC_HPD5_INT_CONTROL, tmp);
5401 }
5402 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5403 tmp = RREG32(DC_HPD5_INT_CONTROL);
5404 tmp |= DC_HPDx_INT_ACK;
5405 WREG32(DC_HPD6_INT_CONTROL, tmp);
5406 }
5407 }
5408
5409 /**
5410 * cik_irq_disable - disable interrupts
5411 *
5412 * @rdev: radeon_device pointer
5413 *
5414 * Disable interrupts on the hw (CIK).
5415 */
5416 static void cik_irq_disable(struct radeon_device *rdev)
5417 {
5418 cik_disable_interrupts(rdev);
5419 /* Wait and acknowledge irq */
5420 mdelay(1);
5421 cik_irq_ack(rdev);
5422 cik_disable_interrupt_state(rdev);
5423 }
5424
5425 /**
5426 * cik_irq_disable - disable interrupts for suspend
5427 *
5428 * @rdev: radeon_device pointer
5429 *
5430 * Disable interrupts and stop the RLC (CIK).
5431 * Used for suspend.
5432 */
5433 static void cik_irq_suspend(struct radeon_device *rdev)
5434 {
5435 cik_irq_disable(rdev);
5436 cik_rlc_stop(rdev);
5437 }
5438
5439 /**
5440 * cik_irq_fini - tear down interrupt support
5441 *
5442 * @rdev: radeon_device pointer
5443 *
5444 * Disable interrupts on the hw and free the IH ring
5445 * buffer (CIK).
5446 * Used for driver unload.
5447 */
5448 static void cik_irq_fini(struct radeon_device *rdev)
5449 {
5450 cik_irq_suspend(rdev);
5451 r600_ih_ring_fini(rdev);
5452 }
5453
5454 /**
5455 * cik_get_ih_wptr - get the IH ring buffer wptr
5456 *
5457 * @rdev: radeon_device pointer
5458 *
5459 * Get the IH ring buffer wptr from either the register
5460 * or the writeback memory buffer (CIK). Also check for
5461 * ring buffer overflow and deal with it.
5462 * Used by cik_irq_process().
5463 * Returns the value of the wptr.
5464 */
5465 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
5466 {
5467 u32 wptr, tmp;
5468
5469 if (rdev->wb.enabled)
5470 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5471 else
5472 wptr = RREG32(IH_RB_WPTR);
5473
5474 if (wptr & RB_OVERFLOW) {
5475 /* When a ring buffer overflow happen start parsing interrupt
5476 * from the last not overwritten vector (wptr + 16). Hopefully
5477 * this should allow us to catchup.
5478 */
5479 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5480 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5481 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5482 tmp = RREG32(IH_RB_CNTL);
5483 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5484 WREG32(IH_RB_CNTL, tmp);
5485 }
5486 return (wptr & rdev->ih.ptr_mask);
5487 }
5488
5489 /* CIK IV Ring
5490 * Each IV ring entry is 128 bits:
5491 * [7:0] - interrupt source id
5492 * [31:8] - reserved
5493 * [59:32] - interrupt source data
5494 * [63:60] - reserved
5495 * [71:64] - RINGID
5496 * CP:
5497 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
5498 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
5499 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
5500 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
5501 * PIPE_ID - ME0 0=3D
5502 * - ME1&2 compute dispatcher (4 pipes each)
5503 * SDMA:
5504 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
5505 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
5506 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
5507 * [79:72] - VMID
5508 * [95:80] - PASID
5509 * [127:96] - reserved
5510 */
5511 /**
5512 * cik_irq_process - interrupt handler
5513 *
5514 * @rdev: radeon_device pointer
5515 *
5516 * Interrupt hander (CIK). Walk the IH ring,
5517 * ack interrupts and schedule work to handle
5518 * interrupt events.
5519 * Returns irq process return code.
5520 */
5521 int cik_irq_process(struct radeon_device *rdev)
5522 {
5523 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5524 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5525 u32 wptr;
5526 u32 rptr;
5527 u32 src_id, src_data, ring_id;
5528 u8 me_id, pipe_id, queue_id;
5529 u32 ring_index;
5530 bool queue_hotplug = false;
5531 bool queue_reset = false;
5532 u32 addr, status, mc_client;
5533
5534 if (!rdev->ih.enabled || rdev->shutdown)
5535 return IRQ_NONE;
5536
5537 wptr = cik_get_ih_wptr(rdev);
5538
5539 restart_ih:
5540 /* is somebody else already processing irqs? */
5541 if (atomic_xchg(&rdev->ih.lock, 1))
5542 return IRQ_NONE;
5543
5544 rptr = rdev->ih.rptr;
5545 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5546
5547 /* Order reading of wptr vs. reading of IH ring data */
5548 rmb();
5549
5550 /* display interrupts */
5551 cik_irq_ack(rdev);
5552
5553 while (rptr != wptr) {
5554 /* wptr/rptr are in bytes! */
5555 ring_index = rptr / 4;
5556 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5557 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5558 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5559
5560 switch (src_id) {
5561 case 1: /* D1 vblank/vline */
5562 switch (src_data) {
5563 case 0: /* D1 vblank */
5564 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
5565 if (rdev->irq.crtc_vblank_int[0]) {
5566 drm_handle_vblank(rdev->ddev, 0);
5567 rdev->pm.vblank_sync = true;
5568 wake_up(&rdev->irq.vblank_queue);
5569 }
5570 if (atomic_read(&rdev->irq.pflip[0]))
5571 radeon_crtc_handle_flip(rdev, 0);
5572 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5573 DRM_DEBUG("IH: D1 vblank\n");
5574 }
5575 break;
5576 case 1: /* D1 vline */
5577 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
5578 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5579 DRM_DEBUG("IH: D1 vline\n");
5580 }
5581 break;
5582 default:
5583 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5584 break;
5585 }
5586 break;
5587 case 2: /* D2 vblank/vline */
5588 switch (src_data) {
5589 case 0: /* D2 vblank */
5590 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5591 if (rdev->irq.crtc_vblank_int[1]) {
5592 drm_handle_vblank(rdev->ddev, 1);
5593 rdev->pm.vblank_sync = true;
5594 wake_up(&rdev->irq.vblank_queue);
5595 }
5596 if (atomic_read(&rdev->irq.pflip[1]))
5597 radeon_crtc_handle_flip(rdev, 1);
5598 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5599 DRM_DEBUG("IH: D2 vblank\n");
5600 }
5601 break;
5602 case 1: /* D2 vline */
5603 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5604 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5605 DRM_DEBUG("IH: D2 vline\n");
5606 }
5607 break;
5608 default:
5609 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5610 break;
5611 }
5612 break;
5613 case 3: /* D3 vblank/vline */
5614 switch (src_data) {
5615 case 0: /* D3 vblank */
5616 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5617 if (rdev->irq.crtc_vblank_int[2]) {
5618 drm_handle_vblank(rdev->ddev, 2);
5619 rdev->pm.vblank_sync = true;
5620 wake_up(&rdev->irq.vblank_queue);
5621 }
5622 if (atomic_read(&rdev->irq.pflip[2]))
5623 radeon_crtc_handle_flip(rdev, 2);
5624 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5625 DRM_DEBUG("IH: D3 vblank\n");
5626 }
5627 break;
5628 case 1: /* D3 vline */
5629 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5630 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5631 DRM_DEBUG("IH: D3 vline\n");
5632 }
5633 break;
5634 default:
5635 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5636 break;
5637 }
5638 break;
5639 case 4: /* D4 vblank/vline */
5640 switch (src_data) {
5641 case 0: /* D4 vblank */
5642 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5643 if (rdev->irq.crtc_vblank_int[3]) {
5644 drm_handle_vblank(rdev->ddev, 3);
5645 rdev->pm.vblank_sync = true;
5646 wake_up(&rdev->irq.vblank_queue);
5647 }
5648 if (atomic_read(&rdev->irq.pflip[3]))
5649 radeon_crtc_handle_flip(rdev, 3);
5650 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5651 DRM_DEBUG("IH: D4 vblank\n");
5652 }
5653 break;
5654 case 1: /* D4 vline */
5655 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5656 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5657 DRM_DEBUG("IH: D4 vline\n");
5658 }
5659 break;
5660 default:
5661 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5662 break;
5663 }
5664 break;
5665 case 5: /* D5 vblank/vline */
5666 switch (src_data) {
5667 case 0: /* D5 vblank */
5668 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5669 if (rdev->irq.crtc_vblank_int[4]) {
5670 drm_handle_vblank(rdev->ddev, 4);
5671 rdev->pm.vblank_sync = true;
5672 wake_up(&rdev->irq.vblank_queue);
5673 }
5674 if (atomic_read(&rdev->irq.pflip[4]))
5675 radeon_crtc_handle_flip(rdev, 4);
5676 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5677 DRM_DEBUG("IH: D5 vblank\n");
5678 }
5679 break;
5680 case 1: /* D5 vline */
5681 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5682 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5683 DRM_DEBUG("IH: D5 vline\n");
5684 }
5685 break;
5686 default:
5687 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5688 break;
5689 }
5690 break;
5691 case 6: /* D6 vblank/vline */
5692 switch (src_data) {
5693 case 0: /* D6 vblank */
5694 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5695 if (rdev->irq.crtc_vblank_int[5]) {
5696 drm_handle_vblank(rdev->ddev, 5);
5697 rdev->pm.vblank_sync = true;
5698 wake_up(&rdev->irq.vblank_queue);
5699 }
5700 if (atomic_read(&rdev->irq.pflip[5]))
5701 radeon_crtc_handle_flip(rdev, 5);
5702 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5703 DRM_DEBUG("IH: D6 vblank\n");
5704 }
5705 break;
5706 case 1: /* D6 vline */
5707 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5708 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5709 DRM_DEBUG("IH: D6 vline\n");
5710 }
5711 break;
5712 default:
5713 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5714 break;
5715 }
5716 break;
5717 case 42: /* HPD hotplug */
5718 switch (src_data) {
5719 case 0:
5720 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
5721 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
5722 queue_hotplug = true;
5723 DRM_DEBUG("IH: HPD1\n");
5724 }
5725 break;
5726 case 1:
5727 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
5728 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5729 queue_hotplug = true;
5730 DRM_DEBUG("IH: HPD2\n");
5731 }
5732 break;
5733 case 2:
5734 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5735 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5736 queue_hotplug = true;
5737 DRM_DEBUG("IH: HPD3\n");
5738 }
5739 break;
5740 case 3:
5741 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5742 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5743 queue_hotplug = true;
5744 DRM_DEBUG("IH: HPD4\n");
5745 }
5746 break;
5747 case 4:
5748 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5749 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5750 queue_hotplug = true;
5751 DRM_DEBUG("IH: HPD5\n");
5752 }
5753 break;
5754 case 5:
5755 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5756 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5757 queue_hotplug = true;
5758 DRM_DEBUG("IH: HPD6\n");
5759 }
5760 break;
5761 default:
5762 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5763 break;
5764 }
5765 break;
5766 case 146:
5767 case 147:
5768 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5769 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5770 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
5771 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5772 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5773 addr);
5774 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5775 status);
5776 cik_vm_decode_fault(rdev, status, addr, mc_client);
5777 /* reset addr and status */
5778 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5779 break;
5780 case 176: /* GFX RB CP_INT */
5781 case 177: /* GFX IB CP_INT */
5782 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5783 break;
5784 case 181: /* CP EOP event */
5785 DRM_DEBUG("IH: CP EOP\n");
5786 /* XXX check the bitfield order! */
5787 me_id = (ring_id & 0x60) >> 5;
5788 pipe_id = (ring_id & 0x18) >> 3;
5789 queue_id = (ring_id & 0x7) >> 0;
5790 switch (me_id) {
5791 case 0:
5792 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5793 break;
5794 case 1:
5795 case 2:
5796 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
5797 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5798 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
5799 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5800 break;
5801 }
5802 break;
5803 case 184: /* CP Privileged reg access */
5804 DRM_ERROR("Illegal register access in command stream\n");
5805 /* XXX check the bitfield order! */
5806 me_id = (ring_id & 0x60) >> 5;
5807 pipe_id = (ring_id & 0x18) >> 3;
5808 queue_id = (ring_id & 0x7) >> 0;
5809 switch (me_id) {
5810 case 0:
5811 /* This results in a full GPU reset, but all we need to do is soft
5812 * reset the CP for gfx
5813 */
5814 queue_reset = true;
5815 break;
5816 case 1:
5817 /* XXX compute */
5818 queue_reset = true;
5819 break;
5820 case 2:
5821 /* XXX compute */
5822 queue_reset = true;
5823 break;
5824 }
5825 break;
5826 case 185: /* CP Privileged inst */
5827 DRM_ERROR("Illegal instruction in command stream\n");
5828 /* XXX check the bitfield order! */
5829 me_id = (ring_id & 0x60) >> 5;
5830 pipe_id = (ring_id & 0x18) >> 3;
5831 queue_id = (ring_id & 0x7) >> 0;
5832 switch (me_id) {
5833 case 0:
5834 /* This results in a full GPU reset, but all we need to do is soft
5835 * reset the CP for gfx
5836 */
5837 queue_reset = true;
5838 break;
5839 case 1:
5840 /* XXX compute */
5841 queue_reset = true;
5842 break;
5843 case 2:
5844 /* XXX compute */
5845 queue_reset = true;
5846 break;
5847 }
5848 break;
5849 case 224: /* SDMA trap event */
5850 /* XXX check the bitfield order! */
5851 me_id = (ring_id & 0x3) >> 0;
5852 queue_id = (ring_id & 0xc) >> 2;
5853 DRM_DEBUG("IH: SDMA trap\n");
5854 switch (me_id) {
5855 case 0:
5856 switch (queue_id) {
5857 case 0:
5858 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5859 break;
5860 case 1:
5861 /* XXX compute */
5862 break;
5863 case 2:
5864 /* XXX compute */
5865 break;
5866 }
5867 break;
5868 case 1:
5869 switch (queue_id) {
5870 case 0:
5871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5872 break;
5873 case 1:
5874 /* XXX compute */
5875 break;
5876 case 2:
5877 /* XXX compute */
5878 break;
5879 }
5880 break;
5881 }
5882 break;
5883 case 241: /* SDMA Privileged inst */
5884 case 247: /* SDMA Privileged inst */
5885 DRM_ERROR("Illegal instruction in SDMA command stream\n");
5886 /* XXX check the bitfield order! */
5887 me_id = (ring_id & 0x3) >> 0;
5888 queue_id = (ring_id & 0xc) >> 2;
5889 switch (me_id) {
5890 case 0:
5891 switch (queue_id) {
5892 case 0:
5893 queue_reset = true;
5894 break;
5895 case 1:
5896 /* XXX compute */
5897 queue_reset = true;
5898 break;
5899 case 2:
5900 /* XXX compute */
5901 queue_reset = true;
5902 break;
5903 }
5904 break;
5905 case 1:
5906 switch (queue_id) {
5907 case 0:
5908 queue_reset = true;
5909 break;
5910 case 1:
5911 /* XXX compute */
5912 queue_reset = true;
5913 break;
5914 case 2:
5915 /* XXX compute */
5916 queue_reset = true;
5917 break;
5918 }
5919 break;
5920 }
5921 break;
5922 case 233: /* GUI IDLE */
5923 DRM_DEBUG("IH: GUI idle\n");
5924 break;
5925 default:
5926 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5927 break;
5928 }
5929
5930 /* wptr/rptr are in bytes! */
5931 rptr += 16;
5932 rptr &= rdev->ih.ptr_mask;
5933 }
5934 if (queue_hotplug)
5935 schedule_work(&rdev->hotplug_work);
5936 if (queue_reset)
5937 schedule_work(&rdev->reset_work);
5938 rdev->ih.rptr = rptr;
5939 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5940 atomic_set(&rdev->ih.lock, 0);
5941
5942 /* make sure wptr hasn't changed while processing */
5943 wptr = cik_get_ih_wptr(rdev);
5944 if (wptr != rptr)
5945 goto restart_ih;
5946
5947 return IRQ_HANDLED;
5948 }
5949
5950 /*
5951 * startup/shutdown callbacks
5952 */
5953 /**
5954 * cik_startup - program the asic to a functional state
5955 *
5956 * @rdev: radeon_device pointer
5957 *
5958 * Programs the asic to a functional state (CIK).
5959 * Called by cik_init() and cik_resume().
5960 * Returns 0 for success, error for failure.
5961 */
5962 static int cik_startup(struct radeon_device *rdev)
5963 {
5964 struct radeon_ring *ring;
5965 int r;
5966
5967 cik_mc_program(rdev);
5968
5969 if (rdev->flags & RADEON_IS_IGP) {
5970 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5971 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
5972 r = cik_init_microcode(rdev);
5973 if (r) {
5974 DRM_ERROR("Failed to load firmware!\n");
5975 return r;
5976 }
5977 }
5978 } else {
5979 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5980 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
5981 !rdev->mc_fw) {
5982 r = cik_init_microcode(rdev);
5983 if (r) {
5984 DRM_ERROR("Failed to load firmware!\n");
5985 return r;
5986 }
5987 }
5988
5989 r = ci_mc_load_microcode(rdev);
5990 if (r) {
5991 DRM_ERROR("Failed to load MC firmware!\n");
5992 return r;
5993 }
5994 }
5995
5996 r = r600_vram_scratch_init(rdev);
5997 if (r)
5998 return r;
5999
6000 r = cik_pcie_gart_enable(rdev);
6001 if (r)
6002 return r;
6003 cik_gpu_init(rdev);
6004
6005 /* allocate rlc buffers */
6006 r = si_rlc_init(rdev);
6007 if (r) {
6008 DRM_ERROR("Failed to init rlc BOs!\n");
6009 return r;
6010 }
6011
6012 /* allocate wb buffer */
6013 r = radeon_wb_init(rdev);
6014 if (r)
6015 return r;
6016
6017 /* allocate mec buffers */
6018 r = cik_mec_init(rdev);
6019 if (r) {
6020 DRM_ERROR("Failed to init MEC BOs!\n");
6021 return r;
6022 }
6023
6024 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6025 if (r) {
6026 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6027 return r;
6028 }
6029
6030 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6031 if (r) {
6032 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6033 return r;
6034 }
6035
6036 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6037 if (r) {
6038 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6039 return r;
6040 }
6041
6042 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6043 if (r) {
6044 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6045 return r;
6046 }
6047
6048 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6049 if (r) {
6050 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6051 return r;
6052 }
6053
6054 r = cik_uvd_resume(rdev);
6055 if (!r) {
6056 r = radeon_fence_driver_start_ring(rdev,
6057 R600_RING_TYPE_UVD_INDEX);
6058 if (r)
6059 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6060 }
6061 if (r)
6062 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6063
6064 /* Enable IRQ */
6065 if (!rdev->irq.installed) {
6066 r = radeon_irq_kms_init(rdev);
6067 if (r)
6068 return r;
6069 }
6070
6071 r = cik_irq_init(rdev);
6072 if (r) {
6073 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6074 radeon_irq_kms_fini(rdev);
6075 return r;
6076 }
6077 cik_irq_set(rdev);
6078
6079 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6080 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6081 CP_RB0_RPTR, CP_RB0_WPTR,
6082 0, 0xfffff, RADEON_CP_PACKET2);
6083 if (r)
6084 return r;
6085
6086 /* set up the compute queues */
6087 /* type-2 packets are deprecated on MEC, use type-3 instead */
6088 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6089 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6090 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
6091 0, 0xfffff, PACKET3(PACKET3_NOP, 0x3FFF));
6092 if (r)
6093 return r;
6094 ring->me = 1; /* first MEC */
6095 ring->pipe = 0; /* first pipe */
6096 ring->queue = 0; /* first queue */
6097 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
6098
6099 /* type-2 packets are deprecated on MEC, use type-3 instead */
6100 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6101 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6102 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
6103 0, 0xffffffff, PACKET3(PACKET3_NOP, 0x3FFF));
6104 if (r)
6105 return r;
6106 /* dGPU only have 1 MEC */
6107 ring->me = 1; /* first MEC */
6108 ring->pipe = 0; /* first pipe */
6109 ring->queue = 1; /* second queue */
6110 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
6111
6112 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6113 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6114 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
6115 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
6116 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
6117 if (r)
6118 return r;
6119
6120 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6121 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6122 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
6123 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
6124 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
6125 if (r)
6126 return r;
6127
6128 r = cik_cp_resume(rdev);
6129 if (r)
6130 return r;
6131
6132 r = cik_sdma_resume(rdev);
6133 if (r)
6134 return r;
6135
6136 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6137 if (ring->ring_size) {
6138 r = radeon_ring_init(rdev, ring, ring->ring_size,
6139 R600_WB_UVD_RPTR_OFFSET,
6140 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6141 0, 0xfffff, RADEON_CP_PACKET2);
6142 if (!r)
6143 r = r600_uvd_init(rdev);
6144 if (r)
6145 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6146 }
6147
6148 r = radeon_ib_pool_init(rdev);
6149 if (r) {
6150 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6151 return r;
6152 }
6153
6154 r = radeon_vm_manager_init(rdev);
6155 if (r) {
6156 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6157 return r;
6158 }
6159
6160 return 0;
6161 }
6162
6163 /**
6164 * cik_resume - resume the asic to a functional state
6165 *
6166 * @rdev: radeon_device pointer
6167 *
6168 * Programs the asic to a functional state (CIK).
6169 * Called at resume.
6170 * Returns 0 for success, error for failure.
6171 */
6172 int cik_resume(struct radeon_device *rdev)
6173 {
6174 int r;
6175
6176 /* post card */
6177 atom_asic_init(rdev->mode_info.atom_context);
6178
6179 /* init golden registers */
6180 cik_init_golden_registers(rdev);
6181
6182 rdev->accel_working = true;
6183 r = cik_startup(rdev);
6184 if (r) {
6185 DRM_ERROR("cik startup failed on resume\n");
6186 rdev->accel_working = false;
6187 return r;
6188 }
6189
6190 return r;
6191
6192 }
6193
6194 /**
6195 * cik_suspend - suspend the asic
6196 *
6197 * @rdev: radeon_device pointer
6198 *
6199 * Bring the chip into a state suitable for suspend (CIK).
6200 * Called at suspend.
6201 * Returns 0 for success.
6202 */
6203 int cik_suspend(struct radeon_device *rdev)
6204 {
6205 radeon_vm_manager_fini(rdev);
6206 cik_cp_enable(rdev, false);
6207 cik_sdma_enable(rdev, false);
6208 r600_uvd_stop(rdev);
6209 radeon_uvd_suspend(rdev);
6210 cik_irq_suspend(rdev);
6211 radeon_wb_disable(rdev);
6212 cik_pcie_gart_disable(rdev);
6213 return 0;
6214 }
6215
6216 /* Plan is to move initialization in that function and use
6217 * helper function so that radeon_device_init pretty much
6218 * do nothing more than calling asic specific function. This
6219 * should also allow to remove a bunch of callback function
6220 * like vram_info.
6221 */
6222 /**
6223 * cik_init - asic specific driver and hw init
6224 *
6225 * @rdev: radeon_device pointer
6226 *
6227 * Setup asic specific driver variables and program the hw
6228 * to a functional state (CIK).
6229 * Called at driver startup.
6230 * Returns 0 for success, errors for failure.
6231 */
6232 int cik_init(struct radeon_device *rdev)
6233 {
6234 struct radeon_ring *ring;
6235 int r;
6236
6237 /* Read BIOS */
6238 if (!radeon_get_bios(rdev)) {
6239 if (ASIC_IS_AVIVO(rdev))
6240 return -EINVAL;
6241 }
6242 /* Must be an ATOMBIOS */
6243 if (!rdev->is_atom_bios) {
6244 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6245 return -EINVAL;
6246 }
6247 r = radeon_atombios_init(rdev);
6248 if (r)
6249 return r;
6250
6251 /* Post card if necessary */
6252 if (!radeon_card_posted(rdev)) {
6253 if (!rdev->bios) {
6254 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6255 return -EINVAL;
6256 }
6257 DRM_INFO("GPU not posted. posting now...\n");
6258 atom_asic_init(rdev->mode_info.atom_context);
6259 }
6260 /* init golden registers */
6261 cik_init_golden_registers(rdev);
6262 /* Initialize scratch registers */
6263 cik_scratch_init(rdev);
6264 /* Initialize surface registers */
6265 radeon_surface_init(rdev);
6266 /* Initialize clocks */
6267 radeon_get_clock_info(rdev->ddev);
6268
6269 /* Fence driver */
6270 r = radeon_fence_driver_init(rdev);
6271 if (r)
6272 return r;
6273
6274 /* initialize memory controller */
6275 r = cik_mc_init(rdev);
6276 if (r)
6277 return r;
6278 /* Memory manager */
6279 r = radeon_bo_init(rdev);
6280 if (r)
6281 return r;
6282
6283 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6284 ring->ring_obj = NULL;
6285 r600_ring_init(rdev, ring, 1024 * 1024);
6286
6287 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6288 ring->ring_obj = NULL;
6289 r600_ring_init(rdev, ring, 1024 * 1024);
6290 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
6291 if (r)
6292 return r;
6293
6294 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6295 ring->ring_obj = NULL;
6296 r600_ring_init(rdev, ring, 1024 * 1024);
6297 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
6298 if (r)
6299 return r;
6300
6301 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6302 ring->ring_obj = NULL;
6303 r600_ring_init(rdev, ring, 256 * 1024);
6304
6305 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6306 ring->ring_obj = NULL;
6307 r600_ring_init(rdev, ring, 256 * 1024);
6308
6309 r = radeon_uvd_init(rdev);
6310 if (!r) {
6311 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6312 ring->ring_obj = NULL;
6313 r600_ring_init(rdev, ring, 4096);
6314 }
6315
6316 rdev->ih.ring_obj = NULL;
6317 r600_ih_ring_init(rdev, 64 * 1024);
6318
6319 r = r600_pcie_gart_init(rdev);
6320 if (r)
6321 return r;
6322
6323 rdev->accel_working = true;
6324 r = cik_startup(rdev);
6325 if (r) {
6326 dev_err(rdev->dev, "disabling GPU acceleration\n");
6327 cik_cp_fini(rdev);
6328 cik_sdma_fini(rdev);
6329 cik_irq_fini(rdev);
6330 si_rlc_fini(rdev);
6331 cik_mec_fini(rdev);
6332 radeon_wb_fini(rdev);
6333 radeon_ib_pool_fini(rdev);
6334 radeon_vm_manager_fini(rdev);
6335 radeon_irq_kms_fini(rdev);
6336 cik_pcie_gart_fini(rdev);
6337 rdev->accel_working = false;
6338 }
6339
6340 /* Don't start up if the MC ucode is missing.
6341 * The default clocks and voltages before the MC ucode
6342 * is loaded are not suffient for advanced operations.
6343 */
6344 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
6345 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6346 return -EINVAL;
6347 }
6348
6349 return 0;
6350 }
6351
6352 /**
6353 * cik_fini - asic specific driver and hw fini
6354 *
6355 * @rdev: radeon_device pointer
6356 *
6357 * Tear down the asic specific driver variables and program the hw
6358 * to an idle state (CIK).
6359 * Called at driver unload.
6360 */
6361 void cik_fini(struct radeon_device *rdev)
6362 {
6363 cik_cp_fini(rdev);
6364 cik_sdma_fini(rdev);
6365 cik_irq_fini(rdev);
6366 si_rlc_fini(rdev);
6367 cik_mec_fini(rdev);
6368 radeon_wb_fini(rdev);
6369 radeon_vm_manager_fini(rdev);
6370 radeon_ib_pool_fini(rdev);
6371 radeon_irq_kms_fini(rdev);
6372 r600_uvd_stop(rdev);
6373 radeon_uvd_fini(rdev);
6374 cik_pcie_gart_fini(rdev);
6375 r600_vram_scratch_fini(rdev);
6376 radeon_gem_fini(rdev);
6377 radeon_fence_driver_fini(rdev);
6378 radeon_bo_fini(rdev);
6379 radeon_atombios_fini(rdev);
6380 kfree(rdev->bios);
6381 rdev->bios = NULL;
6382 }
6383
6384 /* display watermark setup */
6385 /**
6386 * dce8_line_buffer_adjust - Set up the line buffer
6387 *
6388 * @rdev: radeon_device pointer
6389 * @radeon_crtc: the selected display controller
6390 * @mode: the current display mode on the selected display
6391 * controller
6392 *
6393 * Setup up the line buffer allocation for
6394 * the selected display controller (CIK).
6395 * Returns the line buffer size in pixels.
6396 */
6397 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
6398 struct radeon_crtc *radeon_crtc,
6399 struct drm_display_mode *mode)
6400 {
6401 u32 tmp;
6402
6403 /*
6404 * Line Buffer Setup
6405 * There are 6 line buffers, one for each display controllers.
6406 * There are 3 partitions per LB. Select the number of partitions
6407 * to enable based on the display width. For display widths larger
6408 * than 4096, you need use to use 2 display controllers and combine
6409 * them using the stereo blender.
6410 */
6411 if (radeon_crtc->base.enabled && mode) {
6412 if (mode->crtc_hdisplay < 1920)
6413 tmp = 1;
6414 else if (mode->crtc_hdisplay < 2560)
6415 tmp = 2;
6416 else if (mode->crtc_hdisplay < 4096)
6417 tmp = 0;
6418 else {
6419 DRM_DEBUG_KMS("Mode too big for LB!\n");
6420 tmp = 0;
6421 }
6422 } else
6423 tmp = 1;
6424
6425 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
6426 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
6427
6428 if (radeon_crtc->base.enabled && mode) {
6429 switch (tmp) {
6430 case 0:
6431 default:
6432 return 4096 * 2;
6433 case 1:
6434 return 1920 * 2;
6435 case 2:
6436 return 2560 * 2;
6437 }
6438 }
6439
6440 /* controller not enabled, so no lb used */
6441 return 0;
6442 }
6443
6444 /**
6445 * cik_get_number_of_dram_channels - get the number of dram channels
6446 *
6447 * @rdev: radeon_device pointer
6448 *
6449 * Look up the number of video ram channels (CIK).
6450 * Used for display watermark bandwidth calculations
6451 * Returns the number of dram channels
6452 */
6453 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
6454 {
6455 u32 tmp = RREG32(MC_SHARED_CHMAP);
6456
6457 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
6458 case 0:
6459 default:
6460 return 1;
6461 case 1:
6462 return 2;
6463 case 2:
6464 return 4;
6465 case 3:
6466 return 8;
6467 case 4:
6468 return 3;
6469 case 5:
6470 return 6;
6471 case 6:
6472 return 10;
6473 case 7:
6474 return 12;
6475 case 8:
6476 return 16;
6477 }
6478 }
6479
6480 struct dce8_wm_params {
6481 u32 dram_channels; /* number of dram channels */
6482 u32 yclk; /* bandwidth per dram data pin in kHz */
6483 u32 sclk; /* engine clock in kHz */
6484 u32 disp_clk; /* display clock in kHz */
6485 u32 src_width; /* viewport width */
6486 u32 active_time; /* active display time in ns */
6487 u32 blank_time; /* blank time in ns */
6488 bool interlaced; /* mode is interlaced */
6489 fixed20_12 vsc; /* vertical scale ratio */
6490 u32 num_heads; /* number of active crtcs */
6491 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
6492 u32 lb_size; /* line buffer allocated to pipe */
6493 u32 vtaps; /* vertical scaler taps */
6494 };
6495
6496 /**
6497 * dce8_dram_bandwidth - get the dram bandwidth
6498 *
6499 * @wm: watermark calculation data
6500 *
6501 * Calculate the raw dram bandwidth (CIK).
6502 * Used for display watermark bandwidth calculations
6503 * Returns the dram bandwidth in MBytes/s
6504 */
6505 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
6506 {
6507 /* Calculate raw DRAM Bandwidth */
6508 fixed20_12 dram_efficiency; /* 0.7 */
6509 fixed20_12 yclk, dram_channels, bandwidth;
6510 fixed20_12 a;
6511
6512 a.full = dfixed_const(1000);
6513 yclk.full = dfixed_const(wm->yclk);
6514 yclk.full = dfixed_div(yclk, a);
6515 dram_channels.full = dfixed_const(wm->dram_channels * 4);
6516 a.full = dfixed_const(10);
6517 dram_efficiency.full = dfixed_const(7);
6518 dram_efficiency.full = dfixed_div(dram_efficiency, a);
6519 bandwidth.full = dfixed_mul(dram_channels, yclk);
6520 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
6521
6522 return dfixed_trunc(bandwidth);
6523 }
6524
6525 /**
6526 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
6527 *
6528 * @wm: watermark calculation data
6529 *
6530 * Calculate the dram bandwidth used for display (CIK).
6531 * Used for display watermark bandwidth calculations
6532 * Returns the dram bandwidth for display in MBytes/s
6533 */
6534 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
6535 {
6536 /* Calculate DRAM Bandwidth and the part allocated to display. */
6537 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
6538 fixed20_12 yclk, dram_channels, bandwidth;
6539 fixed20_12 a;
6540
6541 a.full = dfixed_const(1000);
6542 yclk.full = dfixed_const(wm->yclk);
6543 yclk.full = dfixed_div(yclk, a);
6544 dram_channels.full = dfixed_const(wm->dram_channels * 4);
6545 a.full = dfixed_const(10);
6546 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
6547 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
6548 bandwidth.full = dfixed_mul(dram_channels, yclk);
6549 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
6550
6551 return dfixed_trunc(bandwidth);
6552 }
6553
6554 /**
6555 * dce8_data_return_bandwidth - get the data return bandwidth
6556 *
6557 * @wm: watermark calculation data
6558 *
6559 * Calculate the data return bandwidth used for display (CIK).
6560 * Used for display watermark bandwidth calculations
6561 * Returns the data return bandwidth in MBytes/s
6562 */
6563 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
6564 {
6565 /* Calculate the display Data return Bandwidth */
6566 fixed20_12 return_efficiency; /* 0.8 */
6567 fixed20_12 sclk, bandwidth;
6568 fixed20_12 a;
6569
6570 a.full = dfixed_const(1000);
6571 sclk.full = dfixed_const(wm->sclk);
6572 sclk.full = dfixed_div(sclk, a);
6573 a.full = dfixed_const(10);
6574 return_efficiency.full = dfixed_const(8);
6575 return_efficiency.full = dfixed_div(return_efficiency, a);
6576 a.full = dfixed_const(32);
6577 bandwidth.full = dfixed_mul(a, sclk);
6578 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
6579
6580 return dfixed_trunc(bandwidth);
6581 }
6582
6583 /**
6584 * dce8_dmif_request_bandwidth - get the dmif bandwidth
6585 *
6586 * @wm: watermark calculation data
6587 *
6588 * Calculate the dmif bandwidth used for display (CIK).
6589 * Used for display watermark bandwidth calculations
6590 * Returns the dmif bandwidth in MBytes/s
6591 */
6592 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
6593 {
6594 /* Calculate the DMIF Request Bandwidth */
6595 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
6596 fixed20_12 disp_clk, bandwidth;
6597 fixed20_12 a, b;
6598
6599 a.full = dfixed_const(1000);
6600 disp_clk.full = dfixed_const(wm->disp_clk);
6601 disp_clk.full = dfixed_div(disp_clk, a);
6602 a.full = dfixed_const(32);
6603 b.full = dfixed_mul(a, disp_clk);
6604
6605 a.full = dfixed_const(10);
6606 disp_clk_request_efficiency.full = dfixed_const(8);
6607 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
6608
6609 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
6610
6611 return dfixed_trunc(bandwidth);
6612 }
6613
6614 /**
6615 * dce8_available_bandwidth - get the min available bandwidth
6616 *
6617 * @wm: watermark calculation data
6618 *
6619 * Calculate the min available bandwidth used for display (CIK).
6620 * Used for display watermark bandwidth calculations
6621 * Returns the min available bandwidth in MBytes/s
6622 */
6623 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
6624 {
6625 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
6626 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
6627 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
6628 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
6629
6630 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
6631 }
6632
6633 /**
6634 * dce8_average_bandwidth - get the average available bandwidth
6635 *
6636 * @wm: watermark calculation data
6637 *
6638 * Calculate the average available bandwidth used for display (CIK).
6639 * Used for display watermark bandwidth calculations
6640 * Returns the average available bandwidth in MBytes/s
6641 */
6642 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
6643 {
6644 /* Calculate the display mode Average Bandwidth
6645 * DisplayMode should contain the source and destination dimensions,
6646 * timing, etc.
6647 */
6648 fixed20_12 bpp;
6649 fixed20_12 line_time;
6650 fixed20_12 src_width;
6651 fixed20_12 bandwidth;
6652 fixed20_12 a;
6653
6654 a.full = dfixed_const(1000);
6655 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
6656 line_time.full = dfixed_div(line_time, a);
6657 bpp.full = dfixed_const(wm->bytes_per_pixel);
6658 src_width.full = dfixed_const(wm->src_width);
6659 bandwidth.full = dfixed_mul(src_width, bpp);
6660 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
6661 bandwidth.full = dfixed_div(bandwidth, line_time);
6662
6663 return dfixed_trunc(bandwidth);
6664 }
6665
6666 /**
6667 * dce8_latency_watermark - get the latency watermark
6668 *
6669 * @wm: watermark calculation data
6670 *
6671 * Calculate the latency watermark (CIK).
6672 * Used for display watermark bandwidth calculations
6673 * Returns the latency watermark in ns
6674 */
6675 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
6676 {
6677 /* First calculate the latency in ns */
6678 u32 mc_latency = 2000; /* 2000 ns. */
6679 u32 available_bandwidth = dce8_available_bandwidth(wm);
6680 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
6681 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
6682 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
6683 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
6684 (wm->num_heads * cursor_line_pair_return_time);
6685 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
6686 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
6687 u32 tmp, dmif_size = 12288;
6688 fixed20_12 a, b, c;
6689
6690 if (wm->num_heads == 0)
6691 return 0;
6692
6693 a.full = dfixed_const(2);
6694 b.full = dfixed_const(1);
6695 if ((wm->vsc.full > a.full) ||
6696 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
6697 (wm->vtaps >= 5) ||
6698 ((wm->vsc.full >= a.full) && wm->interlaced))
6699 max_src_lines_per_dst_line = 4;
6700 else
6701 max_src_lines_per_dst_line = 2;
6702
6703 a.full = dfixed_const(available_bandwidth);
6704 b.full = dfixed_const(wm->num_heads);
6705 a.full = dfixed_div(a, b);
6706
6707 b.full = dfixed_const(mc_latency + 512);
6708 c.full = dfixed_const(wm->disp_clk);
6709 b.full = dfixed_div(b, c);
6710
6711 c.full = dfixed_const(dmif_size);
6712 b.full = dfixed_div(c, b);
6713
6714 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
6715
6716 b.full = dfixed_const(1000);
6717 c.full = dfixed_const(wm->disp_clk);
6718 b.full = dfixed_div(c, b);
6719 c.full = dfixed_const(wm->bytes_per_pixel);
6720 b.full = dfixed_mul(b, c);
6721
6722 lb_fill_bw = min(tmp, dfixed_trunc(b));
6723
6724 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
6725 b.full = dfixed_const(1000);
6726 c.full = dfixed_const(lb_fill_bw);
6727 b.full = dfixed_div(c, b);
6728 a.full = dfixed_div(a, b);
6729 line_fill_time = dfixed_trunc(a);
6730
6731 if (line_fill_time < wm->active_time)
6732 return latency;
6733 else
6734 return latency + (line_fill_time - wm->active_time);
6735
6736 }
6737
6738 /**
6739 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
6740 * average and available dram bandwidth
6741 *
6742 * @wm: watermark calculation data
6743 *
6744 * Check if the display average bandwidth fits in the display
6745 * dram bandwidth (CIK).
6746 * Used for display watermark bandwidth calculations
6747 * Returns true if the display fits, false if not.
6748 */
6749 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
6750 {
6751 if (dce8_average_bandwidth(wm) <=
6752 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
6753 return true;
6754 else
6755 return false;
6756 }
6757
6758 /**
6759 * dce8_average_bandwidth_vs_available_bandwidth - check
6760 * average and available bandwidth
6761 *
6762 * @wm: watermark calculation data
6763 *
6764 * Check if the display average bandwidth fits in the display
6765 * available bandwidth (CIK).
6766 * Used for display watermark bandwidth calculations
6767 * Returns true if the display fits, false if not.
6768 */
6769 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
6770 {
6771 if (dce8_average_bandwidth(wm) <=
6772 (dce8_available_bandwidth(wm) / wm->num_heads))
6773 return true;
6774 else
6775 return false;
6776 }
6777
6778 /**
6779 * dce8_check_latency_hiding - check latency hiding
6780 *
6781 * @wm: watermark calculation data
6782 *
6783 * Check latency hiding (CIK).
6784 * Used for display watermark bandwidth calculations
6785 * Returns true if the display fits, false if not.
6786 */
6787 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
6788 {
6789 u32 lb_partitions = wm->lb_size / wm->src_width;
6790 u32 line_time = wm->active_time + wm->blank_time;
6791 u32 latency_tolerant_lines;
6792 u32 latency_hiding;
6793 fixed20_12 a;
6794
6795 a.full = dfixed_const(1);
6796 if (wm->vsc.full > a.full)
6797 latency_tolerant_lines = 1;
6798 else {
6799 if (lb_partitions <= (wm->vtaps + 1))
6800 latency_tolerant_lines = 1;
6801 else
6802 latency_tolerant_lines = 2;
6803 }
6804
6805 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
6806
6807 if (dce8_latency_watermark(wm) <= latency_hiding)
6808 return true;
6809 else
6810 return false;
6811 }
6812
6813 /**
6814 * dce8_program_watermarks - program display watermarks
6815 *
6816 * @rdev: radeon_device pointer
6817 * @radeon_crtc: the selected display controller
6818 * @lb_size: line buffer size
6819 * @num_heads: number of display controllers in use
6820 *
6821 * Calculate and program the display watermarks for the
6822 * selected display controller (CIK).
6823 */
6824 static void dce8_program_watermarks(struct radeon_device *rdev,
6825 struct radeon_crtc *radeon_crtc,
6826 u32 lb_size, u32 num_heads)
6827 {
6828 struct drm_display_mode *mode = &radeon_crtc->base.mode;
6829 struct dce8_wm_params wm;
6830 u32 pixel_period;
6831 u32 line_time = 0;
6832 u32 latency_watermark_a = 0, latency_watermark_b = 0;
6833 u32 tmp, wm_mask;
6834
6835 if (radeon_crtc->base.enabled && num_heads && mode) {
6836 pixel_period = 1000000 / (u32)mode->clock;
6837 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
6838
6839 wm.yclk = rdev->pm.current_mclk * 10;
6840 wm.sclk = rdev->pm.current_sclk * 10;
6841 wm.disp_clk = mode->clock;
6842 wm.src_width = mode->crtc_hdisplay;
6843 wm.active_time = mode->crtc_hdisplay * pixel_period;
6844 wm.blank_time = line_time - wm.active_time;
6845 wm.interlaced = false;
6846 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
6847 wm.interlaced = true;
6848 wm.vsc = radeon_crtc->vsc;
6849 wm.vtaps = 1;
6850 if (radeon_crtc->rmx_type != RMX_OFF)
6851 wm.vtaps = 2;
6852 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
6853 wm.lb_size = lb_size;
6854 wm.dram_channels = cik_get_number_of_dram_channels(rdev);
6855 wm.num_heads = num_heads;
6856
6857 /* set for high clocks */
6858 latency_watermark_a = min(dce8_latency_watermark(&wm), (u32)65535);
6859 /* set for low clocks */
6860 /* wm.yclk = low clk; wm.sclk = low clk */
6861 latency_watermark_b = min(dce8_latency_watermark(&wm), (u32)65535);
6862
6863 /* possibly force display priority to high */
6864 /* should really do this at mode validation time... */
6865 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
6866 !dce8_average_bandwidth_vs_available_bandwidth(&wm) ||
6867 !dce8_check_latency_hiding(&wm) ||
6868 (rdev->disp_priority == 2)) {
6869 DRM_DEBUG_KMS("force priority to high\n");
6870 }
6871 }
6872
6873 /* select wm A */
6874 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
6875 tmp = wm_mask;
6876 tmp &= ~LATENCY_WATERMARK_MASK(3);
6877 tmp |= LATENCY_WATERMARK_MASK(1);
6878 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
6879 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
6880 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
6881 LATENCY_HIGH_WATERMARK(line_time)));
6882 /* select wm B */
6883 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
6884 tmp &= ~LATENCY_WATERMARK_MASK(3);
6885 tmp |= LATENCY_WATERMARK_MASK(2);
6886 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
6887 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
6888 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
6889 LATENCY_HIGH_WATERMARK(line_time)));
6890 /* restore original selection */
6891 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
6892 }
6893
6894 /**
6895 * dce8_bandwidth_update - program display watermarks
6896 *
6897 * @rdev: radeon_device pointer
6898 *
6899 * Calculate and program the display watermarks and line
6900 * buffer allocation (CIK).
6901 */
6902 void dce8_bandwidth_update(struct radeon_device *rdev)
6903 {
6904 struct drm_display_mode *mode = NULL;
6905 u32 num_heads = 0, lb_size;
6906 int i;
6907
6908 radeon_update_display_priority(rdev);
6909
6910 for (i = 0; i < rdev->num_crtc; i++) {
6911 if (rdev->mode_info.crtcs[i]->base.enabled)
6912 num_heads++;
6913 }
6914 for (i = 0; i < rdev->num_crtc; i++) {
6915 mode = &rdev->mode_info.crtcs[i]->base.mode;
6916 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
6917 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
6918 }
6919 }
6920
6921 /**
6922 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
6923 *
6924 * @rdev: radeon_device pointer
6925 *
6926 * Fetches a GPU clock counter snapshot (SI).
6927 * Returns the 64 bit clock counter snapshot.
6928 */
6929 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
6930 {
6931 uint64_t clock;
6932
6933 mutex_lock(&rdev->gpu_clock_mutex);
6934 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6935 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6936 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6937 mutex_unlock(&rdev->gpu_clock_mutex);
6938 return clock;
6939 }
6940
6941 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
6942 u32 cntl_reg, u32 status_reg)
6943 {
6944 int r, i;
6945 struct atom_clock_dividers dividers;
6946 uint32_t tmp;
6947
6948 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
6949 clock, false, &dividers);
6950 if (r)
6951 return r;
6952
6953 tmp = RREG32_SMC(cntl_reg);
6954 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
6955 tmp |= dividers.post_divider;
6956 WREG32_SMC(cntl_reg, tmp);
6957
6958 for (i = 0; i < 100; i++) {
6959 if (RREG32_SMC(status_reg) & DCLK_STATUS)
6960 break;
6961 mdelay(10);
6962 }
6963 if (i == 100)
6964 return -ETIMEDOUT;
6965
6966 return 0;
6967 }
6968
6969 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6970 {
6971 int r = 0;
6972
6973 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
6974 if (r)
6975 return r;
6976
6977 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
6978 return r;
6979 }
6980
6981 int cik_uvd_resume(struct radeon_device *rdev)
6982 {
6983 uint64_t addr;
6984 uint32_t size;
6985 int r;
6986
6987 r = radeon_uvd_resume(rdev);
6988 if (r)
6989 return r;
6990
6991 /* programm the VCPU memory controller bits 0-27 */
6992 addr = rdev->uvd.gpu_addr >> 3;
6993 size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3;
6994 WREG32(UVD_VCPU_CACHE_OFFSET0, addr);
6995 WREG32(UVD_VCPU_CACHE_SIZE0, size);
6996
6997 addr += size;
6998 size = RADEON_UVD_STACK_SIZE >> 3;
6999 WREG32(UVD_VCPU_CACHE_OFFSET1, addr);
7000 WREG32(UVD_VCPU_CACHE_SIZE1, size);
7001
7002 addr += size;
7003 size = RADEON_UVD_HEAP_SIZE >> 3;
7004 WREG32(UVD_VCPU_CACHE_OFFSET2, addr);
7005 WREG32(UVD_VCPU_CACHE_SIZE2, size);
7006
7007 /* bits 28-31 */
7008 addr = (rdev->uvd.gpu_addr >> 28) & 0xF;
7009 WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0));
7010
7011 /* bits 32-39 */
7012 addr = (rdev->uvd.gpu_addr >> 32) & 0xFF;
7013 WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31));
7014
7015 return 0;
7016 }
Linux kernel - Deliverables
This page took 0.26946 seconds and 6 git commands to generate.