projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: require rlc for dpm
[deliverable/linux.git] / drivers / gpu / drm / radeon / ni.c
CommitLineData
0af62b01
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
0af62b01 25#include <linux/slab.h>
e0cd3608 26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
0af62b01
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0af62b01
AD		 31#include "nid.h"
32#include "atom.h"
33#include "ni_reg.h"
0c88a02e 34#include "cayman_blit_shaders.h"
138e4e16 35#include "radeon_ucode.h"
2948f5e6
AD		 36#include "clearstate_cayman.h"
37
38static u32 tn_rlc_save_restore_register_list[] =
39{
40 0x98fc,
41 0x98f0,
42 0x9834,
43 0x9838,
44 0x9870,
45 0x9874,
46 0x8a14,
47 0x8b24,
48 0x8bcc,
49 0x8b10,
50 0x8c30,
51 0x8d00,
52 0x8d04,
53 0x8c00,
54 0x8c04,
55 0x8c10,
56 0x8c14,
57 0x8d8c,
58 0x8cf0,
59 0x8e38,
60 0x9508,
61 0x9688,
62 0x9608,
63 0x960c,
64 0x9610,
65 0x9614,
66 0x88c4,
67 0x8978,
68 0x88d4,
69 0x900c,
70 0x9100,
71 0x913c,
72 0x90e8,
73 0x9354,
74 0xa008,
75 0x98f8,
76 0x9148,
77 0x914c,
78 0x3f94,
79 0x98f4,
80 0x9b7c,
81 0x3f8c,
82 0x8950,
83 0x8954,
84 0x8a18,
85 0x8b28,
86 0x9144,
87 0x3f90,
88 0x915c,
89 0x9160,
90 0x9178,
91 0x917c,
92 0x9180,
93 0x918c,
94 0x9190,
95 0x9194,
96 0x9198,
97 0x919c,
98 0x91a8,
99 0x91ac,
100 0x91b0,
101 0x91b4,
102 0x91b8,
103 0x91c4,
104 0x91c8,
105 0x91cc,
106 0x91d0,
107 0x91d4,
108 0x91e0,
109 0x91e4,
110 0x91ec,
111 0x91f0,
112 0x91f4,
113 0x9200,
114 0x9204,
115 0x929c,
116 0x8030,
117 0x9150,
118 0x9a60,
119 0x920c,
120 0x9210,
121 0x9228,
122 0x922c,
123 0x9244,
124 0x9248,
125 0x91e8,
126 0x9294,
127 0x9208,
128 0x9224,
129 0x9240,
130 0x9220,
131 0x923c,
132 0x9258,
133 0x9744,
134 0xa200,
135 0xa204,
136 0xa208,
137 0xa20c,
138 0x8d58,
139 0x9030,
140 0x9034,
141 0x9038,
142 0x903c,
143 0x9040,
144 0x9654,
145 0x897c,
146 0xa210,
147 0xa214,
148 0x9868,
149 0xa02c,
150 0x9664,
151 0x9698,
152 0x949c,
153 0x8e10,
154 0x8e18,
155 0x8c50,
156 0x8c58,
157 0x8c60,
158 0x8c68,
159 0x89b4,
160 0x9830,
161 0x802c,
162};
163static u32 tn_rlc_save_restore_register_list_size = ARRAY_SIZE(tn_rlc_save_restore_register_list);
0af62b01 164
168757ea 165extern bool evergreen_is_display_hung(struct radeon_device *rdev);
187e3593 166extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
b9952a8a
AD		 167extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
168extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
169extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
755d819e
AD		 170extern void evergreen_mc_program(struct radeon_device *rdev);
171extern void evergreen_irq_suspend(struct radeon_device *rdev);
172extern int evergreen_mc_init(struct radeon_device *rdev);
d054ac16 173extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
b07759bf 174extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 175extern void evergreen_program_aspm(struct radeon_device *rdev);
2948f5e6
AD		 176extern void sumo_rlc_fini(struct radeon_device *rdev);
177extern int sumo_rlc_init(struct radeon_device *rdev);
b9952a8a 178
0af62b01
AD		 179/* Firmware Names */
180MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
181MODULE_FIRMWARE("radeon/BARTS_me.bin");
182MODULE_FIRMWARE("radeon/BARTS_mc.bin");
6596afd4 183MODULE_FIRMWARE("radeon/BARTS_smc.bin");
0af62b01
AD		 184MODULE_FIRMWARE("radeon/BTC_rlc.bin");
185MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
186MODULE_FIRMWARE("radeon/TURKS_me.bin");
187MODULE_FIRMWARE("radeon/TURKS_mc.bin");
6596afd4 188MODULE_FIRMWARE("radeon/TURKS_smc.bin");
0af62b01
AD		 189MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
190MODULE_FIRMWARE("radeon/CAICOS_me.bin");
191MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
6596afd4 192MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
9b8253ce
AD		 193MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
194MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
195MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
196MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
69e0b57a 197MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
c420c745
AD		 198MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
199MODULE_FIRMWARE("radeon/ARUBA_me.bin");
200MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
0af62b01 201
a2c96a21
AD		 202
203static const u32 cayman_golden_registers2[] =
204{
205 0x3e5c, 0xffffffff, 0x00000000,
206 0x3e48, 0xffffffff, 0x00000000,
207 0x3e4c, 0xffffffff, 0x00000000,
208 0x3e64, 0xffffffff, 0x00000000,
209 0x3e50, 0xffffffff, 0x00000000,
210 0x3e60, 0xffffffff, 0x00000000
211};
212
213static const u32 cayman_golden_registers[] =
214{
215 0x5eb4, 0xffffffff, 0x00000002,
216 0x5e78, 0x8f311ff1, 0x001000f0,
217 0x3f90, 0xffff0000, 0xff000000,
218 0x9148, 0xffff0000, 0xff000000,
219 0x3f94, 0xffff0000, 0xff000000,
220 0x914c, 0xffff0000, 0xff000000,
221 0xc78, 0x00000080, 0x00000080,
222 0xbd4, 0x70073777, 0x00011003,
223 0xd02c, 0xbfffff1f, 0x08421000,
224 0xd0b8, 0x73773777, 0x02011003,
225 0x5bc0, 0x00200000, 0x50100000,
226 0x98f8, 0x33773777, 0x02011003,
227 0x98fc, 0xffffffff, 0x76541032,
228 0x7030, 0x31000311, 0x00000011,
229 0x2f48, 0x33773777, 0x42010001,
230 0x6b28, 0x00000010, 0x00000012,
231 0x7728, 0x00000010, 0x00000012,
232 0x10328, 0x00000010, 0x00000012,
233 0x10f28, 0x00000010, 0x00000012,
234 0x11b28, 0x00000010, 0x00000012,
235 0x12728, 0x00000010, 0x00000012,
236 0x240c, 0x000007ff, 0x00000000,
237 0x8a14, 0xf000001f, 0x00000007,
238 0x8b24, 0x3fff3fff, 0x00ff0fff,
239 0x8b10, 0x0000ff0f, 0x00000000,
240 0x28a4c, 0x07ffffff, 0x06000000,
241 0x10c, 0x00000001, 0x00010003,
242 0xa02c, 0xffffffff, 0x0000009b,
243 0x913c, 0x0000010f, 0x01000100,
244 0x8c04, 0xf8ff00ff, 0x40600060,
245 0x28350, 0x00000f01, 0x00000000,
246 0x9508, 0x3700001f, 0x00000002,
247 0x960c, 0xffffffff, 0x54763210,
248 0x88c4, 0x001f3ae3, 0x00000082,
249 0x88d0, 0xffffffff, 0x0f40df40,
250 0x88d4, 0x0000001f, 0x00000010,
251 0x8974, 0xffffffff, 0x00000000
252};
253
254static const u32 dvst_golden_registers2[] =
255{
256 0x8f8, 0xffffffff, 0,
257 0x8fc, 0x00380000, 0,
258 0x8f8, 0xffffffff, 1,
259 0x8fc, 0x0e000000, 0
260};
261
262static const u32 dvst_golden_registers[] =
263{
264 0x690, 0x3fff3fff, 0x20c00033,
265 0x918c, 0x0fff0fff, 0x00010006,
266 0x91a8, 0x0fff0fff, 0x00010006,
267 0x9150, 0xffffdfff, 0x6e944040,
268 0x917c, 0x0fff0fff, 0x00030002,
269 0x9198, 0x0fff0fff, 0x00030002,
270 0x915c, 0x0fff0fff, 0x00010000,
271 0x3f90, 0xffff0001, 0xff000000,
272 0x9178, 0x0fff0fff, 0x00070000,
273 0x9194, 0x0fff0fff, 0x00070000,
274 0x9148, 0xffff0001, 0xff000000,
275 0x9190, 0x0fff0fff, 0x00090008,
276 0x91ac, 0x0fff0fff, 0x00090008,
277 0x3f94, 0xffff0000, 0xff000000,
278 0x914c, 0xffff0000, 0xff000000,
279 0x929c, 0x00000fff, 0x00000001,
280 0x55e4, 0xff607fff, 0xfc000100,
281 0x8a18, 0xff000fff, 0x00000100,
282 0x8b28, 0xff000fff, 0x00000100,
283 0x9144, 0xfffc0fff, 0x00000100,
284 0x6ed8, 0x00010101, 0x00010000,
285 0x9830, 0xffffffff, 0x00000000,
286 0x9834, 0xf00fffff, 0x00000400,
287 0x9838, 0xfffffffe, 0x00000000,
288 0xd0c0, 0xff000fff, 0x00000100,
289 0xd02c, 0xbfffff1f, 0x08421000,
290 0xd0b8, 0x73773777, 0x12010001,
291 0x5bb0, 0x000000f0, 0x00000070,
292 0x98f8, 0x73773777, 0x12010001,
293 0x98fc, 0xffffffff, 0x00000010,
294 0x9b7c, 0x00ff0000, 0x00fc0000,
295 0x8030, 0x00001f0f, 0x0000100a,
296 0x2f48, 0x73773777, 0x12010001,
297 0x2408, 0x00030000, 0x000c007f,
298 0x8a14, 0xf000003f, 0x00000007,
299 0x8b24, 0x3fff3fff, 0x00ff0fff,
300 0x8b10, 0x0000ff0f, 0x00000000,
301 0x28a4c, 0x07ffffff, 0x06000000,
302 0x4d8, 0x00000fff, 0x00000100,
303 0xa008, 0xffffffff, 0x00010000,
304 0x913c, 0xffff03ff, 0x01000100,
305 0x8c00, 0x000000ff, 0x00000003,
306 0x8c04, 0xf8ff00ff, 0x40600060,
307 0x8cf0, 0x1fff1fff, 0x08e00410,
308 0x28350, 0x00000f01, 0x00000000,
309 0x9508, 0xf700071f, 0x00000002,
310 0x960c, 0xffffffff, 0x54763210,
311 0x20ef8, 0x01ff01ff, 0x00000002,
312 0x20e98, 0xfffffbff, 0x00200000,
313 0x2015c, 0xffffffff, 0x00000f40,
314 0x88c4, 0x001f3ae3, 0x00000082,
315 0x8978, 0x3fffffff, 0x04050140,
316 0x88d4, 0x0000001f, 0x00000010,
317 0x8974, 0xffffffff, 0x00000000
318};
319
320static const u32 scrapper_golden_registers[] =
321{
322 0x690, 0x3fff3fff, 0x20c00033,
323 0x918c, 0x0fff0fff, 0x00010006,
324 0x918c, 0x0fff0fff, 0x00010006,
325 0x91a8, 0x0fff0fff, 0x00010006,
326 0x91a8, 0x0fff0fff, 0x00010006,
327 0x9150, 0xffffdfff, 0x6e944040,
328 0x9150, 0xffffdfff, 0x6e944040,
329 0x917c, 0x0fff0fff, 0x00030002,
330 0x917c, 0x0fff0fff, 0x00030002,
331 0x9198, 0x0fff0fff, 0x00030002,
332 0x9198, 0x0fff0fff, 0x00030002,
333 0x915c, 0x0fff0fff, 0x00010000,
334 0x915c, 0x0fff0fff, 0x00010000,
335 0x3f90, 0xffff0001, 0xff000000,
336 0x3f90, 0xffff0001, 0xff000000,
337 0x9178, 0x0fff0fff, 0x00070000,
338 0x9178, 0x0fff0fff, 0x00070000,
339 0x9194, 0x0fff0fff, 0x00070000,
340 0x9194, 0x0fff0fff, 0x00070000,
341 0x9148, 0xffff0001, 0xff000000,
342 0x9148, 0xffff0001, 0xff000000,
343 0x9190, 0x0fff0fff, 0x00090008,
344 0x9190, 0x0fff0fff, 0x00090008,
345 0x91ac, 0x0fff0fff, 0x00090008,
346 0x91ac, 0x0fff0fff, 0x00090008,
347 0x3f94, 0xffff0000, 0xff000000,
348 0x3f94, 0xffff0000, 0xff000000,
349 0x914c, 0xffff0000, 0xff000000,
350 0x914c, 0xffff0000, 0xff000000,
351 0x929c, 0x00000fff, 0x00000001,
352 0x929c, 0x00000fff, 0x00000001,
353 0x55e4, 0xff607fff, 0xfc000100,
354 0x8a18, 0xff000fff, 0x00000100,
355 0x8a18, 0xff000fff, 0x00000100,
356 0x8b28, 0xff000fff, 0x00000100,
357 0x8b28, 0xff000fff, 0x00000100,
358 0x9144, 0xfffc0fff, 0x00000100,
359 0x9144, 0xfffc0fff, 0x00000100,
360 0x6ed8, 0x00010101, 0x00010000,
361 0x9830, 0xffffffff, 0x00000000,
362 0x9830, 0xffffffff, 0x00000000,
363 0x9834, 0xf00fffff, 0x00000400,
364 0x9834, 0xf00fffff, 0x00000400,
365 0x9838, 0xfffffffe, 0x00000000,
366 0x9838, 0xfffffffe, 0x00000000,
367 0xd0c0, 0xff000fff, 0x00000100,
368 0xd02c, 0xbfffff1f, 0x08421000,
369 0xd02c, 0xbfffff1f, 0x08421000,
370 0xd0b8, 0x73773777, 0x12010001,
371 0xd0b8, 0x73773777, 0x12010001,
372 0x5bb0, 0x000000f0, 0x00000070,
373 0x98f8, 0x73773777, 0x12010001,
374 0x98f8, 0x73773777, 0x12010001,
375 0x98fc, 0xffffffff, 0x00000010,
376 0x98fc, 0xffffffff, 0x00000010,
377 0x9b7c, 0x00ff0000, 0x00fc0000,
378 0x9b7c, 0x00ff0000, 0x00fc0000,
379 0x8030, 0x00001f0f, 0x0000100a,
380 0x8030, 0x00001f0f, 0x0000100a,
381 0x2f48, 0x73773777, 0x12010001,
382 0x2f48, 0x73773777, 0x12010001,
383 0x2408, 0x00030000, 0x000c007f,
384 0x8a14, 0xf000003f, 0x00000007,
385 0x8a14, 0xf000003f, 0x00000007,
386 0x8b24, 0x3fff3fff, 0x00ff0fff,
387 0x8b24, 0x3fff3fff, 0x00ff0fff,
388 0x8b10, 0x0000ff0f, 0x00000000,
389 0x8b10, 0x0000ff0f, 0x00000000,
390 0x28a4c, 0x07ffffff, 0x06000000,
391 0x28a4c, 0x07ffffff, 0x06000000,
392 0x4d8, 0x00000fff, 0x00000100,
393 0x4d8, 0x00000fff, 0x00000100,
394 0xa008, 0xffffffff, 0x00010000,
395 0xa008, 0xffffffff, 0x00010000,
396 0x913c, 0xffff03ff, 0x01000100,
397 0x913c, 0xffff03ff, 0x01000100,
398 0x90e8, 0x001fffff, 0x010400c0,
399 0x8c00, 0x000000ff, 0x00000003,
400 0x8c00, 0x000000ff, 0x00000003,
401 0x8c04, 0xf8ff00ff, 0x40600060,
402 0x8c04, 0xf8ff00ff, 0x40600060,
403 0x8c30, 0x0000000f, 0x00040005,
404 0x8cf0, 0x1fff1fff, 0x08e00410,
405 0x8cf0, 0x1fff1fff, 0x08e00410,
406 0x900c, 0x00ffffff, 0x0017071f,
407 0x28350, 0x00000f01, 0x00000000,
408 0x28350, 0x00000f01, 0x00000000,
409 0x9508, 0xf700071f, 0x00000002,
410 0x9508, 0xf700071f, 0x00000002,
411 0x9688, 0x00300000, 0x0017000f,
412 0x960c, 0xffffffff, 0x54763210,
413 0x960c, 0xffffffff, 0x54763210,
414 0x20ef8, 0x01ff01ff, 0x00000002,
415 0x20e98, 0xfffffbff, 0x00200000,
416 0x2015c, 0xffffffff, 0x00000f40,
417 0x88c4, 0x001f3ae3, 0x00000082,
418 0x88c4, 0x001f3ae3, 0x00000082,
419 0x8978, 0x3fffffff, 0x04050140,
420 0x8978, 0x3fffffff, 0x04050140,
421 0x88d4, 0x0000001f, 0x00000010,
422 0x88d4, 0x0000001f, 0x00000010,
423 0x8974, 0xffffffff, 0x00000000,
424 0x8974, 0xffffffff, 0x00000000
425};
426
427static void ni_init_golden_registers(struct radeon_device *rdev)
428{
429 switch (rdev->family) {
430 case CHIP_CAYMAN:
431 radeon_program_register_sequence(rdev,
432 cayman_golden_registers,
433 (const u32)ARRAY_SIZE(cayman_golden_registers));
434 radeon_program_register_sequence(rdev,
435 cayman_golden_registers2,
436 (const u32)ARRAY_SIZE(cayman_golden_registers2));
437 break;
438 case CHIP_ARUBA:
439 if ((rdev->pdev->device == 0x9900) ||
440 (rdev->pdev->device == 0x9901) ||
441 (rdev->pdev->device == 0x9903) ||
442 (rdev->pdev->device == 0x9904) ||
443 (rdev->pdev->device == 0x9905) ||
444 (rdev->pdev->device == 0x9906) ||
445 (rdev->pdev->device == 0x9907) ||
446 (rdev->pdev->device == 0x9908) ||
447 (rdev->pdev->device == 0x9909) ||
448 (rdev->pdev->device == 0x990A) ||
449 (rdev->pdev->device == 0x990B) ||
450 (rdev->pdev->device == 0x990C) ||
451 (rdev->pdev->device == 0x990D) ||
452 (rdev->pdev->device == 0x990E) ||
453 (rdev->pdev->device == 0x990F) ||
454 (rdev->pdev->device == 0x9910) ||
455 (rdev->pdev->device == 0x9913) ||
456 (rdev->pdev->device == 0x9917) ||
457 (rdev->pdev->device == 0x9918)) {
458 radeon_program_register_sequence(rdev,
459 dvst_golden_registers,
460 (const u32)ARRAY_SIZE(dvst_golden_registers));
461 radeon_program_register_sequence(rdev,
462 dvst_golden_registers2,
463 (const u32)ARRAY_SIZE(dvst_golden_registers2));
464 } else {
465 radeon_program_register_sequence(rdev,
466 scrapper_golden_registers,
467 (const u32)ARRAY_SIZE(scrapper_golden_registers));
468 radeon_program_register_sequence(rdev,
469 dvst_golden_registers2,
470 (const u32)ARRAY_SIZE(dvst_golden_registers2));
471 }
472 break;
473 default:
474 break;
475 }
476}
477
0af62b01
AD		 478#define BTC_IO_MC_REGS_SIZE 29
479
480static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
481 {0x00000077, 0xff010100},
482 {0x00000078, 0x00000000},
483 {0x00000079, 0x00001434},
484 {0x0000007a, 0xcc08ec08},
485 {0x0000007b, 0x00040000},
486 {0x0000007c, 0x000080c0},
487 {0x0000007d, 0x09000000},
488 {0x0000007e, 0x00210404},
489 {0x00000081, 0x08a8e800},
490 {0x00000082, 0x00030444},
491 {0x00000083, 0x00000000},
492 {0x00000085, 0x00000001},
493 {0x00000086, 0x00000002},
494 {0x00000087, 0x48490000},
495 {0x00000088, 0x20244647},
496 {0x00000089, 0x00000005},
497 {0x0000008b, 0x66030000},
498 {0x0000008c, 0x00006603},
499 {0x0000008d, 0x00000100},
500 {0x0000008f, 0x00001c0a},
501 {0x00000090, 0xff000001},
502 {0x00000094, 0x00101101},
503 {0x00000095, 0x00000fff},
504 {0x00000096, 0x00116fff},
505 {0x00000097, 0x60010000},
506 {0x00000098, 0x10010000},
507 {0x00000099, 0x00006000},
508 {0x0000009a, 0x00001000},
509 {0x0000009f, 0x00946a00}
510};
511
512static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
513 {0x00000077, 0xff010100},
514 {0x00000078, 0x00000000},
515 {0x00000079, 0x00001434},
516 {0x0000007a, 0xcc08ec08},
517 {0x0000007b, 0x00040000},
518 {0x0000007c, 0x000080c0},
519 {0x0000007d, 0x09000000},
520 {0x0000007e, 0x00210404},
521 {0x00000081, 0x08a8e800},
522 {0x00000082, 0x00030444},
523 {0x00000083, 0x00000000},
524 {0x00000085, 0x00000001},
525 {0x00000086, 0x00000002},
526 {0x00000087, 0x48490000},
527 {0x00000088, 0x20244647},
528 {0x00000089, 0x00000005},
529 {0x0000008b, 0x66030000},
530 {0x0000008c, 0x00006603},
531 {0x0000008d, 0x00000100},
532 {0x0000008f, 0x00001c0a},
533 {0x00000090, 0xff000001},
534 {0x00000094, 0x00101101},
535 {0x00000095, 0x00000fff},
536 {0x00000096, 0x00116fff},
537 {0x00000097, 0x60010000},
538 {0x00000098, 0x10010000},
539 {0x00000099, 0x00006000},
540 {0x0000009a, 0x00001000},
541 {0x0000009f, 0x00936a00}
542};
543
544static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
545 {0x00000077, 0xff010100},
546 {0x00000078, 0x00000000},
547 {0x00000079, 0x00001434},
548 {0x0000007a, 0xcc08ec08},
549 {0x0000007b, 0x00040000},
550 {0x0000007c, 0x000080c0},
551 {0x0000007d, 0x09000000},
552 {0x0000007e, 0x00210404},
553 {0x00000081, 0x08a8e800},
554 {0x00000082, 0x00030444},
555 {0x00000083, 0x00000000},
556 {0x00000085, 0x00000001},
557 {0x00000086, 0x00000002},
558 {0x00000087, 0x48490000},
559 {0x00000088, 0x20244647},
560 {0x00000089, 0x00000005},
561 {0x0000008b, 0x66030000},
562 {0x0000008c, 0x00006603},
563 {0x0000008d, 0x00000100},
564 {0x0000008f, 0x00001c0a},
565 {0x00000090, 0xff000001},
566 {0x00000094, 0x00101101},
567 {0x00000095, 0x00000fff},
568 {0x00000096, 0x00116fff},
569 {0x00000097, 0x60010000},
570 {0x00000098, 0x10010000},
571 {0x00000099, 0x00006000},
572 {0x0000009a, 0x00001000},
573 {0x0000009f, 0x00916a00}
574};
575
9b8253ce
AD		 576static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
577 {0x00000077, 0xff010100},
578 {0x00000078, 0x00000000},
579 {0x00000079, 0x00001434},
580 {0x0000007a, 0xcc08ec08},
581 {0x0000007b, 0x00040000},
582 {0x0000007c, 0x000080c0},
583 {0x0000007d, 0x09000000},
584 {0x0000007e, 0x00210404},
585 {0x00000081, 0x08a8e800},
586 {0x00000082, 0x00030444},
587 {0x00000083, 0x00000000},
588 {0x00000085, 0x00000001},
589 {0x00000086, 0x00000002},
590 {0x00000087, 0x48490000},
591 {0x00000088, 0x20244647},
592 {0x00000089, 0x00000005},
593 {0x0000008b, 0x66030000},
594 {0x0000008c, 0x00006603},
595 {0x0000008d, 0x00000100},
596 {0x0000008f, 0x00001c0a},
597 {0x00000090, 0xff000001},
598 {0x00000094, 0x00101101},
599 {0x00000095, 0x00000fff},
600 {0x00000096, 0x00116fff},
601 {0x00000097, 0x60010000},
602 {0x00000098, 0x10010000},
603 {0x00000099, 0x00006000},
604 {0x0000009a, 0x00001000},
605 {0x0000009f, 0x00976b00}
606};
607
755d819e 608int ni_mc_load_microcode(struct radeon_device *rdev)
0af62b01
AD		 609{
610 const __be32 *fw_data;
611 u32 mem_type, running, blackout = 0;
612 u32 *io_mc_regs;
9b8253ce 613 int i, ucode_size, regs_size;
0af62b01
AD		 614
615 if (!rdev->mc_fw)
616 return -EINVAL;
617
618 switch (rdev->family) {
619 case CHIP_BARTS:
620 io_mc_regs = (u32 *)&barts_io_mc_regs;
9b8253ce
AD		 621 ucode_size = BTC_MC_UCODE_SIZE;
622 regs_size = BTC_IO_MC_REGS_SIZE;
0af62b01
AD		 623 break;
624 case CHIP_TURKS:
625 io_mc_regs = (u32 *)&turks_io_mc_regs;
9b8253ce
AD		 626 ucode_size = BTC_MC_UCODE_SIZE;
627 regs_size = BTC_IO_MC_REGS_SIZE;
0af62b01
AD		 628 break;
629 case CHIP_CAICOS:
630 default:
631 io_mc_regs = (u32 *)&caicos_io_mc_regs;
9b8253ce
AD		 632 ucode_size = BTC_MC_UCODE_SIZE;
633 regs_size = BTC_IO_MC_REGS_SIZE;
634 break;
635 case CHIP_CAYMAN:
636 io_mc_regs = (u32 *)&cayman_io_mc_regs;
637 ucode_size = CAYMAN_MC_UCODE_SIZE;
638 regs_size = BTC_IO_MC_REGS_SIZE;
0af62b01
AD		 639 break;
640 }
641
642 mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
643 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
644
645 if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
646 if (running) {
647 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
648 WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
649 }
650
651 /* reset the engine and set to writable */
652 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
653 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
654
655 /* load mc io regs */
9b8253ce 656 for (i = 0; i < regs_size; i++) {
0af62b01
AD		 657 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
658 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
659 }
660 /* load the MC ucode */
661 fw_data = (const __be32 *)rdev->mc_fw->data;
9b8253ce 662 for (i = 0; i < ucode_size; i++)
0af62b01
AD		 663 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
664
665 /* put the engine back into the active state */
666 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
667 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
668 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
669
670 /* wait for training to complete */
0e2c978e
AD		 671 for (i = 0; i < rdev->usec_timeout; i++) {
672 if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
673 break;
674 udelay(1);
675 }
0af62b01
AD		 676
677 if (running)
678 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
679 }
680
681 return 0;
682}
683
684int ni_init_microcode(struct radeon_device *rdev)
685{
0af62b01
AD		 686 const char *chip_name;
687 const char *rlc_chip_name;
688 size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
6596afd4 689 size_t smc_req_size = 0;
0af62b01
AD		 690 char fw_name[30];
691 int err;
692
693 DRM_DEBUG("\n");
694
0af62b01
AD		 695 switch (rdev->family) {
696 case CHIP_BARTS:
697 chip_name = "BARTS";
698 rlc_chip_name = "BTC";
9b8253ce
AD		 699 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
700 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
701 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
702 mc_req_size = BTC_MC_UCODE_SIZE * 4;
6596afd4 703 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
0af62b01
AD		 704 break;
705 case CHIP_TURKS:
706 chip_name = "TURKS";
707 rlc_chip_name = "BTC";
9b8253ce
AD		 708 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
709 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
710 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
711 mc_req_size = BTC_MC_UCODE_SIZE * 4;
6596afd4 712 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
0af62b01
AD		 713 break;
714 case CHIP_CAICOS:
715 chip_name = "CAICOS";
716 rlc_chip_name = "BTC";
9b8253ce
AD		 717 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
718 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
719 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
720 mc_req_size = BTC_MC_UCODE_SIZE * 4;
6596afd4 721 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
9b8253ce
AD		 722 break;
723 case CHIP_CAYMAN:
724 chip_name = "CAYMAN";
725 rlc_chip_name = "CAYMAN";
726 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
727 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
728 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
729 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
69e0b57a 730 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
0af62b01 731 break;
c420c745
AD		 732 case CHIP_ARUBA:
733 chip_name = "ARUBA";
734 rlc_chip_name = "ARUBA";
735 /* pfp/me same size as CAYMAN */
736 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
737 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
738 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
739 mc_req_size = 0;
740 break;
0af62b01
AD		 741 default: BUG();
742 }
743
0af62b01
AD		 744 DRM_INFO("Loading %s Microcode\n", chip_name);
745
746 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 747 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0af62b01
AD		 748 if (err)
749 goto out;
750 if (rdev->pfp_fw->size != pfp_req_size) {
751 printk(KERN_ERR
752 "ni_cp: Bogus length %zu in firmware \"%s\"\n",
753 rdev->pfp_fw->size, fw_name);
754 err = -EINVAL;
755 goto out;
756 }
757
758 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 759 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0af62b01
AD		 760 if (err)
761 goto out;
762 if (rdev->me_fw->size != me_req_size) {
763 printk(KERN_ERR
764 "ni_cp: Bogus length %zu in firmware \"%s\"\n",
765 rdev->me_fw->size, fw_name);
766 err = -EINVAL;
767 }
768
769 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 770 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0af62b01
AD		 771 if (err)
772 goto out;
773 if (rdev->rlc_fw->size != rlc_req_size) {
774 printk(KERN_ERR
775 "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
776 rdev->rlc_fw->size, fw_name);
777 err = -EINVAL;
778 }
779
c420c745
AD		 780 /* no MC ucode on TN */
781 if (!(rdev->flags & RADEON_IS_IGP)) {
782 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 783 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
c420c745
AD		 784 if (err)
785 goto out;
786 if (rdev->mc_fw->size != mc_req_size) {
787 printk(KERN_ERR
788 "ni_mc: Bogus length %zu in firmware \"%s\"\n",
789 rdev->mc_fw->size, fw_name);
790 err = -EINVAL;
791 }
0af62b01 792 }
6596afd4 793
69e0b57a 794 if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
6596afd4 795 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 796 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
6596afd4
AD		 797 if (err)
798 goto out;
799 if (rdev->smc_fw->size != smc_req_size) {
800 printk(KERN_ERR
801 "ni_mc: Bogus length %zu in firmware \"%s\"\n",
802 rdev->mc_fw->size, fw_name);
803 err = -EINVAL;
804 }
805 }
806
0af62b01 807out:
0af62b01
AD		 808 if (err) {
809 if (err != -EINVAL)
810 printk(KERN_ERR
811 "ni_cp: Failed to load firmware \"%s\"\n",
812 fw_name);
813 release_firmware(rdev->pfp_fw);
814 rdev->pfp_fw = NULL;
815 release_firmware(rdev->me_fw);
816 rdev->me_fw = NULL;
817 release_firmware(rdev->rlc_fw);
818 rdev->rlc_fw = NULL;
819 release_firmware(rdev->mc_fw);
820 rdev->mc_fw = NULL;
821 }
822 return err;
823}
824
29a15221
AD		 825int tn_get_temp(struct radeon_device *rdev)
826{
827 u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
828 int actual_temp = (temp / 8) - 49;
829
830 return actual_temp * 1000;
831}
832
fecf1d07
AD		 833/*
834 * Core functions
835 */
fecf1d07
AD		 836static void cayman_gpu_init(struct radeon_device *rdev)
837{
fecf1d07
AD		 838 u32 gb_addr_config = 0;
839 u32 mc_shared_chmap, mc_arb_ramcfg;
fecf1d07
AD		 840 u32 cgts_tcc_disable;
841 u32 sx_debug_1;
842 u32 smx_dc_ctl0;
fecf1d07
AD		 843 u32 cgts_sm_ctrl_reg;
844 u32 hdp_host_path_cntl;
845 u32 tmp;
416a2bd2 846 u32 disabled_rb_mask;
fecf1d07
AD		 847 int i, j;
848
849 switch (rdev->family) {
850 case CHIP_CAYMAN:
fecf1d07
AD		 851 rdev->config.cayman.max_shader_engines = 2;
852 rdev->config.cayman.max_pipes_per_simd = 4;
853 rdev->config.cayman.max_tile_pipes = 8;
854 rdev->config.cayman.max_simds_per_se = 12;
855 rdev->config.cayman.max_backends_per_se = 4;
856 rdev->config.cayman.max_texture_channel_caches = 8;
857 rdev->config.cayman.max_gprs = 256;
858 rdev->config.cayman.max_threads = 256;
859 rdev->config.cayman.max_gs_threads = 32;
860 rdev->config.cayman.max_stack_entries = 512;
861 rdev->config.cayman.sx_num_of_sets = 8;
862 rdev->config.cayman.sx_max_export_size = 256;
863 rdev->config.cayman.sx_max_export_pos_size = 64;
864 rdev->config.cayman.sx_max_export_smx_size = 192;
865 rdev->config.cayman.max_hw_contexts = 8;
866 rdev->config.cayman.sq_num_cf_insts = 2;
867
868 rdev->config.cayman.sc_prim_fifo_size = 0x100;
869 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
870 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 871 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
fecf1d07 872 break;
7b76e479
AD		 873 case CHIP_ARUBA:
874 default:
875 rdev->config.cayman.max_shader_engines = 1;
876 rdev->config.cayman.max_pipes_per_simd = 4;
877 rdev->config.cayman.max_tile_pipes = 2;
878 if ((rdev->pdev->device == 0x9900) ||
d430f7db
AD		 879 (rdev->pdev->device == 0x9901) ||
880 (rdev->pdev->device == 0x9905) ||
881 (rdev->pdev->device == 0x9906) ||
882 (rdev->pdev->device == 0x9907) ||
883 (rdev->pdev->device == 0x9908) ||
884 (rdev->pdev->device == 0x9909) ||
e4d17063
AD		 885 (rdev->pdev->device == 0x990B) ||
886 (rdev->pdev->device == 0x990C) ||
887 (rdev->pdev->device == 0x990F) ||
d430f7db 888 (rdev->pdev->device == 0x9910) ||
e4d17063 889 (rdev->pdev->device == 0x9917) ||
62d1f92e
AD		 890 (rdev->pdev->device == 0x9999) ||
891 (rdev->pdev->device == 0x999C)) {
7b76e479
AD		 892 rdev->config.cayman.max_simds_per_se = 6;
893 rdev->config.cayman.max_backends_per_se = 2;
894 } else if ((rdev->pdev->device == 0x9903) ||
d430f7db
AD		 895 (rdev->pdev->device == 0x9904) ||
896 (rdev->pdev->device == 0x990A) ||
e4d17063
AD		 897 (rdev->pdev->device == 0x990D) ||
898 (rdev->pdev->device == 0x990E) ||
d430f7db 899 (rdev->pdev->device == 0x9913) ||
62d1f92e
AD		 900 (rdev->pdev->device == 0x9918) ||
901 (rdev->pdev->device == 0x999D)) {
7b76e479
AD		 902 rdev->config.cayman.max_simds_per_se = 4;
903 rdev->config.cayman.max_backends_per_se = 2;
d430f7db
AD		 904 } else if ((rdev->pdev->device == 0x9919) ||
905 (rdev->pdev->device == 0x9990) ||
906 (rdev->pdev->device == 0x9991) ||
907 (rdev->pdev->device == 0x9994) ||
e4d17063
AD		 908 (rdev->pdev->device == 0x9995) ||
909 (rdev->pdev->device == 0x9996) ||
910 (rdev->pdev->device == 0x999A) ||
d430f7db 911 (rdev->pdev->device == 0x99A0)) {
7b76e479
AD		 912 rdev->config.cayman.max_simds_per_se = 3;
913 rdev->config.cayman.max_backends_per_se = 1;
914 } else {
915 rdev->config.cayman.max_simds_per_se = 2;
916 rdev->config.cayman.max_backends_per_se = 1;
917 }
918 rdev->config.cayman.max_texture_channel_caches = 2;
919 rdev->config.cayman.max_gprs = 256;
920 rdev->config.cayman.max_threads = 256;
921 rdev->config.cayman.max_gs_threads = 32;
922 rdev->config.cayman.max_stack_entries = 512;
923 rdev->config.cayman.sx_num_of_sets = 8;
924 rdev->config.cayman.sx_max_export_size = 256;
925 rdev->config.cayman.sx_max_export_pos_size = 64;
926 rdev->config.cayman.sx_max_export_smx_size = 192;
927 rdev->config.cayman.max_hw_contexts = 8;
928 rdev->config.cayman.sq_num_cf_insts = 2;
929
930 rdev->config.cayman.sc_prim_fifo_size = 0x40;
931 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
932 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 933 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
7b76e479 934 break;
fecf1d07
AD		 935 }
936
937 /* Initialize HDP */
938 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
939 WREG32((0x2c14 + j), 0x00000000);
940 WREG32((0x2c18 + j), 0x00000000);
941 WREG32((0x2c1c + j), 0x00000000);
942 WREG32((0x2c20 + j), 0x00000000);
943 WREG32((0x2c24 + j), 0x00000000);
944 }
945
946 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
947
d054ac16
AD		 948 evergreen_fix_pci_max_read_req_size(rdev);
949
fecf1d07
AD		 950 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
951 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
952
fecf1d07
AD		 953 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
954 rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
955 if (rdev->config.cayman.mem_row_size_in_kb > 4)
956 rdev->config.cayman.mem_row_size_in_kb = 4;
957 /* XXX use MC settings? */
958 rdev->config.cayman.shader_engine_tile_size = 32;
959 rdev->config.cayman.num_gpus = 1;
960 rdev->config.cayman.multi_gpu_tile_size = 64;
961
fecf1d07
AD		 962 tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
963 rdev->config.cayman.num_tile_pipes = (1 << tmp);
964 tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
965 rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
966 tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
967 rdev->config.cayman.num_shader_engines = tmp + 1;
968 tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
969 rdev->config.cayman.num_gpus = tmp + 1;
970 tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
971 rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
972 tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
973 rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
974
416a2bd2 975
fecf1d07
AD		 976 /* setup tiling info dword. gb_addr_config is not adequate since it does
977 * not have bank info, so create a custom tiling dword.
978 * bits 3:0 num_pipes
979 * bits 7:4 num_banks
980 * bits 11:8 group_size
981 * bits 15:12 row_size
982 */
983 rdev->config.cayman.tile_config = 0;
984 switch (rdev->config.cayman.num_tile_pipes) {
985 case 1:
986 default:
987 rdev->config.cayman.tile_config |= (0 << 0);
988 break;
989 case 2:
990 rdev->config.cayman.tile_config |= (1 << 0);
991 break;
992 case 4:
993 rdev->config.cayman.tile_config |= (2 << 0);
994 break;
995 case 8:
996 rdev->config.cayman.tile_config |= (3 << 0);
997 break;
998 }
7b76e479
AD		 999
1000 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1001 if (rdev->flags & RADEON_IS_IGP)
1f73cca7 1002 rdev->config.cayman.tile_config |= 1 << 4;
29d65406 1003 else {
5b23c904
AD		 1004 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1005 case 0: /* four banks */
29d65406 1006 rdev->config.cayman.tile_config |= 0 << 4;
5b23c904
AD		 1007 break;
1008 case 1: /* eight banks */
1009 rdev->config.cayman.tile_config |= 1 << 4;
1010 break;
1011 case 2: /* sixteen banks */
1012 default:
1013 rdev->config.cayman.tile_config |= 2 << 4;
1014 break;
1015 }
29d65406 1016 }
fecf1d07 1017 rdev->config.cayman.tile_config |=
cde5083b 1018 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
fecf1d07
AD		 1019 rdev->config.cayman.tile_config |=
1020 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1021
416a2bd2
AD		 1022 tmp = 0;
1023 for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1024 u32 rb_disable_bitmap;
1025
1026 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1027 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1028 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1029 tmp <<= 4;
1030 tmp |= rb_disable_bitmap;
1031 }
1032 /* enabled rb are just the one not disabled :) */
1033 disabled_rb_mask = tmp;
cedb655a
AD		 1034 tmp = 0;
1035 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1036 tmp |= (1 << i);
1037 /* if all the backends are disabled, fix it up here */
1038 if ((disabled_rb_mask & tmp) == tmp) {
1039 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1040 disabled_rb_mask &= ~(1 << i);
1041 }
416a2bd2
AD		 1042
1043 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1044 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1045
fecf1d07
AD		 1046 WREG32(GB_ADDR_CONFIG, gb_addr_config);
1047 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18
AD		 1048 if (ASIC_IS_DCE6(rdev))
1049 WREG32(DMIF_ADDR_CALC, gb_addr_config);
fecf1d07 1050 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
f60cbd11
AD		 1051 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1052 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
9a21059d
CK		 1053 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1054 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1055 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
fecf1d07 1056
8f612b23
AD		 1057 if ((rdev->config.cayman.max_backends_per_se == 1) &&
1058 (rdev->flags & RADEON_IS_IGP)) {
1059 if ((disabled_rb_mask & 3) == 1) {
1060 /* RB0 disabled, RB1 enabled */
1061 tmp = 0x11111111;
1062 } else {
1063 /* RB1 disabled, RB0 enabled */
1064 tmp = 0x00000000;
1065 }
1066 } else {
1067 tmp = gb_addr_config & NUM_PIPES_MASK;
1068 tmp = r6xx_remap_render_backend(rdev, tmp,
1069 rdev->config.cayman.max_backends_per_se *
1070 rdev->config.cayman.max_shader_engines,
1071 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1072 }
416a2bd2 1073 WREG32(GB_BACKEND_MAP, tmp);
fecf1d07 1074
416a2bd2
AD		 1075 cgts_tcc_disable = 0xffff0000;
1076 for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1077 cgts_tcc_disable &= ~(1 << (16 + i));
fecf1d07
AD		 1078 WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1079 WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
fecf1d07
AD		 1080 WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1081 WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1082
1083 /* reprogram the shader complex */
1084 cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1085 for (i = 0; i < 16; i++)
1086 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1087 WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1088
1089 /* set HW defaults for 3D engine */
1090 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1091
1092 sx_debug_1 = RREG32(SX_DEBUG_1);
1093 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1094 WREG32(SX_DEBUG_1, sx_debug_1);
1095
1096 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1097 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
285e042d 1098 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
fecf1d07
AD		 1099 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1100
1101 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1102
1103 /* need to be explicitly zero-ed */
1104 WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1105 WREG32(SQ_LSTMP_RING_BASE, 0);
1106 WREG32(SQ_HSTMP_RING_BASE, 0);
1107 WREG32(SQ_ESTMP_RING_BASE, 0);
1108 WREG32(SQ_GSTMP_RING_BASE, 0);
1109 WREG32(SQ_VSTMP_RING_BASE, 0);
1110 WREG32(SQ_PSTMP_RING_BASE, 0);
1111
1112 WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1113
285e042d
DA		 1114 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1115 POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1116 SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
fecf1d07 1117
285e042d
DA		 1118 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1119 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1120 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
fecf1d07
AD		 1121
1122
1123 WREG32(VGT_NUM_INSTANCES, 1);
1124
1125 WREG32(CP_PERFMON_CNTL, 0);
1126
285e042d 1127 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
fecf1d07
AD		 1128 FETCH_FIFO_HIWATER(0x4) |
1129 DONE_FIFO_HIWATER(0xe0) |
1130 ALU_UPDATE_FIFO_HIWATER(0x8)));
1131
1132 WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1133 WREG32(SQ_CONFIG, (VC_ENABLE |
1134 EXPORT_SRC_C |
1135 GFX_PRIO(0) |
1136 CS1_PRIO(0) |
1137 CS2_PRIO(1)));
1138 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1139
1140 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1141 FORCE_EOV_MAX_REZ_CNT(255)));
1142
1143 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1144 AUTO_INVLD_EN(ES_AND_GS_AUTO));
1145
1146 WREG32(VGT_GS_VERTEX_REUSE, 16);
1147 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1148
1149 WREG32(CB_PERF_CTR0_SEL_0, 0);
1150 WREG32(CB_PERF_CTR0_SEL_1, 0);
1151 WREG32(CB_PERF_CTR1_SEL_0, 0);
1152 WREG32(CB_PERF_CTR1_SEL_1, 0);
1153 WREG32(CB_PERF_CTR2_SEL_0, 0);
1154 WREG32(CB_PERF_CTR2_SEL_1, 0);
1155 WREG32(CB_PERF_CTR3_SEL_0, 0);
1156 WREG32(CB_PERF_CTR3_SEL_1, 0);
1157
0b65f83f
DA		 1158 tmp = RREG32(HDP_MISC_CNTL);
1159 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1160 WREG32(HDP_MISC_CNTL, tmp);
1161
fecf1d07
AD		 1162 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1163 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1164
1165 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1166
1167 udelay(50);
8ba10463
AD		 1168
1169 /* set clockgating golden values on TN */
1170 if (rdev->family == CHIP_ARUBA) {
1171 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1172 tmp &= ~0x00380000;
1173 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1174 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1175 tmp &= ~0x0e000000;
1176 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1177 }
fecf1d07
AD		 1178}
1179
fa8198ea
AD		 1180/*
1181 * GART
1182 */
1183void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1184{
1185 /* flush hdp cache */
1186 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1187
1188 /* bits 0-7 are the VM contexts0-7 */
1189 WREG32(VM_INVALIDATE_REQUEST, 1);
1190}
1191
1109ca09 1192static int cayman_pcie_gart_enable(struct radeon_device *rdev)
fa8198ea 1193{
721604a1 1194 int i, r;
fa8198ea 1195
c9a1be96 1196 if (rdev->gart.robj == NULL) {
fa8198ea
AD		 1197 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1198 return -EINVAL;
1199 }
1200 r = radeon_gart_table_vram_pin(rdev);
1201 if (r)
1202 return r;
1203 radeon_gart_restore(rdev);
1204 /* Setup TLB control */
721604a1
JG		 1205 WREG32(MC_VM_MX_L1_TLB_CNTL,
1206 (0xA << 7) |
1207 ENABLE_L1_TLB |
fa8198ea
AD		 1208 ENABLE_L1_FRAGMENT_PROCESSING |
1209 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
721604a1 1210 ENABLE_ADVANCED_DRIVER_MODEL |
fa8198ea
AD		 1211 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1212 /* Setup L2 cache */
1213 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1214 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1215 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1216 EFFECTIVE_L2_QUEUE_SIZE(7) |
1217 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1218 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1219 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1220 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1221 /* setup context0 */
1222 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1223 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1224 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1225 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1226 (u32)(rdev->dummy_page.addr >> 12));
1227 WREG32(VM_CONTEXT0_CNTL2, 0);
1228 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1229 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
721604a1
JG		 1230
1231 WREG32(0x15D4, 0);
1232 WREG32(0x15D8, 0);
1233 WREG32(0x15DC, 0);
1234
1235 /* empty context1-7 */
23d4f1f2
AD		 1236 /* Assign the pt base to something valid for now; the pts used for
1237 * the VMs are determined by the application and setup and assigned
1238 * on the fly in the vm part of radeon_gart.c
1239 */
721604a1
JG		 1240 for (i = 1; i < 8; i++) {
1241 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
c1a7ca0d 1242 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
721604a1
JG		 1243 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1244 rdev->gart.table_addr >> 12);
1245 }
1246
1247 /* enable context1-7 */
1248 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1249 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 1250 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 1251 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 1252 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1253 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1254 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1255 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1256 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1257 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1258 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1259 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1260 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1261 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1262 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1263 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
fa8198ea
AD		 1264
1265 cayman_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 1266 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1267 (unsigned)(rdev->mc.gtt_size >> 20),
1268 (unsigned long long)rdev->gart.table_addr);
fa8198ea
AD		 1269 rdev->gart.ready = true;
1270 return 0;
1271}
1272
1109ca09 1273static void cayman_pcie_gart_disable(struct radeon_device *rdev)
fa8198ea 1274{
fa8198ea
AD		 1275 /* Disable all tables */
1276 WREG32(VM_CONTEXT0_CNTL, 0);
1277 WREG32(VM_CONTEXT1_CNTL, 0);
1278 /* Setup TLB control */
1279 WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1280 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1281 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1282 /* Setup L2 cache */
1283 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1284 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1285 EFFECTIVE_L2_QUEUE_SIZE(7) |
1286 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1287 WREG32(VM_L2_CNTL2, 0);
1288 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1289 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
c9a1be96 1290 radeon_gart_table_vram_unpin(rdev);
fa8198ea
AD		 1291}
1292
1109ca09 1293static void cayman_pcie_gart_fini(struct radeon_device *rdev)
fa8198ea
AD		 1294{
1295 cayman_pcie_gart_disable(rdev);
1296 radeon_gart_table_vram_free(rdev);
1297 radeon_gart_fini(rdev);
1298}
1299
1b37078b
AD		 1300void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1301 int ring, u32 cp_int_cntl)
1302{
1303 u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1304
1305 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1306 WREG32(CP_INT_CNTL, cp_int_cntl);
1307}
1308
0c88a02e
AD		 1309/*
1310 * CP.
1311 */
b40e7e16
AD		 1312void cayman_fence_ring_emit(struct radeon_device *rdev,
1313 struct radeon_fence *fence)
1314{
1315 struct radeon_ring *ring = &rdev->ring[fence->ring];
1316 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1317
721604a1
JG		 1318 /* flush read cache over gart for this vmid */
1319 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1320 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1321 radeon_ring_write(ring, 0);
b40e7e16
AD		 1322 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1323 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1324 radeon_ring_write(ring, 0xFFFFFFFF);
1325 radeon_ring_write(ring, 0);
1326 radeon_ring_write(ring, 10); /* poll interval */
1327 /* EVENT_WRITE_EOP - flush caches, send int */
1328 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1329 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1330 radeon_ring_write(ring, addr & 0xffffffff);
1331 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1332 radeon_ring_write(ring, fence->seq);
1333 radeon_ring_write(ring, 0);
1334}
1335
721604a1
JG		 1336void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1337{
876dc9f3 1338 struct radeon_ring *ring = &rdev->ring[ib->ring];
721604a1
JG		 1339
1340 /* set to DX10/11 mode */
1341 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1342 radeon_ring_write(ring, 1);
45df6803
CK		 1343
1344 if (ring->rptr_save_reg) {
1345 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1346 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1347 radeon_ring_write(ring, ((ring->rptr_save_reg -
1348 PACKET3_SET_CONFIG_REG_START) >> 2));
1349 radeon_ring_write(ring, next_rptr);
1350 }
1351
721604a1
JG		 1352 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1353 radeon_ring_write(ring,
1354#ifdef __BIG_ENDIAN
1355 (2 << 0) |
1356#endif
1357 (ib->gpu_addr & 0xFFFFFFFC));
1358 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
4bf3dd92
CK		 1359 radeon_ring_write(ring, ib->length_dw |
1360 (ib->vm ? (ib->vm->id << 24) : 0));
721604a1
JG		 1361
1362 /* flush read cache over gart for this vmid */
1363 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1364 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 1365 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
721604a1
JG		 1366 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1367 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1368 radeon_ring_write(ring, 0xFFFFFFFF);
1369 radeon_ring_write(ring, 0);
1370 radeon_ring_write(ring, 10); /* poll interval */
1371}
1372
f2ba57b5
CK		 1373void cayman_uvd_semaphore_emit(struct radeon_device *rdev,
1374 struct radeon_ring *ring,
1375 struct radeon_semaphore *semaphore,
1376 bool emit_wait)
1377{
1378 uint64_t addr = semaphore->gpu_addr;
1379
1380 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
1381 radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
1382
1383 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
1384 radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
1385
1386 radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
1387 radeon_ring_write(ring, 0x80 | (emit_wait ? 1 : 0));
1388}
1389
0c88a02e
AD		 1390static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1391{
1392 if (enable)
1393 WREG32(CP_ME_CNTL, 0);
1394 else {
38f1cff0 1395 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
0c88a02e
AD		 1396 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1397 WREG32(SCRATCH_UMSK, 0);
f60cbd11 1398 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
0c88a02e
AD		 1399 }
1400}
1401
1402static int cayman_cp_load_microcode(struct radeon_device *rdev)
1403{
1404 const __be32 *fw_data;
1405 int i;
1406
1407 if (!rdev->me_fw || !rdev->pfp_fw)
1408 return -EINVAL;
1409
1410 cayman_cp_enable(rdev, false);
1411
1412 fw_data = (const __be32 *)rdev->pfp_fw->data;
1413 WREG32(CP_PFP_UCODE_ADDR, 0);
1414 for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1415 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1416 WREG32(CP_PFP_UCODE_ADDR, 0);
1417
1418 fw_data = (const __be32 *)rdev->me_fw->data;
1419 WREG32(CP_ME_RAM_WADDR, 0);
1420 for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1421 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1422
1423 WREG32(CP_PFP_UCODE_ADDR, 0);
1424 WREG32(CP_ME_RAM_WADDR, 0);
1425 WREG32(CP_ME_RAM_RADDR, 0);
1426 return 0;
1427}
1428
1429static int cayman_cp_start(struct radeon_device *rdev)
1430{
e32eb50d 1431 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
0c88a02e
AD		 1432 int r, i;
1433
e32eb50d 1434 r = radeon_ring_lock(rdev, ring, 7);
0c88a02e
AD		 1435 if (r) {
1436 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1437 return r;
1438 }
e32eb50d
CK		 1439 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1440 radeon_ring_write(ring, 0x1);
1441 radeon_ring_write(ring, 0x0);
1442 radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1443 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1444 radeon_ring_write(ring, 0);
1445 radeon_ring_write(ring, 0);
1446 radeon_ring_unlock_commit(rdev, ring);
0c88a02e
AD		 1447
1448 cayman_cp_enable(rdev, true);
1449
e32eb50d 1450 r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
0c88a02e
AD		 1451 if (r) {
1452 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1453 return r;
1454 }
1455
1456 /* setup clear context state */
e32eb50d
CK		 1457 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1458 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
0c88a02e
AD		 1459
1460 for (i = 0; i < cayman_default_size; i++)
e32eb50d 1461 radeon_ring_write(ring, cayman_default_state[i]);
0c88a02e 1462
e32eb50d
CK		 1463 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1464 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
0c88a02e
AD		 1465
1466 /* set clear context state */
e32eb50d
CK		 1467 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1468 radeon_ring_write(ring, 0);
0c88a02e
AD		 1469
1470 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 1471 radeon_ring_write(ring, 0xc0026f00);
1472 radeon_ring_write(ring, 0x00000000);
1473 radeon_ring_write(ring, 0x00000000);
1474 radeon_ring_write(ring, 0x00000000);
0c88a02e
AD		 1475
1476 /* Clear consts */
e32eb50d
CK		 1477 radeon_ring_write(ring, 0xc0036f00);
1478 radeon_ring_write(ring, 0x00000bc4);
1479 radeon_ring_write(ring, 0xffffffff);
1480 radeon_ring_write(ring, 0xffffffff);
1481 radeon_ring_write(ring, 0xffffffff);
0c88a02e 1482
e32eb50d
CK		 1483 radeon_ring_write(ring, 0xc0026900);
1484 radeon_ring_write(ring, 0x00000316);
1485 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1486 radeon_ring_write(ring, 0x00000010); /* */
9b91d18d 1487
e32eb50d 1488 radeon_ring_unlock_commit(rdev, ring);
0c88a02e
AD		 1489
1490 /* XXX init other rings */
1491
1492 return 0;
1493}
1494
755d819e
AD		 1495static void cayman_cp_fini(struct radeon_device *rdev)
1496{
45df6803 1497 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
755d819e 1498 cayman_cp_enable(rdev, false);
45df6803
CK		 1499 radeon_ring_fini(rdev, ring);
1500 radeon_scratch_free(rdev, ring->rptr_save_reg);
755d819e
AD		 1501}
1502
1109ca09 1503static int cayman_cp_resume(struct radeon_device *rdev)
0c88a02e 1504{
b90ca986
CK		 1505 static const int ridx[] = {
1506 RADEON_RING_TYPE_GFX_INDEX,
1507 CAYMAN_RING_TYPE_CP1_INDEX,
1508 CAYMAN_RING_TYPE_CP2_INDEX
1509 };
1510 static const unsigned cp_rb_cntl[] = {
1511 CP_RB0_CNTL,
1512 CP_RB1_CNTL,
1513 CP_RB2_CNTL,
1514 };
1515 static const unsigned cp_rb_rptr_addr[] = {
1516 CP_RB0_RPTR_ADDR,
1517 CP_RB1_RPTR_ADDR,
1518 CP_RB2_RPTR_ADDR
1519 };
1520 static const unsigned cp_rb_rptr_addr_hi[] = {
1521 CP_RB0_RPTR_ADDR_HI,
1522 CP_RB1_RPTR_ADDR_HI,
1523 CP_RB2_RPTR_ADDR_HI
1524 };
1525 static const unsigned cp_rb_base[] = {
1526 CP_RB0_BASE,
1527 CP_RB1_BASE,
1528 CP_RB2_BASE
1529 };
e32eb50d 1530 struct radeon_ring *ring;
b90ca986 1531 int i, r;
0c88a02e
AD		 1532
1533 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1534 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1535 SOFT_RESET_PA |
1536 SOFT_RESET_SH |
1537 SOFT_RESET_VGT |
a49a50da 1538 SOFT_RESET_SPI |
0c88a02e
AD		 1539 SOFT_RESET_SX));
1540 RREG32(GRBM_SOFT_RESET);
1541 mdelay(15);
1542 WREG32(GRBM_SOFT_RESET, 0);
1543 RREG32(GRBM_SOFT_RESET);
1544
15d3332f 1545 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 1546 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
0c88a02e
AD		 1547
1548 /* Set the write pointer delay */
1549 WREG32(CP_RB_WPTR_DELAY, 0);
1550
1551 WREG32(CP_DEBUG, (1 << 27));
1552
48fc7f7e 1553 /* set the wb address whether it's enabled or not */
0c88a02e 1554 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
b90ca986 1555 WREG32(SCRATCH_UMSK, 0xff);
0c88a02e 1556
b90ca986
CK		 1557 for (i = 0; i < 3; ++i) {
1558 uint32_t rb_cntl;
1559 uint64_t addr;
0c88a02e 1560
b90ca986
CK		 1561 /* Set ring buffer size */
1562 ring = &rdev->ring[ridx[i]];
1563 rb_cntl = drm_order(ring->ring_size / 8);
1564 rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
0c88a02e 1565#ifdef __BIG_ENDIAN
b90ca986 1566 rb_cntl |= BUF_SWAP_32BIT;
0c88a02e 1567#endif
b90ca986 1568 WREG32(cp_rb_cntl[i], rb_cntl);
0c88a02e 1569
48fc7f7e 1570 /* set the wb address whether it's enabled or not */
b90ca986
CK		 1571 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1572 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1573 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1574 }
0c88a02e 1575
b90ca986
CK		 1576 /* set the rb base addr, this causes an internal reset of ALL rings */
1577 for (i = 0; i < 3; ++i) {
1578 ring = &rdev->ring[ridx[i]];
1579 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1580 }
0c88a02e 1581
b90ca986
CK		 1582 for (i = 0; i < 3; ++i) {
1583 /* Initialize the ring buffer's read and write pointers */
1584 ring = &rdev->ring[ridx[i]];
1585 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
0c88a02e 1586
b90ca986
CK		 1587 ring->rptr = ring->wptr = 0;
1588 WREG32(ring->rptr_reg, ring->rptr);
1589 WREG32(ring->wptr_reg, ring->wptr);
0c88a02e 1590
b90ca986
CK		 1591 mdelay(1);
1592 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1593 }
0c88a02e
AD		 1594
1595 /* start the rings */
1596 cayman_cp_start(rdev);
e32eb50d
CK		 1597 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1598 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1599 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
0c88a02e 1600 /* this only test cp0 */
f712812e 1601 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
0c88a02e 1602 if (r) {
e32eb50d
CK		 1603 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1604 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1605 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
0c88a02e
AD		 1606 return r;
1607 }
1608
1609 return 0;
1610}
1611
f60cbd11
AD		 1612/*
1613 * DMA
1614 * Starting with R600, the GPU has an asynchronous
1615 * DMA engine. The programming model is very similar
1616 * to the 3D engine (ring buffer, IBs, etc.), but the
1617 * DMA controller has it's own packet format that is
1618 * different form the PM4 format used by the 3D engine.
1619 * It supports copying data, writing embedded data,
1620 * solid fills, and a number of other things. It also
1621 * has support for tiling/detiling of buffers.
1622 * Cayman and newer support two asynchronous DMA engines.
1623 */
1624/**
1625 * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
1626 *
1627 * @rdev: radeon_device pointer
1628 * @ib: IB object to schedule
1629 *
1630 * Schedule an IB in the DMA ring (cayman-SI).
1631 */
1632void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
1633 struct radeon_ib *ib)
1634{
1635 struct radeon_ring *ring = &rdev->ring[ib->ring];
1636
1637 if (rdev->wb.enabled) {
1638 u32 next_rptr = ring->wptr + 4;
1639 while ((next_rptr & 7) != 5)
1640 next_rptr++;
1641 next_rptr += 3;
1642 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
1643 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1644 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
1645 radeon_ring_write(ring, next_rptr);
1646 }
1647
1648 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
1649 * Pad as necessary with NOPs.
1650 */
1651 while ((ring->wptr & 7) != 5)
1652 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1653 radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));
1654 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
1655 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
1656
1657}
1658
1659/**
1660 * cayman_dma_stop - stop the async dma engines
1661 *
1662 * @rdev: radeon_device pointer
1663 *
1664 * Stop the async dma engines (cayman-SI).
1665 */
1666void cayman_dma_stop(struct radeon_device *rdev)
1667{
1668 u32 rb_cntl;
1669
1670 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1671
1672 /* dma0 */
1673 rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1674 rb_cntl &= ~DMA_RB_ENABLE;
1675 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
1676
1677 /* dma1 */
1678 rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1679 rb_cntl &= ~DMA_RB_ENABLE;
1680 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
1681
1682 rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
1683 rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
1684}
1685
1686/**
1687 * cayman_dma_resume - setup and start the async dma engines
1688 *
1689 * @rdev: radeon_device pointer
1690 *
1691 * Set up the DMA ring buffers and enable them. (cayman-SI).
1692 * Returns 0 for success, error for failure.
1693 */
1694int cayman_dma_resume(struct radeon_device *rdev)
1695{
1696 struct radeon_ring *ring;
b3dfcb20 1697 u32 rb_cntl, dma_cntl, ib_cntl;
f60cbd11
AD		 1698 u32 rb_bufsz;
1699 u32 reg_offset, wb_offset;
1700 int i, r;
1701
1702 /* Reset dma */
1703 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1);
1704 RREG32(SRBM_SOFT_RESET);
1705 udelay(50);
1706 WREG32(SRBM_SOFT_RESET, 0);
1707
1708 for (i = 0; i < 2; i++) {
1709 if (i == 0) {
1710 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1711 reg_offset = DMA0_REGISTER_OFFSET;
1712 wb_offset = R600_WB_DMA_RPTR_OFFSET;
1713 } else {
1714 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1715 reg_offset = DMA1_REGISTER_OFFSET;
1716 wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
1717 }
1718
1719 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
1720 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
1721
1722 /* Set ring buffer size in dwords */
1723 rb_bufsz = drm_order(ring->ring_size / 4);
1724 rb_cntl = rb_bufsz << 1;
1725#ifdef __BIG_ENDIAN
1726 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
1727#endif
1728 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
1729
1730 /* Initialize the ring buffer's read and write pointers */
1731 WREG32(DMA_RB_RPTR + reg_offset, 0);
1732 WREG32(DMA_RB_WPTR + reg_offset, 0);
1733
1734 /* set the wb address whether it's enabled or not */
1735 WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
1736 upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
1737 WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
1738 ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
1739
1740 if (rdev->wb.enabled)
1741 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
1742
1743 WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
1744
1745 /* enable DMA IBs */
b3dfcb20
MD		 1746 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
1747#ifdef __BIG_ENDIAN
1748 ib_cntl |= DMA_IB_SWAP_ENABLE;
1749#endif
1750 WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
f60cbd11
AD		 1751
1752 dma_cntl = RREG32(DMA_CNTL + reg_offset);
1753 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
1754 WREG32(DMA_CNTL + reg_offset, dma_cntl);
1755
1756 ring->wptr = 0;
1757 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
1758
1759 ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
1760
1761 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
1762
1763 ring->ready = true;
1764
1765 r = radeon_ring_test(rdev, ring->idx, ring);
1766 if (r) {
1767 ring->ready = false;
1768 return r;
1769 }
1770 }
1771
1772 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1773
1774 return 0;
1775}
1776
1777/**
1778 * cayman_dma_fini - tear down the async dma engines
1779 *
1780 * @rdev: radeon_device pointer
1781 *
1782 * Stop the async dma engines and free the rings (cayman-SI).
1783 */
1784void cayman_dma_fini(struct radeon_device *rdev)
1785{
1786 cayman_dma_stop(rdev);
1787 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
1788 radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
1789}
1790
168757ea 1791static u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
271d6fed 1792{
168757ea 1793 u32 reset_mask = 0;
187e3593 1794 u32 tmp;
271d6fed 1795
168757ea
AD		 1796 /* GRBM_STATUS */
1797 tmp = RREG32(GRBM_STATUS);
1798 if (tmp & (PA_BUSY | SC_BUSY |
1799 SH_BUSY | SX_BUSY |
1800 TA_BUSY | VGT_BUSY |
1801 DB_BUSY | CB_BUSY |
1802 GDS_BUSY | SPI_BUSY |
1803 IA_BUSY | IA_BUSY_NO_DMA))
1804 reset_mask |= RADEON_RESET_GFX;
1805
1806 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1807 CP_BUSY | CP_COHERENCY_BUSY))
1808 reset_mask |= RADEON_RESET_CP;
1809
1810 if (tmp & GRBM_EE_BUSY)
1811 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1812
1813 /* DMA_STATUS_REG 0 */
1814 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1815 if (!(tmp & DMA_IDLE))
1816 reset_mask |= RADEON_RESET_DMA;
1817
1818 /* DMA_STATUS_REG 1 */
1819 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1820 if (!(tmp & DMA_IDLE))
1821 reset_mask |= RADEON_RESET_DMA1;
1822
1823 /* SRBM_STATUS2 */
1824 tmp = RREG32(SRBM_STATUS2);
1825 if (tmp & DMA_BUSY)
1826 reset_mask |= RADEON_RESET_DMA;
1827
1828 if (tmp & DMA1_BUSY)
1829 reset_mask |= RADEON_RESET_DMA1;
1830
1831 /* SRBM_STATUS */
1832 tmp = RREG32(SRBM_STATUS);
1833 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1834 reset_mask |= RADEON_RESET_RLC;
1835
1836 if (tmp & IH_BUSY)
1837 reset_mask |= RADEON_RESET_IH;
1838
1839 if (tmp & SEM_BUSY)
1840 reset_mask |= RADEON_RESET_SEM;
1841
1842 if (tmp & GRBM_RQ_PENDING)
1843 reset_mask |= RADEON_RESET_GRBM;
1844
1845 if (tmp & VMC_BUSY)
1846 reset_mask |= RADEON_RESET_VMC;
19fc42ed 1847
168757ea
AD		 1848 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1849 MCC_BUSY | MCD_BUSY))
1850 reset_mask |= RADEON_RESET_MC;
1851
1852 if (evergreen_is_display_hung(rdev))
1853 reset_mask |= RADEON_RESET_DISPLAY;
1854
1855 /* VM_L2_STATUS */
1856 tmp = RREG32(VM_L2_STATUS);
1857 if (tmp & L2_BUSY)
1858 reset_mask |= RADEON_RESET_VMC;
1859
d808fc88
AD		 1860 /* Skip MC reset as it's mostly likely not hung, just busy */
1861 if (reset_mask & RADEON_RESET_MC) {
1862 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1863 reset_mask &= ~RADEON_RESET_MC;
1864 }
1865
168757ea
AD		 1866 return reset_mask;
1867}
1868
1869static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1870{
1871 struct evergreen_mc_save save;
1872 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1873 u32 tmp;
19fc42ed 1874
271d6fed 1875 if (reset_mask == 0)
168757ea 1876 return;
271d6fed
AD		 1877
1878 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1879
187e3593 1880 evergreen_print_gpu_status_regs(rdev);
271d6fed
AD		 1881 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
1882 RREG32(0x14F8));
1883 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1884 RREG32(0x14D8));
1885 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1886 RREG32(0x14FC));
1887 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1888 RREG32(0x14DC));
1889
187e3593
AD		 1890 /* Disable CP parsing/prefetching */
1891 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1892
1893 if (reset_mask & RADEON_RESET_DMA) {
1894 /* dma0 */
1895 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1896 tmp &= ~DMA_RB_ENABLE;
1897 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
168757ea 1898 }
187e3593 1899
168757ea 1900 if (reset_mask & RADEON_RESET_DMA1) {
187e3593
AD		 1901 /* dma1 */
1902 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1903 tmp &= ~DMA_RB_ENABLE;
1904 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1905 }
1906
90fb8779
AD		 1907 udelay(50);
1908
1909 evergreen_mc_stop(rdev, &save);
1910 if (evergreen_mc_wait_for_idle(rdev)) {
1911 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1912 }
1913
187e3593
AD		 1914 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1915 grbm_soft_reset = SOFT_RESET_CB |
1916 SOFT_RESET_DB |
1917 SOFT_RESET_GDS |
1918 SOFT_RESET_PA |
1919 SOFT_RESET_SC |
1920 SOFT_RESET_SPI |
1921 SOFT_RESET_SH |
1922 SOFT_RESET_SX |
1923 SOFT_RESET_TC |
1924 SOFT_RESET_TA |
1925 SOFT_RESET_VGT |
1926 SOFT_RESET_IA;
1927 }
1928
1929 if (reset_mask & RADEON_RESET_CP) {
1930 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1931
1932 srbm_soft_reset |= SOFT_RESET_GRBM;
1933 }
271d6fed
AD		 1934
1935 if (reset_mask & RADEON_RESET_DMA)
168757ea
AD		 1936 srbm_soft_reset |= SOFT_RESET_DMA;
1937
1938 if (reset_mask & RADEON_RESET_DMA1)
1939 srbm_soft_reset |= SOFT_RESET_DMA1;
1940
1941 if (reset_mask & RADEON_RESET_DISPLAY)
1942 srbm_soft_reset |= SOFT_RESET_DC;
1943
1944 if (reset_mask & RADEON_RESET_RLC)
1945 srbm_soft_reset |= SOFT_RESET_RLC;
1946
1947 if (reset_mask & RADEON_RESET_SEM)
1948 srbm_soft_reset |= SOFT_RESET_SEM;
1949
1950 if (reset_mask & RADEON_RESET_IH)
1951 srbm_soft_reset |= SOFT_RESET_IH;
1952
1953 if (reset_mask & RADEON_RESET_GRBM)
1954 srbm_soft_reset |= SOFT_RESET_GRBM;
1955
1956 if (reset_mask & RADEON_RESET_VMC)
1957 srbm_soft_reset |= SOFT_RESET_VMC;
1958
24178ec4
AD		 1959 if (!(rdev->flags & RADEON_IS_IGP)) {
1960 if (reset_mask & RADEON_RESET_MC)
1961 srbm_soft_reset |= SOFT_RESET_MC;
1962 }
187e3593
AD		 1963
1964 if (grbm_soft_reset) {
1965 tmp = RREG32(GRBM_SOFT_RESET);
1966 tmp |= grbm_soft_reset;
1967 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1968 WREG32(GRBM_SOFT_RESET, tmp);
1969 tmp = RREG32(GRBM_SOFT_RESET);
1970
1971 udelay(50);
1972
1973 tmp &= ~grbm_soft_reset;
1974 WREG32(GRBM_SOFT_RESET, tmp);
1975 tmp = RREG32(GRBM_SOFT_RESET);
1976 }
1977
1978 if (srbm_soft_reset) {
1979 tmp = RREG32(SRBM_SOFT_RESET);
1980 tmp |= srbm_soft_reset;
1981 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1982 WREG32(SRBM_SOFT_RESET, tmp);
1983 tmp = RREG32(SRBM_SOFT_RESET);
1984
1985 udelay(50);
1986
1987 tmp &= ~srbm_soft_reset;
1988 WREG32(SRBM_SOFT_RESET, tmp);
1989 tmp = RREG32(SRBM_SOFT_RESET);
1990 }
271d6fed
AD		 1991
1992 /* Wait a little for things to settle down */
1993 udelay(50);
1994
b9952a8a 1995 evergreen_mc_resume(rdev, &save);
187e3593
AD		 1996 udelay(50);
1997
187e3593 1998 evergreen_print_gpu_status_regs(rdev);
b9952a8a
AD		 1999}
2000
2001int cayman_asic_reset(struct radeon_device *rdev)
2002{
168757ea
AD		 2003 u32 reset_mask;
2004
2005 reset_mask = cayman_gpu_check_soft_reset(rdev);
2006
2007 if (reset_mask)
2008 r600_set_bios_scratch_engine_hung(rdev, true);
2009
2010 cayman_gpu_soft_reset(rdev, reset_mask);
2011
2012 reset_mask = cayman_gpu_check_soft_reset(rdev);
2013
2014 if (!reset_mask)
2015 r600_set_bios_scratch_engine_hung(rdev, false);
2016
2017 return 0;
b9952a8a
AD		 2018}
2019
123bc183
AD		 2020/**
2021 * cayman_gfx_is_lockup - Check if the GFX engine is locked up
2022 *
2023 * @rdev: radeon_device pointer
2024 * @ring: radeon_ring structure holding ring information
2025 *
2026 * Check if the GFX engine is locked up.
2027 * Returns true if the engine appears to be locked up, false if not.
2028 */
2029bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2030{
2031 u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2032
2033 if (!(reset_mask & (RADEON_RESET_GFX |
2034 RADEON_RESET_COMPUTE |
2035 RADEON_RESET_CP))) {
2036 radeon_ring_lockup_update(ring);
2037 return false;
2038 }
2039 /* force CP activities */
2040 radeon_ring_force_activity(rdev, ring);
2041 return radeon_ring_test_lockup(rdev, ring);
2042}
2043
f60cbd11
AD		 2044/**
2045 * cayman_dma_is_lockup - Check if the DMA engine is locked up
2046 *
2047 * @rdev: radeon_device pointer
2048 * @ring: radeon_ring structure holding ring information
2049 *
123bc183 2050 * Check if the async DMA engine is locked up.
f60cbd11
AD		 2051 * Returns true if the engine appears to be locked up, false if not.
2052 */
2053bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2054{
123bc183
AD		 2055 u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2056 u32 mask;
f60cbd11
AD		 2057
2058 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
123bc183 2059 mask = RADEON_RESET_DMA;
f60cbd11 2060 else
123bc183
AD		 2061 mask = RADEON_RESET_DMA1;
2062
2063 if (!(reset_mask & mask)) {
f60cbd11
AD		 2064 radeon_ring_lockup_update(ring);
2065 return false;
2066 }
2067 /* force ring activities */
2068 radeon_ring_force_activity(rdev, ring);
2069 return radeon_ring_test_lockup(rdev, ring);
2070}
2071
755d819e
AD		 2072static int cayman_startup(struct radeon_device *rdev)
2073{
e32eb50d 2074 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
755d819e
AD		 2075 int r;
2076
b07759bf
IH		 2077 /* enable pcie gen2 link */
2078 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 2079 /* enable aspm */
2080 evergreen_program_aspm(rdev);
b07759bf 2081
6fab3feb
AD		 2082 evergreen_mc_program(rdev);
2083
c420c745
AD		 2084 if (rdev->flags & RADEON_IS_IGP) {
2085 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2086 r = ni_init_microcode(rdev);
2087 if (r) {
2088 DRM_ERROR("Failed to load firmware!\n");
2089 return r;
2090 }
2091 }
2092 } else {
2093 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2094 r = ni_init_microcode(rdev);
2095 if (r) {
2096 DRM_ERROR("Failed to load firmware!\n");
2097 return r;
2098 }
2099 }
2100
2101 r = ni_mc_load_microcode(rdev);
755d819e 2102 if (r) {
c420c745 2103 DRM_ERROR("Failed to load MC firmware!\n");
755d819e
AD		 2104 return r;
2105 }
2106 }
755d819e 2107
16cdf04d
AD		 2108 r = r600_vram_scratch_init(rdev);
2109 if (r)
2110 return r;
2111
755d819e
AD		 2112 r = cayman_pcie_gart_enable(rdev);
2113 if (r)
2114 return r;
2115 cayman_gpu_init(rdev);
2116
cb92d452 2117 r = evergreen_blit_init(rdev);
755d819e 2118 if (r) {
fb3d9e97 2119 r600_blit_fini(rdev);
27cd7769 2120 rdev->asic->copy.copy = NULL;
755d819e
AD		 2121 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2122 }
755d819e 2123
c420c745
AD		 2124 /* allocate rlc buffers */
2125 if (rdev->flags & RADEON_IS_IGP) {
2948f5e6
AD		 2126 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2127 rdev->rlc.reg_list_size = tn_rlc_save_restore_register_list_size;
2128 rdev->rlc.cs_data = cayman_cs_data;
2129 r = sumo_rlc_init(rdev);
c420c745
AD		 2130 if (r) {
2131 DRM_ERROR("Failed to init rlc BOs!\n");
2132 return r;
2133 }
2134 }
2135
755d819e
AD		 2136 /* allocate wb buffer */
2137 r = radeon_wb_init(rdev);
2138 if (r)
2139 return r;
2140
30eb77f4
JG		 2141 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2142 if (r) {
2143 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2144 return r;
2145 }
2146
f2ba57b5
CK		 2147 r = rv770_uvd_resume(rdev);
2148 if (!r) {
2149 r = radeon_fence_driver_start_ring(rdev,
2150 R600_RING_TYPE_UVD_INDEX);
2151 if (r)
2152 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2153 }
2154 if (r)
2155 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2156
30eb77f4
JG		 2157 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2158 if (r) {
2159 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2160 return r;
2161 }
2162
2163 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2164 if (r) {
2165 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2166 return r;
2167 }
2168
f60cbd11
AD		 2169 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2170 if (r) {
2171 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2172 return r;
2173 }
2174
2175 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2176 if (r) {
2177 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2178 return r;
2179 }
2180
755d819e 2181 /* Enable IRQ */
e49f3959
AH		 2182 if (!rdev->irq.installed) {
2183 r = radeon_irq_kms_init(rdev);
2184 if (r)
2185 return r;
2186 }
2187
755d819e
AD		 2188 r = r600_irq_init(rdev);
2189 if (r) {
2190 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2191 radeon_irq_kms_fini(rdev);
2192 return r;
2193 }
2194 evergreen_irq_set(rdev);
2195
e32eb50d 2196 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 2197 CP_RB0_RPTR, CP_RB0_WPTR,
2198 0, 0xfffff, RADEON_CP_PACKET2);
755d819e
AD		 2199 if (r)
2200 return r;
f60cbd11
AD		 2201
2202 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2203 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2204 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
2205 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2206 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2207 if (r)
2208 return r;
2209
2210 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2211 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2212 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
2213 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2214 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2215 if (r)
2216 return r;
2217
755d819e
AD		 2218 r = cayman_cp_load_microcode(rdev);
2219 if (r)
2220 return r;
2221 r = cayman_cp_resume(rdev);
2222 if (r)
2223 return r;
2224
f60cbd11
AD		 2225 r = cayman_dma_resume(rdev);
2226 if (r)
2227 return r;
2228
f2ba57b5
CK		 2229 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2230 if (ring->ring_size) {
2231 r = radeon_ring_init(rdev, ring, ring->ring_size,
2232 R600_WB_UVD_RPTR_OFFSET,
2233 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2234 0, 0xfffff, RADEON_CP_PACKET2);
2235 if (!r)
2236 r = r600_uvd_init(rdev);
2237 if (r)
2238 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2239 }
2240
2898c348
CK		 2241 r = radeon_ib_pool_init(rdev);
2242 if (r) {
2243 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 2244 return r;
2898c348 2245 }
b15ba512 2246
c6105f24
CK		 2247 r = radeon_vm_manager_init(rdev);
2248 if (r) {
2249 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
721604a1 2250 return r;
c6105f24 2251 }
721604a1 2252
6b53a050
RM		 2253 r = r600_audio_init(rdev);
2254 if (r)
2255 return r;
2256
755d819e
AD		 2257 return 0;
2258}
2259
2260int cayman_resume(struct radeon_device *rdev)
2261{
2262 int r;
2263
2264 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2265 * posting will perform necessary task to bring back GPU into good
2266 * shape.
2267 */
2268 /* post card */
2269 atom_asic_init(rdev->mode_info.atom_context);
2270
a2c96a21
AD		 2271 /* init golden registers */
2272 ni_init_golden_registers(rdev);
2273
b15ba512 2274 rdev->accel_working = true;
755d819e
AD		 2275 r = cayman_startup(rdev);
2276 if (r) {
2277 DRM_ERROR("cayman startup failed on resume\n");
6b7746e8 2278 rdev->accel_working = false;
755d819e
AD		 2279 return r;
2280 }
755d819e 2281 return r;
755d819e
AD		 2282}
2283
2284int cayman_suspend(struct radeon_device *rdev)
2285{
6b53a050 2286 r600_audio_fini(rdev);
fa3daf9a 2287 radeon_vm_manager_fini(rdev);
755d819e 2288 cayman_cp_enable(rdev, false);
f60cbd11 2289 cayman_dma_stop(rdev);
2858c00d 2290 r600_uvd_stop(rdev);
f2ba57b5 2291 radeon_uvd_suspend(rdev);
755d819e
AD		 2292 evergreen_irq_suspend(rdev);
2293 radeon_wb_disable(rdev);
2294 cayman_pcie_gart_disable(rdev);
755d819e
AD		 2295 return 0;
2296}
2297
2298/* Plan is to move initialization in that function and use
2299 * helper function so that radeon_device_init pretty much
2300 * do nothing more than calling asic specific function. This
2301 * should also allow to remove a bunch of callback function
2302 * like vram_info.
2303 */
2304int cayman_init(struct radeon_device *rdev)
2305{
e32eb50d 2306 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
755d819e
AD		 2307 int r;
2308
755d819e
AD		 2309 /* Read BIOS */
2310 if (!radeon_get_bios(rdev)) {
2311 if (ASIC_IS_AVIVO(rdev))
2312 return -EINVAL;
2313 }
2314 /* Must be an ATOMBIOS */
2315 if (!rdev->is_atom_bios) {
2316 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2317 return -EINVAL;
2318 }
2319 r = radeon_atombios_init(rdev);
2320 if (r)
2321 return r;
2322
2323 /* Post card if necessary */
2324 if (!radeon_card_posted(rdev)) {
2325 if (!rdev->bios) {
2326 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2327 return -EINVAL;
2328 }
2329 DRM_INFO("GPU not posted. posting now...\n");
2330 atom_asic_init(rdev->mode_info.atom_context);
2331 }
a2c96a21
AD		 2332 /* init golden registers */
2333 ni_init_golden_registers(rdev);
755d819e
AD		 2334 /* Initialize scratch registers */
2335 r600_scratch_init(rdev);
2336 /* Initialize surface registers */
2337 radeon_surface_init(rdev);
2338 /* Initialize clocks */
2339 radeon_get_clock_info(rdev->ddev);
2340 /* Fence driver */
30eb77f4 2341 r = radeon_fence_driver_init(rdev);
755d819e
AD		 2342 if (r)
2343 return r;
2344 /* initialize memory controller */
2345 r = evergreen_mc_init(rdev);
2346 if (r)
2347 return r;
2348 /* Memory manager */
2349 r = radeon_bo_init(rdev);
2350 if (r)
2351 return r;
2352
e32eb50d
CK		 2353 ring->ring_obj = NULL;
2354 r600_ring_init(rdev, ring, 1024 * 1024);
755d819e 2355
f60cbd11
AD		 2356 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2357 ring->ring_obj = NULL;
2358 r600_ring_init(rdev, ring, 64 * 1024);
2359
2360 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2361 ring->ring_obj = NULL;
2362 r600_ring_init(rdev, ring, 64 * 1024);
2363
f2ba57b5
CK		 2364 r = radeon_uvd_init(rdev);
2365 if (!r) {
2366 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2367 ring->ring_obj = NULL;
2368 r600_ring_init(rdev, ring, 4096);
2369 }
2370
755d819e
AD		 2371 rdev->ih.ring_obj = NULL;
2372 r600_ih_ring_init(rdev, 64 * 1024);
2373
2374 r = r600_pcie_gart_init(rdev);
2375 if (r)
2376 return r;
2377
2378 rdev->accel_working = true;
2379 r = cayman_startup(rdev);
2380 if (r) {
2381 dev_err(rdev->dev, "disabling GPU acceleration\n");
2382 cayman_cp_fini(rdev);
f60cbd11 2383 cayman_dma_fini(rdev);
755d819e 2384 r600_irq_fini(rdev);
c420c745 2385 if (rdev->flags & RADEON_IS_IGP)
2948f5e6 2386 sumo_rlc_fini(rdev);
755d819e 2387 radeon_wb_fini(rdev);
2898c348 2388 radeon_ib_pool_fini(rdev);
721604a1 2389 radeon_vm_manager_fini(rdev);
755d819e
AD		 2390 radeon_irq_kms_fini(rdev);
2391 cayman_pcie_gart_fini(rdev);
2392 rdev->accel_working = false;
2393 }
755d819e
AD		 2394
2395 /* Don't start up if the MC ucode is missing.
2396 * The default clocks and voltages before the MC ucode
2397 * is loaded are not suffient for advanced operations.
c420c745
AD		 2398 *
2399 * We can skip this check for TN, because there is no MC
2400 * ucode.
755d819e 2401 */
c420c745 2402 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
755d819e
AD		 2403 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2404 return -EINVAL;
2405 }
2406
2407 return 0;
2408}
2409
2410void cayman_fini(struct radeon_device *rdev)
2411{
fb3d9e97 2412 r600_blit_fini(rdev);
755d819e 2413 cayman_cp_fini(rdev);
f60cbd11 2414 cayman_dma_fini(rdev);
755d819e 2415 r600_irq_fini(rdev);
c420c745 2416 if (rdev->flags & RADEON_IS_IGP)
2948f5e6 2417 sumo_rlc_fini(rdev);
755d819e 2418 radeon_wb_fini(rdev);
721604a1 2419 radeon_vm_manager_fini(rdev);
2898c348 2420 radeon_ib_pool_fini(rdev);
755d819e 2421 radeon_irq_kms_fini(rdev);
2858c00d 2422 r600_uvd_stop(rdev);
f2ba57b5 2423 radeon_uvd_fini(rdev);
755d819e 2424 cayman_pcie_gart_fini(rdev);
16cdf04d 2425 r600_vram_scratch_fini(rdev);
755d819e
AD		 2426 radeon_gem_fini(rdev);
2427 radeon_fence_driver_fini(rdev);
2428 radeon_bo_fini(rdev);
2429 radeon_atombios_fini(rdev);
2430 kfree(rdev->bios);
2431 rdev->bios = NULL;
2432}
2433
721604a1
JG		 2434/*
2435 * vm
2436 */
2437int cayman_vm_init(struct radeon_device *rdev)
2438{
2439 /* number of VMs */
2440 rdev->vm_manager.nvm = 8;
2441 /* base offset of vram pages */
e71270fd
AD		 2442 if (rdev->flags & RADEON_IS_IGP) {
2443 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2444 tmp <<= 22;
2445 rdev->vm_manager.vram_base_offset = tmp;
2446 } else
2447 rdev->vm_manager.vram_base_offset = 0;
721604a1
JG		 2448 return 0;
2449}
2450
2451void cayman_vm_fini(struct radeon_device *rdev)
2452{
2453}
2454
54e2e49c
AD		 2455/**
2456 * cayman_vm_decode_fault - print human readable fault info
2457 *
2458 * @rdev: radeon_device pointer
2459 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2460 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2461 *
2462 * Print human readable fault information (cayman/TN).
2463 */
2464void cayman_vm_decode_fault(struct radeon_device *rdev,
2465 u32 status, u32 addr)
2466{
2467 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2468 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2469 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2470 char *block;
2471
2472 switch (mc_id) {
2473 case 32:
2474 case 16:
2475 case 96:
2476 case 80:
2477 case 160:
2478 case 144:
2479 case 224:
2480 case 208:
2481 block = "CB";
2482 break;
2483 case 33:
2484 case 17:
2485 case 97:
2486 case 81:
2487 case 161:
2488 case 145:
2489 case 225:
2490 case 209:
2491 block = "CB_FMASK";
2492 break;
2493 case 34:
2494 case 18:
2495 case 98:
2496 case 82:
2497 case 162:
2498 case 146:
2499 case 226:
2500 case 210:
2501 block = "CB_CMASK";
2502 break;
2503 case 35:
2504 case 19:
2505 case 99:
2506 case 83:
2507 case 163:
2508 case 147:
2509 case 227:
2510 case 211:
2511 block = "CB_IMMED";
2512 break;
2513 case 36:
2514 case 20:
2515 case 100:
2516 case 84:
2517 case 164:
2518 case 148:
2519 case 228:
2520 case 212:
2521 block = "DB";
2522 break;
2523 case 37:
2524 case 21:
2525 case 101:
2526 case 85:
2527 case 165:
2528 case 149:
2529 case 229:
2530 case 213:
2531 block = "DB_HTILE";
2532 break;
2533 case 38:
2534 case 22:
2535 case 102:
2536 case 86:
2537 case 166:
2538 case 150:
2539 case 230:
2540 case 214:
2541 block = "SX";
2542 break;
2543 case 39:
2544 case 23:
2545 case 103:
2546 case 87:
2547 case 167:
2548 case 151:
2549 case 231:
2550 case 215:
2551 block = "DB_STEN";
2552 break;
2553 case 40:
2554 case 24:
2555 case 104:
2556 case 88:
2557 case 232:
2558 case 216:
2559 case 168:
2560 case 152:
2561 block = "TC_TFETCH";
2562 break;
2563 case 41:
2564 case 25:
2565 case 105:
2566 case 89:
2567 case 233:
2568 case 217:
2569 case 169:
2570 case 153:
2571 block = "TC_VFETCH";
2572 break;
2573 case 42:
2574 case 26:
2575 case 106:
2576 case 90:
2577 case 234:
2578 case 218:
2579 case 170:
2580 case 154:
2581 block = "VC";
2582 break;
2583 case 112:
2584 block = "CP";
2585 break;
2586 case 113:
2587 case 114:
2588 block = "SH";
2589 break;
2590 case 115:
2591 block = "VGT";
2592 break;
2593 case 178:
2594 block = "IH";
2595 break;
2596 case 51:
2597 block = "RLC";
2598 break;
2599 case 55:
2600 block = "DMA";
2601 break;
2602 case 56:
2603 block = "HDP";
2604 break;
2605 default:
2606 block = "unknown";
2607 break;
2608 }
2609
2610 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2611 protections, vmid, addr,
2612 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2613 block, mc_id);
2614}
2615
dce34bfd 2616#define R600_ENTRY_VALID (1 << 0)
721604a1
JG		 2617#define R600_PTE_SYSTEM (1 << 1)
2618#define R600_PTE_SNOOPED (1 << 2)
2619#define R600_PTE_READABLE (1 << 5)
2620#define R600_PTE_WRITEABLE (1 << 6)
2621
089a786e 2622uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
721604a1
JG		 2623{
2624 uint32_t r600_flags = 0;
dce34bfd 2625 r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
721604a1
JG		 2626 r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
2627 r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
2628 if (flags & RADEON_VM_PAGE_SYSTEM) {
2629 r600_flags |= R600_PTE_SYSTEM;
2630 r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
2631 }
2632 return r600_flags;
2633}
2634
7a083293
AD		 2635/**
2636 * cayman_vm_set_page - update the page tables using the CP
2637 *
2638 * @rdev: radeon_device pointer
43f1214a 2639 * @ib: indirect buffer to fill with commands
dce34bfd
CK		 2640 * @pe: addr of the page entry
2641 * @addr: dst addr to write into pe
2642 * @count: number of page entries to update
2643 * @incr: increase next addr by incr bytes
2644 * @flags: access flags
7a083293 2645 *
43f1214a 2646 * Update the page tables using the CP (cayman/TN).
7a083293 2647 */
43f1214a
AD		 2648void cayman_vm_set_page(struct radeon_device *rdev,
2649 struct radeon_ib *ib,
2650 uint64_t pe,
dce34bfd
CK		 2651 uint64_t addr, unsigned count,
2652 uint32_t incr, uint32_t flags)
721604a1 2653{
dce34bfd 2654 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
3b6b59b6
AD		 2655 uint64_t value;
2656 unsigned ndw;
2657
2658 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
2659 while (count) {
2660 ndw = 1 + count * 2;
2661 if (ndw > 0x3FFF)
2662 ndw = 0x3FFF;
2663
43f1214a
AD		 2664 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw);
2665 ib->ptr[ib->length_dw++] = pe;
2666 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3b6b59b6
AD		 2667 for (; ndw > 1; ndw -= 2, --count, pe += 8) {
2668 if (flags & RADEON_VM_PAGE_SYSTEM) {
2669 value = radeon_vm_map_gart(rdev, addr);
2670 value &= 0xFFFFFFFFFFFFF000ULL;
2671 } else if (flags & RADEON_VM_PAGE_VALID) {
2672 value = addr;
2673 } else {
2674 value = 0;
2675 }
f9fdffa5 2676 addr += incr;
3b6b59b6 2677 value |= r600_flags;
43f1214a
AD		 2678 ib->ptr[ib->length_dw++] = value;
2679 ib->ptr[ib->length_dw++] = upper_32_bits(value);
3b6b59b6
AD		 2680 }
2681 }
2682 } else {
2ab91ada
AD		 2683 if ((flags & RADEON_VM_PAGE_SYSTEM) ||
2684 (count == 1)) {
2685 while (count) {
2686 ndw = count * 2;
2687 if (ndw > 0xFFFFE)
2688 ndw = 0xFFFFE;
2689
2690 /* for non-physically contiguous pages (system) */
2691 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw);
2692 ib->ptr[ib->length_dw++] = pe;
2693 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2694 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
2695 if (flags & RADEON_VM_PAGE_SYSTEM) {
2696 value = radeon_vm_map_gart(rdev, addr);
2697 value &= 0xFFFFFFFFFFFFF000ULL;
2698 } else if (flags & RADEON_VM_PAGE_VALID) {
2699 value = addr;
2700 } else {
2701 value = 0;
2702 }
2703 addr += incr;
2704 value |= r600_flags;
2705 ib->ptr[ib->length_dw++] = value;
2706 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2707 }
2708 }
2709 while (ib->length_dw & 0x7)
2710 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2711 } else {
2712 while (count) {
2713 ndw = count * 2;
2714 if (ndw > 0xFFFFE)
2715 ndw = 0xFFFFE;
3b6b59b6 2716
2ab91ada 2717 if (flags & RADEON_VM_PAGE_VALID)
3b6b59b6 2718 value = addr;
2ab91ada 2719 else
3b6b59b6 2720 value = 0;
2ab91ada
AD		 2721 /* for physically contiguous pages (vram) */
2722 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
2723 ib->ptr[ib->length_dw++] = pe; /* dst addr */
2724 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2725 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
2726 ib->ptr[ib->length_dw++] = 0;
2727 ib->ptr[ib->length_dw++] = value; /* value */
43f1214a 2728 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2ab91ada
AD		 2729 ib->ptr[ib->length_dw++] = incr; /* increment size */
2730 ib->ptr[ib->length_dw++] = 0;
2731 pe += ndw * 4;
2732 addr += (ndw / 2) * incr;
2733 count -= ndw / 2;
f9fdffa5 2734 }
f9fdffa5 2735 }
43f1214a
AD		 2736 while (ib->length_dw & 0x7)
2737 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2a6f1abb 2738 }
721604a1 2739}
9b40e5d8 2740
7a083293
AD		 2741/**
2742 * cayman_vm_flush - vm flush using the CP
2743 *
2744 * @rdev: radeon_device pointer
2745 *
2746 * Update the page table base and flush the VM TLB
2747 * using the CP (cayman-si).
2748 */
498522b4 2749void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
721604a1 2750{
498522b4 2751 struct radeon_ring *ring = &rdev->ring[ridx];
9b40e5d8 2752
ee60e29f 2753 if (vm == NULL)
9b40e5d8
CK		 2754 return;
2755
ee60e29f 2756 radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
fa87e62d 2757 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 2758
9b40e5d8
CK		 2759 /* flush hdp cache */
2760 radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2761 radeon_ring_write(ring, 0x1);
2762
2763 /* bits 0-7 are the VM contexts0-7 */
2764 radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
498522b4 2765 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 2766
2767 /* sync PFP to ME, otherwise we might get invalid PFP reads */
2768 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2769 radeon_ring_write(ring, 0x0);
721604a1 2770}
f60cbd11
AD		 2771
2772void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2773{
2774 struct radeon_ring *ring = &rdev->ring[ridx];
2775
2776 if (vm == NULL)
2777 return;
2778
2779 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2780 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
2781 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2782
2783 /* flush hdp cache */
2784 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2785 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
2786 radeon_ring_write(ring, 1);
2787
2788 /* bits 0-7 are the VM contexts0-7 */
2789 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2790 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
2791 radeon_ring_write(ring, 1 << vm->id);
2792}
2793
Linux kernel - Deliverables
This page took 0.355167 seconds and 5 git commands to generate.