projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add si tile mode array query v3
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
bcc1c2a1 36
fe251e2f
AD		 37#define EVERGREEN_PFP_UCODE_SIZE 1120
38#define EVERGREEN_PM4_UCODE_SIZE 1376
39
4a15903d
AD		 40static const u32 crtc_offsets[6] =
41{
42 EVERGREEN_CRTC0_REGISTER_OFFSET,
43 EVERGREEN_CRTC1_REGISTER_OFFSET,
44 EVERGREEN_CRTC2_REGISTER_OFFSET,
45 EVERGREEN_CRTC3_REGISTER_OFFSET,
46 EVERGREEN_CRTC4_REGISTER_OFFSET,
47 EVERGREEN_CRTC5_REGISTER_OFFSET
48};
49
bcc1c2a1
AD		 50static void evergreen_gpu_init(struct radeon_device *rdev);
51void evergreen_fini(struct radeon_device *rdev);
b07759bf 52void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1b37078b
AD		 53extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
54 int ring, u32 cp_int_cntl);
bcc1c2a1 55
285484e2
JG		 56void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
57 unsigned *bankh, unsigned *mtaspect,
58 unsigned *tile_split)
59{
60 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
61 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
62 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
63 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
64 switch (*bankw) {
65 default:
66 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
67 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
68 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
69 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
70 }
71 switch (*bankh) {
72 default:
73 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
74 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
75 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
76 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
77 }
78 switch (*mtaspect) {
79 default:
80 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
81 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
82 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
83 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
84 }
85}
86
23d33ba3
AD		 87static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
88 u32 cntl_reg, u32 status_reg)
89{
90 int r, i;
91 struct atom_clock_dividers dividers;
92
93 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
94 clock, false, &dividers);
95 if (r)
96 return r;
97
98 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
99
100 for (i = 0; i < 100; i++) {
101 if (RREG32(status_reg) & DCLK_STATUS)
102 break;
103 mdelay(10);
104 }
105 if (i == 100)
106 return -ETIMEDOUT;
107
108 return 0;
109}
110
111int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
112{
113 int r = 0;
114 u32 cg_scratch = RREG32(CG_SCRATCH1);
115
116 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
117 if (r)
118 goto done;
119 cg_scratch &= 0xffff0000;
120 cg_scratch |= vclk / 100; /* Mhz */
121
122 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
123 if (r)
124 goto done;
125 cg_scratch &= 0x0000ffff;
126 cg_scratch |= (dclk / 100) << 16; /* Mhz */
127
128done:
129 WREG32(CG_SCRATCH1, cg_scratch);
130
131 return r;
132}
133
a8b4925c
AD		 134static int evergreen_uvd_calc_post_div(unsigned target_freq,
135 unsigned vco_freq,
136 unsigned *div)
137{
138 /* target larger than vco frequency ? */
139 if (vco_freq < target_freq)
140 return -1; /* forget it */
141
142 /* Fclk = Fvco / PDIV */
143 *div = vco_freq / target_freq;
144
145 /* we alway need a frequency less than or equal the target */
146 if ((vco_freq / *div) > target_freq)
147 *div += 1;
148
149 /* dividers above 5 must be even */
150 if (*div > 5 && *div % 2)
151 *div += 1;
152
153 /* out of range ? */
154 if (*div >= 128)
155 return -1; /* forget it */
156
157 return vco_freq / *div;
158}
159
160static int evergreen_uvd_send_upll_ctlreq(struct radeon_device *rdev)
161{
162 unsigned i;
163
164 /* assert UPLL_CTLREQ */
165 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
166
167 /* wait for CTLACK and CTLACK2 to get asserted */
168 for (i = 0; i < 100; ++i) {
169 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
170 if ((RREG32(CG_UPLL_FUNC_CNTL) & mask) == mask)
171 break;
172 mdelay(10);
173 }
174 if (i == 100)
175 return -ETIMEDOUT;
176
177 /* deassert UPLL_CTLREQ */
178 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
179
180 return 0;
181}
182
183int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
184{
185 /* start off with something large */
186 int optimal_diff_score = 0x7FFFFFF;
187 unsigned optimal_fb_div = 0, optimal_vclk_div = 0;
188 unsigned optimal_dclk_div = 0, optimal_vco_freq = 0;
189 unsigned vco_freq;
190 int r;
191
192 /* loop through vco from low to high */
193 for (vco_freq = 125000; vco_freq <= 250000; vco_freq += 100) {
194 unsigned fb_div = vco_freq / rdev->clock.spll.reference_freq * 16384;
195 int calc_clk, diff_score, diff_vclk, diff_dclk;
196 unsigned vclk_div, dclk_div;
197
198 /* fb div out of range ? */
199 if (fb_div > 0x03FFFFFF)
200 break; /* it can oly get worse */
201
202 /* calc vclk with current vco freq. */
203 calc_clk = evergreen_uvd_calc_post_div(vclk, vco_freq, &vclk_div);
204 if (calc_clk == -1)
205 break; /* vco is too big, it has to stop. */
206 diff_vclk = vclk - calc_clk;
207
208 /* calc dclk with current vco freq. */
209 calc_clk = evergreen_uvd_calc_post_div(dclk, vco_freq, &dclk_div);
210 if (calc_clk == -1)
211 break; /* vco is too big, it has to stop. */
212 diff_dclk = dclk - calc_clk;
213
214 /* determine if this vco setting is better than current optimal settings */
215 diff_score = abs(diff_vclk) + abs(diff_dclk);
216 if (diff_score < optimal_diff_score) {
217 optimal_fb_div = fb_div;
218 optimal_vclk_div = vclk_div;
219 optimal_dclk_div = dclk_div;
220 optimal_vco_freq = vco_freq;
221 optimal_diff_score = diff_score;
222 if (optimal_diff_score == 0)
223 break; /* it can't get better than this */
224 }
225 }
226
227 /* set VCO_MODE to 1 */
228 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
229
230 /* toggle UPLL_SLEEP to 1 then back to 0 */
231 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
232 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
233
234 /* deassert UPLL_RESET */
235 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
236
237 mdelay(1);
238
239 /* bypass vclk and dclk with bclk */
240 WREG32_P(CG_UPLL_FUNC_CNTL_2,
241 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
242 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
243
244 /* put PLL in bypass mode */
245 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
246
247 r = evergreen_uvd_send_upll_ctlreq(rdev);
248 if (r)
249 return r;
250
251 /* assert UPLL_RESET again */
252 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
253
254 /* disable spread spectrum. */
255 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
256
257 /* set feedback divider */
258 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(optimal_fb_div), ~UPLL_FB_DIV_MASK);
259
260 /* set ref divider to 0 */
261 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
262
263 if (optimal_vco_freq < 187500)
264 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
265 else
266 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
267
268 /* set PDIV_A and PDIV_B */
269 WREG32_P(CG_UPLL_FUNC_CNTL_2,
270 UPLL_PDIV_A(optimal_vclk_div) | UPLL_PDIV_B(optimal_dclk_div),
271 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
272
273 /* give the PLL some time to settle */
274 mdelay(15);
275
276 /* deassert PLL_RESET */
277 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
278
279 mdelay(15);
280
281 /* switch from bypass mode to normal mode */
282 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
283
284 r = evergreen_uvd_send_upll_ctlreq(rdev);
285 if (r)
286 return r;
287
288 /* switch VCLK and DCLK selection */
289 WREG32_P(CG_UPLL_FUNC_CNTL_2,
290 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
291 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
292
293 mdelay(100);
294
295 return 0;
296}
297
d054ac16
AD		 298void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
299{
300 u16 ctl, v;
32195aec 301 int err;
d054ac16 302
32195aec 303 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 304 if (err)
305 return;
306
307 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
308
309 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
310 * to avoid hangs or perfomance issues
311 */
312 if ((v == 0) || (v == 6) || (v == 7)) {
313 ctl &= ~PCI_EXP_DEVCTL_READRQ;
314 ctl |= (2 << 12);
32195aec 315 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 316 }
317}
318
377edc8b
AD		 319/**
320 * dce4_wait_for_vblank - vblank wait asic callback.
321 *
322 * @rdev: radeon_device pointer
323 * @crtc: crtc to wait for vblank on
324 *
325 * Wait for vblank on the requested crtc (evergreen+).
326 */
3ae19b75
AD		 327void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
328{
3ae19b75
AD		 329 int i;
330
4a15903d
AD		 331 if (crtc >= rdev->num_crtc)
332 return;
333
334 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
3ae19b75 335 for (i = 0; i < rdev->usec_timeout; i++) {
4a15903d 336 if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
3ae19b75
AD		 337 break;
338 udelay(1);
339 }
340 for (i = 0; i < rdev->usec_timeout; i++) {
4a15903d 341 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
3ae19b75
AD		 342 break;
343 udelay(1);
344 }
345 }
346}
347
377edc8b
AD		 348/**
349 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
350 *
351 * @rdev: radeon_device pointer
352 * @crtc: crtc to prepare for pageflip on
353 *
354 * Pre-pageflip callback (evergreen+).
355 * Enables the pageflip irq (vblank irq).
356 */
6f34be50
AD		 357void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
358{
6f34be50
AD		 359 /* enable the pflip int */
360 radeon_irq_kms_pflip_irq_get(rdev, crtc);
361}
362
377edc8b
AD		 363/**
364 * evergreen_post_page_flip - pos-pageflip callback.
365 *
366 * @rdev: radeon_device pointer
367 * @crtc: crtc to cleanup pageflip on
368 *
369 * Post-pageflip callback (evergreen+).
370 * Disables the pageflip irq (vblank irq).
371 */
6f34be50
AD		 372void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
373{
374 /* disable the pflip int */
375 radeon_irq_kms_pflip_irq_put(rdev, crtc);
376}
377
377edc8b
AD		 378/**
379 * evergreen_page_flip - pageflip callback.
380 *
381 * @rdev: radeon_device pointer
382 * @crtc_id: crtc to cleanup pageflip on
383 * @crtc_base: new address of the crtc (GPU MC address)
384 *
385 * Does the actual pageflip (evergreen+).
386 * During vblank we take the crtc lock and wait for the update_pending
387 * bit to go high, when it does, we release the lock, and allow the
388 * double buffered update to take place.
389 * Returns the current update pending status.
390 */
6f34be50
AD		 391u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
392{
393 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
394 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 395 int i;
6f34be50
AD		 396
397 /* Lock the graphics update lock */
398 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
399 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
400
401 /* update the scanout addresses */
402 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
403 upper_32_bits(crtc_base));
404 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
405 (u32)crtc_base);
406
407 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
408 upper_32_bits(crtc_base));
409 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
410 (u32)crtc_base);
411
412 /* Wait for update_pending to go high. */
f6496479
AD		 413 for (i = 0; i < rdev->usec_timeout; i++) {
414 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
415 break;
416 udelay(1);
417 }
6f34be50
AD		 418 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
419
420 /* Unlock the lock, so double-buffering can take place inside vblank */
421 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
422 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
423
424 /* Return current update_pending status: */
425 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
426}
427
21a8122a 428/* get temperature in millidegrees */
20d391d7 429int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 430{
1c88d74f
AD		 431 u32 temp, toffset;
432 int actual_temp = 0;
67b3f823
AD		 433
434 if (rdev->family == CHIP_JUNIPER) {
435 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
436 TOFFSET_SHIFT;
437 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
438 TS0_ADC_DOUT_SHIFT;
439
440 if (toffset & 0x100)
441 actual_temp = temp / 2 - (0x200 - toffset);
442 else
443 actual_temp = temp / 2 + toffset;
444
445 actual_temp = actual_temp * 1000;
446
447 } else {
448 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
449 ASIC_T_SHIFT;
450
451 if (temp & 0x400)
452 actual_temp = -256;
453 else if (temp & 0x200)
454 actual_temp = 255;
455 else if (temp & 0x100) {
456 actual_temp = temp & 0x1ff;
457 actual_temp |= ~0x1ff;
458 } else
459 actual_temp = temp & 0xff;
460
461 actual_temp = (actual_temp * 1000) / 2;
462 }
21a8122a 463
67b3f823 464 return actual_temp;
21a8122a
AD		 465}
466
20d391d7 467int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 468{
469 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 470 int actual_temp = temp - 49;
e33df25f
AD		 471
472 return actual_temp * 1000;
473}
474
377edc8b
AD		 475/**
476 * sumo_pm_init_profile - Initialize power profiles callback.
477 *
478 * @rdev: radeon_device pointer
479 *
480 * Initialize the power states used in profile mode
481 * (sumo, trinity, SI).
482 * Used for profile mode only.
483 */
a4c9e2ee
AD		 484void sumo_pm_init_profile(struct radeon_device *rdev)
485{
486 int idx;
487
488 /* default */
489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
492 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
493
494 /* low,mid sh/mh */
495 if (rdev->flags & RADEON_IS_MOBILITY)
496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
497 else
498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
499
500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
504
505 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
506 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
507 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
508 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
509
510 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
511 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
512 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
513 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
514
515 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
516 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
517 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
518 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
519
520 /* high sh/mh */
521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
522 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
523 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
524 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
525 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
526 rdev->pm.power_state[idx].num_clock_modes - 1;
527
528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
530 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
531 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
532 rdev->pm.power_state[idx].num_clock_modes - 1;
533}
534
27810fb2
AD		 535/**
536 * btc_pm_init_profile - Initialize power profiles callback.
537 *
538 * @rdev: radeon_device pointer
539 *
540 * Initialize the power states used in profile mode
541 * (BTC, cayman).
542 * Used for profile mode only.
543 */
544void btc_pm_init_profile(struct radeon_device *rdev)
545{
546 int idx;
547
548 /* default */
549 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
550 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
551 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
552 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
553 /* starting with BTC, there is one state that is used for both
554 * MH and SH. Difference is that we always use the high clock index for
555 * mclk.
556 */
557 if (rdev->flags & RADEON_IS_MOBILITY)
558 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
559 else
560 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
561 /* low sh */
562 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
563 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
564 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
565 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
566 /* mid sh */
567 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
568 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
569 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
570 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
571 /* high sh */
572 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
573 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
574 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
575 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
576 /* low mh */
577 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
578 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
579 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
580 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
581 /* mid mh */
582 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
583 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
584 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
585 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
586 /* high mh */
587 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
588 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
589 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
590 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
591}
592
377edc8b
AD		 593/**
594 * evergreen_pm_misc - set additional pm hw parameters callback.
595 *
596 * @rdev: radeon_device pointer
597 *
598 * Set non-clock parameters associated with a power state
599 * (voltage, etc.) (evergreen+).
600 */
49e02b73
AD		 601void evergreen_pm_misc(struct radeon_device *rdev)
602{
a081a9d6
RM		 603 int req_ps_idx = rdev->pm.requested_power_state_index;
604 int req_cm_idx = rdev->pm.requested_clock_mode_index;
605 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
606 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 607
2feea49a 608 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 609 /* 0xff01 is a flag rather then an actual voltage */
610 if (voltage->voltage == 0xff01)
611 return;
2feea49a 612 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 613 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 614 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 615 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
616 }
7ae764b1
AD		 617
618 /* starting with BTC, there is one state that is used for both
619 * MH and SH. Difference is that we always use the high clock index for
620 * mclk and vddci.
621 */
622 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
623 (rdev->family >= CHIP_BARTS) &&
624 rdev->pm.active_crtc_count &&
625 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
626 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
627 voltage = &rdev->pm.power_state[req_ps_idx].
628 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
629
a377e187
AD		 630 /* 0xff01 is a flag rather then an actual voltage */
631 if (voltage->vddci == 0xff01)
632 return;
2feea49a
AD		 633 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
634 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
635 rdev->pm.current_vddci = voltage->vddci;
636 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 637 }
638 }
49e02b73
AD		 639}
640
377edc8b
AD		 641/**
642 * evergreen_pm_prepare - pre-power state change callback.
643 *
644 * @rdev: radeon_device pointer
645 *
646 * Prepare for a power state change (evergreen+).
647 */
49e02b73
AD		 648void evergreen_pm_prepare(struct radeon_device *rdev)
649{
650 struct drm_device *ddev = rdev->ddev;
651 struct drm_crtc *crtc;
652 struct radeon_crtc *radeon_crtc;
653 u32 tmp;
654
655 /* disable any active CRTCs */
656 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
657 radeon_crtc = to_radeon_crtc(crtc);
658 if (radeon_crtc->enabled) {
659 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
660 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
661 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
662 }
663 }
664}
665
377edc8b
AD		 666/**
667 * evergreen_pm_finish - post-power state change callback.
668 *
669 * @rdev: radeon_device pointer
670 *
671 * Clean up after a power state change (evergreen+).
672 */
49e02b73
AD		 673void evergreen_pm_finish(struct radeon_device *rdev)
674{
675 struct drm_device *ddev = rdev->ddev;
676 struct drm_crtc *crtc;
677 struct radeon_crtc *radeon_crtc;
678 u32 tmp;
679
680 /* enable any active CRTCs */
681 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
682 radeon_crtc = to_radeon_crtc(crtc);
683 if (radeon_crtc->enabled) {
684 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
685 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
686 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
687 }
688 }
689}
690
377edc8b
AD		 691/**
692 * evergreen_hpd_sense - hpd sense callback.
693 *
694 * @rdev: radeon_device pointer
695 * @hpd: hpd (hotplug detect) pin
696 *
697 * Checks if a digital monitor is connected (evergreen+).
698 * Returns true if connected, false if not connected.
699 */
bcc1c2a1
AD		 700bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
701{
702 bool connected = false;
0ca2ab52
AD		 703
704 switch (hpd) {
705 case RADEON_HPD_1:
706 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
707 connected = true;
708 break;
709 case RADEON_HPD_2:
710 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
711 connected = true;
712 break;
713 case RADEON_HPD_3:
714 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
715 connected = true;
716 break;
717 case RADEON_HPD_4:
718 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
719 connected = true;
720 break;
721 case RADEON_HPD_5:
722 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
723 connected = true;
724 break;
725 case RADEON_HPD_6:
726 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
727 connected = true;
728 break;
729 default:
730 break;
731 }
732
bcc1c2a1
AD		 733 return connected;
734}
735
377edc8b
AD		 736/**
737 * evergreen_hpd_set_polarity - hpd set polarity callback.
738 *
739 * @rdev: radeon_device pointer
740 * @hpd: hpd (hotplug detect) pin
741 *
742 * Set the polarity of the hpd pin (evergreen+).
743 */
bcc1c2a1
AD		 744void evergreen_hpd_set_polarity(struct radeon_device *rdev,
745 enum radeon_hpd_id hpd)
746{
0ca2ab52
AD		 747 u32 tmp;
748 bool connected = evergreen_hpd_sense(rdev, hpd);
749
750 switch (hpd) {
751 case RADEON_HPD_1:
752 tmp = RREG32(DC_HPD1_INT_CONTROL);
753 if (connected)
754 tmp &= ~DC_HPDx_INT_POLARITY;
755 else
756 tmp |= DC_HPDx_INT_POLARITY;
757 WREG32(DC_HPD1_INT_CONTROL, tmp);
758 break;
759 case RADEON_HPD_2:
760 tmp = RREG32(DC_HPD2_INT_CONTROL);
761 if (connected)
762 tmp &= ~DC_HPDx_INT_POLARITY;
763 else
764 tmp |= DC_HPDx_INT_POLARITY;
765 WREG32(DC_HPD2_INT_CONTROL, tmp);
766 break;
767 case RADEON_HPD_3:
768 tmp = RREG32(DC_HPD3_INT_CONTROL);
769 if (connected)
770 tmp &= ~DC_HPDx_INT_POLARITY;
771 else
772 tmp |= DC_HPDx_INT_POLARITY;
773 WREG32(DC_HPD3_INT_CONTROL, tmp);
774 break;
775 case RADEON_HPD_4:
776 tmp = RREG32(DC_HPD4_INT_CONTROL);
777 if (connected)
778 tmp &= ~DC_HPDx_INT_POLARITY;
779 else
780 tmp |= DC_HPDx_INT_POLARITY;
781 WREG32(DC_HPD4_INT_CONTROL, tmp);
782 break;
783 case RADEON_HPD_5:
784 tmp = RREG32(DC_HPD5_INT_CONTROL);
785 if (connected)
786 tmp &= ~DC_HPDx_INT_POLARITY;
787 else
788 tmp |= DC_HPDx_INT_POLARITY;
789 WREG32(DC_HPD5_INT_CONTROL, tmp);
790 break;
791 case RADEON_HPD_6:
792 tmp = RREG32(DC_HPD6_INT_CONTROL);
793 if (connected)
794 tmp &= ~DC_HPDx_INT_POLARITY;
795 else
796 tmp |= DC_HPDx_INT_POLARITY;
797 WREG32(DC_HPD6_INT_CONTROL, tmp);
798 break;
799 default:
800 break;
801 }
bcc1c2a1
AD		 802}
803
377edc8b
AD		 804/**
805 * evergreen_hpd_init - hpd setup callback.
806 *
807 * @rdev: radeon_device pointer
808 *
809 * Setup the hpd pins used by the card (evergreen+).
810 * Enable the pin, set the polarity, and enable the hpd interrupts.
811 */
bcc1c2a1
AD		 812void evergreen_hpd_init(struct radeon_device *rdev)
813{
0ca2ab52
AD		 814 struct drm_device *dev = rdev->ddev;
815 struct drm_connector *connector;
fb98257a 816 unsigned enabled = 0;
0ca2ab52
AD		 817 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
818 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 819
0ca2ab52
AD		 820 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
821 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
822 switch (radeon_connector->hpd.hpd) {
823 case RADEON_HPD_1:
824 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 825 break;
826 case RADEON_HPD_2:
827 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 828 break;
829 case RADEON_HPD_3:
830 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 831 break;
832 case RADEON_HPD_4:
833 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 834 break;
835 case RADEON_HPD_5:
836 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 837 break;
838 case RADEON_HPD_6:
839 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 840 break;
841 default:
842 break;
843 }
64912e99 844 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 845 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 846 }
fb98257a 847 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 848}
849
377edc8b
AD		 850/**
851 * evergreen_hpd_fini - hpd tear down callback.
852 *
853 * @rdev: radeon_device pointer
854 *
855 * Tear down the hpd pins used by the card (evergreen+).
856 * Disable the hpd interrupts.
857 */
0ca2ab52 858void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 859{
0ca2ab52
AD		 860 struct drm_device *dev = rdev->ddev;
861 struct drm_connector *connector;
fb98257a 862 unsigned disabled = 0;
0ca2ab52
AD		 863
864 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
865 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
866 switch (radeon_connector->hpd.hpd) {
867 case RADEON_HPD_1:
868 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 869 break;
870 case RADEON_HPD_2:
871 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 872 break;
873 case RADEON_HPD_3:
874 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 875 break;
876 case RADEON_HPD_4:
877 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 878 break;
879 case RADEON_HPD_5:
880 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 881 break;
882 case RADEON_HPD_6:
883 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 884 break;
885 default:
886 break;
887 }
fb98257a 888 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 889 }
fb98257a 890 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 891}
892
f9d9c362
AD		 893/* watermark setup */
894
895static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
896 struct radeon_crtc *radeon_crtc,
897 struct drm_display_mode *mode,
898 struct drm_display_mode *other_mode)
899{
12dfc843 900 u32 tmp;
f9d9c362
AD		 901 /*
902 * Line Buffer Setup
903 * There are 3 line buffers, each one shared by 2 display controllers.
904 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
905 * the display controllers. The paritioning is done via one of four
906 * preset allocations specified in bits 2:0:
907 * first display controller
908 * 0 - first half of lb (3840 * 2)
909 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 910 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 911 * 3 - first 1/4 of lb (1920 * 2)
912 * second display controller
913 * 4 - second half of lb (3840 * 2)
914 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 915 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 916 * 7 - last 1/4 of lb (1920 * 2)
917 */
12dfc843
AD		 918 /* this can get tricky if we have two large displays on a paired group
919 * of crtcs. Ideally for multiple large displays we'd assign them to
920 * non-linked crtcs for maximum line buffer allocation.
921 */
922 if (radeon_crtc->base.enabled && mode) {
923 if (other_mode)
f9d9c362 924 tmp = 0; /* 1/2 */
12dfc843
AD		 925 else
926 tmp = 2; /* whole */
927 } else
928 tmp = 0;
f9d9c362
AD		 929
930 /* second controller of the pair uses second half of the lb */
931 if (radeon_crtc->crtc_id % 2)
932 tmp += 4;
933 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
934
12dfc843
AD		 935 if (radeon_crtc->base.enabled && mode) {
936 switch (tmp) {
937 case 0:
938 case 4:
939 default:
940 if (ASIC_IS_DCE5(rdev))
941 return 4096 * 2;
942 else
943 return 3840 * 2;
944 case 1:
945 case 5:
946 if (ASIC_IS_DCE5(rdev))
947 return 6144 * 2;
948 else
949 return 5760 * 2;
950 case 2:
951 case 6:
952 if (ASIC_IS_DCE5(rdev))
953 return 8192 * 2;
954 else
955 return 7680 * 2;
956 case 3:
957 case 7:
958 if (ASIC_IS_DCE5(rdev))
959 return 2048 * 2;
960 else
961 return 1920 * 2;
962 }
f9d9c362 963 }
12dfc843
AD		 964
965 /* controller not enabled, so no lb used */
966 return 0;
f9d9c362
AD		 967}
968
ca7db22b 969u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 970{
971 u32 tmp = RREG32(MC_SHARED_CHMAP);
972
973 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
974 case 0:
975 default:
976 return 1;
977 case 1:
978 return 2;
979 case 2:
980 return 4;
981 case 3:
982 return 8;
983 }
984}
985
986struct evergreen_wm_params {
987 u32 dram_channels; /* number of dram channels */
988 u32 yclk; /* bandwidth per dram data pin in kHz */
989 u32 sclk; /* engine clock in kHz */
990 u32 disp_clk; /* display clock in kHz */
991 u32 src_width; /* viewport width */
992 u32 active_time; /* active display time in ns */
993 u32 blank_time; /* blank time in ns */
994 bool interlaced; /* mode is interlaced */
995 fixed20_12 vsc; /* vertical scale ratio */
996 u32 num_heads; /* number of active crtcs */
997 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
998 u32 lb_size; /* line buffer allocated to pipe */
999 u32 vtaps; /* vertical scaler taps */
1000};
1001
1002static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1003{
1004 /* Calculate DRAM Bandwidth and the part allocated to display. */
1005 fixed20_12 dram_efficiency; /* 0.7 */
1006 fixed20_12 yclk, dram_channels, bandwidth;
1007 fixed20_12 a;
1008
1009 a.full = dfixed_const(1000);
1010 yclk.full = dfixed_const(wm->yclk);
1011 yclk.full = dfixed_div(yclk, a);
1012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1013 a.full = dfixed_const(10);
1014 dram_efficiency.full = dfixed_const(7);
1015 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1016 bandwidth.full = dfixed_mul(dram_channels, yclk);
1017 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1018
1019 return dfixed_trunc(bandwidth);
1020}
1021
1022static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1023{
1024 /* Calculate DRAM Bandwidth and the part allocated to display. */
1025 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1026 fixed20_12 yclk, dram_channels, bandwidth;
1027 fixed20_12 a;
1028
1029 a.full = dfixed_const(1000);
1030 yclk.full = dfixed_const(wm->yclk);
1031 yclk.full = dfixed_div(yclk, a);
1032 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1033 a.full = dfixed_const(10);
1034 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1035 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1036 bandwidth.full = dfixed_mul(dram_channels, yclk);
1037 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1038
1039 return dfixed_trunc(bandwidth);
1040}
1041
1042static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1043{
1044 /* Calculate the display Data return Bandwidth */
1045 fixed20_12 return_efficiency; /* 0.8 */
1046 fixed20_12 sclk, bandwidth;
1047 fixed20_12 a;
1048
1049 a.full = dfixed_const(1000);
1050 sclk.full = dfixed_const(wm->sclk);
1051 sclk.full = dfixed_div(sclk, a);
1052 a.full = dfixed_const(10);
1053 return_efficiency.full = dfixed_const(8);
1054 return_efficiency.full = dfixed_div(return_efficiency, a);
1055 a.full = dfixed_const(32);
1056 bandwidth.full = dfixed_mul(a, sclk);
1057 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1058
1059 return dfixed_trunc(bandwidth);
1060}
1061
1062static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1063{
1064 /* Calculate the DMIF Request Bandwidth */
1065 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1066 fixed20_12 disp_clk, bandwidth;
1067 fixed20_12 a;
1068
1069 a.full = dfixed_const(1000);
1070 disp_clk.full = dfixed_const(wm->disp_clk);
1071 disp_clk.full = dfixed_div(disp_clk, a);
1072 a.full = dfixed_const(10);
1073 disp_clk_request_efficiency.full = dfixed_const(8);
1074 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1075 a.full = dfixed_const(32);
1076 bandwidth.full = dfixed_mul(a, disp_clk);
1077 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1078
1079 return dfixed_trunc(bandwidth);
1080}
1081
1082static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1083{
1084 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1085 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1086 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1087 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1088
1089 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1090}
1091
1092static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
1093{
1094 /* Calculate the display mode Average Bandwidth
1095 * DisplayMode should contain the source and destination dimensions,
1096 * timing, etc.
1097 */
1098 fixed20_12 bpp;
1099 fixed20_12 line_time;
1100 fixed20_12 src_width;
1101 fixed20_12 bandwidth;
1102 fixed20_12 a;
1103
1104 a.full = dfixed_const(1000);
1105 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1106 line_time.full = dfixed_div(line_time, a);
1107 bpp.full = dfixed_const(wm->bytes_per_pixel);
1108 src_width.full = dfixed_const(wm->src_width);
1109 bandwidth.full = dfixed_mul(src_width, bpp);
1110 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1111 bandwidth.full = dfixed_div(bandwidth, line_time);
1112
1113 return dfixed_trunc(bandwidth);
1114}
1115
1116static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
1117{
1118 /* First calcualte the latency in ns */
1119 u32 mc_latency = 2000; /* 2000 ns. */
1120 u32 available_bandwidth = evergreen_available_bandwidth(wm);
1121 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1122 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1123 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1124 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1125 (wm->num_heads * cursor_line_pair_return_time);
1126 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1127 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1128 fixed20_12 a, b, c;
1129
1130 if (wm->num_heads == 0)
1131 return 0;
1132
1133 a.full = dfixed_const(2);
1134 b.full = dfixed_const(1);
1135 if ((wm->vsc.full > a.full) ||
1136 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1137 (wm->vtaps >= 5) ||
1138 ((wm->vsc.full >= a.full) && wm->interlaced))
1139 max_src_lines_per_dst_line = 4;
1140 else
1141 max_src_lines_per_dst_line = 2;
1142
1143 a.full = dfixed_const(available_bandwidth);
1144 b.full = dfixed_const(wm->num_heads);
1145 a.full = dfixed_div(a, b);
1146
1147 b.full = dfixed_const(1000);
1148 c.full = dfixed_const(wm->disp_clk);
1149 b.full = dfixed_div(c, b);
1150 c.full = dfixed_const(wm->bytes_per_pixel);
1151 b.full = dfixed_mul(b, c);
1152
1153 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
1154
1155 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1156 b.full = dfixed_const(1000);
1157 c.full = dfixed_const(lb_fill_bw);
1158 b.full = dfixed_div(c, b);
1159 a.full = dfixed_div(a, b);
1160 line_fill_time = dfixed_trunc(a);
1161
1162 if (line_fill_time < wm->active_time)
1163 return latency;
1164 else
1165 return latency + (line_fill_time - wm->active_time);
1166
1167}
1168
1169static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1170{
1171 if (evergreen_average_bandwidth(wm) <=
1172 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
1173 return true;
1174 else
1175 return false;
1176};
1177
1178static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
1179{
1180 if (evergreen_average_bandwidth(wm) <=
1181 (evergreen_available_bandwidth(wm) / wm->num_heads))
1182 return true;
1183 else
1184 return false;
1185};
1186
1187static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
1188{
1189 u32 lb_partitions = wm->lb_size / wm->src_width;
1190 u32 line_time = wm->active_time + wm->blank_time;
1191 u32 latency_tolerant_lines;
1192 u32 latency_hiding;
1193 fixed20_12 a;
1194
1195 a.full = dfixed_const(1);
1196 if (wm->vsc.full > a.full)
1197 latency_tolerant_lines = 1;
1198 else {
1199 if (lb_partitions <= (wm->vtaps + 1))
1200 latency_tolerant_lines = 1;
1201 else
1202 latency_tolerant_lines = 2;
1203 }
1204
1205 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1206
1207 if (evergreen_latency_watermark(wm) <= latency_hiding)
1208 return true;
1209 else
1210 return false;
1211}
1212
1213static void evergreen_program_watermarks(struct radeon_device *rdev,
1214 struct radeon_crtc *radeon_crtc,
1215 u32 lb_size, u32 num_heads)
1216{
1217 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1218 struct evergreen_wm_params wm;
1219 u32 pixel_period;
1220 u32 line_time = 0;
1221 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1222 u32 priority_a_mark = 0, priority_b_mark = 0;
1223 u32 priority_a_cnt = PRIORITY_OFF;
1224 u32 priority_b_cnt = PRIORITY_OFF;
1225 u32 pipe_offset = radeon_crtc->crtc_id * 16;
1226 u32 tmp, arb_control3;
1227 fixed20_12 a, b, c;
1228
1229 if (radeon_crtc->base.enabled && num_heads && mode) {
1230 pixel_period = 1000000 / (u32)mode->clock;
1231 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1232 priority_a_cnt = 0;
1233 priority_b_cnt = 0;
1234
1235 wm.yclk = rdev->pm.current_mclk * 10;
1236 wm.sclk = rdev->pm.current_sclk * 10;
1237 wm.disp_clk = mode->clock;
1238 wm.src_width = mode->crtc_hdisplay;
1239 wm.active_time = mode->crtc_hdisplay * pixel_period;
1240 wm.blank_time = line_time - wm.active_time;
1241 wm.interlaced = false;
1242 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1243 wm.interlaced = true;
1244 wm.vsc = radeon_crtc->vsc;
1245 wm.vtaps = 1;
1246 if (radeon_crtc->rmx_type != RMX_OFF)
1247 wm.vtaps = 2;
1248 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1249 wm.lb_size = lb_size;
1250 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1251 wm.num_heads = num_heads;
1252
1253 /* set for high clocks */
1254 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
1255 /* set for low clocks */
1256 /* wm.yclk = low clk; wm.sclk = low clk */
1257 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
1258
1259 /* possibly force display priority to high */
1260 /* should really do this at mode validation time... */
1261 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1262 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
1263 !evergreen_check_latency_hiding(&wm) ||
1264 (rdev->disp_priority == 2)) {
92bdfd4a 1265 DRM_DEBUG_KMS("force priority to high\n");
f9d9c362
AD		 1266 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1267 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1268 }
1269
1270 a.full = dfixed_const(1000);
1271 b.full = dfixed_const(mode->clock);
1272 b.full = dfixed_div(b, a);
1273 c.full = dfixed_const(latency_watermark_a);
1274 c.full = dfixed_mul(c, b);
1275 c.full = dfixed_mul(c, radeon_crtc->hsc);
1276 c.full = dfixed_div(c, a);
1277 a.full = dfixed_const(16);
1278 c.full = dfixed_div(c, a);
1279 priority_a_mark = dfixed_trunc(c);
1280 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1281
1282 a.full = dfixed_const(1000);
1283 b.full = dfixed_const(mode->clock);
1284 b.full = dfixed_div(b, a);
1285 c.full = dfixed_const(latency_watermark_b);
1286 c.full = dfixed_mul(c, b);
1287 c.full = dfixed_mul(c, radeon_crtc->hsc);
1288 c.full = dfixed_div(c, a);
1289 a.full = dfixed_const(16);
1290 c.full = dfixed_div(c, a);
1291 priority_b_mark = dfixed_trunc(c);
1292 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1293 }
1294
1295 /* select wm A */
1296 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1297 tmp = arb_control3;
1298 tmp &= ~LATENCY_WATERMARK_MASK(3);
1299 tmp |= LATENCY_WATERMARK_MASK(1);
1300 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1301 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1302 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1303 LATENCY_HIGH_WATERMARK(line_time)));
1304 /* select wm B */
1305 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1306 tmp &= ~LATENCY_WATERMARK_MASK(3);
1307 tmp |= LATENCY_WATERMARK_MASK(2);
1308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1309 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1310 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1311 LATENCY_HIGH_WATERMARK(line_time)));
1312 /* restore original selection */
1313 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
1314
1315 /* write the priority marks */
1316 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1317 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1318
1319}
1320
377edc8b
AD		 1321/**
1322 * evergreen_bandwidth_update - update display watermarks callback.
1323 *
1324 * @rdev: radeon_device pointer
1325 *
1326 * Update the display watermarks based on the requested mode(s)
1327 * (evergreen+).
1328 */
0ca2ab52 1329void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 1330{
f9d9c362
AD		 1331 struct drm_display_mode *mode0 = NULL;
1332 struct drm_display_mode *mode1 = NULL;
1333 u32 num_heads = 0, lb_size;
1334 int i;
1335
1336 radeon_update_display_priority(rdev);
1337
1338 for (i = 0; i < rdev->num_crtc; i++) {
1339 if (rdev->mode_info.crtcs[i]->base.enabled)
1340 num_heads++;
1341 }
1342 for (i = 0; i < rdev->num_crtc; i += 2) {
1343 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1344 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1347 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1348 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1349 }
bcc1c2a1
AD		 1350}
1351
377edc8b
AD		 1352/**
1353 * evergreen_mc_wait_for_idle - wait for MC idle callback.
1354 *
1355 * @rdev: radeon_device pointer
1356 *
1357 * Wait for the MC (memory controller) to be idle.
1358 * (evergreen+).
1359 * Returns 0 if the MC is idle, -1 if not.
1360 */
b9952a8a 1361int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 1362{
1363 unsigned i;
1364 u32 tmp;
1365
1366 for (i = 0; i < rdev->usec_timeout; i++) {
1367 /* read MC_STATUS */
1368 tmp = RREG32(SRBM_STATUS) & 0x1F00;
1369 if (!tmp)
1370 return 0;
1371 udelay(1);
1372 }
1373 return -1;
1374}
1375
1376/*
1377 * GART
1378 */
0fcdb61e
AD		 1379void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
1380{
1381 unsigned i;
1382 u32 tmp;
1383
6f2f48a9
AD		 1384 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1385
0fcdb61e
AD		 1386 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
1387 for (i = 0; i < rdev->usec_timeout; i++) {
1388 /* read MC_STATUS */
1389 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
1390 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
1391 if (tmp == 2) {
1392 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
1393 return;
1394 }
1395 if (tmp) {
1396 return;
1397 }
1398 udelay(1);
1399 }
1400}
1401
1109ca09 1402static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 1403{
1404 u32 tmp;
0fcdb61e 1405 int r;
bcc1c2a1 1406
c9a1be96 1407 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 1408 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1409 return -EINVAL;
1410 }
1411 r = radeon_gart_table_vram_pin(rdev);
1412 if (r)
1413 return r;
82568565 1414 radeon_gart_restore(rdev);
bcc1c2a1
AD		 1415 /* Setup L2 cache */
1416 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1417 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1418 EFFECTIVE_L2_QUEUE_SIZE(7));
1419 WREG32(VM_L2_CNTL2, 0);
1420 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1421 /* Setup TLB control */
1422 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1423 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1424 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1425 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 1426 if (rdev->flags & RADEON_IS_IGP) {
1427 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
1428 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
1429 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
1430 } else {
1431 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1432 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1433 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 1434 if ((rdev->family == CHIP_JUNIPER) ||
1435 (rdev->family == CHIP_CYPRESS) ||
1436 (rdev->family == CHIP_HEMLOCK) ||
1437 (rdev->family == CHIP_BARTS))
1438 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 1439 }
bcc1c2a1
AD		 1440 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1441 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1442 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1443 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1444 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1445 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1446 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1447 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1448 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1449 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1450 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 1451 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 1452
0fcdb61e 1453 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 1454 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1455 (unsigned)(rdev->mc.gtt_size >> 20),
1456 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 1457 rdev->gart.ready = true;
1458 return 0;
1459}
1460
1109ca09 1461static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 1462{
1463 u32 tmp;
bcc1c2a1
AD		 1464
1465 /* Disable all tables */
0fcdb61e
AD		 1466 WREG32(VM_CONTEXT0_CNTL, 0);
1467 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 1468
1469 /* Setup L2 cache */
1470 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
1471 EFFECTIVE_L2_QUEUE_SIZE(7));
1472 WREG32(VM_L2_CNTL2, 0);
1473 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1474 /* Setup TLB control */
1475 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1476 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1477 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1478 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1479 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1480 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1481 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1482 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 1483 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 1484}
1485
1109ca09 1486static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 1487{
1488 evergreen_pcie_gart_disable(rdev);
1489 radeon_gart_table_vram_free(rdev);
1490 radeon_gart_fini(rdev);
1491}
1492
1493
1109ca09 1494static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 1495{
1496 u32 tmp;
bcc1c2a1
AD		 1497
1498 /* Setup L2 cache */
1499 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1500 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1501 EFFECTIVE_L2_QUEUE_SIZE(7));
1502 WREG32(VM_L2_CNTL2, 0);
1503 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1504 /* Setup TLB control */
1505 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1506 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1507 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1508 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1509 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1510 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1511 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1512 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1513 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1514 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1515 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 1516 WREG32(VM_CONTEXT0_CNTL, 0);
1517 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 1518}
1519
b9952a8a 1520void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 1521{
62444b74
AD		 1522 u32 crtc_enabled, tmp, frame_count, blackout;
1523 int i, j;
1524
bcc1c2a1
AD		 1525 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1526 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 1527
62444b74 1528 /* disable VGA render */
bcc1c2a1 1529 WREG32(VGA_RENDER_CONTROL, 0);
62444b74
AD		 1530 /* blank the display controllers */
1531 for (i = 0; i < rdev->num_crtc; i++) {
1532 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
1533 if (crtc_enabled) {
1534 save->crtc_enabled[i] = true;
1535 if (ASIC_IS_DCE6(rdev)) {
1536 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1537 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
1538 radeon_wait_for_vblank(rdev, i);
1539 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 1540 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 1541 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 1542 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 1543 }
1544 } else {
1545 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1546 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
1547 radeon_wait_for_vblank(rdev, i);
1548 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 1549 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 1550 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 1551 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 1552 }
1553 }
1554 /* wait for the next frame */
1555 frame_count = radeon_get_vblank_counter(rdev, i);
1556 for (j = 0; j < rdev->usec_timeout; j++) {
1557 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1558 break;
1559 udelay(1);
1560 }
804cc4a0
AD		 1561 } else {
1562 save->crtc_enabled[i] = false;
62444b74 1563 }
18007401 1564 }
bcc1c2a1 1565
62444b74
AD		 1566 radeon_mc_wait_for_idle(rdev);
1567
1568 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1569 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
1570 /* Block CPU access */
1571 WREG32(BIF_FB_EN, 0);
1572 /* blackout the MC */
1573 blackout &= ~BLACKOUT_MODE_MASK;
1574 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 1575 }
ed39fadd
AD		 1576 /* wait for the MC to settle */
1577 udelay(100);
bcc1c2a1
AD		 1578}
1579
b9952a8a 1580void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 1581{
62444b74
AD		 1582 u32 tmp, frame_count;
1583 int i, j;
18007401 1584
62444b74
AD		 1585 /* update crtc base addresses */
1586 for (i = 0; i < rdev->num_crtc; i++) {
1587 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 1588 upper_32_bits(rdev->mc.vram_start));
62444b74 1589 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 1590 upper_32_bits(rdev->mc.vram_start));
62444b74 1591 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 1592 (u32)rdev->mc.vram_start);
62444b74 1593 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 1594 (u32)rdev->mc.vram_start);
1595 }
bcc1c2a1
AD		 1596 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1597 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
62444b74
AD		 1598
1599 /* unblackout the MC */
1600 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
1601 tmp &= ~BLACKOUT_MODE_MASK;
1602 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
1603 /* allow CPU access */
1604 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1605
1606 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 1607 if (save->crtc_enabled[i]) {
62444b74
AD		 1608 if (ASIC_IS_DCE6(rdev)) {
1609 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1610 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 1611 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 1612 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 1613 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 1614 } else {
1615 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1616 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 1617 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 1618 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 1619 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 1620 }
1621 /* wait for the next frame */
1622 frame_count = radeon_get_vblank_counter(rdev, i);
1623 for (j = 0; j < rdev->usec_timeout; j++) {
1624 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1625 break;
1626 udelay(1);
1627 }
1628 }
1629 }
1630 /* Unlock vga access */
bcc1c2a1
AD		 1631 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1632 mdelay(1);
bcc1c2a1
AD		 1633 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1634}
1635
755d819e 1636void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 1637{
1638 struct evergreen_mc_save save;
1639 u32 tmp;
1640 int i, j;
1641
1642 /* Initialize HDP */
1643 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1644 WREG32((0x2c14 + j), 0x00000000);
1645 WREG32((0x2c18 + j), 0x00000000);
1646 WREG32((0x2c1c + j), 0x00000000);
1647 WREG32((0x2c20 + j), 0x00000000);
1648 WREG32((0x2c24 + j), 0x00000000);
1649 }
1650 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1651
1652 evergreen_mc_stop(rdev, &save);
1653 if (evergreen_mc_wait_for_idle(rdev)) {
1654 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1655 }
1656 /* Lockout access through VGA aperture*/
1657 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1658 /* Update configuration */
1659 if (rdev->flags & RADEON_IS_AGP) {
1660 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1661 /* VRAM before AGP */
1662 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1663 rdev->mc.vram_start >> 12);
1664 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1665 rdev->mc.gtt_end >> 12);
1666 } else {
1667 /* VRAM after AGP */
1668 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1669 rdev->mc.gtt_start >> 12);
1670 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1671 rdev->mc.vram_end >> 12);
1672 }
1673 } else {
1674 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1675 rdev->mc.vram_start >> 12);
1676 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1677 rdev->mc.vram_end >> 12);
1678 }
3b9832f6 1679 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 1680 /* llano/ontario only */
1681 if ((rdev->family == CHIP_PALM) ||
1682 (rdev->family == CHIP_SUMO) ||
1683 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 1684 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1685 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1686 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1687 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1688 }
bcc1c2a1
AD		 1689 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1690 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1691 WREG32(MC_VM_FB_LOCATION, tmp);
1692 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 1693 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 1694 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 1695 if (rdev->flags & RADEON_IS_AGP) {
1696 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1697 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1698 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1699 } else {
1700 WREG32(MC_VM_AGP_BASE, 0);
1701 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1702 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1703 }
1704 if (evergreen_mc_wait_for_idle(rdev)) {
1705 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1706 }
1707 evergreen_mc_resume(rdev, &save);
1708 /* we need to own VRAM, so turn off the VGA renderer here
1709 * to stop it overwriting our objects */
1710 rv515_vga_render_disable(rdev);
1711}
1712
bcc1c2a1
AD		 1713/*
1714 * CP.
1715 */
12920591
AD		 1716void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1717{
876dc9f3 1718 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 1719 u32 next_rptr;
7b1f2485 1720
12920591 1721 /* set to DX10/11 mode */
e32eb50d
CK		 1722 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1723 radeon_ring_write(ring, 1);
45df6803
CK		 1724
1725 if (ring->rptr_save_reg) {
89d35807 1726 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 1727 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1728 radeon_ring_write(ring, ((ring->rptr_save_reg -
1729 PACKET3_SET_CONFIG_REG_START) >> 2));
1730 radeon_ring_write(ring, next_rptr);
89d35807
AD		 1731 } else if (rdev->wb.enabled) {
1732 next_rptr = ring->wptr + 5 + 4;
1733 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
1734 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1735 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
1736 radeon_ring_write(ring, next_rptr);
1737 radeon_ring_write(ring, 0);
45df6803
CK		 1738 }
1739
e32eb50d
CK		 1740 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1741 radeon_ring_write(ring,
0f234f5f
AD		 1742#ifdef __BIG_ENDIAN
1743 (2 << 0) |
1744#endif
1745 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 1746 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1747 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 1748}
1749
bcc1c2a1
AD		 1750
1751static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1752{
fe251e2f
AD		 1753 const __be32 *fw_data;
1754 int i;
1755
1756 if (!rdev->me_fw || !rdev->pfp_fw)
1757 return -EINVAL;
bcc1c2a1 1758
fe251e2f 1759 r700_cp_stop(rdev);
0f234f5f
AD		 1760 WREG32(CP_RB_CNTL,
1761#ifdef __BIG_ENDIAN
1762 BUF_SWAP_32BIT |
1763#endif
1764 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 1765
1766 fw_data = (const __be32 *)rdev->pfp_fw->data;
1767 WREG32(CP_PFP_UCODE_ADDR, 0);
1768 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1769 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1770 WREG32(CP_PFP_UCODE_ADDR, 0);
1771
1772 fw_data = (const __be32 *)rdev->me_fw->data;
1773 WREG32(CP_ME_RAM_WADDR, 0);
1774 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1775 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1776
1777 WREG32(CP_PFP_UCODE_ADDR, 0);
1778 WREG32(CP_ME_RAM_WADDR, 0);
1779 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 1780 return 0;
1781}
1782
7e7b41d2
AD		 1783static int evergreen_cp_start(struct radeon_device *rdev)
1784{
e32eb50d 1785 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 1786 int r, i;
7e7b41d2
AD		 1787 uint32_t cp_me;
1788
e32eb50d 1789 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 1790 if (r) {
1791 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1792 return r;
1793 }
e32eb50d
CK		 1794 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1795 radeon_ring_write(ring, 0x1);
1796 radeon_ring_write(ring, 0x0);
1797 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
1798 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1799 radeon_ring_write(ring, 0);
1800 radeon_ring_write(ring, 0);
1801 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 1802
1803 cp_me = 0xff;
1804 WREG32(CP_ME_CNTL, cp_me);
1805
e32eb50d 1806 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 1807 if (r) {
1808 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1809 return r;
1810 }
2281a378
AD		 1811
1812 /* setup clear context state */
e32eb50d
CK		 1813 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1814 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 1815
1816 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 1817 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 1818
e32eb50d
CK		 1819 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1820 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 1821
1822 /* set clear context state */
e32eb50d
CK		 1823 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1824 radeon_ring_write(ring, 0);
2281a378
AD		 1825
1826 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 1827 radeon_ring_write(ring, 0xc0026f00);
1828 radeon_ring_write(ring, 0x00000000);
1829 radeon_ring_write(ring, 0x00000000);
1830 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 1831
1832 /* Clear consts */
e32eb50d
CK		 1833 radeon_ring_write(ring, 0xc0036f00);
1834 radeon_ring_write(ring, 0x00000bc4);
1835 radeon_ring_write(ring, 0xffffffff);
1836 radeon_ring_write(ring, 0xffffffff);
1837 radeon_ring_write(ring, 0xffffffff);
2281a378 1838
e32eb50d
CK		 1839 radeon_ring_write(ring, 0xc0026900);
1840 radeon_ring_write(ring, 0x00000316);
1841 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1842 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 1843
e32eb50d 1844 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 1845
1846 return 0;
1847}
1848
1109ca09 1849static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 1850{
e32eb50d 1851 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 1852 u32 tmp;
1853 u32 rb_bufsz;
1854 int r;
1855
1856 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1857 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1858 SOFT_RESET_PA |
1859 SOFT_RESET_SH |
1860 SOFT_RESET_VGT |
a49a50da 1861 SOFT_RESET_SPI |
fe251e2f
AD		 1862 SOFT_RESET_SX));
1863 RREG32(GRBM_SOFT_RESET);
1864 mdelay(15);
1865 WREG32(GRBM_SOFT_RESET, 0);
1866 RREG32(GRBM_SOFT_RESET);
1867
1868 /* Set ring buffer size */
e32eb50d 1869 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 1870 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 1871#ifdef __BIG_ENDIAN
1872 tmp |= BUF_SWAP_32BIT;
32fcdbf4 1873#endif
fe251e2f 1874 WREG32(CP_RB_CNTL, tmp);
15d3332f 1875 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 1876 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 1877
1878 /* Set the write pointer delay */
1879 WREG32(CP_RB_WPTR_DELAY, 0);
1880
1881 /* Initialize the ring buffer's read and write pointers */
1882 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1883 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 1884 ring->wptr = 0;
1885 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 1886
48fc7f7e 1887 /* set the wb address whether it's enabled or not */
0f234f5f 1888 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 1889 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 1890 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1891 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1892
1893 if (rdev->wb.enabled)
1894 WREG32(SCRATCH_UMSK, 0xff);
1895 else {
1896 tmp |= RB_NO_UPDATE;
1897 WREG32(SCRATCH_UMSK, 0);
1898 }
1899
fe251e2f
AD		 1900 mdelay(1);
1901 WREG32(CP_RB_CNTL, tmp);
1902
e32eb50d 1903 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 1904 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1905
e32eb50d 1906 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 1907
7e7b41d2 1908 evergreen_cp_start(rdev);
e32eb50d 1909 ring->ready = true;
f712812e 1910 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 1911 if (r) {
e32eb50d 1912 ring->ready = false;
fe251e2f
AD		 1913 return r;
1914 }
1915 return 0;
1916}
bcc1c2a1
AD		 1917
1918/*
1919 * Core functions
1920 */
bcc1c2a1
AD		 1921static void evergreen_gpu_init(struct radeon_device *rdev)
1922{
416a2bd2 1923 u32 gb_addr_config;
32fcdbf4 1924 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 1925 u32 sx_debug_1;
1926 u32 smx_dc_ctl0;
1927 u32 sq_config;
1928 u32 sq_lds_resource_mgmt;
1929 u32 sq_gpr_resource_mgmt_1;
1930 u32 sq_gpr_resource_mgmt_2;
1931 u32 sq_gpr_resource_mgmt_3;
1932 u32 sq_thread_resource_mgmt;
1933 u32 sq_thread_resource_mgmt_2;
1934 u32 sq_stack_resource_mgmt_1;
1935 u32 sq_stack_resource_mgmt_2;
1936 u32 sq_stack_resource_mgmt_3;
1937 u32 vgt_cache_invalidation;
f25a5c63 1938 u32 hdp_host_path_cntl, tmp;
416a2bd2 1939 u32 disabled_rb_mask;
32fcdbf4
AD		 1940 int i, j, num_shader_engines, ps_thread_count;
1941
1942 switch (rdev->family) {
1943 case CHIP_CYPRESS:
1944 case CHIP_HEMLOCK:
1945 rdev->config.evergreen.num_ses = 2;
1946 rdev->config.evergreen.max_pipes = 4;
1947 rdev->config.evergreen.max_tile_pipes = 8;
1948 rdev->config.evergreen.max_simds = 10;
1949 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1950 rdev->config.evergreen.max_gprs = 256;
1951 rdev->config.evergreen.max_threads = 248;
1952 rdev->config.evergreen.max_gs_threads = 32;
1953 rdev->config.evergreen.max_stack_entries = 512;
1954 rdev->config.evergreen.sx_num_of_sets = 4;
1955 rdev->config.evergreen.sx_max_export_size = 256;
1956 rdev->config.evergreen.sx_max_export_pos_size = 64;
1957 rdev->config.evergreen.sx_max_export_smx_size = 192;
1958 rdev->config.evergreen.max_hw_contexts = 8;
1959 rdev->config.evergreen.sq_num_cf_insts = 2;
1960
1961 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1962 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1963 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 1964 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 1965 break;
1966 case CHIP_JUNIPER:
1967 rdev->config.evergreen.num_ses = 1;
1968 rdev->config.evergreen.max_pipes = 4;
1969 rdev->config.evergreen.max_tile_pipes = 4;
1970 rdev->config.evergreen.max_simds = 10;
1971 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1972 rdev->config.evergreen.max_gprs = 256;
1973 rdev->config.evergreen.max_threads = 248;
1974 rdev->config.evergreen.max_gs_threads = 32;
1975 rdev->config.evergreen.max_stack_entries = 512;
1976 rdev->config.evergreen.sx_num_of_sets = 4;
1977 rdev->config.evergreen.sx_max_export_size = 256;
1978 rdev->config.evergreen.sx_max_export_pos_size = 64;
1979 rdev->config.evergreen.sx_max_export_smx_size = 192;
1980 rdev->config.evergreen.max_hw_contexts = 8;
1981 rdev->config.evergreen.sq_num_cf_insts = 2;
1982
1983 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1984 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1985 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 1986 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 1987 break;
1988 case CHIP_REDWOOD:
1989 rdev->config.evergreen.num_ses = 1;
1990 rdev->config.evergreen.max_pipes = 4;
1991 rdev->config.evergreen.max_tile_pipes = 4;
1992 rdev->config.evergreen.max_simds = 5;
1993 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1994 rdev->config.evergreen.max_gprs = 256;
1995 rdev->config.evergreen.max_threads = 248;
1996 rdev->config.evergreen.max_gs_threads = 32;
1997 rdev->config.evergreen.max_stack_entries = 256;
1998 rdev->config.evergreen.sx_num_of_sets = 4;
1999 rdev->config.evergreen.sx_max_export_size = 256;
2000 rdev->config.evergreen.sx_max_export_pos_size = 64;
2001 rdev->config.evergreen.sx_max_export_smx_size = 192;
2002 rdev->config.evergreen.max_hw_contexts = 8;
2003 rdev->config.evergreen.sq_num_cf_insts = 2;
2004
2005 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2006 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2007 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2008 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2009 break;
2010 case CHIP_CEDAR:
2011 default:
2012 rdev->config.evergreen.num_ses = 1;
2013 rdev->config.evergreen.max_pipes = 2;
2014 rdev->config.evergreen.max_tile_pipes = 2;
2015 rdev->config.evergreen.max_simds = 2;
2016 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2017 rdev->config.evergreen.max_gprs = 256;
2018 rdev->config.evergreen.max_threads = 192;
2019 rdev->config.evergreen.max_gs_threads = 16;
2020 rdev->config.evergreen.max_stack_entries = 256;
2021 rdev->config.evergreen.sx_num_of_sets = 4;
2022 rdev->config.evergreen.sx_max_export_size = 128;
2023 rdev->config.evergreen.sx_max_export_pos_size = 32;
2024 rdev->config.evergreen.sx_max_export_smx_size = 96;
2025 rdev->config.evergreen.max_hw_contexts = 4;
2026 rdev->config.evergreen.sq_num_cf_insts = 1;
2027
d5e455e4
AD		 2028 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2029 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2030 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2031 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 2032 break;
2033 case CHIP_PALM:
2034 rdev->config.evergreen.num_ses = 1;
2035 rdev->config.evergreen.max_pipes = 2;
2036 rdev->config.evergreen.max_tile_pipes = 2;
2037 rdev->config.evergreen.max_simds = 2;
2038 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2039 rdev->config.evergreen.max_gprs = 256;
2040 rdev->config.evergreen.max_threads = 192;
2041 rdev->config.evergreen.max_gs_threads = 16;
2042 rdev->config.evergreen.max_stack_entries = 256;
2043 rdev->config.evergreen.sx_num_of_sets = 4;
2044 rdev->config.evergreen.sx_max_export_size = 128;
2045 rdev->config.evergreen.sx_max_export_pos_size = 32;
2046 rdev->config.evergreen.sx_max_export_smx_size = 96;
2047 rdev->config.evergreen.max_hw_contexts = 4;
2048 rdev->config.evergreen.sq_num_cf_insts = 1;
2049
d5c5a72f
AD		 2050 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2051 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2052 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2053 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 2054 break;
2055 case CHIP_SUMO:
2056 rdev->config.evergreen.num_ses = 1;
2057 rdev->config.evergreen.max_pipes = 4;
bd25f078 2058 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 2059 if (rdev->pdev->device == 0x9648)
2060 rdev->config.evergreen.max_simds = 3;
2061 else if ((rdev->pdev->device == 0x9647) ||
2062 (rdev->pdev->device == 0x964a))
2063 rdev->config.evergreen.max_simds = 4;
2064 else
2065 rdev->config.evergreen.max_simds = 5;
2066 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2067 rdev->config.evergreen.max_gprs = 256;
2068 rdev->config.evergreen.max_threads = 248;
2069 rdev->config.evergreen.max_gs_threads = 32;
2070 rdev->config.evergreen.max_stack_entries = 256;
2071 rdev->config.evergreen.sx_num_of_sets = 4;
2072 rdev->config.evergreen.sx_max_export_size = 256;
2073 rdev->config.evergreen.sx_max_export_pos_size = 64;
2074 rdev->config.evergreen.sx_max_export_smx_size = 192;
2075 rdev->config.evergreen.max_hw_contexts = 8;
2076 rdev->config.evergreen.sq_num_cf_insts = 2;
2077
2078 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2079 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2080 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 2081 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 2082 break;
2083 case CHIP_SUMO2:
2084 rdev->config.evergreen.num_ses = 1;
2085 rdev->config.evergreen.max_pipes = 4;
2086 rdev->config.evergreen.max_tile_pipes = 4;
2087 rdev->config.evergreen.max_simds = 2;
2088 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2089 rdev->config.evergreen.max_gprs = 256;
2090 rdev->config.evergreen.max_threads = 248;
2091 rdev->config.evergreen.max_gs_threads = 32;
2092 rdev->config.evergreen.max_stack_entries = 512;
2093 rdev->config.evergreen.sx_num_of_sets = 4;
2094 rdev->config.evergreen.sx_max_export_size = 256;
2095 rdev->config.evergreen.sx_max_export_pos_size = 64;
2096 rdev->config.evergreen.sx_max_export_smx_size = 192;
2097 rdev->config.evergreen.max_hw_contexts = 8;
2098 rdev->config.evergreen.sq_num_cf_insts = 2;
2099
adb68fa2
AD		 2100 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2101 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2102 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 2103 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 2104 break;
2105 case CHIP_BARTS:
2106 rdev->config.evergreen.num_ses = 2;
2107 rdev->config.evergreen.max_pipes = 4;
2108 rdev->config.evergreen.max_tile_pipes = 8;
2109 rdev->config.evergreen.max_simds = 7;
2110 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2111 rdev->config.evergreen.max_gprs = 256;
2112 rdev->config.evergreen.max_threads = 248;
2113 rdev->config.evergreen.max_gs_threads = 32;
2114 rdev->config.evergreen.max_stack_entries = 512;
2115 rdev->config.evergreen.sx_num_of_sets = 4;
2116 rdev->config.evergreen.sx_max_export_size = 256;
2117 rdev->config.evergreen.sx_max_export_pos_size = 64;
2118 rdev->config.evergreen.sx_max_export_smx_size = 192;
2119 rdev->config.evergreen.max_hw_contexts = 8;
2120 rdev->config.evergreen.sq_num_cf_insts = 2;
2121
2122 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2123 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2124 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2125 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 2126 break;
2127 case CHIP_TURKS:
2128 rdev->config.evergreen.num_ses = 1;
2129 rdev->config.evergreen.max_pipes = 4;
2130 rdev->config.evergreen.max_tile_pipes = 4;
2131 rdev->config.evergreen.max_simds = 6;
2132 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2133 rdev->config.evergreen.max_gprs = 256;
2134 rdev->config.evergreen.max_threads = 248;
2135 rdev->config.evergreen.max_gs_threads = 32;
2136 rdev->config.evergreen.max_stack_entries = 256;
2137 rdev->config.evergreen.sx_num_of_sets = 4;
2138 rdev->config.evergreen.sx_max_export_size = 256;
2139 rdev->config.evergreen.sx_max_export_pos_size = 64;
2140 rdev->config.evergreen.sx_max_export_smx_size = 192;
2141 rdev->config.evergreen.max_hw_contexts = 8;
2142 rdev->config.evergreen.sq_num_cf_insts = 2;
2143
2144 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2145 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2146 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2147 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 2148 break;
2149 case CHIP_CAICOS:
2150 rdev->config.evergreen.num_ses = 1;
bd25f078 2151 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 2152 rdev->config.evergreen.max_tile_pipes = 2;
2153 rdev->config.evergreen.max_simds = 2;
2154 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2155 rdev->config.evergreen.max_gprs = 256;
2156 rdev->config.evergreen.max_threads = 192;
2157 rdev->config.evergreen.max_gs_threads = 16;
2158 rdev->config.evergreen.max_stack_entries = 256;
2159 rdev->config.evergreen.sx_num_of_sets = 4;
2160 rdev->config.evergreen.sx_max_export_size = 128;
2161 rdev->config.evergreen.sx_max_export_pos_size = 32;
2162 rdev->config.evergreen.sx_max_export_smx_size = 96;
2163 rdev->config.evergreen.max_hw_contexts = 4;
2164 rdev->config.evergreen.sq_num_cf_insts = 1;
2165
32fcdbf4
AD		 2166 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2167 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2168 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2169 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2170 break;
2171 }
2172
2173 /* Initialize HDP */
2174 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2175 WREG32((0x2c14 + j), 0x00000000);
2176 WREG32((0x2c18 + j), 0x00000000);
2177 WREG32((0x2c1c + j), 0x00000000);
2178 WREG32((0x2c20 + j), 0x00000000);
2179 WREG32((0x2c24 + j), 0x00000000);
2180 }
2181
2182 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2183
d054ac16
AD		 2184 evergreen_fix_pci_max_read_req_size(rdev);
2185
32fcdbf4 2186 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 2187 if ((rdev->family == CHIP_PALM) ||
2188 (rdev->family == CHIP_SUMO) ||
2189 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 2190 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
2191 else
2192 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 2193
1aa52bd3
AD		 2194 /* setup tiling info dword. gb_addr_config is not adequate since it does
2195 * not have bank info, so create a custom tiling dword.
2196 * bits 3:0 num_pipes
2197 * bits 7:4 num_banks
2198 * bits 11:8 group_size
2199 * bits 15:12 row_size
2200 */
2201 rdev->config.evergreen.tile_config = 0;
2202 switch (rdev->config.evergreen.max_tile_pipes) {
2203 case 1:
2204 default:
2205 rdev->config.evergreen.tile_config |= (0 << 0);
2206 break;
2207 case 2:
2208 rdev->config.evergreen.tile_config |= (1 << 0);
2209 break;
2210 case 4:
2211 rdev->config.evergreen.tile_config |= (2 << 0);
2212 break;
2213 case 8:
2214 rdev->config.evergreen.tile_config |= (3 << 0);
2215 break;
2216 }
d698a34d 2217 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 2218 if (rdev->flags & RADEON_IS_IGP)
d698a34d 2219 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 2220 else {
c8d15edc
AD		 2221 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2222 case 0: /* four banks */
29d65406 2223 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 2224 break;
2225 case 1: /* eight banks */
2226 rdev->config.evergreen.tile_config |= 1 << 4;
2227 break;
2228 case 2: /* sixteen banks */
2229 default:
2230 rdev->config.evergreen.tile_config |= 2 << 4;
2231 break;
2232 }
29d65406 2233 }
416a2bd2 2234 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 2235 rdev->config.evergreen.tile_config |=
2236 ((gb_addr_config & 0x30000000) >> 28) << 12;
2237
416a2bd2 2238 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 2239
416a2bd2
AD		 2240 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
2241 u32 efuse_straps_4;
2242 u32 efuse_straps_3;
32fcdbf4 2243
416a2bd2
AD		 2244 WREG32(RCU_IND_INDEX, 0x204);
2245 efuse_straps_4 = RREG32(RCU_IND_DATA);
2246 WREG32(RCU_IND_INDEX, 0x203);
2247 efuse_straps_3 = RREG32(RCU_IND_DATA);
2248 tmp = (((efuse_straps_4 & 0xf) << 4) |
2249 ((efuse_straps_3 & 0xf0000000) >> 28));
2250 } else {
2251 tmp = 0;
2252 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
2253 u32 rb_disable_bitmap;
2254
2255 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2256 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2257 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
2258 tmp <<= 4;
2259 tmp |= rb_disable_bitmap;
32fcdbf4 2260 }
416a2bd2
AD		 2261 }
2262 /* enabled rb are just the one not disabled :) */
2263 disabled_rb_mask = tmp;
cedb655a
AD		 2264 tmp = 0;
2265 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
2266 tmp |= (1 << i);
2267 /* if all the backends are disabled, fix it up here */
2268 if ((disabled_rb_mask & tmp) == tmp) {
2269 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
2270 disabled_rb_mask &= ~(1 << i);
2271 }
32fcdbf4 2272
416a2bd2
AD		 2273 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
2274 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 2275
416a2bd2
AD		 2276 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2277 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2278 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 2279 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 2280 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2281 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2282 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 2283
f7eb9730
AD		 2284 if ((rdev->config.evergreen.max_backends == 1) &&
2285 (rdev->flags & RADEON_IS_IGP)) {
2286 if ((disabled_rb_mask & 3) == 1) {
2287 /* RB0 disabled, RB1 enabled */
2288 tmp = 0x11111111;
2289 } else {
2290 /* RB1 disabled, RB0 enabled */
2291 tmp = 0x00000000;
2292 }
2293 } else {
2294 tmp = gb_addr_config & NUM_PIPES_MASK;
2295 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
2296 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
2297 }
416a2bd2 2298 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 2299
2300 WREG32(CGTS_SYS_TCC_DISABLE, 0);
2301 WREG32(CGTS_TCC_DISABLE, 0);
2302 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2303 WREG32(CGTS_USER_TCC_DISABLE, 0);
2304
2305 /* set HW defaults for 3D engine */
2306 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2307 ROQ_IB2_START(0x2b)));
2308
2309 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2310
2311 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2312 SYNC_GRADIENT |
2313 SYNC_WALKER |
2314 SYNC_ALIGNER));
2315
2316 sx_debug_1 = RREG32(SX_DEBUG_1);
2317 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2318 WREG32(SX_DEBUG_1, sx_debug_1);
2319
2320
2321 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2322 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2323 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2324 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2325
b866d133
AD		 2326 if (rdev->family <= CHIP_SUMO2)
2327 WREG32(SMX_SAR_CTL0, 0x00010000);
2328
32fcdbf4
AD		 2329 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2330 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2331 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2332
2333 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2334 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2335 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2336
2337 WREG32(VGT_NUM_INSTANCES, 1);
2338 WREG32(SPI_CONFIG_CNTL, 0);
2339 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2340 WREG32(CP_PERFMON_CNTL, 0);
2341
2342 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2343 FETCH_FIFO_HIWATER(0x4) |
2344 DONE_FIFO_HIWATER(0xe0) |
2345 ALU_UPDATE_FIFO_HIWATER(0x8)));
2346
2347 sq_config = RREG32(SQ_CONFIG);
2348 sq_config &= ~(PS_PRIO(3) |
2349 VS_PRIO(3) |
2350 GS_PRIO(3) |
2351 ES_PRIO(3));
2352 sq_config |= (VC_ENABLE |
2353 EXPORT_SRC_C |
2354 PS_PRIO(0) |
2355 VS_PRIO(1) |
2356 GS_PRIO(2) |
2357 ES_PRIO(3));
2358
d5e455e4
AD		 2359 switch (rdev->family) {
2360 case CHIP_CEDAR:
2361 case CHIP_PALM:
d5c5a72f
AD		 2362 case CHIP_SUMO:
2363 case CHIP_SUMO2:
adb68fa2 2364 case CHIP_CAICOS:
32fcdbf4
AD		 2365 /* no vertex cache */
2366 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 2367 break;
2368 default:
2369 break;
2370 }
32fcdbf4
AD		 2371
2372 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2373
2374 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2375 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2376 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2377 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2378 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2379 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2380 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2381
d5e455e4
AD		 2382 switch (rdev->family) {
2383 case CHIP_CEDAR:
2384 case CHIP_PALM:
d5c5a72f
AD		 2385 case CHIP_SUMO:
2386 case CHIP_SUMO2:
32fcdbf4 2387 ps_thread_count = 96;
d5e455e4
AD		 2388 break;
2389 default:
32fcdbf4 2390 ps_thread_count = 128;
d5e455e4
AD		 2391 break;
2392 }
32fcdbf4
AD		 2393
2394 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 2395 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2396 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2397 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2398 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2399 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 2400
2401 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2402 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2403 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2404 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2405 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2406 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2407
2408 WREG32(SQ_CONFIG, sq_config);
2409 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2410 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2411 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2412 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2413 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2414 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2415 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2416 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2417 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2418 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2419
2420 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2421 FORCE_EOV_MAX_REZ_CNT(255)));
2422
d5e455e4
AD		 2423 switch (rdev->family) {
2424 case CHIP_CEDAR:
2425 case CHIP_PALM:
d5c5a72f
AD		 2426 case CHIP_SUMO:
2427 case CHIP_SUMO2:
adb68fa2 2428 case CHIP_CAICOS:
32fcdbf4 2429 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 2430 break;
2431 default:
32fcdbf4 2432 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 2433 break;
2434 }
32fcdbf4
AD		 2435 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2436 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2437
2438 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 2439 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 2440 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2441
60a4a3e0
AD		 2442 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2443 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2444
32fcdbf4
AD		 2445 WREG32(CB_PERF_CTR0_SEL_0, 0);
2446 WREG32(CB_PERF_CTR0_SEL_1, 0);
2447 WREG32(CB_PERF_CTR1_SEL_0, 0);
2448 WREG32(CB_PERF_CTR1_SEL_1, 0);
2449 WREG32(CB_PERF_CTR2_SEL_0, 0);
2450 WREG32(CB_PERF_CTR2_SEL_1, 0);
2451 WREG32(CB_PERF_CTR3_SEL_0, 0);
2452 WREG32(CB_PERF_CTR3_SEL_1, 0);
2453
60a4a3e0
AD		 2454 /* clear render buffer base addresses */
2455 WREG32(CB_COLOR0_BASE, 0);
2456 WREG32(CB_COLOR1_BASE, 0);
2457 WREG32(CB_COLOR2_BASE, 0);
2458 WREG32(CB_COLOR3_BASE, 0);
2459 WREG32(CB_COLOR4_BASE, 0);
2460 WREG32(CB_COLOR5_BASE, 0);
2461 WREG32(CB_COLOR6_BASE, 0);
2462 WREG32(CB_COLOR7_BASE, 0);
2463 WREG32(CB_COLOR8_BASE, 0);
2464 WREG32(CB_COLOR9_BASE, 0);
2465 WREG32(CB_COLOR10_BASE, 0);
2466 WREG32(CB_COLOR11_BASE, 0);
2467
2468 /* set the shader const cache sizes to 0 */
2469 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2470 WREG32(i, 0);
2471 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2472 WREG32(i, 0);
2473
f25a5c63
AD		 2474 tmp = RREG32(HDP_MISC_CNTL);
2475 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2476 WREG32(HDP_MISC_CNTL, tmp);
2477
32fcdbf4
AD		 2478 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2479 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2480
2481 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2482
2483 udelay(50);
2484
bcc1c2a1
AD		 2485}
2486
2487int evergreen_mc_init(struct radeon_device *rdev)
2488{
bcc1c2a1
AD		 2489 u32 tmp;
2490 int chansize, numchan;
bcc1c2a1
AD		 2491
2492 /* Get VRAM informations */
2493 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 2494 if ((rdev->family == CHIP_PALM) ||
2495 (rdev->family == CHIP_SUMO) ||
2496 (rdev->family == CHIP_SUMO2))
8208441b
AD		 2497 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2498 else
2499 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 2500 if (tmp & CHANSIZE_OVERRIDE) {
2501 chansize = 16;
2502 } else if (tmp & CHANSIZE_MASK) {
2503 chansize = 64;
2504 } else {
2505 chansize = 32;
2506 }
2507 tmp = RREG32(MC_SHARED_CHMAP);
2508 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2509 case 0:
2510 default:
2511 numchan = 1;
2512 break;
2513 case 1:
2514 numchan = 2;
2515 break;
2516 case 2:
2517 numchan = 4;
2518 break;
2519 case 3:
2520 numchan = 8;
2521 break;
2522 }
2523 rdev->mc.vram_width = numchan * chansize;
2524 /* Could aper size report 0 ? */
01d73a69
JC		 2525 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2526 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 2527 /* Setup GPU memory space */
05b3ef69
AD		 2528 if ((rdev->family == CHIP_PALM) ||
2529 (rdev->family == CHIP_SUMO) ||
2530 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 2531 /* size in bytes on fusion */
2532 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2533 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2534 } else {
05b3ef69 2535 /* size in MB on evergreen/cayman/tn */
6eb18f8b
AD		 2536 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2537 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2538 }
51e5fcd3 2539 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 2540 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 2541 radeon_update_bandwidth_info(rdev);
2542
bcc1c2a1
AD		 2543 return 0;
2544}
d594e46a 2545
187e3593 2546void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 2547{
64c56e8c 2548 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 2549 RREG32(GRBM_STATUS));
64c56e8c 2550 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 2551 RREG32(GRBM_STATUS_SE0));
64c56e8c 2552 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 2553 RREG32(GRBM_STATUS_SE1));
64c56e8c 2554 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 2555 RREG32(SRBM_STATUS));
a65a4369
AD		 2556 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
2557 RREG32(SRBM_STATUS2));
440a7cd8
JG		 2558 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
2559 RREG32(CP_STALLED_STAT1));
2560 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
2561 RREG32(CP_STALLED_STAT2));
2562 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
2563 RREG32(CP_BUSY_STAT));
2564 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
2565 RREG32(CP_STAT));
eaaa6983
JG		 2566 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
2567 RREG32(DMA_STATUS_REG));
168757ea
AD		 2568 if (rdev->family >= CHIP_CAYMAN) {
2569 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
2570 RREG32(DMA_STATUS_REG + 0x800));
2571 }
0ecebb9e
AD		 2572}
2573
168757ea 2574bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 2575{
a65a4369
AD		 2576 u32 crtc_hung = 0;
2577 u32 crtc_status[6];
2578 u32 i, j, tmp;
2579
2580 for (i = 0; i < rdev->num_crtc; i++) {
2581 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
2582 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2583 crtc_hung |= (1 << i);
2584 }
2585 }
2586
2587 for (j = 0; j < 10; j++) {
2588 for (i = 0; i < rdev->num_crtc; i++) {
2589 if (crtc_hung & (1 << i)) {
2590 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2591 if (tmp != crtc_status[i])
2592 crtc_hung &= ~(1 << i);
2593 }
2594 }
2595 if (crtc_hung == 0)
2596 return false;
2597 udelay(100);
2598 }
2599
2600 return true;
2601}
2602
2603static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
2604{
2605 u32 reset_mask = 0;
b7630473 2606 u32 tmp;
0ecebb9e 2607
a65a4369
AD		 2608 /* GRBM_STATUS */
2609 tmp = RREG32(GRBM_STATUS);
2610 if (tmp & (PA_BUSY | SC_BUSY |
2611 SH_BUSY | SX_BUSY |
2612 TA_BUSY | VGT_BUSY |
2613 DB_BUSY | CB_BUSY |
2614 SPI_BUSY | VGT_BUSY_NO_DMA))
2615 reset_mask |= RADEON_RESET_GFX;
2616
2617 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
2618 CP_BUSY | CP_COHERENCY_BUSY))
2619 reset_mask |= RADEON_RESET_CP;
2620
2621 if (tmp & GRBM_EE_BUSY)
2622 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 2623
a65a4369
AD		 2624 /* DMA_STATUS_REG */
2625 tmp = RREG32(DMA_STATUS_REG);
2626 if (!(tmp & DMA_IDLE))
2627 reset_mask |= RADEON_RESET_DMA;
2628
2629 /* SRBM_STATUS2 */
2630 tmp = RREG32(SRBM_STATUS2);
2631 if (tmp & DMA_BUSY)
2632 reset_mask |= RADEON_RESET_DMA;
2633
2634 /* SRBM_STATUS */
2635 tmp = RREG32(SRBM_STATUS);
2636 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
2637 reset_mask |= RADEON_RESET_RLC;
2638
2639 if (tmp & IH_BUSY)
2640 reset_mask |= RADEON_RESET_IH;
2641
2642 if (tmp & SEM_BUSY)
2643 reset_mask |= RADEON_RESET_SEM;
2644
2645 if (tmp & GRBM_RQ_PENDING)
2646 reset_mask |= RADEON_RESET_GRBM;
2647
2648 if (tmp & VMC_BUSY)
2649 reset_mask |= RADEON_RESET_VMC;
2650
2651 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
2652 MCC_BUSY | MCD_BUSY))
2653 reset_mask |= RADEON_RESET_MC;
2654
2655 if (evergreen_is_display_hung(rdev))
2656 reset_mask |= RADEON_RESET_DISPLAY;
2657
2658 /* VM_L2_STATUS */
2659 tmp = RREG32(VM_L2_STATUS);
2660 if (tmp & L2_BUSY)
2661 reset_mask |= RADEON_RESET_VMC;
2662
d808fc88
AD		 2663 /* Skip MC reset as it's mostly likely not hung, just busy */
2664 if (reset_mask & RADEON_RESET_MC) {
2665 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
2666 reset_mask &= ~RADEON_RESET_MC;
2667 }
2668
a65a4369
AD		 2669 return reset_mask;
2670}
2671
2672static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2673{
2674 struct evergreen_mc_save save;
2675 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
2676 u32 tmp;
19fc42ed 2677
0ecebb9e 2678 if (reset_mask == 0)
a65a4369 2679 return;
0ecebb9e
AD		 2680
2681 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
2682
b7630473
AD		 2683 evergreen_print_gpu_status_regs(rdev);
2684
b7630473
AD		 2685 /* Disable CP parsing/prefetching */
2686 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2687
2688 if (reset_mask & RADEON_RESET_DMA) {
2689 /* Disable DMA */
2690 tmp = RREG32(DMA_RB_CNTL);
2691 tmp &= ~DMA_RB_ENABLE;
2692 WREG32(DMA_RB_CNTL, tmp);
2693 }
2694
b21b6e7a
AD		 2695 udelay(50);
2696
2697 evergreen_mc_stop(rdev, &save);
2698 if (evergreen_mc_wait_for_idle(rdev)) {
2699 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2700 }
2701
b7630473
AD		 2702 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
2703 grbm_soft_reset |= SOFT_RESET_DB |
2704 SOFT_RESET_CB |
2705 SOFT_RESET_PA |
2706 SOFT_RESET_SC |
2707 SOFT_RESET_SPI |
2708 SOFT_RESET_SX |
2709 SOFT_RESET_SH |
2710 SOFT_RESET_TC |
2711 SOFT_RESET_TA |
2712 SOFT_RESET_VC |
2713 SOFT_RESET_VGT;
2714 }
2715
2716 if (reset_mask & RADEON_RESET_CP) {
2717 grbm_soft_reset |= SOFT_RESET_CP |
2718 SOFT_RESET_VGT;
2719
2720 srbm_soft_reset |= SOFT_RESET_GRBM;
2721 }
0ecebb9e
AD		 2722
2723 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 2724 srbm_soft_reset |= SOFT_RESET_DMA;
2725
a65a4369
AD		 2726 if (reset_mask & RADEON_RESET_DISPLAY)
2727 srbm_soft_reset |= SOFT_RESET_DC;
2728
2729 if (reset_mask & RADEON_RESET_RLC)
2730 srbm_soft_reset |= SOFT_RESET_RLC;
2731
2732 if (reset_mask & RADEON_RESET_SEM)
2733 srbm_soft_reset |= SOFT_RESET_SEM;
2734
2735 if (reset_mask & RADEON_RESET_IH)
2736 srbm_soft_reset |= SOFT_RESET_IH;
2737
2738 if (reset_mask & RADEON_RESET_GRBM)
2739 srbm_soft_reset |= SOFT_RESET_GRBM;
2740
2741 if (reset_mask & RADEON_RESET_VMC)
2742 srbm_soft_reset |= SOFT_RESET_VMC;
2743
24178ec4
AD		 2744 if (!(rdev->flags & RADEON_IS_IGP)) {
2745 if (reset_mask & RADEON_RESET_MC)
2746 srbm_soft_reset |= SOFT_RESET_MC;
2747 }
a65a4369 2748
b7630473
AD		 2749 if (grbm_soft_reset) {
2750 tmp = RREG32(GRBM_SOFT_RESET);
2751 tmp |= grbm_soft_reset;
2752 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2753 WREG32(GRBM_SOFT_RESET, tmp);
2754 tmp = RREG32(GRBM_SOFT_RESET);
2755
2756 udelay(50);
2757
2758 tmp &= ~grbm_soft_reset;
2759 WREG32(GRBM_SOFT_RESET, tmp);
2760 tmp = RREG32(GRBM_SOFT_RESET);
2761 }
2762
2763 if (srbm_soft_reset) {
2764 tmp = RREG32(SRBM_SOFT_RESET);
2765 tmp |= srbm_soft_reset;
2766 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
2767 WREG32(SRBM_SOFT_RESET, tmp);
2768 tmp = RREG32(SRBM_SOFT_RESET);
2769
2770 udelay(50);
2771
2772 tmp &= ~srbm_soft_reset;
2773 WREG32(SRBM_SOFT_RESET, tmp);
2774 tmp = RREG32(SRBM_SOFT_RESET);
2775 }
0ecebb9e
AD		 2776
2777 /* Wait a little for things to settle down */
2778 udelay(50);
2779
747943ea 2780 evergreen_mc_resume(rdev, &save);
b7630473
AD		 2781 udelay(50);
2782
b7630473 2783 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 2784}
2785
a2d07b74 2786int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 2787{
a65a4369
AD		 2788 u32 reset_mask;
2789
2790 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2791
2792 if (reset_mask)
2793 r600_set_bios_scratch_engine_hung(rdev, true);
2794
2795 evergreen_gpu_soft_reset(rdev, reset_mask);
2796
2797 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2798
2799 if (!reset_mask)
2800 r600_set_bios_scratch_engine_hung(rdev, false);
2801
2802 return 0;
747943ea
AD		 2803}
2804
123bc183
AD		 2805/**
2806 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
2807 *
2808 * @rdev: radeon_device pointer
2809 * @ring: radeon_ring structure holding ring information
2810 *
2811 * Check if the GFX engine is locked up.
2812 * Returns true if the engine appears to be locked up, false if not.
2813 */
2814bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2815{
2816 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2817
2818 if (!(reset_mask & (RADEON_RESET_GFX |
2819 RADEON_RESET_COMPUTE |
2820 RADEON_RESET_CP))) {
2821 radeon_ring_lockup_update(ring);
2822 return false;
2823 }
2824 /* force CP activities */
2825 radeon_ring_force_activity(rdev, ring);
2826 return radeon_ring_test_lockup(rdev, ring);
2827}
2828
2829/**
2830 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
2831 *
2832 * @rdev: radeon_device pointer
2833 * @ring: radeon_ring structure holding ring information
2834 *
2835 * Check if the async DMA engine is locked up.
2836 * Returns true if the engine appears to be locked up, false if not.
2837 */
2838bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2839{
2840 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2841
2842 if (!(reset_mask & RADEON_RESET_DMA)) {
2843 radeon_ring_lockup_update(ring);
2844 return false;
2845 }
2846 /* force ring activities */
2847 radeon_ring_force_activity(rdev, ring);
2848 return radeon_ring_test_lockup(rdev, ring);
2849}
2850
45f9a39b
AD		 2851/* Interrupts */
2852
2853u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2854{
46437057 2855 if (crtc >= rdev->num_crtc)
45f9a39b 2856 return 0;
46437057
AD		 2857 else
2858 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 2859}
2860
2861void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2862{
2863 u32 tmp;
2864
1b37078b
AD		 2865 if (rdev->family >= CHIP_CAYMAN) {
2866 cayman_cp_int_cntl_setup(rdev, 0,
2867 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2868 cayman_cp_int_cntl_setup(rdev, 1, 0);
2869 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 2870 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2871 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 2872 } else
2873 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 2874 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2875 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 2876 WREG32(GRBM_INT_CNTL, 0);
2877 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2878 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 2879 if (rdev->num_crtc >= 4) {
18007401
AD		 2880 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2881 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 2882 }
2883 if (rdev->num_crtc >= 6) {
18007401
AD		 2884 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2885 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2886 }
45f9a39b
AD		 2887
2888 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2889 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 2890 if (rdev->num_crtc >= 4) {
18007401
AD		 2891 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2892 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 2893 }
2894 if (rdev->num_crtc >= 6) {
18007401
AD		 2895 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2896 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2897 }
45f9a39b 2898
05b3ef69
AD		 2899 /* only one DAC on DCE6 */
2900 if (!ASIC_IS_DCE6(rdev))
2901 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 2902 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2903
2904 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2905 WREG32(DC_HPD1_INT_CONTROL, tmp);
2906 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2907 WREG32(DC_HPD2_INT_CONTROL, tmp);
2908 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2909 WREG32(DC_HPD3_INT_CONTROL, tmp);
2910 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2911 WREG32(DC_HPD4_INT_CONTROL, tmp);
2912 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2913 WREG32(DC_HPD5_INT_CONTROL, tmp);
2914 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2915 WREG32(DC_HPD6_INT_CONTROL, tmp);
2916
2917}
2918
2919int evergreen_irq_set(struct radeon_device *rdev)
2920{
2921 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 2922 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 2923 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2924 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 2925 u32 grbm_int_cntl = 0;
6f34be50 2926 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 2927 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 2928 u32 dma_cntl, dma_cntl1 = 0;
45f9a39b
AD		 2929
2930 if (!rdev->irq.installed) {
fce7d61b 2931 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 2932 return -EINVAL;
2933 }
2934 /* don't enable anything if the ih is disabled */
2935 if (!rdev->ih.enabled) {
2936 r600_disable_interrupts(rdev);
2937 /* force the active interrupt state to all disabled */
2938 evergreen_disable_interrupt_state(rdev);
2939 return 0;
2940 }
2941
2942 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2943 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2944 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2945 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2946 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2947 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2948
f122c610
AD		 2949 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2950 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2951 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2952 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2953 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2954 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2955
233d1ad5
AD		 2956 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2957
1b37078b
AD		 2958 if (rdev->family >= CHIP_CAYMAN) {
2959 /* enable CP interrupts on all rings */
736fc37f 2960 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 2961 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2962 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2963 }
736fc37f 2964 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 2965 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
2966 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
2967 }
736fc37f 2968 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 2969 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
2970 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
2971 }
2972 } else {
736fc37f 2973 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 2974 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2975 cp_int_cntl |= RB_INT_ENABLE;
2976 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2977 }
45f9a39b 2978 }
1b37078b 2979
233d1ad5
AD		 2980 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
2981 DRM_DEBUG("r600_irq_set: sw int dma\n");
2982 dma_cntl |= TRAP_ENABLE;
2983 }
2984
f60cbd11
AD		 2985 if (rdev->family >= CHIP_CAYMAN) {
2986 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2987 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
2988 DRM_DEBUG("r600_irq_set: sw int dma1\n");
2989 dma_cntl1 |= TRAP_ENABLE;
2990 }
2991 }
2992
6f34be50 2993 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 2994 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 2995 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2996 crtc1 |= VBLANK_INT_MASK;
2997 }
6f34be50 2998 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 2999 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 3000 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
3001 crtc2 |= VBLANK_INT_MASK;
3002 }
6f34be50 3003 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 3004 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 3005 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
3006 crtc3 |= VBLANK_INT_MASK;
3007 }
6f34be50 3008 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 3009 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 3010 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
3011 crtc4 |= VBLANK_INT_MASK;
3012 }
6f34be50 3013 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 3014 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 3015 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
3016 crtc5 |= VBLANK_INT_MASK;
3017 }
6f34be50 3018 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 3019 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 3020 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
3021 crtc6 |= VBLANK_INT_MASK;
3022 }
3023 if (rdev->irq.hpd[0]) {
3024 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
3025 hpd1 |= DC_HPDx_INT_EN;
3026 }
3027 if (rdev->irq.hpd[1]) {
3028 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
3029 hpd2 |= DC_HPDx_INT_EN;
3030 }
3031 if (rdev->irq.hpd[2]) {
3032 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
3033 hpd3 |= DC_HPDx_INT_EN;
3034 }
3035 if (rdev->irq.hpd[3]) {
3036 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
3037 hpd4 |= DC_HPDx_INT_EN;
3038 }
3039 if (rdev->irq.hpd[4]) {
3040 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
3041 hpd5 |= DC_HPDx_INT_EN;
3042 }
3043 if (rdev->irq.hpd[5]) {
3044 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
3045 hpd6 |= DC_HPDx_INT_EN;
3046 }
f122c610
AD		 3047 if (rdev->irq.afmt[0]) {
3048 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
3049 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3050 }
3051 if (rdev->irq.afmt[1]) {
3052 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
3053 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3054 }
3055 if (rdev->irq.afmt[2]) {
3056 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
3057 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3058 }
3059 if (rdev->irq.afmt[3]) {
3060 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
3061 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3062 }
3063 if (rdev->irq.afmt[4]) {
3064 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
3065 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3066 }
3067 if (rdev->irq.afmt[5]) {
3068 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
3069 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3070 }
45f9a39b 3071
1b37078b
AD		 3072 if (rdev->family >= CHIP_CAYMAN) {
3073 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
3074 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
3075 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
3076 } else
3077 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 3078
3079 WREG32(DMA_CNTL, dma_cntl);
3080
f60cbd11
AD		 3081 if (rdev->family >= CHIP_CAYMAN)
3082 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
3083
2031f77c 3084 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 3085
3086 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
3087 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 3088 if (rdev->num_crtc >= 4) {
18007401
AD		 3089 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
3090 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 3091 }
3092 if (rdev->num_crtc >= 6) {
18007401
AD		 3093 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
3094 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
3095 }
45f9a39b 3096
6f34be50
AD		 3097 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
3098 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 3099 if (rdev->num_crtc >= 4) {
3100 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
3101 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
3102 }
3103 if (rdev->num_crtc >= 6) {
3104 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
3105 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
3106 }
6f34be50 3107
45f9a39b
AD		 3108 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3109 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3110 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3111 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3112 WREG32(DC_HPD5_INT_CONTROL, hpd5);
3113 WREG32(DC_HPD6_INT_CONTROL, hpd6);
3114
f122c610
AD		 3115 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
3116 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
3117 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
3118 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
3119 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
3120 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
3121
bcc1c2a1
AD		 3122 return 0;
3123}
3124
cbdd4501 3125static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 3126{
3127 u32 tmp;
3128
6f34be50
AD		 3129 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3130 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3131 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
3132 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
3133 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
3134 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
3135 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3136 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 3137 if (rdev->num_crtc >= 4) {
3138 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3139 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3140 }
3141 if (rdev->num_crtc >= 6) {
3142 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3143 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3144 }
6f34be50 3145
f122c610
AD		 3146 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3147 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
3148 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3149 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3150 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3151 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3152
6f34be50
AD		 3153 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
3154 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3155 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
3156 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 3157 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 3158 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 3159 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 3160 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 3161 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 3162 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 3163 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 3164 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
3165
b7eff394
AD		 3166 if (rdev->num_crtc >= 4) {
3167 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
3168 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3169 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
3170 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3171 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
3172 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
3173 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
3174 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
3175 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
3176 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
3177 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
3178 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
3179 }
3180
3181 if (rdev->num_crtc >= 6) {
3182 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
3183 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3184 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
3185 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3186 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
3187 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
3188 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
3189 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
3190 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
3191 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
3192 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
3193 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
3194 }
45f9a39b 3195
6f34be50 3196 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 3197 tmp = RREG32(DC_HPD1_INT_CONTROL);
3198 tmp |= DC_HPDx_INT_ACK;
3199 WREG32(DC_HPD1_INT_CONTROL, tmp);
3200 }
6f34be50 3201 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 3202 tmp = RREG32(DC_HPD2_INT_CONTROL);
3203 tmp |= DC_HPDx_INT_ACK;
3204 WREG32(DC_HPD2_INT_CONTROL, tmp);
3205 }
6f34be50 3206 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 3207 tmp = RREG32(DC_HPD3_INT_CONTROL);
3208 tmp |= DC_HPDx_INT_ACK;
3209 WREG32(DC_HPD3_INT_CONTROL, tmp);
3210 }
6f34be50 3211 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 3212 tmp = RREG32(DC_HPD4_INT_CONTROL);
3213 tmp |= DC_HPDx_INT_ACK;
3214 WREG32(DC_HPD4_INT_CONTROL, tmp);
3215 }
6f34be50 3216 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 3217 tmp = RREG32(DC_HPD5_INT_CONTROL);
3218 tmp |= DC_HPDx_INT_ACK;
3219 WREG32(DC_HPD5_INT_CONTROL, tmp);
3220 }
6f34be50 3221 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 3222 tmp = RREG32(DC_HPD5_INT_CONTROL);
3223 tmp |= DC_HPDx_INT_ACK;
3224 WREG32(DC_HPD6_INT_CONTROL, tmp);
3225 }
f122c610
AD		 3226 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3227 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
3228 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3229 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
3230 }
3231 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3232 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
3233 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3234 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
3235 }
3236 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3237 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
3238 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3239 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
3240 }
3241 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3242 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
3243 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3244 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
3245 }
3246 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3247 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
3248 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3249 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
3250 }
3251 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3252 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
3253 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3254 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
3255 }
45f9a39b
AD		 3256}
3257
1109ca09 3258static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 3259{
45f9a39b
AD		 3260 r600_disable_interrupts(rdev);
3261 /* Wait and acknowledge irq */
3262 mdelay(1);
6f34be50 3263 evergreen_irq_ack(rdev);
45f9a39b
AD		 3264 evergreen_disable_interrupt_state(rdev);
3265}
3266
755d819e 3267void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 3268{
3269 evergreen_irq_disable(rdev);
3270 r600_rlc_stop(rdev);
3271}
3272
cbdd4501 3273static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 3274{
3275 u32 wptr, tmp;
3276
724c80e1 3277 if (rdev->wb.enabled)
204ae24d 3278 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 3279 else
3280 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 3281
3282 if (wptr & RB_OVERFLOW) {
3283 /* When a ring buffer overflow happen start parsing interrupt
3284 * from the last not overwritten vector (wptr + 16). Hopefully
3285 * this should allow us to catchup.
3286 */
3287 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3288 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3289 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3290 tmp = RREG32(IH_RB_CNTL);
3291 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3292 WREG32(IH_RB_CNTL, tmp);
3293 }
3294 return (wptr & rdev->ih.ptr_mask);
3295}
3296
3297int evergreen_irq_process(struct radeon_device *rdev)
3298{
682f1a54
DA		 3299 u32 wptr;
3300 u32 rptr;
45f9a39b
AD		 3301 u32 src_id, src_data;
3302 u32 ring_index;
45f9a39b 3303 bool queue_hotplug = false;
f122c610 3304 bool queue_hdmi = false;
45f9a39b 3305
682f1a54 3306 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 3307 return IRQ_NONE;
3308
682f1a54 3309 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 3310
3311restart_ih:
3312 /* is somebody else already processing irqs? */
3313 if (atomic_xchg(&rdev->ih.lock, 1))
3314 return IRQ_NONE;
3315
682f1a54
DA		 3316 rptr = rdev->ih.rptr;
3317 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 3318
964f6645
BH		 3319 /* Order reading of wptr vs. reading of IH ring data */
3320 rmb();
3321
45f9a39b 3322 /* display interrupts */
6f34be50 3323 evergreen_irq_ack(rdev);
45f9a39b 3324
45f9a39b
AD		 3325 while (rptr != wptr) {
3326 /* wptr/rptr are in bytes! */
3327 ring_index = rptr / 4;
0f234f5f
AD		 3328 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3329 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 3330
3331 switch (src_id) {
3332 case 1: /* D1 vblank/vline */
3333 switch (src_data) {
3334 case 0: /* D1 vblank */
6f34be50 3335 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 3336 if (rdev->irq.crtc_vblank_int[0]) {
3337 drm_handle_vblank(rdev->ddev, 0);
3338 rdev->pm.vblank_sync = true;
3339 wake_up(&rdev->irq.vblank_queue);
3340 }
736fc37f 3341 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 3342 radeon_crtc_handle_flip(rdev, 0);
6f34be50 3343 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 3344 DRM_DEBUG("IH: D1 vblank\n");
3345 }
3346 break;
3347 case 1: /* D1 vline */
6f34be50
AD		 3348 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
3349 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 3350 DRM_DEBUG("IH: D1 vline\n");
3351 }
3352 break;
3353 default:
3354 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3355 break;
3356 }
3357 break;
3358 case 2: /* D2 vblank/vline */
3359 switch (src_data) {
3360 case 0: /* D2 vblank */
6f34be50 3361 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 3362 if (rdev->irq.crtc_vblank_int[1]) {
3363 drm_handle_vblank(rdev->ddev, 1);
3364 rdev->pm.vblank_sync = true;
3365 wake_up(&rdev->irq.vblank_queue);
3366 }
736fc37f 3367 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 3368 radeon_crtc_handle_flip(rdev, 1);
6f34be50 3369 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 3370 DRM_DEBUG("IH: D2 vblank\n");
3371 }
3372 break;
3373 case 1: /* D2 vline */
6f34be50
AD		 3374 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
3375 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 3376 DRM_DEBUG("IH: D2 vline\n");
3377 }
3378 break;
3379 default:
3380 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3381 break;
3382 }
3383 break;
3384 case 3: /* D3 vblank/vline */
3385 switch (src_data) {
3386 case 0: /* D3 vblank */
6f34be50
AD		 3387 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
3388 if (rdev->irq.crtc_vblank_int[2]) {
3389 drm_handle_vblank(rdev->ddev, 2);
3390 rdev->pm.vblank_sync = true;
3391 wake_up(&rdev->irq.vblank_queue);
3392 }
736fc37f 3393 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 3394 radeon_crtc_handle_flip(rdev, 2);
3395 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 3396 DRM_DEBUG("IH: D3 vblank\n");
3397 }
3398 break;
3399 case 1: /* D3 vline */
6f34be50
AD		 3400 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
3401 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 3402 DRM_DEBUG("IH: D3 vline\n");
3403 }
3404 break;
3405 default:
3406 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3407 break;
3408 }
3409 break;
3410 case 4: /* D4 vblank/vline */
3411 switch (src_data) {
3412 case 0: /* D4 vblank */
6f34be50
AD		 3413 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
3414 if (rdev->irq.crtc_vblank_int[3]) {
3415 drm_handle_vblank(rdev->ddev, 3);
3416 rdev->pm.vblank_sync = true;
3417 wake_up(&rdev->irq.vblank_queue);
3418 }
736fc37f 3419 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 3420 radeon_crtc_handle_flip(rdev, 3);
3421 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 3422 DRM_DEBUG("IH: D4 vblank\n");
3423 }
3424 break;
3425 case 1: /* D4 vline */
6f34be50
AD		 3426 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
3427 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 3428 DRM_DEBUG("IH: D4 vline\n");
3429 }
3430 break;
3431 default:
3432 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3433 break;
3434 }
3435 break;
3436 case 5: /* D5 vblank/vline */
3437 switch (src_data) {
3438 case 0: /* D5 vblank */
6f34be50
AD		 3439 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
3440 if (rdev->irq.crtc_vblank_int[4]) {
3441 drm_handle_vblank(rdev->ddev, 4);
3442 rdev->pm.vblank_sync = true;
3443 wake_up(&rdev->irq.vblank_queue);
3444 }
736fc37f 3445 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 3446 radeon_crtc_handle_flip(rdev, 4);
3447 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 3448 DRM_DEBUG("IH: D5 vblank\n");
3449 }
3450 break;
3451 case 1: /* D5 vline */
6f34be50
AD		 3452 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
3453 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 3454 DRM_DEBUG("IH: D5 vline\n");
3455 }
3456 break;
3457 default:
3458 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3459 break;
3460 }
3461 break;
3462 case 6: /* D6 vblank/vline */
3463 switch (src_data) {
3464 case 0: /* D6 vblank */
6f34be50
AD		 3465 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
3466 if (rdev->irq.crtc_vblank_int[5]) {
3467 drm_handle_vblank(rdev->ddev, 5);
3468 rdev->pm.vblank_sync = true;
3469 wake_up(&rdev->irq.vblank_queue);
3470 }
736fc37f 3471 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 3472 radeon_crtc_handle_flip(rdev, 5);
3473 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 3474 DRM_DEBUG("IH: D6 vblank\n");
3475 }
3476 break;
3477 case 1: /* D6 vline */
6f34be50
AD		 3478 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
3479 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 3480 DRM_DEBUG("IH: D6 vline\n");
3481 }
3482 break;
3483 default:
3484 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3485 break;
3486 }
3487 break;
3488 case 42: /* HPD hotplug */
3489 switch (src_data) {
3490 case 0:
6f34be50
AD		 3491 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3492 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 3493 queue_hotplug = true;
3494 DRM_DEBUG("IH: HPD1\n");
3495 }
3496 break;
3497 case 1:
6f34be50
AD		 3498 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3499 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 3500 queue_hotplug = true;
3501 DRM_DEBUG("IH: HPD2\n");
3502 }
3503 break;
3504 case 2:
6f34be50
AD		 3505 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3506 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 3507 queue_hotplug = true;
3508 DRM_DEBUG("IH: HPD3\n");
3509 }
3510 break;
3511 case 3:
6f34be50
AD		 3512 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3513 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 3514 queue_hotplug = true;
3515 DRM_DEBUG("IH: HPD4\n");
3516 }
3517 break;
3518 case 4:
6f34be50
AD		 3519 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3520 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 3521 queue_hotplug = true;
3522 DRM_DEBUG("IH: HPD5\n");
3523 }
3524 break;
3525 case 5:
6f34be50
AD		 3526 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3527 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 3528 queue_hotplug = true;
3529 DRM_DEBUG("IH: HPD6\n");
3530 }
3531 break;
3532 default:
3533 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3534 break;
3535 }
3536 break;
f122c610
AD		 3537 case 44: /* hdmi */
3538 switch (src_data) {
3539 case 0:
3540 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3541 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
3542 queue_hdmi = true;
3543 DRM_DEBUG("IH: HDMI0\n");
3544 }
3545 break;
3546 case 1:
3547 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3548 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
3549 queue_hdmi = true;
3550 DRM_DEBUG("IH: HDMI1\n");
3551 }
3552 break;
3553 case 2:
3554 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3555 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
3556 queue_hdmi = true;
3557 DRM_DEBUG("IH: HDMI2\n");
3558 }
3559 break;
3560 case 3:
3561 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3562 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
3563 queue_hdmi = true;
3564 DRM_DEBUG("IH: HDMI3\n");
3565 }
3566 break;
3567 case 4:
3568 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3569 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
3570 queue_hdmi = true;
3571 DRM_DEBUG("IH: HDMI4\n");
3572 }
3573 break;
3574 case 5:
3575 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3576 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
3577 queue_hdmi = true;
3578 DRM_DEBUG("IH: HDMI5\n");
3579 }
3580 break;
3581 default:
3582 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
3583 break;
3584 }
f2ba57b5
CK		 3585 case 124: /* UVD */
3586 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
3587 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 3588 break;
ae133a11
CK		 3589 case 146:
3590 case 147:
3591 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
3592 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3593 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3594 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3595 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3596 /* reset addr and status */
3597 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
3598 break;
45f9a39b
AD		 3599 case 176: /* CP_INT in ring buffer */
3600 case 177: /* CP_INT in IB1 */
3601 case 178: /* CP_INT in IB2 */
3602 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 3603 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 3604 break;
3605 case 181: /* CP EOP event */
3606 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 3607 if (rdev->family >= CHIP_CAYMAN) {
3608 switch (src_data) {
3609 case 0:
3610 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3611 break;
3612 case 1:
3613 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3614 break;
3615 case 2:
3616 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3617 break;
3618 }
3619 } else
3620 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 3621 break;
233d1ad5
AD		 3622 case 224: /* DMA trap event */
3623 DRM_DEBUG("IH: DMA trap\n");
3624 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
3625 break;
2031f77c 3626 case 233: /* GUI IDLE */
303c805c 3627 DRM_DEBUG("IH: GUI idle\n");
2031f77c 3628 break;
f60cbd11
AD		 3629 case 244: /* DMA trap event */
3630 if (rdev->family >= CHIP_CAYMAN) {
3631 DRM_DEBUG("IH: DMA1 trap\n");
3632 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
3633 }
3634 break;
45f9a39b
AD		 3635 default:
3636 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3637 break;
3638 }
3639
3640 /* wptr/rptr are in bytes! */
3641 rptr += 16;
3642 rptr &= rdev->ih.ptr_mask;
3643 }
45f9a39b 3644 if (queue_hotplug)
32c87fca 3645 schedule_work(&rdev->hotplug_work);
f122c610
AD		 3646 if (queue_hdmi)
3647 schedule_work(&rdev->audio_work);
45f9a39b
AD		 3648 rdev->ih.rptr = rptr;
3649 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 3650 atomic_set(&rdev->ih.lock, 0);
3651
3652 /* make sure wptr hasn't changed while processing */
3653 wptr = evergreen_get_ih_wptr(rdev);
3654 if (wptr != rptr)
3655 goto restart_ih;
3656
45f9a39b
AD		 3657 return IRQ_HANDLED;
3658}
3659
233d1ad5
AD		 3660/**
3661 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
3662 *
3663 * @rdev: radeon_device pointer
3664 * @fence: radeon fence object
3665 *
3666 * Add a DMA fence packet to the ring to write
3667 * the fence seq number and DMA trap packet to generate
3668 * an interrupt if needed (evergreen-SI).
3669 */
3670void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
3671 struct radeon_fence *fence)
3672{
3673 struct radeon_ring *ring = &rdev->ring[fence->ring];
3674 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3675 /* write the fence */
0fcb6155 3676 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 3677 radeon_ring_write(ring, addr & 0xfffffffc);
3678 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
3679 radeon_ring_write(ring, fence->seq);
3680 /* generate an interrupt */
0fcb6155 3681 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 3682 /* flush HDP */
0fcb6155 3683 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 3684 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 3685 radeon_ring_write(ring, 1);
3686}
3687
3688/**
3689 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
3690 *
3691 * @rdev: radeon_device pointer
3692 * @ib: IB object to schedule
3693 *
3694 * Schedule an IB in the DMA ring (evergreen).
3695 */
3696void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
3697 struct radeon_ib *ib)
3698{
3699 struct radeon_ring *ring = &rdev->ring[ib->ring];
3700
3701 if (rdev->wb.enabled) {
3702 u32 next_rptr = ring->wptr + 4;
3703 while ((next_rptr & 7) != 5)
3704 next_rptr++;
3705 next_rptr += 3;
0fcb6155 3706 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 3707 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3708 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3709 radeon_ring_write(ring, next_rptr);
3710 }
3711
3712 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3713 * Pad as necessary with NOPs.
3714 */
3715 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 3716 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3717 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 3718 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3719 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3720
3721}
3722
3723/**
3724 * evergreen_copy_dma - copy pages using the DMA engine
3725 *
3726 * @rdev: radeon_device pointer
3727 * @src_offset: src GPU address
3728 * @dst_offset: dst GPU address
3729 * @num_gpu_pages: number of GPU pages to xfer
3730 * @fence: radeon fence object
3731 *
3732 * Copy GPU paging using the DMA engine (evergreen-cayman).
3733 * Used by the radeon ttm implementation to move pages if
3734 * registered as the asic copy callback.
3735 */
3736int evergreen_copy_dma(struct radeon_device *rdev,
3737 uint64_t src_offset, uint64_t dst_offset,
3738 unsigned num_gpu_pages,
3739 struct radeon_fence **fence)
3740{
3741 struct radeon_semaphore *sem = NULL;
3742 int ring_index = rdev->asic->copy.dma_ring_index;
3743 struct radeon_ring *ring = &rdev->ring[ring_index];
3744 u32 size_in_dw, cur_size_in_dw;
3745 int i, num_loops;
3746 int r = 0;
3747
3748 r = radeon_semaphore_create(rdev, &sem);
3749 if (r) {
3750 DRM_ERROR("radeon: moving bo (%d).\n", r);
3751 return r;
3752 }
3753
3754 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
3755 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
3756 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
3757 if (r) {
3758 DRM_ERROR("radeon: moving bo (%d).\n", r);
3759 radeon_semaphore_free(rdev, &sem, NULL);
3760 return r;
3761 }
3762
3763 if (radeon_fence_need_sync(*fence, ring->idx)) {
3764 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3765 ring->idx);
3766 radeon_fence_note_sync(*fence, ring->idx);
3767 } else {
3768 radeon_semaphore_free(rdev, &sem, NULL);
3769 }
3770
3771 for (i = 0; i < num_loops; i++) {
3772 cur_size_in_dw = size_in_dw;
3773 if (cur_size_in_dw > 0xFFFFF)
3774 cur_size_in_dw = 0xFFFFF;
3775 size_in_dw -= cur_size_in_dw;
0fcb6155 3776 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 3777 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3778 radeon_ring_write(ring, src_offset & 0xfffffffc);
3779 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
3780 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
3781 src_offset += cur_size_in_dw * 4;
3782 dst_offset += cur_size_in_dw * 4;
3783 }
3784
3785 r = radeon_fence_emit(rdev, fence, ring->idx);
3786 if (r) {
3787 radeon_ring_unlock_undo(rdev, ring);
3788 return r;
3789 }
3790
3791 radeon_ring_unlock_commit(rdev, ring);
3792 radeon_semaphore_free(rdev, &sem, *fence);
3793
3794 return r;
3795}
3796
bcc1c2a1
AD		 3797static int evergreen_startup(struct radeon_device *rdev)
3798{
f2ba57b5 3799 struct radeon_ring *ring;
bcc1c2a1
AD		 3800 int r;
3801
9e46a48d 3802 /* enable pcie gen2 link */
cd54033a 3803 evergreen_pcie_gen2_enable(rdev);
9e46a48d 3804
0af62b01
AD		 3805 if (ASIC_IS_DCE5(rdev)) {
3806 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3807 r = ni_init_microcode(rdev);
3808 if (r) {
3809 DRM_ERROR("Failed to load firmware!\n");
3810 return r;
3811 }
3812 }
755d819e 3813 r = ni_mc_load_microcode(rdev);
bcc1c2a1 3814 if (r) {
0af62b01 3815 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 3816 return r;
3817 }
0af62b01
AD		 3818 } else {
3819 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3820 r = r600_init_microcode(rdev);
3821 if (r) {
3822 DRM_ERROR("Failed to load firmware!\n");
3823 return r;
3824 }
3825 }
bcc1c2a1 3826 }
fe251e2f 3827
16cdf04d
AD		 3828 r = r600_vram_scratch_init(rdev);
3829 if (r)
3830 return r;
3831
bcc1c2a1 3832 evergreen_mc_program(rdev);
bcc1c2a1 3833 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 3834 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 3835 } else {
3836 r = evergreen_pcie_gart_enable(rdev);
3837 if (r)
3838 return r;
3839 }
bcc1c2a1 3840 evergreen_gpu_init(rdev);
bcc1c2a1 3841
d7ccd8fc 3842 r = evergreen_blit_init(rdev);
bcc1c2a1 3843 if (r) {
fb3d9e97 3844 r600_blit_fini(rdev);
27cd7769 3845 rdev->asic->copy.copy = NULL;
d7ccd8fc 3846 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 3847 }
3848
724c80e1
AD		 3849 /* allocate wb buffer */
3850 r = radeon_wb_init(rdev);
3851 if (r)
3852 return r;
3853
30eb77f4
JG		 3854 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3855 if (r) {
3856 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3857 return r;
3858 }
3859
233d1ad5
AD		 3860 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3861 if (r) {
3862 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3863 return r;
3864 }
3865
f2ba57b5
CK		 3866 r = rv770_uvd_resume(rdev);
3867 if (!r) {
3868 r = radeon_fence_driver_start_ring(rdev,
3869 R600_RING_TYPE_UVD_INDEX);
3870 if (r)
3871 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
3872 }
3873
3874 if (r)
3875 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
3876
bcc1c2a1
AD		 3877 /* Enable IRQ */
3878 r = r600_irq_init(rdev);
3879 if (r) {
3880 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3881 radeon_irq_kms_fini(rdev);
3882 return r;
3883 }
45f9a39b 3884 evergreen_irq_set(rdev);
bcc1c2a1 3885
f2ba57b5 3886 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 3887 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 3888 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3889 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 3890 if (r)
3891 return r;
233d1ad5
AD		 3892
3893 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3894 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3895 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 3896 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 3897 if (r)
3898 return r;
3899
bcc1c2a1
AD		 3900 r = evergreen_cp_load_microcode(rdev);
3901 if (r)
3902 return r;
fe251e2f 3903 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 3904 if (r)
3905 return r;
3906 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 3907 if (r)
3908 return r;
fe251e2f 3909
f2ba57b5
CK		 3910 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
3911 if (ring->ring_size) {
3912 r = radeon_ring_init(rdev, ring, ring->ring_size,
3913 R600_WB_UVD_RPTR_OFFSET,
3914 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
3915 0, 0xfffff, RADEON_CP_PACKET2);
3916 if (!r)
3917 r = r600_uvd_init(rdev);
3918
3919 if (r)
3920 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
3921 }
3922
2898c348
CK		 3923 r = radeon_ib_pool_init(rdev);
3924 if (r) {
3925 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 3926 return r;
2898c348 3927 }
b15ba512 3928
69d2ae57
RM		 3929 r = r600_audio_init(rdev);
3930 if (r) {
3931 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 3932 return r;
3933 }
3934
bcc1c2a1
AD		 3935 return 0;
3936}
3937
3938int evergreen_resume(struct radeon_device *rdev)
3939{
3940 int r;
3941
86f5c9ed
AD		 3942 /* reset the asic, the gfx blocks are often in a bad state
3943 * after the driver is unloaded or after a resume
3944 */
3945 if (radeon_asic_reset(rdev))
3946 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 3947 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3948 * posting will perform necessary task to bring back GPU into good
3949 * shape.
3950 */
3951 /* post card */
3952 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 3953
b15ba512 3954 rdev->accel_working = true;
bcc1c2a1
AD		 3955 r = evergreen_startup(rdev);
3956 if (r) {
755d819e 3957 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 3958 rdev->accel_working = false;
bcc1c2a1
AD		 3959 return r;
3960 }
fe251e2f 3961
bcc1c2a1
AD		 3962 return r;
3963
3964}
3965
3966int evergreen_suspend(struct radeon_device *rdev)
3967{
69d2ae57 3968 r600_audio_fini(rdev);
f2ba57b5 3969 radeon_uvd_suspend(rdev);
bcc1c2a1 3970 r700_cp_stop(rdev);
233d1ad5 3971 r600_dma_stop(rdev);
f2ba57b5 3972 r600_uvd_rbc_stop(rdev);
45f9a39b 3973 evergreen_irq_suspend(rdev);
724c80e1 3974 radeon_wb_disable(rdev);
bcc1c2a1 3975 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 3976
3977 return 0;
3978}
3979
bcc1c2a1
AD		 3980/* Plan is to move initialization in that function and use
3981 * helper function so that radeon_device_init pretty much
3982 * do nothing more than calling asic specific function. This
3983 * should also allow to remove a bunch of callback function
3984 * like vram_info.
3985 */
3986int evergreen_init(struct radeon_device *rdev)
3987{
3988 int r;
3989
bcc1c2a1
AD		 3990 /* Read BIOS */
3991 if (!radeon_get_bios(rdev)) {
3992 if (ASIC_IS_AVIVO(rdev))
3993 return -EINVAL;
3994 }
3995 /* Must be an ATOMBIOS */
3996 if (!rdev->is_atom_bios) {
755d819e 3997 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 3998 return -EINVAL;
3999 }
4000 r = radeon_atombios_init(rdev);
4001 if (r)
4002 return r;
86f5c9ed
AD		 4003 /* reset the asic, the gfx blocks are often in a bad state
4004 * after the driver is unloaded or after a resume
4005 */
4006 if (radeon_asic_reset(rdev))
4007 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 4008 /* Post card if necessary */
fd909c37 4009 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 4010 if (!rdev->bios) {
4011 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
4012 return -EINVAL;
4013 }
4014 DRM_INFO("GPU not posted. posting now...\n");
4015 atom_asic_init(rdev->mode_info.atom_context);
4016 }
4017 /* Initialize scratch registers */
4018 r600_scratch_init(rdev);
4019 /* Initialize surface registers */
4020 radeon_surface_init(rdev);
4021 /* Initialize clocks */
4022 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 4023 /* Fence driver */
4024 r = radeon_fence_driver_init(rdev);
4025 if (r)
4026 return r;
d594e46a
JG		 4027 /* initialize AGP */
4028 if (rdev->flags & RADEON_IS_AGP) {
4029 r = radeon_agp_init(rdev);
4030 if (r)
4031 radeon_agp_disable(rdev);
4032 }
4033 /* initialize memory controller */
bcc1c2a1
AD		 4034 r = evergreen_mc_init(rdev);
4035 if (r)
4036 return r;
4037 /* Memory manager */
4038 r = radeon_bo_init(rdev);
4039 if (r)
4040 return r;
45f9a39b 4041
bcc1c2a1
AD		 4042 r = radeon_irq_kms_init(rdev);
4043 if (r)
4044 return r;
4045
e32eb50d
CK		 4046 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
4047 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 4048
233d1ad5
AD		 4049 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
4050 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
4051
f2ba57b5
CK		 4052 r = radeon_uvd_init(rdev);
4053 if (!r) {
4054 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4055 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
4056 4096);
4057 }
4058
bcc1c2a1
AD		 4059 rdev->ih.ring_obj = NULL;
4060 r600_ih_ring_init(rdev, 64 * 1024);
4061
4062 r = r600_pcie_gart_init(rdev);
4063 if (r)
4064 return r;
0fcdb61e 4065
148a03bc 4066 rdev->accel_working = true;
bcc1c2a1
AD		 4067 r = evergreen_startup(rdev);
4068 if (r) {
fe251e2f
AD		 4069 dev_err(rdev->dev, "disabling GPU acceleration\n");
4070 r700_cp_fini(rdev);
233d1ad5 4071 r600_dma_fini(rdev);
fe251e2f 4072 r600_irq_fini(rdev);
724c80e1 4073 radeon_wb_fini(rdev);
2898c348 4074 radeon_ib_pool_fini(rdev);
fe251e2f 4075 radeon_irq_kms_fini(rdev);
0fcdb61e 4076 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 4077 rdev->accel_working = false;
4078 }
77e00f2e
AD		 4079
4080 /* Don't start up if the MC ucode is missing on BTC parts.
4081 * The default clocks and voltages before the MC ucode
4082 * is loaded are not suffient for advanced operations.
4083 */
4084 if (ASIC_IS_DCE5(rdev)) {
4085 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
4086 DRM_ERROR("radeon: MC ucode required for NI+.\n");
4087 return -EINVAL;
4088 }
4089 }
4090
bcc1c2a1
AD		 4091 return 0;
4092}
4093
4094void evergreen_fini(struct radeon_device *rdev)
4095{
69d2ae57 4096 r600_audio_fini(rdev);
fb3d9e97 4097 r600_blit_fini(rdev);
45f9a39b 4098 r700_cp_fini(rdev);
233d1ad5 4099 r600_dma_fini(rdev);
bcc1c2a1 4100 r600_irq_fini(rdev);
724c80e1 4101 radeon_wb_fini(rdev);
2898c348 4102 radeon_ib_pool_fini(rdev);
bcc1c2a1 4103 radeon_irq_kms_fini(rdev);
bcc1c2a1 4104 evergreen_pcie_gart_fini(rdev);
f2ba57b5 4105 radeon_uvd_fini(rdev);
16cdf04d 4106 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 4107 radeon_gem_fini(rdev);
4108 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 4109 radeon_agp_fini(rdev);
4110 radeon_bo_fini(rdev);
4111 radeon_atombios_fini(rdev);
4112 kfree(rdev->bios);
4113 rdev->bios = NULL;
bcc1c2a1 4114}
9e46a48d 4115
b07759bf 4116void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 4117{
197bbb3d
DA		 4118 u32 link_width_cntl, speed_cntl, mask;
4119 int ret;
9e46a48d 4120
d42dd579
AD		 4121 if (radeon_pcie_gen2 == 0)
4122 return;
4123
9e46a48d
AD		 4124 if (rdev->flags & RADEON_IS_IGP)
4125 return;
4126
4127 if (!(rdev->flags & RADEON_IS_PCIE))
4128 return;
4129
4130 /* x2 cards have a special sequence */
4131 if (ASIC_IS_X2(rdev))
4132 return;
4133
197bbb3d
DA		 4134 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4135 if (ret != 0)
4136 return;
4137
4138 if (!(mask & DRM_PCIE_SPEED_50))
4139 return;
4140
492d2b61 4141 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 4142 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4143 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4144 return;
4145 }
4146
197bbb3d
DA		 4147 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4148
9e46a48d
AD		 4149 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
4150 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4151
492d2b61 4152 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 4153 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 4154 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 4155
492d2b61 4156 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4157 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 4158 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 4159
492d2b61 4160 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4161 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 4162 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 4163
492d2b61 4164 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4165 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 4166 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 4167
492d2b61 4168 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4169 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 4170 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4171
4172 } else {
492d2b61 4173 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 4174 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4175 if (1)
4176 link_width_cntl |= LC_UPCONFIGURE_DIS;
4177 else
4178 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 4179 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4180 }
4181}
Linux kernel - Deliverables
This page took 0.474824 seconds and 5 git commands to generate.