2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
37 #define EVERGREEN_PFP_UCODE_SIZE 1120
38 #define EVERGREEN_PM4_UCODE_SIZE 1376
40 static const u32 crtc_offsets
[6] =
42 EVERGREEN_CRTC0_REGISTER_OFFSET
,
43 EVERGREEN_CRTC1_REGISTER_OFFSET
,
44 EVERGREEN_CRTC2_REGISTER_OFFSET
,
45 EVERGREEN_CRTC3_REGISTER_OFFSET
,
46 EVERGREEN_CRTC4_REGISTER_OFFSET
,
47 EVERGREEN_CRTC5_REGISTER_OFFSET
50 static void evergreen_gpu_init(struct radeon_device
*rdev
);
51 void evergreen_fini(struct radeon_device
*rdev
);
52 void evergreen_pcie_gen2_enable(struct radeon_device
*rdev
);
53 extern void cayman_cp_int_cntl_setup(struct radeon_device
*rdev
,
54 int ring
, u32 cp_int_cntl
);
56 void evergreen_tiling_fields(unsigned tiling_flags
, unsigned *bankw
,
57 unsigned *bankh
, unsigned *mtaspect
,
60 *bankw
= (tiling_flags
>> RADEON_TILING_EG_BANKW_SHIFT
) & RADEON_TILING_EG_BANKW_MASK
;
61 *bankh
= (tiling_flags
>> RADEON_TILING_EG_BANKH_SHIFT
) & RADEON_TILING_EG_BANKH_MASK
;
62 *mtaspect
= (tiling_flags
>> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT
) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK
;
63 *tile_split
= (tiling_flags
>> RADEON_TILING_EG_TILE_SPLIT_SHIFT
) & RADEON_TILING_EG_TILE_SPLIT_MASK
;
66 case 1: *bankw
= EVERGREEN_ADDR_SURF_BANK_WIDTH_1
; break;
67 case 2: *bankw
= EVERGREEN_ADDR_SURF_BANK_WIDTH_2
; break;
68 case 4: *bankw
= EVERGREEN_ADDR_SURF_BANK_WIDTH_4
; break;
69 case 8: *bankw
= EVERGREEN_ADDR_SURF_BANK_WIDTH_8
; break;
73 case 1: *bankh
= EVERGREEN_ADDR_SURF_BANK_HEIGHT_1
; break;
74 case 2: *bankh
= EVERGREEN_ADDR_SURF_BANK_HEIGHT_2
; break;
75 case 4: *bankh
= EVERGREEN_ADDR_SURF_BANK_HEIGHT_4
; break;
76 case 8: *bankh
= EVERGREEN_ADDR_SURF_BANK_HEIGHT_8
; break;
80 case 1: *mtaspect
= EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1
; break;
81 case 2: *mtaspect
= EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2
; break;
82 case 4: *mtaspect
= EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4
; break;
83 case 8: *mtaspect
= EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8
; break;
87 void evergreen_fix_pci_max_read_req_size(struct radeon_device
*rdev
)
92 err
= pcie_capability_read_word(rdev
->pdev
, PCI_EXP_DEVCTL
, &ctl
);
96 v
= (ctl
& PCI_EXP_DEVCTL_READRQ
) >> 12;
98 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
99 * to avoid hangs or perfomance issues
101 if ((v
== 0) || (v
== 6) || (v
== 7)) {
102 ctl
&= ~PCI_EXP_DEVCTL_READRQ
;
104 pcie_capability_write_word(rdev
->pdev
, PCI_EXP_DEVCTL
, ctl
);
109 * dce4_wait_for_vblank - vblank wait asic callback.
111 * @rdev: radeon_device pointer
112 * @crtc: crtc to wait for vblank on
114 * Wait for vblank on the requested crtc (evergreen+).
116 void dce4_wait_for_vblank(struct radeon_device
*rdev
, int crtc
)
120 if (crtc
>= rdev
->num_crtc
)
123 if (RREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[crtc
]) & EVERGREEN_CRTC_MASTER_EN
) {
124 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
125 if (!(RREG32(EVERGREEN_CRTC_STATUS
+ crtc_offsets
[crtc
]) & EVERGREEN_CRTC_V_BLANK
))
129 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
130 if (RREG32(EVERGREEN_CRTC_STATUS
+ crtc_offsets
[crtc
]) & EVERGREEN_CRTC_V_BLANK
)
138 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
140 * @rdev: radeon_device pointer
141 * @crtc: crtc to prepare for pageflip on
143 * Pre-pageflip callback (evergreen+).
144 * Enables the pageflip irq (vblank irq).
146 void evergreen_pre_page_flip(struct radeon_device
*rdev
, int crtc
)
148 /* enable the pflip int */
149 radeon_irq_kms_pflip_irq_get(rdev
, crtc
);
153 * evergreen_post_page_flip - pos-pageflip callback.
155 * @rdev: radeon_device pointer
156 * @crtc: crtc to cleanup pageflip on
158 * Post-pageflip callback (evergreen+).
159 * Disables the pageflip irq (vblank irq).
161 void evergreen_post_page_flip(struct radeon_device
*rdev
, int crtc
)
163 /* disable the pflip int */
164 radeon_irq_kms_pflip_irq_put(rdev
, crtc
);
168 * evergreen_page_flip - pageflip callback.
170 * @rdev: radeon_device pointer
171 * @crtc_id: crtc to cleanup pageflip on
172 * @crtc_base: new address of the crtc (GPU MC address)
174 * Does the actual pageflip (evergreen+).
175 * During vblank we take the crtc lock and wait for the update_pending
176 * bit to go high, when it does, we release the lock, and allow the
177 * double buffered update to take place.
178 * Returns the current update pending status.
180 u32
evergreen_page_flip(struct radeon_device
*rdev
, int crtc_id
, u64 crtc_base
)
182 struct radeon_crtc
*radeon_crtc
= rdev
->mode_info
.crtcs
[crtc_id
];
183 u32 tmp
= RREG32(EVERGREEN_GRPH_UPDATE
+ radeon_crtc
->crtc_offset
);
186 /* Lock the graphics update lock */
187 tmp
|= EVERGREEN_GRPH_UPDATE_LOCK
;
188 WREG32(EVERGREEN_GRPH_UPDATE
+ radeon_crtc
->crtc_offset
, tmp
);
190 /* update the scanout addresses */
191 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH
+ radeon_crtc
->crtc_offset
,
192 upper_32_bits(crtc_base
));
193 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS
+ radeon_crtc
->crtc_offset
,
196 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH
+ radeon_crtc
->crtc_offset
,
197 upper_32_bits(crtc_base
));
198 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS
+ radeon_crtc
->crtc_offset
,
201 /* Wait for update_pending to go high. */
202 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
203 if (RREG32(EVERGREEN_GRPH_UPDATE
+ radeon_crtc
->crtc_offset
) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING
)
207 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
209 /* Unlock the lock, so double-buffering can take place inside vblank */
210 tmp
&= ~EVERGREEN_GRPH_UPDATE_LOCK
;
211 WREG32(EVERGREEN_GRPH_UPDATE
+ radeon_crtc
->crtc_offset
, tmp
);
213 /* Return current update_pending status: */
214 return RREG32(EVERGREEN_GRPH_UPDATE
+ radeon_crtc
->crtc_offset
) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING
;
217 /* get temperature in millidegrees */
218 int evergreen_get_temp(struct radeon_device
*rdev
)
223 if (rdev
->family
== CHIP_JUNIPER
) {
224 toffset
= (RREG32(CG_THERMAL_CTRL
) & TOFFSET_MASK
) >>
226 temp
= (RREG32(CG_TS0_STATUS
) & TS0_ADC_DOUT_MASK
) >>
230 actual_temp
= temp
/ 2 - (0x200 - toffset
);
232 actual_temp
= temp
/ 2 + toffset
;
234 actual_temp
= actual_temp
* 1000;
237 temp
= (RREG32(CG_MULT_THERMAL_STATUS
) & ASIC_T_MASK
) >>
242 else if (temp
& 0x200)
244 else if (temp
& 0x100) {
245 actual_temp
= temp
& 0x1ff;
246 actual_temp
|= ~0x1ff;
248 actual_temp
= temp
& 0xff;
250 actual_temp
= (actual_temp
* 1000) / 2;
256 int sumo_get_temp(struct radeon_device
*rdev
)
258 u32 temp
= RREG32(CG_THERMAL_STATUS
) & 0xff;
259 int actual_temp
= temp
- 49;
261 return actual_temp
* 1000;
265 * sumo_pm_init_profile - Initialize power profiles callback.
267 * @rdev: radeon_device pointer
269 * Initialize the power states used in profile mode
270 * (sumo, trinity, SI).
271 * Used for profile mode only.
273 void sumo_pm_init_profile(struct radeon_device
*rdev
)
278 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_off_ps_idx
= rdev
->pm
.default_power_state_index
;
279 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_on_ps_idx
= rdev
->pm
.default_power_state_index
;
280 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_off_cm_idx
= 0;
281 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_on_cm_idx
= 0;
284 if (rdev
->flags
& RADEON_IS_MOBILITY
)
285 idx
= radeon_pm_get_type_index(rdev
, POWER_STATE_TYPE_BATTERY
, 0);
287 idx
= radeon_pm_get_type_index(rdev
, POWER_STATE_TYPE_PERFORMANCE
, 0);
289 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_off_ps_idx
= idx
;
290 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_on_ps_idx
= idx
;
291 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_off_cm_idx
= 0;
292 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_on_cm_idx
= 0;
294 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_off_ps_idx
= idx
;
295 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_on_ps_idx
= idx
;
296 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_off_cm_idx
= 0;
297 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_on_cm_idx
= 0;
299 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_off_ps_idx
= idx
;
300 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_on_ps_idx
= idx
;
301 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_off_cm_idx
= 0;
302 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_on_cm_idx
= 0;
304 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_off_ps_idx
= idx
;
305 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_on_ps_idx
= idx
;
306 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_off_cm_idx
= 0;
307 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_on_cm_idx
= 0;
310 idx
= radeon_pm_get_type_index(rdev
, POWER_STATE_TYPE_PERFORMANCE
, 0);
311 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_off_ps_idx
= idx
;
312 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_on_ps_idx
= idx
;
313 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_off_cm_idx
= 0;
314 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_on_cm_idx
=
315 rdev
->pm
.power_state
[idx
].num_clock_modes
- 1;
317 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_off_ps_idx
= idx
;
318 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_on_ps_idx
= idx
;
319 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_off_cm_idx
= 0;
320 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_on_cm_idx
=
321 rdev
->pm
.power_state
[idx
].num_clock_modes
- 1;
325 * btc_pm_init_profile - Initialize power profiles callback.
327 * @rdev: radeon_device pointer
329 * Initialize the power states used in profile mode
331 * Used for profile mode only.
333 void btc_pm_init_profile(struct radeon_device
*rdev
)
338 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_off_ps_idx
= rdev
->pm
.default_power_state_index
;
339 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_on_ps_idx
= rdev
->pm
.default_power_state_index
;
340 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_off_cm_idx
= 0;
341 rdev
->pm
.profiles
[PM_PROFILE_DEFAULT_IDX
].dpms_on_cm_idx
= 2;
342 /* starting with BTC, there is one state that is used for both
343 * MH and SH. Difference is that we always use the high clock index for
346 if (rdev
->flags
& RADEON_IS_MOBILITY
)
347 idx
= radeon_pm_get_type_index(rdev
, POWER_STATE_TYPE_BATTERY
, 0);
349 idx
= radeon_pm_get_type_index(rdev
, POWER_STATE_TYPE_PERFORMANCE
, 0);
351 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_off_ps_idx
= idx
;
352 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_on_ps_idx
= idx
;
353 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_off_cm_idx
= 0;
354 rdev
->pm
.profiles
[PM_PROFILE_LOW_SH_IDX
].dpms_on_cm_idx
= 0;
356 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_off_ps_idx
= idx
;
357 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_on_ps_idx
= idx
;
358 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_off_cm_idx
= 0;
359 rdev
->pm
.profiles
[PM_PROFILE_MID_SH_IDX
].dpms_on_cm_idx
= 1;
361 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_off_ps_idx
= idx
;
362 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_on_ps_idx
= idx
;
363 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_off_cm_idx
= 0;
364 rdev
->pm
.profiles
[PM_PROFILE_HIGH_SH_IDX
].dpms_on_cm_idx
= 2;
366 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_off_ps_idx
= idx
;
367 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_on_ps_idx
= idx
;
368 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_off_cm_idx
= 0;
369 rdev
->pm
.profiles
[PM_PROFILE_LOW_MH_IDX
].dpms_on_cm_idx
= 0;
371 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_off_ps_idx
= idx
;
372 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_on_ps_idx
= idx
;
373 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_off_cm_idx
= 0;
374 rdev
->pm
.profiles
[PM_PROFILE_MID_MH_IDX
].dpms_on_cm_idx
= 1;
376 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_off_ps_idx
= idx
;
377 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_on_ps_idx
= idx
;
378 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_off_cm_idx
= 0;
379 rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_on_cm_idx
= 2;
383 * evergreen_pm_misc - set additional pm hw parameters callback.
385 * @rdev: radeon_device pointer
387 * Set non-clock parameters associated with a power state
388 * (voltage, etc.) (evergreen+).
390 void evergreen_pm_misc(struct radeon_device
*rdev
)
392 int req_ps_idx
= rdev
->pm
.requested_power_state_index
;
393 int req_cm_idx
= rdev
->pm
.requested_clock_mode_index
;
394 struct radeon_power_state
*ps
= &rdev
->pm
.power_state
[req_ps_idx
];
395 struct radeon_voltage
*voltage
= &ps
->clock_info
[req_cm_idx
].voltage
;
397 if (voltage
->type
== VOLTAGE_SW
) {
398 /* 0xff01 is a flag rather then an actual voltage */
399 if (voltage
->voltage
== 0xff01)
401 if (voltage
->voltage
&& (voltage
->voltage
!= rdev
->pm
.current_vddc
)) {
402 radeon_atom_set_voltage(rdev
, voltage
->voltage
, SET_VOLTAGE_TYPE_ASIC_VDDC
);
403 rdev
->pm
.current_vddc
= voltage
->voltage
;
404 DRM_DEBUG("Setting: vddc: %d\n", voltage
->voltage
);
407 /* starting with BTC, there is one state that is used for both
408 * MH and SH. Difference is that we always use the high clock index for
411 if ((rdev
->pm
.pm_method
== PM_METHOD_PROFILE
) &&
412 (rdev
->family
>= CHIP_BARTS
) &&
413 rdev
->pm
.active_crtc_count
&&
414 ((rdev
->pm
.profile_index
== PM_PROFILE_MID_MH_IDX
) ||
415 (rdev
->pm
.profile_index
== PM_PROFILE_LOW_MH_IDX
)))
416 voltage
= &rdev
->pm
.power_state
[req_ps_idx
].
417 clock_info
[rdev
->pm
.profiles
[PM_PROFILE_HIGH_MH_IDX
].dpms_on_cm_idx
].voltage
;
419 /* 0xff01 is a flag rather then an actual voltage */
420 if (voltage
->vddci
== 0xff01)
422 if (voltage
->vddci
&& (voltage
->vddci
!= rdev
->pm
.current_vddci
)) {
423 radeon_atom_set_voltage(rdev
, voltage
->vddci
, SET_VOLTAGE_TYPE_ASIC_VDDCI
);
424 rdev
->pm
.current_vddci
= voltage
->vddci
;
425 DRM_DEBUG("Setting: vddci: %d\n", voltage
->vddci
);
431 * evergreen_pm_prepare - pre-power state change callback.
433 * @rdev: radeon_device pointer
435 * Prepare for a power state change (evergreen+).
437 void evergreen_pm_prepare(struct radeon_device
*rdev
)
439 struct drm_device
*ddev
= rdev
->ddev
;
440 struct drm_crtc
*crtc
;
441 struct radeon_crtc
*radeon_crtc
;
444 /* disable any active CRTCs */
445 list_for_each_entry(crtc
, &ddev
->mode_config
.crtc_list
, head
) {
446 radeon_crtc
= to_radeon_crtc(crtc
);
447 if (radeon_crtc
->enabled
) {
448 tmp
= RREG32(EVERGREEN_CRTC_CONTROL
+ radeon_crtc
->crtc_offset
);
449 tmp
|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE
;
450 WREG32(EVERGREEN_CRTC_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
456 * evergreen_pm_finish - post-power state change callback.
458 * @rdev: radeon_device pointer
460 * Clean up after a power state change (evergreen+).
462 void evergreen_pm_finish(struct radeon_device
*rdev
)
464 struct drm_device
*ddev
= rdev
->ddev
;
465 struct drm_crtc
*crtc
;
466 struct radeon_crtc
*radeon_crtc
;
469 /* enable any active CRTCs */
470 list_for_each_entry(crtc
, &ddev
->mode_config
.crtc_list
, head
) {
471 radeon_crtc
= to_radeon_crtc(crtc
);
472 if (radeon_crtc
->enabled
) {
473 tmp
= RREG32(EVERGREEN_CRTC_CONTROL
+ radeon_crtc
->crtc_offset
);
474 tmp
&= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE
;
475 WREG32(EVERGREEN_CRTC_CONTROL
+ radeon_crtc
->crtc_offset
, tmp
);
481 * evergreen_hpd_sense - hpd sense callback.
483 * @rdev: radeon_device pointer
484 * @hpd: hpd (hotplug detect) pin
486 * Checks if a digital monitor is connected (evergreen+).
487 * Returns true if connected, false if not connected.
489 bool evergreen_hpd_sense(struct radeon_device
*rdev
, enum radeon_hpd_id hpd
)
491 bool connected
= false;
495 if (RREG32(DC_HPD1_INT_STATUS
) & DC_HPDx_SENSE
)
499 if (RREG32(DC_HPD2_INT_STATUS
) & DC_HPDx_SENSE
)
503 if (RREG32(DC_HPD3_INT_STATUS
) & DC_HPDx_SENSE
)
507 if (RREG32(DC_HPD4_INT_STATUS
) & DC_HPDx_SENSE
)
511 if (RREG32(DC_HPD5_INT_STATUS
) & DC_HPDx_SENSE
)
515 if (RREG32(DC_HPD6_INT_STATUS
) & DC_HPDx_SENSE
)
526 * evergreen_hpd_set_polarity - hpd set polarity callback.
528 * @rdev: radeon_device pointer
529 * @hpd: hpd (hotplug detect) pin
531 * Set the polarity of the hpd pin (evergreen+).
533 void evergreen_hpd_set_polarity(struct radeon_device
*rdev
,
534 enum radeon_hpd_id hpd
)
537 bool connected
= evergreen_hpd_sense(rdev
, hpd
);
541 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
543 tmp
&= ~DC_HPDx_INT_POLARITY
;
545 tmp
|= DC_HPDx_INT_POLARITY
;
546 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
549 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
551 tmp
&= ~DC_HPDx_INT_POLARITY
;
553 tmp
|= DC_HPDx_INT_POLARITY
;
554 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
557 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
559 tmp
&= ~DC_HPDx_INT_POLARITY
;
561 tmp
|= DC_HPDx_INT_POLARITY
;
562 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
565 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
567 tmp
&= ~DC_HPDx_INT_POLARITY
;
569 tmp
|= DC_HPDx_INT_POLARITY
;
570 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
573 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
575 tmp
&= ~DC_HPDx_INT_POLARITY
;
577 tmp
|= DC_HPDx_INT_POLARITY
;
578 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
581 tmp
= RREG32(DC_HPD6_INT_CONTROL
);
583 tmp
&= ~DC_HPDx_INT_POLARITY
;
585 tmp
|= DC_HPDx_INT_POLARITY
;
586 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
594 * evergreen_hpd_init - hpd setup callback.
596 * @rdev: radeon_device pointer
598 * Setup the hpd pins used by the card (evergreen+).
599 * Enable the pin, set the polarity, and enable the hpd interrupts.
601 void evergreen_hpd_init(struct radeon_device
*rdev
)
603 struct drm_device
*dev
= rdev
->ddev
;
604 struct drm_connector
*connector
;
605 unsigned enabled
= 0;
606 u32 tmp
= DC_HPDx_CONNECTION_TIMER(0x9c4) |
607 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN
;
609 list_for_each_entry(connector
, &dev
->mode_config
.connector_list
, head
) {
610 struct radeon_connector
*radeon_connector
= to_radeon_connector(connector
);
611 switch (radeon_connector
->hpd
.hpd
) {
613 WREG32(DC_HPD1_CONTROL
, tmp
);
616 WREG32(DC_HPD2_CONTROL
, tmp
);
619 WREG32(DC_HPD3_CONTROL
, tmp
);
622 WREG32(DC_HPD4_CONTROL
, tmp
);
625 WREG32(DC_HPD5_CONTROL
, tmp
);
628 WREG32(DC_HPD6_CONTROL
, tmp
);
633 radeon_hpd_set_polarity(rdev
, radeon_connector
->hpd
.hpd
);
634 enabled
|= 1 << radeon_connector
->hpd
.hpd
;
636 radeon_irq_kms_enable_hpd(rdev
, enabled
);
640 * evergreen_hpd_fini - hpd tear down callback.
642 * @rdev: radeon_device pointer
644 * Tear down the hpd pins used by the card (evergreen+).
645 * Disable the hpd interrupts.
647 void evergreen_hpd_fini(struct radeon_device
*rdev
)
649 struct drm_device
*dev
= rdev
->ddev
;
650 struct drm_connector
*connector
;
651 unsigned disabled
= 0;
653 list_for_each_entry(connector
, &dev
->mode_config
.connector_list
, head
) {
654 struct radeon_connector
*radeon_connector
= to_radeon_connector(connector
);
655 switch (radeon_connector
->hpd
.hpd
) {
657 WREG32(DC_HPD1_CONTROL
, 0);
660 WREG32(DC_HPD2_CONTROL
, 0);
663 WREG32(DC_HPD3_CONTROL
, 0);
666 WREG32(DC_HPD4_CONTROL
, 0);
669 WREG32(DC_HPD5_CONTROL
, 0);
672 WREG32(DC_HPD6_CONTROL
, 0);
677 disabled
|= 1 << radeon_connector
->hpd
.hpd
;
679 radeon_irq_kms_disable_hpd(rdev
, disabled
);
682 /* watermark setup */
684 static u32
evergreen_line_buffer_adjust(struct radeon_device
*rdev
,
685 struct radeon_crtc
*radeon_crtc
,
686 struct drm_display_mode
*mode
,
687 struct drm_display_mode
*other_mode
)
692 * There are 3 line buffers, each one shared by 2 display controllers.
693 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
694 * the display controllers. The paritioning is done via one of four
695 * preset allocations specified in bits 2:0:
696 * first display controller
697 * 0 - first half of lb (3840 * 2)
698 * 1 - first 3/4 of lb (5760 * 2)
699 * 2 - whole lb (7680 * 2), other crtc must be disabled
700 * 3 - first 1/4 of lb (1920 * 2)
701 * second display controller
702 * 4 - second half of lb (3840 * 2)
703 * 5 - second 3/4 of lb (5760 * 2)
704 * 6 - whole lb (7680 * 2), other crtc must be disabled
705 * 7 - last 1/4 of lb (1920 * 2)
707 /* this can get tricky if we have two large displays on a paired group
708 * of crtcs. Ideally for multiple large displays we'd assign them to
709 * non-linked crtcs for maximum line buffer allocation.
711 if (radeon_crtc
->base
.enabled
&& mode
) {
719 /* second controller of the pair uses second half of the lb */
720 if (radeon_crtc
->crtc_id
% 2)
722 WREG32(DC_LB_MEMORY_SPLIT
+ radeon_crtc
->crtc_offset
, tmp
);
724 if (radeon_crtc
->base
.enabled
&& mode
) {
729 if (ASIC_IS_DCE5(rdev
))
735 if (ASIC_IS_DCE5(rdev
))
741 if (ASIC_IS_DCE5(rdev
))
747 if (ASIC_IS_DCE5(rdev
))
754 /* controller not enabled, so no lb used */
758 u32
evergreen_get_number_of_dram_channels(struct radeon_device
*rdev
)
760 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
762 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
775 struct evergreen_wm_params
{
776 u32 dram_channels
; /* number of dram channels */
777 u32 yclk
; /* bandwidth per dram data pin in kHz */
778 u32 sclk
; /* engine clock in kHz */
779 u32 disp_clk
; /* display clock in kHz */
780 u32 src_width
; /* viewport width */
781 u32 active_time
; /* active display time in ns */
782 u32 blank_time
; /* blank time in ns */
783 bool interlaced
; /* mode is interlaced */
784 fixed20_12 vsc
; /* vertical scale ratio */
785 u32 num_heads
; /* number of active crtcs */
786 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
787 u32 lb_size
; /* line buffer allocated to pipe */
788 u32 vtaps
; /* vertical scaler taps */
791 static u32
evergreen_dram_bandwidth(struct evergreen_wm_params
*wm
)
793 /* Calculate DRAM Bandwidth and the part allocated to display. */
794 fixed20_12 dram_efficiency
; /* 0.7 */
795 fixed20_12 yclk
, dram_channels
, bandwidth
;
798 a
.full
= dfixed_const(1000);
799 yclk
.full
= dfixed_const(wm
->yclk
);
800 yclk
.full
= dfixed_div(yclk
, a
);
801 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
802 a
.full
= dfixed_const(10);
803 dram_efficiency
.full
= dfixed_const(7);
804 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
805 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
806 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
808 return dfixed_trunc(bandwidth
);
811 static u32
evergreen_dram_bandwidth_for_display(struct evergreen_wm_params
*wm
)
813 /* Calculate DRAM Bandwidth and the part allocated to display. */
814 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
815 fixed20_12 yclk
, dram_channels
, bandwidth
;
818 a
.full
= dfixed_const(1000);
819 yclk
.full
= dfixed_const(wm
->yclk
);
820 yclk
.full
= dfixed_div(yclk
, a
);
821 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
822 a
.full
= dfixed_const(10);
823 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
824 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
825 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
826 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
828 return dfixed_trunc(bandwidth
);
831 static u32
evergreen_data_return_bandwidth(struct evergreen_wm_params
*wm
)
833 /* Calculate the display Data return Bandwidth */
834 fixed20_12 return_efficiency
; /* 0.8 */
835 fixed20_12 sclk
, bandwidth
;
838 a
.full
= dfixed_const(1000);
839 sclk
.full
= dfixed_const(wm
->sclk
);
840 sclk
.full
= dfixed_div(sclk
, a
);
841 a
.full
= dfixed_const(10);
842 return_efficiency
.full
= dfixed_const(8);
843 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
844 a
.full
= dfixed_const(32);
845 bandwidth
.full
= dfixed_mul(a
, sclk
);
846 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
848 return dfixed_trunc(bandwidth
);
851 static u32
evergreen_dmif_request_bandwidth(struct evergreen_wm_params
*wm
)
853 /* Calculate the DMIF Request Bandwidth */
854 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
855 fixed20_12 disp_clk
, bandwidth
;
858 a
.full
= dfixed_const(1000);
859 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
860 disp_clk
.full
= dfixed_div(disp_clk
, a
);
861 a
.full
= dfixed_const(10);
862 disp_clk_request_efficiency
.full
= dfixed_const(8);
863 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
864 a
.full
= dfixed_const(32);
865 bandwidth
.full
= dfixed_mul(a
, disp_clk
);
866 bandwidth
.full
= dfixed_mul(bandwidth
, disp_clk_request_efficiency
);
868 return dfixed_trunc(bandwidth
);
871 static u32
evergreen_available_bandwidth(struct evergreen_wm_params
*wm
)
873 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
874 u32 dram_bandwidth
= evergreen_dram_bandwidth(wm
);
875 u32 data_return_bandwidth
= evergreen_data_return_bandwidth(wm
);
876 u32 dmif_req_bandwidth
= evergreen_dmif_request_bandwidth(wm
);
878 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
881 static u32
evergreen_average_bandwidth(struct evergreen_wm_params
*wm
)
883 /* Calculate the display mode Average Bandwidth
884 * DisplayMode should contain the source and destination dimensions,
888 fixed20_12 line_time
;
889 fixed20_12 src_width
;
890 fixed20_12 bandwidth
;
893 a
.full
= dfixed_const(1000);
894 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
895 line_time
.full
= dfixed_div(line_time
, a
);
896 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
897 src_width
.full
= dfixed_const(wm
->src_width
);
898 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
899 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
900 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
902 return dfixed_trunc(bandwidth
);
905 static u32
evergreen_latency_watermark(struct evergreen_wm_params
*wm
)
907 /* First calcualte the latency in ns */
908 u32 mc_latency
= 2000; /* 2000 ns. */
909 u32 available_bandwidth
= evergreen_available_bandwidth(wm
);
910 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
911 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
912 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
913 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
914 (wm
->num_heads
* cursor_line_pair_return_time
);
915 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
916 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
919 if (wm
->num_heads
== 0)
922 a
.full
= dfixed_const(2);
923 b
.full
= dfixed_const(1);
924 if ((wm
->vsc
.full
> a
.full
) ||
925 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
927 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
928 max_src_lines_per_dst_line
= 4;
930 max_src_lines_per_dst_line
= 2;
932 a
.full
= dfixed_const(available_bandwidth
);
933 b
.full
= dfixed_const(wm
->num_heads
);
934 a
.full
= dfixed_div(a
, b
);
936 b
.full
= dfixed_const(1000);
937 c
.full
= dfixed_const(wm
->disp_clk
);
938 b
.full
= dfixed_div(c
, b
);
939 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
940 b
.full
= dfixed_mul(b
, c
);
942 lb_fill_bw
= min(dfixed_trunc(a
), dfixed_trunc(b
));
944 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
945 b
.full
= dfixed_const(1000);
946 c
.full
= dfixed_const(lb_fill_bw
);
947 b
.full
= dfixed_div(c
, b
);
948 a
.full
= dfixed_div(a
, b
);
949 line_fill_time
= dfixed_trunc(a
);
951 if (line_fill_time
< wm
->active_time
)
954 return latency
+ (line_fill_time
- wm
->active_time
);
958 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params
*wm
)
960 if (evergreen_average_bandwidth(wm
) <=
961 (evergreen_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
967 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params
*wm
)
969 if (evergreen_average_bandwidth(wm
) <=
970 (evergreen_available_bandwidth(wm
) / wm
->num_heads
))
976 static bool evergreen_check_latency_hiding(struct evergreen_wm_params
*wm
)
978 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
979 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
980 u32 latency_tolerant_lines
;
984 a
.full
= dfixed_const(1);
985 if (wm
->vsc
.full
> a
.full
)
986 latency_tolerant_lines
= 1;
988 if (lb_partitions
<= (wm
->vtaps
+ 1))
989 latency_tolerant_lines
= 1;
991 latency_tolerant_lines
= 2;
994 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
996 if (evergreen_latency_watermark(wm
) <= latency_hiding
)
1002 static void evergreen_program_watermarks(struct radeon_device
*rdev
,
1003 struct radeon_crtc
*radeon_crtc
,
1004 u32 lb_size
, u32 num_heads
)
1006 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
1007 struct evergreen_wm_params wm
;
1010 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
1011 u32 priority_a_mark
= 0, priority_b_mark
= 0;
1012 u32 priority_a_cnt
= PRIORITY_OFF
;
1013 u32 priority_b_cnt
= PRIORITY_OFF
;
1014 u32 pipe_offset
= radeon_crtc
->crtc_id
* 16;
1015 u32 tmp
, arb_control3
;
1018 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
1019 pixel_period
= 1000000 / (u32
)mode
->clock
;
1020 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
1024 wm
.yclk
= rdev
->pm
.current_mclk
* 10;
1025 wm
.sclk
= rdev
->pm
.current_sclk
* 10;
1026 wm
.disp_clk
= mode
->clock
;
1027 wm
.src_width
= mode
->crtc_hdisplay
;
1028 wm
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
1029 wm
.blank_time
= line_time
- wm
.active_time
;
1030 wm
.interlaced
= false;
1031 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
1032 wm
.interlaced
= true;
1033 wm
.vsc
= radeon_crtc
->vsc
;
1035 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
1037 wm
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
1038 wm
.lb_size
= lb_size
;
1039 wm
.dram_channels
= evergreen_get_number_of_dram_channels(rdev
);
1040 wm
.num_heads
= num_heads
;
1042 /* set for high clocks */
1043 latency_watermark_a
= min(evergreen_latency_watermark(&wm
), (u32
)65535);
1044 /* set for low clocks */
1045 /* wm.yclk = low clk; wm.sclk = low clk */
1046 latency_watermark_b
= min(evergreen_latency_watermark(&wm
), (u32
)65535);
1048 /* possibly force display priority to high */
1049 /* should really do this at mode validation time... */
1050 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm
) ||
1051 !evergreen_average_bandwidth_vs_available_bandwidth(&wm
) ||
1052 !evergreen_check_latency_hiding(&wm
) ||
1053 (rdev
->disp_priority
== 2)) {
1054 DRM_DEBUG_KMS("force priority to high\n");
1055 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
1056 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
1059 a
.full
= dfixed_const(1000);
1060 b
.full
= dfixed_const(mode
->clock
);
1061 b
.full
= dfixed_div(b
, a
);
1062 c
.full
= dfixed_const(latency_watermark_a
);
1063 c
.full
= dfixed_mul(c
, b
);
1064 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
1065 c
.full
= dfixed_div(c
, a
);
1066 a
.full
= dfixed_const(16);
1067 c
.full
= dfixed_div(c
, a
);
1068 priority_a_mark
= dfixed_trunc(c
);
1069 priority_a_cnt
|= priority_a_mark
& PRIORITY_MARK_MASK
;
1071 a
.full
= dfixed_const(1000);
1072 b
.full
= dfixed_const(mode
->clock
);
1073 b
.full
= dfixed_div(b
, a
);
1074 c
.full
= dfixed_const(latency_watermark_b
);
1075 c
.full
= dfixed_mul(c
, b
);
1076 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
1077 c
.full
= dfixed_div(c
, a
);
1078 a
.full
= dfixed_const(16);
1079 c
.full
= dfixed_div(c
, a
);
1080 priority_b_mark
= dfixed_trunc(c
);
1081 priority_b_cnt
|= priority_b_mark
& PRIORITY_MARK_MASK
;
1085 arb_control3
= RREG32(PIPE0_ARBITRATION_CONTROL3
+ pipe_offset
);
1087 tmp
&= ~LATENCY_WATERMARK_MASK(3);
1088 tmp
|= LATENCY_WATERMARK_MASK(1);
1089 WREG32(PIPE0_ARBITRATION_CONTROL3
+ pipe_offset
, tmp
);
1090 WREG32(PIPE0_LATENCY_CONTROL
+ pipe_offset
,
1091 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
1092 LATENCY_HIGH_WATERMARK(line_time
)));
1094 tmp
= RREG32(PIPE0_ARBITRATION_CONTROL3
+ pipe_offset
);
1095 tmp
&= ~LATENCY_WATERMARK_MASK(3);
1096 tmp
|= LATENCY_WATERMARK_MASK(2);
1097 WREG32(PIPE0_ARBITRATION_CONTROL3
+ pipe_offset
, tmp
);
1098 WREG32(PIPE0_LATENCY_CONTROL
+ pipe_offset
,
1099 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
1100 LATENCY_HIGH_WATERMARK(line_time
)));
1101 /* restore original selection */
1102 WREG32(PIPE0_ARBITRATION_CONTROL3
+ pipe_offset
, arb_control3
);
1104 /* write the priority marks */
1105 WREG32(PRIORITY_A_CNT
+ radeon_crtc
->crtc_offset
, priority_a_cnt
);
1106 WREG32(PRIORITY_B_CNT
+ radeon_crtc
->crtc_offset
, priority_b_cnt
);
1111 * evergreen_bandwidth_update - update display watermarks callback.
1113 * @rdev: radeon_device pointer
1115 * Update the display watermarks based on the requested mode(s)
1118 void evergreen_bandwidth_update(struct radeon_device
*rdev
)
1120 struct drm_display_mode
*mode0
= NULL
;
1121 struct drm_display_mode
*mode1
= NULL
;
1122 u32 num_heads
= 0, lb_size
;
1125 radeon_update_display_priority(rdev
);
1127 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
1128 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
1131 for (i
= 0; i
< rdev
->num_crtc
; i
+= 2) {
1132 mode0
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
1133 mode1
= &rdev
->mode_info
.crtcs
[i
+1]->base
.mode
;
1134 lb_size
= evergreen_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode0
, mode1
);
1135 evergreen_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
1136 lb_size
= evergreen_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
+1], mode1
, mode0
);
1137 evergreen_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
+1], lb_size
, num_heads
);
1142 * evergreen_mc_wait_for_idle - wait for MC idle callback.
1144 * @rdev: radeon_device pointer
1146 * Wait for the MC (memory controller) to be idle.
1148 * Returns 0 if the MC is idle, -1 if not.
1150 int evergreen_mc_wait_for_idle(struct radeon_device
*rdev
)
1155 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1156 /* read MC_STATUS */
1157 tmp
= RREG32(SRBM_STATUS
) & 0x1F00;
1168 void evergreen_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
1173 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0x1);
1175 WREG32(VM_CONTEXT0_REQUEST_RESPONSE
, REQUEST_TYPE(1));
1176 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1177 /* read MC_STATUS */
1178 tmp
= RREG32(VM_CONTEXT0_REQUEST_RESPONSE
);
1179 tmp
= (tmp
& RESPONSE_TYPE_MASK
) >> RESPONSE_TYPE_SHIFT
;
1181 printk(KERN_WARNING
"[drm] r600 flush TLB failed\n");
1191 static int evergreen_pcie_gart_enable(struct radeon_device
*rdev
)
1196 if (rdev
->gart
.robj
== NULL
) {
1197 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
1200 r
= radeon_gart_table_vram_pin(rdev
);
1203 radeon_gart_restore(rdev
);
1204 /* Setup L2 cache */
1205 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
| ENABLE_L2_FRAGMENT_PROCESSING
|
1206 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
1207 EFFECTIVE_L2_QUEUE_SIZE(7));
1208 WREG32(VM_L2_CNTL2
, 0);
1209 WREG32(VM_L2_CNTL3
, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1210 /* Setup TLB control */
1211 tmp
= ENABLE_L1_TLB
| ENABLE_L1_FRAGMENT_PROCESSING
|
1212 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
1213 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
|
1214 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1215 if (rdev
->flags
& RADEON_IS_IGP
) {
1216 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL
, tmp
);
1217 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL
, tmp
);
1218 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL
, tmp
);
1220 WREG32(MC_VM_MD_L1_TLB0_CNTL
, tmp
);
1221 WREG32(MC_VM_MD_L1_TLB1_CNTL
, tmp
);
1222 WREG32(MC_VM_MD_L1_TLB2_CNTL
, tmp
);
1223 if ((rdev
->family
== CHIP_JUNIPER
) ||
1224 (rdev
->family
== CHIP_CYPRESS
) ||
1225 (rdev
->family
== CHIP_HEMLOCK
) ||
1226 (rdev
->family
== CHIP_BARTS
))
1227 WREG32(MC_VM_MD_L1_TLB3_CNTL
, tmp
);
1229 WREG32(MC_VM_MB_L1_TLB0_CNTL
, tmp
);
1230 WREG32(MC_VM_MB_L1_TLB1_CNTL
, tmp
);
1231 WREG32(MC_VM_MB_L1_TLB2_CNTL
, tmp
);
1232 WREG32(MC_VM_MB_L1_TLB3_CNTL
, tmp
);
1233 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
1234 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
1235 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
1236 WREG32(VM_CONTEXT0_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
1237 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
);
1238 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
1239 (u32
)(rdev
->dummy_page
.addr
>> 12));
1240 WREG32(VM_CONTEXT1_CNTL
, 0);
1242 evergreen_pcie_gart_tlb_flush(rdev
);
1243 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1244 (unsigned)(rdev
->mc
.gtt_size
>> 20),
1245 (unsigned long long)rdev
->gart
.table_addr
);
1246 rdev
->gart
.ready
= true;
1250 static void evergreen_pcie_gart_disable(struct radeon_device
*rdev
)
1254 /* Disable all tables */
1255 WREG32(VM_CONTEXT0_CNTL
, 0);
1256 WREG32(VM_CONTEXT1_CNTL
, 0);
1258 /* Setup L2 cache */
1259 WREG32(VM_L2_CNTL
, ENABLE_L2_FRAGMENT_PROCESSING
|
1260 EFFECTIVE_L2_QUEUE_SIZE(7));
1261 WREG32(VM_L2_CNTL2
, 0);
1262 WREG32(VM_L2_CNTL3
, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1263 /* Setup TLB control */
1264 tmp
= EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1265 WREG32(MC_VM_MD_L1_TLB0_CNTL
, tmp
);
1266 WREG32(MC_VM_MD_L1_TLB1_CNTL
, tmp
);
1267 WREG32(MC_VM_MD_L1_TLB2_CNTL
, tmp
);
1268 WREG32(MC_VM_MB_L1_TLB0_CNTL
, tmp
);
1269 WREG32(MC_VM_MB_L1_TLB1_CNTL
, tmp
);
1270 WREG32(MC_VM_MB_L1_TLB2_CNTL
, tmp
);
1271 WREG32(MC_VM_MB_L1_TLB3_CNTL
, tmp
);
1272 radeon_gart_table_vram_unpin(rdev
);
1275 static void evergreen_pcie_gart_fini(struct radeon_device
*rdev
)
1277 evergreen_pcie_gart_disable(rdev
);
1278 radeon_gart_table_vram_free(rdev
);
1279 radeon_gart_fini(rdev
);
1283 static void evergreen_agp_enable(struct radeon_device
*rdev
)
1287 /* Setup L2 cache */
1288 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
| ENABLE_L2_FRAGMENT_PROCESSING
|
1289 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
1290 EFFECTIVE_L2_QUEUE_SIZE(7));
1291 WREG32(VM_L2_CNTL2
, 0);
1292 WREG32(VM_L2_CNTL3
, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1293 /* Setup TLB control */
1294 tmp
= ENABLE_L1_TLB
| ENABLE_L1_FRAGMENT_PROCESSING
|
1295 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
1296 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
|
1297 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1298 WREG32(MC_VM_MD_L1_TLB0_CNTL
, tmp
);
1299 WREG32(MC_VM_MD_L1_TLB1_CNTL
, tmp
);
1300 WREG32(MC_VM_MD_L1_TLB2_CNTL
, tmp
);
1301 WREG32(MC_VM_MB_L1_TLB0_CNTL
, tmp
);
1302 WREG32(MC_VM_MB_L1_TLB1_CNTL
, tmp
);
1303 WREG32(MC_VM_MB_L1_TLB2_CNTL
, tmp
);
1304 WREG32(MC_VM_MB_L1_TLB3_CNTL
, tmp
);
1305 WREG32(VM_CONTEXT0_CNTL
, 0);
1306 WREG32(VM_CONTEXT1_CNTL
, 0);
1309 void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
)
1311 u32 crtc_enabled
, tmp
, frame_count
, blackout
;
1314 save
->vga_render_control
= RREG32(VGA_RENDER_CONTROL
);
1315 save
->vga_hdp_control
= RREG32(VGA_HDP_CONTROL
);
1317 /* disable VGA render */
1318 WREG32(VGA_RENDER_CONTROL
, 0);
1319 /* blank the display controllers */
1320 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
1321 crtc_enabled
= RREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
]) & EVERGREEN_CRTC_MASTER_EN
;
1323 save
->crtc_enabled
[i
] = true;
1324 if (ASIC_IS_DCE6(rdev
)) {
1325 tmp
= RREG32(EVERGREEN_CRTC_BLANK_CONTROL
+ crtc_offsets
[i
]);
1326 if (!(tmp
& EVERGREEN_CRTC_BLANK_DATA_EN
)) {
1327 radeon_wait_for_vblank(rdev
, i
);
1328 tmp
|= EVERGREEN_CRTC_BLANK_DATA_EN
;
1329 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 1);
1330 WREG32(EVERGREEN_CRTC_BLANK_CONTROL
+ crtc_offsets
[i
], tmp
);
1331 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 0);
1334 tmp
= RREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
]);
1335 if (!(tmp
& EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE
)) {
1336 radeon_wait_for_vblank(rdev
, i
);
1337 tmp
|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE
;
1338 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 1);
1339 WREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
], tmp
);
1340 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 0);
1343 /* wait for the next frame */
1344 frame_count
= radeon_get_vblank_counter(rdev
, i
);
1345 for (j
= 0; j
< rdev
->usec_timeout
; j
++) {
1346 if (radeon_get_vblank_counter(rdev
, i
) != frame_count
)
1351 save
->crtc_enabled
[i
] = false;
1355 radeon_mc_wait_for_idle(rdev
);
1357 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1358 if ((blackout
& BLACKOUT_MODE_MASK
) != 1) {
1359 /* Block CPU access */
1360 WREG32(BIF_FB_EN
, 0);
1361 /* blackout the MC */
1362 blackout
&= ~BLACKOUT_MODE_MASK
;
1363 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1365 /* wait for the MC to settle */
1369 void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
)
1371 u32 tmp
, frame_count
;
1374 /* update crtc base addresses */
1375 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
1376 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH
+ crtc_offsets
[i
],
1377 upper_32_bits(rdev
->mc
.vram_start
));
1378 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH
+ crtc_offsets
[i
],
1379 upper_32_bits(rdev
->mc
.vram_start
));
1380 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS
+ crtc_offsets
[i
],
1381 (u32
)rdev
->mc
.vram_start
);
1382 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS
+ crtc_offsets
[i
],
1383 (u32
)rdev
->mc
.vram_start
);
1385 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH
, upper_32_bits(rdev
->mc
.vram_start
));
1386 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS
, (u32
)rdev
->mc
.vram_start
);
1388 /* unblackout the MC */
1389 tmp
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1390 tmp
&= ~BLACKOUT_MODE_MASK
;
1391 WREG32(MC_SHARED_BLACKOUT_CNTL
, tmp
);
1392 /* allow CPU access */
1393 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
1395 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
1396 if (save
->crtc_enabled
[i
]) {
1397 if (ASIC_IS_DCE6(rdev
)) {
1398 tmp
= RREG32(EVERGREEN_CRTC_BLANK_CONTROL
+ crtc_offsets
[i
]);
1399 tmp
|= EVERGREEN_CRTC_BLANK_DATA_EN
;
1400 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 1);
1401 WREG32(EVERGREEN_CRTC_BLANK_CONTROL
+ crtc_offsets
[i
], tmp
);
1402 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 0);
1404 tmp
= RREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
]);
1405 tmp
&= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE
;
1406 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 1);
1407 WREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
], tmp
);
1408 WREG32(EVERGREEN_CRTC_UPDATE_LOCK
+ crtc_offsets
[i
], 0);
1410 /* wait for the next frame */
1411 frame_count
= radeon_get_vblank_counter(rdev
, i
);
1412 for (j
= 0; j
< rdev
->usec_timeout
; j
++) {
1413 if (radeon_get_vblank_counter(rdev
, i
) != frame_count
)
1419 /* Unlock vga access */
1420 WREG32(VGA_HDP_CONTROL
, save
->vga_hdp_control
);
1422 WREG32(VGA_RENDER_CONTROL
, save
->vga_render_control
);
1425 void evergreen_mc_program(struct radeon_device
*rdev
)
1427 struct evergreen_mc_save save
;
1431 /* Initialize HDP */
1432 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
1433 WREG32((0x2c14 + j
), 0x00000000);
1434 WREG32((0x2c18 + j
), 0x00000000);
1435 WREG32((0x2c1c + j
), 0x00000000);
1436 WREG32((0x2c20 + j
), 0x00000000);
1437 WREG32((0x2c24 + j
), 0x00000000);
1439 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
1441 evergreen_mc_stop(rdev
, &save
);
1442 if (evergreen_mc_wait_for_idle(rdev
)) {
1443 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
1445 /* Lockout access through VGA aperture*/
1446 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
1447 /* Update configuration */
1448 if (rdev
->flags
& RADEON_IS_AGP
) {
1449 if (rdev
->mc
.vram_start
< rdev
->mc
.gtt_start
) {
1450 /* VRAM before AGP */
1451 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
1452 rdev
->mc
.vram_start
>> 12);
1453 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
1454 rdev
->mc
.gtt_end
>> 12);
1456 /* VRAM after AGP */
1457 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
1458 rdev
->mc
.gtt_start
>> 12);
1459 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
1460 rdev
->mc
.vram_end
>> 12);
1463 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
1464 rdev
->mc
.vram_start
>> 12);
1465 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
1466 rdev
->mc
.vram_end
>> 12);
1468 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
, rdev
->vram_scratch
.gpu_addr
>> 12);
1469 /* llano/ontario only */
1470 if ((rdev
->family
== CHIP_PALM
) ||
1471 (rdev
->family
== CHIP_SUMO
) ||
1472 (rdev
->family
== CHIP_SUMO2
)) {
1473 tmp
= RREG32(MC_FUS_VM_FB_OFFSET
) & 0x000FFFFF;
1474 tmp
|= ((rdev
->mc
.vram_end
>> 20) & 0xF) << 24;
1475 tmp
|= ((rdev
->mc
.vram_start
>> 20) & 0xF) << 20;
1476 WREG32(MC_FUS_VM_FB_OFFSET
, tmp
);
1478 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
1479 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
1480 WREG32(MC_VM_FB_LOCATION
, tmp
);
1481 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
1482 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
1483 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
1484 if (rdev
->flags
& RADEON_IS_AGP
) {
1485 WREG32(MC_VM_AGP_TOP
, rdev
->mc
.gtt_end
>> 16);
1486 WREG32(MC_VM_AGP_BOT
, rdev
->mc
.gtt_start
>> 16);
1487 WREG32(MC_VM_AGP_BASE
, rdev
->mc
.agp_base
>> 22);
1489 WREG32(MC_VM_AGP_BASE
, 0);
1490 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
1491 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
1493 if (evergreen_mc_wait_for_idle(rdev
)) {
1494 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
1496 evergreen_mc_resume(rdev
, &save
);
1497 /* we need to own VRAM, so turn off the VGA renderer here
1498 * to stop it overwriting our objects */
1499 rv515_vga_render_disable(rdev
);
1505 void evergreen_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
1507 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
1510 /* set to DX10/11 mode */
1511 radeon_ring_write(ring
, PACKET3(PACKET3_MODE_CONTROL
, 0));
1512 radeon_ring_write(ring
, 1);
1514 if (ring
->rptr_save_reg
) {
1515 next_rptr
= ring
->wptr
+ 3 + 4;
1516 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
1517 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
1518 PACKET3_SET_CONFIG_REG_START
) >> 2));
1519 radeon_ring_write(ring
, next_rptr
);
1520 } else if (rdev
->wb
.enabled
) {
1521 next_rptr
= ring
->wptr
+ 5 + 4;
1522 radeon_ring_write(ring
, PACKET3(PACKET3_MEM_WRITE
, 3));
1523 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
1524 radeon_ring_write(ring
, (upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xff) | (1 << 18));
1525 radeon_ring_write(ring
, next_rptr
);
1526 radeon_ring_write(ring
, 0);
1529 radeon_ring_write(ring
, PACKET3(PACKET3_INDIRECT_BUFFER
, 2));
1530 radeon_ring_write(ring
,
1534 (ib
->gpu_addr
& 0xFFFFFFFC));
1535 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFF);
1536 radeon_ring_write(ring
, ib
->length_dw
);
1540 static int evergreen_cp_load_microcode(struct radeon_device
*rdev
)
1542 const __be32
*fw_data
;
1545 if (!rdev
->me_fw
|| !rdev
->pfp_fw
)
1553 RB_NO_UPDATE
| RB_BLKSZ(15) | RB_BUFSZ(3));
1555 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
1556 WREG32(CP_PFP_UCODE_ADDR
, 0);
1557 for (i
= 0; i
< EVERGREEN_PFP_UCODE_SIZE
; i
++)
1558 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
1559 WREG32(CP_PFP_UCODE_ADDR
, 0);
1561 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
1562 WREG32(CP_ME_RAM_WADDR
, 0);
1563 for (i
= 0; i
< EVERGREEN_PM4_UCODE_SIZE
; i
++)
1564 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
1566 WREG32(CP_PFP_UCODE_ADDR
, 0);
1567 WREG32(CP_ME_RAM_WADDR
, 0);
1568 WREG32(CP_ME_RAM_RADDR
, 0);
1572 static int evergreen_cp_start(struct radeon_device
*rdev
)
1574 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
1578 r
= radeon_ring_lock(rdev
, ring
, 7);
1580 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
1583 radeon_ring_write(ring
, PACKET3(PACKET3_ME_INITIALIZE
, 5));
1584 radeon_ring_write(ring
, 0x1);
1585 radeon_ring_write(ring
, 0x0);
1586 radeon_ring_write(ring
, rdev
->config
.evergreen
.max_hw_contexts
- 1);
1587 radeon_ring_write(ring
, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1588 radeon_ring_write(ring
, 0);
1589 radeon_ring_write(ring
, 0);
1590 radeon_ring_unlock_commit(rdev
, ring
);
1593 WREG32(CP_ME_CNTL
, cp_me
);
1595 r
= radeon_ring_lock(rdev
, ring
, evergreen_default_size
+ 19);
1597 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
1601 /* setup clear context state */
1602 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
1603 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
1605 for (i
= 0; i
< evergreen_default_size
; i
++)
1606 radeon_ring_write(ring
, evergreen_default_state
[i
]);
1608 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
1609 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
1611 /* set clear context state */
1612 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
1613 radeon_ring_write(ring
, 0);
1615 /* SQ_VTX_BASE_VTX_LOC */
1616 radeon_ring_write(ring
, 0xc0026f00);
1617 radeon_ring_write(ring
, 0x00000000);
1618 radeon_ring_write(ring
, 0x00000000);
1619 radeon_ring_write(ring
, 0x00000000);
1622 radeon_ring_write(ring
, 0xc0036f00);
1623 radeon_ring_write(ring
, 0x00000bc4);
1624 radeon_ring_write(ring
, 0xffffffff);
1625 radeon_ring_write(ring
, 0xffffffff);
1626 radeon_ring_write(ring
, 0xffffffff);
1628 radeon_ring_write(ring
, 0xc0026900);
1629 radeon_ring_write(ring
, 0x00000316);
1630 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1631 radeon_ring_write(ring
, 0x00000010); /* */
1633 radeon_ring_unlock_commit(rdev
, ring
);
1638 static int evergreen_cp_resume(struct radeon_device
*rdev
)
1640 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
1645 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1646 WREG32(GRBM_SOFT_RESET
, (SOFT_RESET_CP
|
1652 RREG32(GRBM_SOFT_RESET
);
1654 WREG32(GRBM_SOFT_RESET
, 0);
1655 RREG32(GRBM_SOFT_RESET
);
1657 /* Set ring buffer size */
1658 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
1659 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
1661 tmp
|= BUF_SWAP_32BIT
;
1663 WREG32(CP_RB_CNTL
, tmp
);
1664 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
1665 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
1667 /* Set the write pointer delay */
1668 WREG32(CP_RB_WPTR_DELAY
, 0);
1670 /* Initialize the ring buffer's read and write pointers */
1671 WREG32(CP_RB_CNTL
, tmp
| RB_RPTR_WR_ENA
);
1672 WREG32(CP_RB_RPTR_WR
, 0);
1674 WREG32(CP_RB_WPTR
, ring
->wptr
);
1676 /* set the wb address whether it's enabled or not */
1677 WREG32(CP_RB_RPTR_ADDR
,
1678 ((rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC));
1679 WREG32(CP_RB_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
1680 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
1682 if (rdev
->wb
.enabled
)
1683 WREG32(SCRATCH_UMSK
, 0xff);
1685 tmp
|= RB_NO_UPDATE
;
1686 WREG32(SCRATCH_UMSK
, 0);
1690 WREG32(CP_RB_CNTL
, tmp
);
1692 WREG32(CP_RB_BASE
, ring
->gpu_addr
>> 8);
1693 WREG32(CP_DEBUG
, (1 << 27) | (1 << 28));
1695 ring
->rptr
= RREG32(CP_RB_RPTR
);
1697 evergreen_cp_start(rdev
);
1699 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, ring
);
1701 ring
->ready
= false;
1710 static void evergreen_gpu_init(struct radeon_device
*rdev
)
1713 u32 mc_shared_chmap
, mc_arb_ramcfg
;
1717 u32 sq_lds_resource_mgmt
;
1718 u32 sq_gpr_resource_mgmt_1
;
1719 u32 sq_gpr_resource_mgmt_2
;
1720 u32 sq_gpr_resource_mgmt_3
;
1721 u32 sq_thread_resource_mgmt
;
1722 u32 sq_thread_resource_mgmt_2
;
1723 u32 sq_stack_resource_mgmt_1
;
1724 u32 sq_stack_resource_mgmt_2
;
1725 u32 sq_stack_resource_mgmt_3
;
1726 u32 vgt_cache_invalidation
;
1727 u32 hdp_host_path_cntl
, tmp
;
1728 u32 disabled_rb_mask
;
1729 int i
, j
, num_shader_engines
, ps_thread_count
;
1731 switch (rdev
->family
) {
1734 rdev
->config
.evergreen
.num_ses
= 2;
1735 rdev
->config
.evergreen
.max_pipes
= 4;
1736 rdev
->config
.evergreen
.max_tile_pipes
= 8;
1737 rdev
->config
.evergreen
.max_simds
= 10;
1738 rdev
->config
.evergreen
.max_backends
= 4 * rdev
->config
.evergreen
.num_ses
;
1739 rdev
->config
.evergreen
.max_gprs
= 256;
1740 rdev
->config
.evergreen
.max_threads
= 248;
1741 rdev
->config
.evergreen
.max_gs_threads
= 32;
1742 rdev
->config
.evergreen
.max_stack_entries
= 512;
1743 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1744 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1745 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1746 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1747 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1748 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1750 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x100;
1751 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1752 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1753 gb_addr_config
= CYPRESS_GB_ADDR_CONFIG_GOLDEN
;
1756 rdev
->config
.evergreen
.num_ses
= 1;
1757 rdev
->config
.evergreen
.max_pipes
= 4;
1758 rdev
->config
.evergreen
.max_tile_pipes
= 4;
1759 rdev
->config
.evergreen
.max_simds
= 10;
1760 rdev
->config
.evergreen
.max_backends
= 4 * rdev
->config
.evergreen
.num_ses
;
1761 rdev
->config
.evergreen
.max_gprs
= 256;
1762 rdev
->config
.evergreen
.max_threads
= 248;
1763 rdev
->config
.evergreen
.max_gs_threads
= 32;
1764 rdev
->config
.evergreen
.max_stack_entries
= 512;
1765 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1766 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1767 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1768 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1769 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1770 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1772 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x100;
1773 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1774 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1775 gb_addr_config
= JUNIPER_GB_ADDR_CONFIG_GOLDEN
;
1778 rdev
->config
.evergreen
.num_ses
= 1;
1779 rdev
->config
.evergreen
.max_pipes
= 4;
1780 rdev
->config
.evergreen
.max_tile_pipes
= 4;
1781 rdev
->config
.evergreen
.max_simds
= 5;
1782 rdev
->config
.evergreen
.max_backends
= 2 * rdev
->config
.evergreen
.num_ses
;
1783 rdev
->config
.evergreen
.max_gprs
= 256;
1784 rdev
->config
.evergreen
.max_threads
= 248;
1785 rdev
->config
.evergreen
.max_gs_threads
= 32;
1786 rdev
->config
.evergreen
.max_stack_entries
= 256;
1787 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1788 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1789 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1790 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1791 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1792 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1794 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x100;
1795 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1796 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1797 gb_addr_config
= REDWOOD_GB_ADDR_CONFIG_GOLDEN
;
1801 rdev
->config
.evergreen
.num_ses
= 1;
1802 rdev
->config
.evergreen
.max_pipes
= 2;
1803 rdev
->config
.evergreen
.max_tile_pipes
= 2;
1804 rdev
->config
.evergreen
.max_simds
= 2;
1805 rdev
->config
.evergreen
.max_backends
= 1 * rdev
->config
.evergreen
.num_ses
;
1806 rdev
->config
.evergreen
.max_gprs
= 256;
1807 rdev
->config
.evergreen
.max_threads
= 192;
1808 rdev
->config
.evergreen
.max_gs_threads
= 16;
1809 rdev
->config
.evergreen
.max_stack_entries
= 256;
1810 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1811 rdev
->config
.evergreen
.sx_max_export_size
= 128;
1812 rdev
->config
.evergreen
.sx_max_export_pos_size
= 32;
1813 rdev
->config
.evergreen
.sx_max_export_smx_size
= 96;
1814 rdev
->config
.evergreen
.max_hw_contexts
= 4;
1815 rdev
->config
.evergreen
.sq_num_cf_insts
= 1;
1817 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x40;
1818 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1819 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1820 gb_addr_config
= CEDAR_GB_ADDR_CONFIG_GOLDEN
;
1823 rdev
->config
.evergreen
.num_ses
= 1;
1824 rdev
->config
.evergreen
.max_pipes
= 2;
1825 rdev
->config
.evergreen
.max_tile_pipes
= 2;
1826 rdev
->config
.evergreen
.max_simds
= 2;
1827 rdev
->config
.evergreen
.max_backends
= 1 * rdev
->config
.evergreen
.num_ses
;
1828 rdev
->config
.evergreen
.max_gprs
= 256;
1829 rdev
->config
.evergreen
.max_threads
= 192;
1830 rdev
->config
.evergreen
.max_gs_threads
= 16;
1831 rdev
->config
.evergreen
.max_stack_entries
= 256;
1832 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1833 rdev
->config
.evergreen
.sx_max_export_size
= 128;
1834 rdev
->config
.evergreen
.sx_max_export_pos_size
= 32;
1835 rdev
->config
.evergreen
.sx_max_export_smx_size
= 96;
1836 rdev
->config
.evergreen
.max_hw_contexts
= 4;
1837 rdev
->config
.evergreen
.sq_num_cf_insts
= 1;
1839 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x40;
1840 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1841 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1842 gb_addr_config
= CEDAR_GB_ADDR_CONFIG_GOLDEN
;
1845 rdev
->config
.evergreen
.num_ses
= 1;
1846 rdev
->config
.evergreen
.max_pipes
= 4;
1847 rdev
->config
.evergreen
.max_tile_pipes
= 4;
1848 if (rdev
->pdev
->device
== 0x9648)
1849 rdev
->config
.evergreen
.max_simds
= 3;
1850 else if ((rdev
->pdev
->device
== 0x9647) ||
1851 (rdev
->pdev
->device
== 0x964a))
1852 rdev
->config
.evergreen
.max_simds
= 4;
1854 rdev
->config
.evergreen
.max_simds
= 5;
1855 rdev
->config
.evergreen
.max_backends
= 2 * rdev
->config
.evergreen
.num_ses
;
1856 rdev
->config
.evergreen
.max_gprs
= 256;
1857 rdev
->config
.evergreen
.max_threads
= 248;
1858 rdev
->config
.evergreen
.max_gs_threads
= 32;
1859 rdev
->config
.evergreen
.max_stack_entries
= 256;
1860 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1861 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1862 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1863 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1864 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1865 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1867 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x40;
1868 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1869 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1870 gb_addr_config
= SUMO_GB_ADDR_CONFIG_GOLDEN
;
1873 rdev
->config
.evergreen
.num_ses
= 1;
1874 rdev
->config
.evergreen
.max_pipes
= 4;
1875 rdev
->config
.evergreen
.max_tile_pipes
= 4;
1876 rdev
->config
.evergreen
.max_simds
= 2;
1877 rdev
->config
.evergreen
.max_backends
= 1 * rdev
->config
.evergreen
.num_ses
;
1878 rdev
->config
.evergreen
.max_gprs
= 256;
1879 rdev
->config
.evergreen
.max_threads
= 248;
1880 rdev
->config
.evergreen
.max_gs_threads
= 32;
1881 rdev
->config
.evergreen
.max_stack_entries
= 512;
1882 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1883 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1884 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1885 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1886 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1887 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1889 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x40;
1890 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1891 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1892 gb_addr_config
= SUMO2_GB_ADDR_CONFIG_GOLDEN
;
1895 rdev
->config
.evergreen
.num_ses
= 2;
1896 rdev
->config
.evergreen
.max_pipes
= 4;
1897 rdev
->config
.evergreen
.max_tile_pipes
= 8;
1898 rdev
->config
.evergreen
.max_simds
= 7;
1899 rdev
->config
.evergreen
.max_backends
= 4 * rdev
->config
.evergreen
.num_ses
;
1900 rdev
->config
.evergreen
.max_gprs
= 256;
1901 rdev
->config
.evergreen
.max_threads
= 248;
1902 rdev
->config
.evergreen
.max_gs_threads
= 32;
1903 rdev
->config
.evergreen
.max_stack_entries
= 512;
1904 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1905 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1906 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1907 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1908 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1909 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1911 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x100;
1912 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1913 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1914 gb_addr_config
= BARTS_GB_ADDR_CONFIG_GOLDEN
;
1917 rdev
->config
.evergreen
.num_ses
= 1;
1918 rdev
->config
.evergreen
.max_pipes
= 4;
1919 rdev
->config
.evergreen
.max_tile_pipes
= 4;
1920 rdev
->config
.evergreen
.max_simds
= 6;
1921 rdev
->config
.evergreen
.max_backends
= 2 * rdev
->config
.evergreen
.num_ses
;
1922 rdev
->config
.evergreen
.max_gprs
= 256;
1923 rdev
->config
.evergreen
.max_threads
= 248;
1924 rdev
->config
.evergreen
.max_gs_threads
= 32;
1925 rdev
->config
.evergreen
.max_stack_entries
= 256;
1926 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1927 rdev
->config
.evergreen
.sx_max_export_size
= 256;
1928 rdev
->config
.evergreen
.sx_max_export_pos_size
= 64;
1929 rdev
->config
.evergreen
.sx_max_export_smx_size
= 192;
1930 rdev
->config
.evergreen
.max_hw_contexts
= 8;
1931 rdev
->config
.evergreen
.sq_num_cf_insts
= 2;
1933 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x100;
1934 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1935 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1936 gb_addr_config
= TURKS_GB_ADDR_CONFIG_GOLDEN
;
1939 rdev
->config
.evergreen
.num_ses
= 1;
1940 rdev
->config
.evergreen
.max_pipes
= 2;
1941 rdev
->config
.evergreen
.max_tile_pipes
= 2;
1942 rdev
->config
.evergreen
.max_simds
= 2;
1943 rdev
->config
.evergreen
.max_backends
= 1 * rdev
->config
.evergreen
.num_ses
;
1944 rdev
->config
.evergreen
.max_gprs
= 256;
1945 rdev
->config
.evergreen
.max_threads
= 192;
1946 rdev
->config
.evergreen
.max_gs_threads
= 16;
1947 rdev
->config
.evergreen
.max_stack_entries
= 256;
1948 rdev
->config
.evergreen
.sx_num_of_sets
= 4;
1949 rdev
->config
.evergreen
.sx_max_export_size
= 128;
1950 rdev
->config
.evergreen
.sx_max_export_pos_size
= 32;
1951 rdev
->config
.evergreen
.sx_max_export_smx_size
= 96;
1952 rdev
->config
.evergreen
.max_hw_contexts
= 4;
1953 rdev
->config
.evergreen
.sq_num_cf_insts
= 1;
1955 rdev
->config
.evergreen
.sc_prim_fifo_size
= 0x40;
1956 rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
= 0x30;
1957 rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
= 0x130;
1958 gb_addr_config
= CAICOS_GB_ADDR_CONFIG_GOLDEN
;
1962 /* Initialize HDP */
1963 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
1964 WREG32((0x2c14 + j
), 0x00000000);
1965 WREG32((0x2c18 + j
), 0x00000000);
1966 WREG32((0x2c1c + j
), 0x00000000);
1967 WREG32((0x2c20 + j
), 0x00000000);
1968 WREG32((0x2c24 + j
), 0x00000000);
1971 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
1973 evergreen_fix_pci_max_read_req_size(rdev
);
1975 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
1976 if ((rdev
->family
== CHIP_PALM
) ||
1977 (rdev
->family
== CHIP_SUMO
) ||
1978 (rdev
->family
== CHIP_SUMO2
))
1979 mc_arb_ramcfg
= RREG32(FUS_MC_ARB_RAMCFG
);
1981 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
1983 /* setup tiling info dword. gb_addr_config is not adequate since it does
1984 * not have bank info, so create a custom tiling dword.
1985 * bits 3:0 num_pipes
1986 * bits 7:4 num_banks
1987 * bits 11:8 group_size
1988 * bits 15:12 row_size
1990 rdev
->config
.evergreen
.tile_config
= 0;
1991 switch (rdev
->config
.evergreen
.max_tile_pipes
) {
1994 rdev
->config
.evergreen
.tile_config
|= (0 << 0);
1997 rdev
->config
.evergreen
.tile_config
|= (1 << 0);
2000 rdev
->config
.evergreen
.tile_config
|= (2 << 0);
2003 rdev
->config
.evergreen
.tile_config
|= (3 << 0);
2006 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2007 if (rdev
->flags
& RADEON_IS_IGP
)
2008 rdev
->config
.evergreen
.tile_config
|= 1 << 4;
2010 switch ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) {
2011 case 0: /* four banks */
2012 rdev
->config
.evergreen
.tile_config
|= 0 << 4;
2014 case 1: /* eight banks */
2015 rdev
->config
.evergreen
.tile_config
|= 1 << 4;
2017 case 2: /* sixteen banks */
2019 rdev
->config
.evergreen
.tile_config
|= 2 << 4;
2023 rdev
->config
.evergreen
.tile_config
|= 0 << 8;
2024 rdev
->config
.evergreen
.tile_config
|=
2025 ((gb_addr_config
& 0x30000000) >> 28) << 12;
2027 num_shader_engines
= (gb_addr_config
& NUM_SHADER_ENGINES(3) >> 12) + 1;
2029 if ((rdev
->family
>= CHIP_CEDAR
) && (rdev
->family
<= CHIP_HEMLOCK
)) {
2033 WREG32(RCU_IND_INDEX
, 0x204);
2034 efuse_straps_4
= RREG32(RCU_IND_DATA
);
2035 WREG32(RCU_IND_INDEX
, 0x203);
2036 efuse_straps_3
= RREG32(RCU_IND_DATA
);
2037 tmp
= (((efuse_straps_4
& 0xf) << 4) |
2038 ((efuse_straps_3
& 0xf0000000) >> 28));
2041 for (i
= (rdev
->config
.evergreen
.num_ses
- 1); i
>= 0; i
--) {
2042 u32 rb_disable_bitmap
;
2044 WREG32(GRBM_GFX_INDEX
, INSTANCE_BROADCAST_WRITES
| SE_INDEX(i
));
2045 WREG32(RLC_GFX_INDEX
, INSTANCE_BROADCAST_WRITES
| SE_INDEX(i
));
2046 rb_disable_bitmap
= (RREG32(CC_RB_BACKEND_DISABLE
) & 0x00ff0000) >> 16;
2048 tmp
|= rb_disable_bitmap
;
2051 /* enabled rb are just the one not disabled :) */
2052 disabled_rb_mask
= tmp
;
2054 WREG32(GRBM_GFX_INDEX
, INSTANCE_BROADCAST_WRITES
| SE_BROADCAST_WRITES
);
2055 WREG32(RLC_GFX_INDEX
, INSTANCE_BROADCAST_WRITES
| SE_BROADCAST_WRITES
);
2057 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
2058 WREG32(DMIF_ADDR_CONFIG
, gb_addr_config
);
2059 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
2060 WREG32(DMA_TILING_CONFIG
, gb_addr_config
);
2062 if ((rdev
->config
.evergreen
.max_backends
== 1) &&
2063 (rdev
->flags
& RADEON_IS_IGP
)) {
2064 if ((disabled_rb_mask
& 3) == 1) {
2065 /* RB0 disabled, RB1 enabled */
2068 /* RB1 disabled, RB0 enabled */
2072 tmp
= gb_addr_config
& NUM_PIPES_MASK
;
2073 tmp
= r6xx_remap_render_backend(rdev
, tmp
, rdev
->config
.evergreen
.max_backends
,
2074 EVERGREEN_MAX_BACKENDS
, disabled_rb_mask
);
2076 WREG32(GB_BACKEND_MAP
, tmp
);
2078 WREG32(CGTS_SYS_TCC_DISABLE
, 0);
2079 WREG32(CGTS_TCC_DISABLE
, 0);
2080 WREG32(CGTS_USER_SYS_TCC_DISABLE
, 0);
2081 WREG32(CGTS_USER_TCC_DISABLE
, 0);
2083 /* set HW defaults for 3D engine */
2084 WREG32(CP_QUEUE_THRESHOLDS
, (ROQ_IB1_START(0x16) |
2085 ROQ_IB2_START(0x2b)));
2087 WREG32(CP_MEQ_THRESHOLDS
, STQ_SPLIT(0x30));
2089 WREG32(TA_CNTL_AUX
, (DISABLE_CUBE_ANISO
|
2094 sx_debug_1
= RREG32(SX_DEBUG_1
);
2095 sx_debug_1
|= ENABLE_NEW_SMX_ADDRESS
;
2096 WREG32(SX_DEBUG_1
, sx_debug_1
);
2099 smx_dc_ctl0
= RREG32(SMX_DC_CTL0
);
2100 smx_dc_ctl0
&= ~NUMBER_OF_SETS(0x1ff);
2101 smx_dc_ctl0
|= NUMBER_OF_SETS(rdev
->config
.evergreen
.sx_num_of_sets
);
2102 WREG32(SMX_DC_CTL0
, smx_dc_ctl0
);
2104 if (rdev
->family
<= CHIP_SUMO2
)
2105 WREG32(SMX_SAR_CTL0
, 0x00010000);
2107 WREG32(SX_EXPORT_BUFFER_SIZES
, (COLOR_BUFFER_SIZE((rdev
->config
.evergreen
.sx_max_export_size
/ 4) - 1) |
2108 POSITION_BUFFER_SIZE((rdev
->config
.evergreen
.sx_max_export_pos_size
/ 4) - 1) |
2109 SMX_BUFFER_SIZE((rdev
->config
.evergreen
.sx_max_export_smx_size
/ 4) - 1)));
2111 WREG32(PA_SC_FIFO_SIZE
, (SC_PRIM_FIFO_SIZE(rdev
->config
.evergreen
.sc_prim_fifo_size
) |
2112 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.evergreen
.sc_hiz_tile_fifo_size
) |
2113 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.evergreen
.sc_earlyz_tile_fifo_size
)));
2115 WREG32(VGT_NUM_INSTANCES
, 1);
2116 WREG32(SPI_CONFIG_CNTL
, 0);
2117 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
2118 WREG32(CP_PERFMON_CNTL
, 0);
2120 WREG32(SQ_MS_FIFO_SIZES
, (CACHE_FIFO_SIZE(16 * rdev
->config
.evergreen
.sq_num_cf_insts
) |
2121 FETCH_FIFO_HIWATER(0x4) |
2122 DONE_FIFO_HIWATER(0xe0) |
2123 ALU_UPDATE_FIFO_HIWATER(0x8)));
2125 sq_config
= RREG32(SQ_CONFIG
);
2126 sq_config
&= ~(PS_PRIO(3) |
2130 sq_config
|= (VC_ENABLE
|
2137 switch (rdev
->family
) {
2143 /* no vertex cache */
2144 sq_config
&= ~VC_ENABLE
;
2150 sq_lds_resource_mgmt
= RREG32(SQ_LDS_RESOURCE_MGMT
);
2152 sq_gpr_resource_mgmt_1
= NUM_PS_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2))* 12 / 32);
2153 sq_gpr_resource_mgmt_1
|= NUM_VS_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2)) * 6 / 32);
2154 sq_gpr_resource_mgmt_1
|= NUM_CLAUSE_TEMP_GPRS(4);
2155 sq_gpr_resource_mgmt_2
= NUM_GS_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2)) * 4 / 32);
2156 sq_gpr_resource_mgmt_2
|= NUM_ES_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2)) * 4 / 32);
2157 sq_gpr_resource_mgmt_3
= NUM_HS_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2)) * 3 / 32);
2158 sq_gpr_resource_mgmt_3
|= NUM_LS_GPRS((rdev
->config
.evergreen
.max_gprs
- (4 * 2)) * 3 / 32);
2160 switch (rdev
->family
) {
2165 ps_thread_count
= 96;
2168 ps_thread_count
= 128;
2172 sq_thread_resource_mgmt
= NUM_PS_THREADS(ps_thread_count
);
2173 sq_thread_resource_mgmt
|= NUM_VS_THREADS((((rdev
->config
.evergreen
.max_threads
- ps_thread_count
) / 6) / 8) * 8);
2174 sq_thread_resource_mgmt
|= NUM_GS_THREADS((((rdev
->config
.evergreen
.max_threads
- ps_thread_count
) / 6) / 8) * 8);
2175 sq_thread_resource_mgmt
|= NUM_ES_THREADS((((rdev
->config
.evergreen
.max_threads
- ps_thread_count
) / 6) / 8) * 8);
2176 sq_thread_resource_mgmt_2
= NUM_HS_THREADS((((rdev
->config
.evergreen
.max_threads
- ps_thread_count
) / 6) / 8) * 8);
2177 sq_thread_resource_mgmt_2
|= NUM_LS_THREADS((((rdev
->config
.evergreen
.max_threads
- ps_thread_count
) / 6) / 8) * 8);
2179 sq_stack_resource_mgmt_1
= NUM_PS_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2180 sq_stack_resource_mgmt_1
|= NUM_VS_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2181 sq_stack_resource_mgmt_2
= NUM_GS_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2182 sq_stack_resource_mgmt_2
|= NUM_ES_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2183 sq_stack_resource_mgmt_3
= NUM_HS_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2184 sq_stack_resource_mgmt_3
|= NUM_LS_STACK_ENTRIES((rdev
->config
.evergreen
.max_stack_entries
* 1) / 6);
2186 WREG32(SQ_CONFIG
, sq_config
);
2187 WREG32(SQ_GPR_RESOURCE_MGMT_1
, sq_gpr_resource_mgmt_1
);
2188 WREG32(SQ_GPR_RESOURCE_MGMT_2
, sq_gpr_resource_mgmt_2
);
2189 WREG32(SQ_GPR_RESOURCE_MGMT_3
, sq_gpr_resource_mgmt_3
);
2190 WREG32(SQ_THREAD_RESOURCE_MGMT
, sq_thread_resource_mgmt
);
2191 WREG32(SQ_THREAD_RESOURCE_MGMT_2
, sq_thread_resource_mgmt_2
);
2192 WREG32(SQ_STACK_RESOURCE_MGMT_1
, sq_stack_resource_mgmt_1
);
2193 WREG32(SQ_STACK_RESOURCE_MGMT_2
, sq_stack_resource_mgmt_2
);
2194 WREG32(SQ_STACK_RESOURCE_MGMT_3
, sq_stack_resource_mgmt_3
);
2195 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ
, 0);
2196 WREG32(SQ_LDS_RESOURCE_MGMT
, sq_lds_resource_mgmt
);
2198 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
2199 FORCE_EOV_MAX_REZ_CNT(255)));
2201 switch (rdev
->family
) {
2207 vgt_cache_invalidation
= CACHE_INVALIDATION(TC_ONLY
);
2210 vgt_cache_invalidation
= CACHE_INVALIDATION(VC_AND_TC
);
2213 vgt_cache_invalidation
|= AUTO_INVLD_EN(ES_AND_GS_AUTO
);
2214 WREG32(VGT_CACHE_INVALIDATION
, vgt_cache_invalidation
);
2216 WREG32(VGT_GS_VERTEX_REUSE
, 16);
2217 WREG32(PA_SU_LINE_STIPPLE_VALUE
, 0);
2218 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
2220 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL
, 14);
2221 WREG32(VGT_OUT_DEALLOC_CNTL
, 16);
2223 WREG32(CB_PERF_CTR0_SEL_0
, 0);
2224 WREG32(CB_PERF_CTR0_SEL_1
, 0);
2225 WREG32(CB_PERF_CTR1_SEL_0
, 0);
2226 WREG32(CB_PERF_CTR1_SEL_1
, 0);
2227 WREG32(CB_PERF_CTR2_SEL_0
, 0);
2228 WREG32(CB_PERF_CTR2_SEL_1
, 0);
2229 WREG32(CB_PERF_CTR3_SEL_0
, 0);
2230 WREG32(CB_PERF_CTR3_SEL_1
, 0);
2232 /* clear render buffer base addresses */
2233 WREG32(CB_COLOR0_BASE
, 0);
2234 WREG32(CB_COLOR1_BASE
, 0);
2235 WREG32(CB_COLOR2_BASE
, 0);
2236 WREG32(CB_COLOR3_BASE
, 0);
2237 WREG32(CB_COLOR4_BASE
, 0);
2238 WREG32(CB_COLOR5_BASE
, 0);
2239 WREG32(CB_COLOR6_BASE
, 0);
2240 WREG32(CB_COLOR7_BASE
, 0);
2241 WREG32(CB_COLOR8_BASE
, 0);
2242 WREG32(CB_COLOR9_BASE
, 0);
2243 WREG32(CB_COLOR10_BASE
, 0);
2244 WREG32(CB_COLOR11_BASE
, 0);
2246 /* set the shader const cache sizes to 0 */
2247 for (i
= SQ_ALU_CONST_BUFFER_SIZE_PS_0
; i
< 0x28200; i
+= 4)
2249 for (i
= SQ_ALU_CONST_BUFFER_SIZE_HS_0
; i
< 0x29000; i
+= 4)
2252 tmp
= RREG32(HDP_MISC_CNTL
);
2253 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
2254 WREG32(HDP_MISC_CNTL
, tmp
);
2256 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
2257 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
2259 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
2265 int evergreen_mc_init(struct radeon_device
*rdev
)
2268 int chansize
, numchan
;
2270 /* Get VRAM informations */
2271 rdev
->mc
.vram_is_ddr
= true;
2272 if ((rdev
->family
== CHIP_PALM
) ||
2273 (rdev
->family
== CHIP_SUMO
) ||
2274 (rdev
->family
== CHIP_SUMO2
))
2275 tmp
= RREG32(FUS_MC_ARB_RAMCFG
);
2277 tmp
= RREG32(MC_ARB_RAMCFG
);
2278 if (tmp
& CHANSIZE_OVERRIDE
) {
2280 } else if (tmp
& CHANSIZE_MASK
) {
2285 tmp
= RREG32(MC_SHARED_CHMAP
);
2286 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
2301 rdev
->mc
.vram_width
= numchan
* chansize
;
2302 /* Could aper size report 0 ? */
2303 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
2304 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
2305 /* Setup GPU memory space */
2306 if ((rdev
->family
== CHIP_PALM
) ||
2307 (rdev
->family
== CHIP_SUMO
) ||
2308 (rdev
->family
== CHIP_SUMO2
)) {
2309 /* size in bytes on fusion */
2310 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
);
2311 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
);
2313 /* size in MB on evergreen/cayman/tn */
2314 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024 * 1024;
2315 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024 * 1024;
2317 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
2318 r700_vram_gtt_location(rdev
, &rdev
->mc
);
2319 radeon_update_bandwidth_info(rdev
);
2324 void evergreen_print_gpu_status_regs(struct radeon_device
*rdev
)
2326 dev_info(rdev
->dev
, " GRBM_STATUS = 0x%08X\n",
2327 RREG32(GRBM_STATUS
));
2328 dev_info(rdev
->dev
, " GRBM_STATUS_SE0 = 0x%08X\n",
2329 RREG32(GRBM_STATUS_SE0
));
2330 dev_info(rdev
->dev
, " GRBM_STATUS_SE1 = 0x%08X\n",
2331 RREG32(GRBM_STATUS_SE1
));
2332 dev_info(rdev
->dev
, " SRBM_STATUS = 0x%08X\n",
2333 RREG32(SRBM_STATUS
));
2334 dev_info(rdev
->dev
, " SRBM_STATUS2 = 0x%08X\n",
2335 RREG32(SRBM_STATUS2
));
2336 dev_info(rdev
->dev
, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
2337 RREG32(CP_STALLED_STAT1
));
2338 dev_info(rdev
->dev
, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
2339 RREG32(CP_STALLED_STAT2
));
2340 dev_info(rdev
->dev
, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
2341 RREG32(CP_BUSY_STAT
));
2342 dev_info(rdev
->dev
, " R_008680_CP_STAT = 0x%08X\n",
2344 dev_info(rdev
->dev
, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
2345 RREG32(DMA_STATUS_REG
));
2346 if (rdev
->family
>= CHIP_CAYMAN
) {
2347 dev_info(rdev
->dev
, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
2348 RREG32(DMA_STATUS_REG
+ 0x800));
2352 bool evergreen_is_display_hung(struct radeon_device
*rdev
)
2358 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
2359 if (RREG32(EVERGREEN_CRTC_CONTROL
+ crtc_offsets
[i
]) & EVERGREEN_CRTC_MASTER_EN
) {
2360 crtc_status
[i
] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT
+ crtc_offsets
[i
]);
2361 crtc_hung
|= (1 << i
);
2365 for (j
= 0; j
< 10; j
++) {
2366 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
2367 if (crtc_hung
& (1 << i
)) {
2368 tmp
= RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT
+ crtc_offsets
[i
]);
2369 if (tmp
!= crtc_status
[i
])
2370 crtc_hung
&= ~(1 << i
);
2381 static u32
evergreen_gpu_check_soft_reset(struct radeon_device
*rdev
)
2387 tmp
= RREG32(GRBM_STATUS
);
2388 if (tmp
& (PA_BUSY
| SC_BUSY
|
2390 TA_BUSY
| VGT_BUSY
|
2392 SPI_BUSY
| VGT_BUSY_NO_DMA
))
2393 reset_mask
|= RADEON_RESET_GFX
;
2395 if (tmp
& (CF_RQ_PENDING
| PF_RQ_PENDING
|
2396 CP_BUSY
| CP_COHERENCY_BUSY
))
2397 reset_mask
|= RADEON_RESET_CP
;
2399 if (tmp
& GRBM_EE_BUSY
)
2400 reset_mask
|= RADEON_RESET_GRBM
| RADEON_RESET_GFX
| RADEON_RESET_CP
;
2402 /* DMA_STATUS_REG */
2403 tmp
= RREG32(DMA_STATUS_REG
);
2404 if (!(tmp
& DMA_IDLE
))
2405 reset_mask
|= RADEON_RESET_DMA
;
2408 tmp
= RREG32(SRBM_STATUS2
);
2410 reset_mask
|= RADEON_RESET_DMA
;
2413 tmp
= RREG32(SRBM_STATUS
);
2414 if (tmp
& (RLC_RQ_PENDING
| RLC_BUSY
))
2415 reset_mask
|= RADEON_RESET_RLC
;
2418 reset_mask
|= RADEON_RESET_IH
;
2421 reset_mask
|= RADEON_RESET_SEM
;
2423 if (tmp
& GRBM_RQ_PENDING
)
2424 reset_mask
|= RADEON_RESET_GRBM
;
2427 reset_mask
|= RADEON_RESET_VMC
;
2429 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
2430 MCC_BUSY
| MCD_BUSY
))
2431 reset_mask
|= RADEON_RESET_MC
;
2433 if (evergreen_is_display_hung(rdev
))
2434 reset_mask
|= RADEON_RESET_DISPLAY
;
2437 tmp
= RREG32(VM_L2_STATUS
);
2439 reset_mask
|= RADEON_RESET_VMC
;
2444 static void evergreen_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
2446 struct evergreen_mc_save save
;
2447 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
2450 if (reset_mask
== 0)
2453 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
2455 evergreen_print_gpu_status_regs(rdev
);
2457 /* Disable CP parsing/prefetching */
2458 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
);
2460 if (reset_mask
& RADEON_RESET_DMA
) {
2462 tmp
= RREG32(DMA_RB_CNTL
);
2463 tmp
&= ~DMA_RB_ENABLE
;
2464 WREG32(DMA_RB_CNTL
, tmp
);
2469 evergreen_mc_stop(rdev
, &save
);
2470 if (evergreen_mc_wait_for_idle(rdev
)) {
2471 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
2474 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
)) {
2475 grbm_soft_reset
|= SOFT_RESET_DB
|
2488 if (reset_mask
& RADEON_RESET_CP
) {
2489 grbm_soft_reset
|= SOFT_RESET_CP
|
2492 srbm_soft_reset
|= SOFT_RESET_GRBM
;
2495 if (reset_mask
& RADEON_RESET_DMA
)
2496 srbm_soft_reset
|= SOFT_RESET_DMA
;
2498 if (reset_mask
& RADEON_RESET_DISPLAY
)
2499 srbm_soft_reset
|= SOFT_RESET_DC
;
2501 if (reset_mask
& RADEON_RESET_RLC
)
2502 srbm_soft_reset
|= SOFT_RESET_RLC
;
2504 if (reset_mask
& RADEON_RESET_SEM
)
2505 srbm_soft_reset
|= SOFT_RESET_SEM
;
2507 if (reset_mask
& RADEON_RESET_IH
)
2508 srbm_soft_reset
|= SOFT_RESET_IH
;
2510 if (reset_mask
& RADEON_RESET_GRBM
)
2511 srbm_soft_reset
|= SOFT_RESET_GRBM
;
2513 if (reset_mask
& RADEON_RESET_VMC
)
2514 srbm_soft_reset
|= SOFT_RESET_VMC
;
2516 if (!(rdev
->flags
& RADEON_IS_IGP
)) {
2517 if (reset_mask
& RADEON_RESET_MC
)
2518 srbm_soft_reset
|= SOFT_RESET_MC
;
2521 if (grbm_soft_reset
) {
2522 tmp
= RREG32(GRBM_SOFT_RESET
);
2523 tmp
|= grbm_soft_reset
;
2524 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
2525 WREG32(GRBM_SOFT_RESET
, tmp
);
2526 tmp
= RREG32(GRBM_SOFT_RESET
);
2530 tmp
&= ~grbm_soft_reset
;
2531 WREG32(GRBM_SOFT_RESET
, tmp
);
2532 tmp
= RREG32(GRBM_SOFT_RESET
);
2535 if (srbm_soft_reset
) {
2536 tmp
= RREG32(SRBM_SOFT_RESET
);
2537 tmp
|= srbm_soft_reset
;
2538 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
2539 WREG32(SRBM_SOFT_RESET
, tmp
);
2540 tmp
= RREG32(SRBM_SOFT_RESET
);
2544 tmp
&= ~srbm_soft_reset
;
2545 WREG32(SRBM_SOFT_RESET
, tmp
);
2546 tmp
= RREG32(SRBM_SOFT_RESET
);
2549 /* Wait a little for things to settle down */
2552 evergreen_mc_resume(rdev
, &save
);
2555 evergreen_print_gpu_status_regs(rdev
);
2558 int evergreen_asic_reset(struct radeon_device
*rdev
)
2562 reset_mask
= evergreen_gpu_check_soft_reset(rdev
);
2565 r600_set_bios_scratch_engine_hung(rdev
, true);
2567 evergreen_gpu_soft_reset(rdev
, reset_mask
);
2569 reset_mask
= evergreen_gpu_check_soft_reset(rdev
);
2572 r600_set_bios_scratch_engine_hung(rdev
, false);
2578 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
2580 * @rdev: radeon_device pointer
2581 * @ring: radeon_ring structure holding ring information
2583 * Check if the GFX engine is locked up.
2584 * Returns true if the engine appears to be locked up, false if not.
2586 bool evergreen_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
2588 u32 reset_mask
= evergreen_gpu_check_soft_reset(rdev
);
2590 if (!(reset_mask
& (RADEON_RESET_GFX
|
2591 RADEON_RESET_COMPUTE
|
2592 RADEON_RESET_CP
))) {
2593 radeon_ring_lockup_update(ring
);
2596 /* force CP activities */
2597 radeon_ring_force_activity(rdev
, ring
);
2598 return radeon_ring_test_lockup(rdev
, ring
);
2602 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
2604 * @rdev: radeon_device pointer
2605 * @ring: radeon_ring structure holding ring information
2607 * Check if the async DMA engine is locked up.
2608 * Returns true if the engine appears to be locked up, false if not.
2610 bool evergreen_dma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
2612 u32 reset_mask
= evergreen_gpu_check_soft_reset(rdev
);
2614 if (!(reset_mask
& RADEON_RESET_DMA
)) {
2615 radeon_ring_lockup_update(ring
);
2618 /* force ring activities */
2619 radeon_ring_force_activity(rdev
, ring
);
2620 return radeon_ring_test_lockup(rdev
, ring
);
2625 u32
evergreen_get_vblank_counter(struct radeon_device
*rdev
, int crtc
)
2627 if (crtc
>= rdev
->num_crtc
)
2630 return RREG32(CRTC_STATUS_FRAME_COUNT
+ crtc_offsets
[crtc
]);
2633 void evergreen_disable_interrupt_state(struct radeon_device
*rdev
)
2637 if (rdev
->family
>= CHIP_CAYMAN
) {
2638 cayman_cp_int_cntl_setup(rdev
, 0,
2639 CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
2640 cayman_cp_int_cntl_setup(rdev
, 1, 0);
2641 cayman_cp_int_cntl_setup(rdev
, 2, 0);
2642 tmp
= RREG32(CAYMAN_DMA1_CNTL
) & ~TRAP_ENABLE
;
2643 WREG32(CAYMAN_DMA1_CNTL
, tmp
);
2645 WREG32(CP_INT_CNTL
, CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
2646 tmp
= RREG32(DMA_CNTL
) & ~TRAP_ENABLE
;
2647 WREG32(DMA_CNTL
, tmp
);
2648 WREG32(GRBM_INT_CNTL
, 0);
2649 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
2650 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
2651 if (rdev
->num_crtc
>= 4) {
2652 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
2653 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
2655 if (rdev
->num_crtc
>= 6) {
2656 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
2657 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
2660 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
2661 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
2662 if (rdev
->num_crtc
>= 4) {
2663 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
2664 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
2666 if (rdev
->num_crtc
>= 6) {
2667 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
2668 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
2671 /* only one DAC on DCE6 */
2672 if (!ASIC_IS_DCE6(rdev
))
2673 WREG32(DACA_AUTODETECT_INT_CONTROL
, 0);
2674 WREG32(DACB_AUTODETECT_INT_CONTROL
, 0);
2676 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2677 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
2678 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2679 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
2680 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2681 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
2682 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2683 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
2684 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2685 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
2686 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
2687 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
2691 int evergreen_irq_set(struct radeon_device
*rdev
)
2693 u32 cp_int_cntl
= CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
;
2694 u32 cp_int_cntl1
= 0, cp_int_cntl2
= 0;
2695 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
2696 u32 hpd1
, hpd2
, hpd3
, hpd4
, hpd5
, hpd6
;
2697 u32 grbm_int_cntl
= 0;
2698 u32 grph1
= 0, grph2
= 0, grph3
= 0, grph4
= 0, grph5
= 0, grph6
= 0;
2699 u32 afmt1
= 0, afmt2
= 0, afmt3
= 0, afmt4
= 0, afmt5
= 0, afmt6
= 0;
2700 u32 dma_cntl
, dma_cntl1
= 0;
2702 if (!rdev
->irq
.installed
) {
2703 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2706 /* don't enable anything if the ih is disabled */
2707 if (!rdev
->ih
.enabled
) {
2708 r600_disable_interrupts(rdev
);
2709 /* force the active interrupt state to all disabled */
2710 evergreen_disable_interrupt_state(rdev
);
2714 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2715 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2716 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2717 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2718 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2719 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
2721 afmt1
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2722 afmt2
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2723 afmt3
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2724 afmt4
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2725 afmt5
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2726 afmt6
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
) & ~AFMT_AZ_FORMAT_WTRIG_MASK
;
2728 dma_cntl
= RREG32(DMA_CNTL
) & ~TRAP_ENABLE
;
2730 if (rdev
->family
>= CHIP_CAYMAN
) {
2731 /* enable CP interrupts on all rings */
2732 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
2733 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2734 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
2736 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
2737 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
2738 cp_int_cntl1
|= TIME_STAMP_INT_ENABLE
;
2740 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
2741 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
2742 cp_int_cntl2
|= TIME_STAMP_INT_ENABLE
;
2745 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
2746 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2747 cp_int_cntl
|= RB_INT_ENABLE
;
2748 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
2752 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
2753 DRM_DEBUG("r600_irq_set: sw int dma\n");
2754 dma_cntl
|= TRAP_ENABLE
;
2757 if (rdev
->family
>= CHIP_CAYMAN
) {
2758 dma_cntl1
= RREG32(CAYMAN_DMA1_CNTL
) & ~TRAP_ENABLE
;
2759 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
2760 DRM_DEBUG("r600_irq_set: sw int dma1\n");
2761 dma_cntl1
|= TRAP_ENABLE
;
2765 if (rdev
->irq
.crtc_vblank_int
[0] ||
2766 atomic_read(&rdev
->irq
.pflip
[0])) {
2767 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2768 crtc1
|= VBLANK_INT_MASK
;
2770 if (rdev
->irq
.crtc_vblank_int
[1] ||
2771 atomic_read(&rdev
->irq
.pflip
[1])) {
2772 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2773 crtc2
|= VBLANK_INT_MASK
;
2775 if (rdev
->irq
.crtc_vblank_int
[2] ||
2776 atomic_read(&rdev
->irq
.pflip
[2])) {
2777 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2778 crtc3
|= VBLANK_INT_MASK
;
2780 if (rdev
->irq
.crtc_vblank_int
[3] ||
2781 atomic_read(&rdev
->irq
.pflip
[3])) {
2782 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2783 crtc4
|= VBLANK_INT_MASK
;
2785 if (rdev
->irq
.crtc_vblank_int
[4] ||
2786 atomic_read(&rdev
->irq
.pflip
[4])) {
2787 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2788 crtc5
|= VBLANK_INT_MASK
;
2790 if (rdev
->irq
.crtc_vblank_int
[5] ||
2791 atomic_read(&rdev
->irq
.pflip
[5])) {
2792 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2793 crtc6
|= VBLANK_INT_MASK
;
2795 if (rdev
->irq
.hpd
[0]) {
2796 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2797 hpd1
|= DC_HPDx_INT_EN
;
2799 if (rdev
->irq
.hpd
[1]) {
2800 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2801 hpd2
|= DC_HPDx_INT_EN
;
2803 if (rdev
->irq
.hpd
[2]) {
2804 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2805 hpd3
|= DC_HPDx_INT_EN
;
2807 if (rdev
->irq
.hpd
[3]) {
2808 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2809 hpd4
|= DC_HPDx_INT_EN
;
2811 if (rdev
->irq
.hpd
[4]) {
2812 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2813 hpd5
|= DC_HPDx_INT_EN
;
2815 if (rdev
->irq
.hpd
[5]) {
2816 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2817 hpd6
|= DC_HPDx_INT_EN
;
2819 if (rdev
->irq
.afmt
[0]) {
2820 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
2821 afmt1
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2823 if (rdev
->irq
.afmt
[1]) {
2824 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
2825 afmt2
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2827 if (rdev
->irq
.afmt
[2]) {
2828 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
2829 afmt3
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2831 if (rdev
->irq
.afmt
[3]) {
2832 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
2833 afmt4
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2835 if (rdev
->irq
.afmt
[4]) {
2836 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
2837 afmt5
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2839 if (rdev
->irq
.afmt
[5]) {
2840 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
2841 afmt6
|= AFMT_AZ_FORMAT_WTRIG_MASK
;
2844 if (rdev
->family
>= CHIP_CAYMAN
) {
2845 cayman_cp_int_cntl_setup(rdev
, 0, cp_int_cntl
);
2846 cayman_cp_int_cntl_setup(rdev
, 1, cp_int_cntl1
);
2847 cayman_cp_int_cntl_setup(rdev
, 2, cp_int_cntl2
);
2849 WREG32(CP_INT_CNTL
, cp_int_cntl
);
2851 WREG32(DMA_CNTL
, dma_cntl
);
2853 if (rdev
->family
>= CHIP_CAYMAN
)
2854 WREG32(CAYMAN_DMA1_CNTL
, dma_cntl1
);
2856 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
2858 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
2859 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
2860 if (rdev
->num_crtc
>= 4) {
2861 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
2862 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
2864 if (rdev
->num_crtc
>= 6) {
2865 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
2866 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
2869 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, grph1
);
2870 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, grph2
);
2871 if (rdev
->num_crtc
>= 4) {
2872 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, grph3
);
2873 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, grph4
);
2875 if (rdev
->num_crtc
>= 6) {
2876 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, grph5
);
2877 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, grph6
);
2880 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
2881 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
2882 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
2883 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
2884 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
2885 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
2887 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, afmt1
);
2888 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, afmt2
);
2889 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, afmt3
);
2890 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, afmt4
);
2891 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, afmt5
);
2892 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, afmt6
);
2897 static void evergreen_irq_ack(struct radeon_device
*rdev
)
2901 rdev
->irq
.stat_regs
.evergreen
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
2902 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
2903 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
2904 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
2905 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
2906 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
2907 rdev
->irq
.stat_regs
.evergreen
.d1grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
2908 rdev
->irq
.stat_regs
.evergreen
.d2grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
2909 if (rdev
->num_crtc
>= 4) {
2910 rdev
->irq
.stat_regs
.evergreen
.d3grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
2911 rdev
->irq
.stat_regs
.evergreen
.d4grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
2913 if (rdev
->num_crtc
>= 6) {
2914 rdev
->irq
.stat_regs
.evergreen
.d5grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
2915 rdev
->irq
.stat_regs
.evergreen
.d6grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
2918 rdev
->irq
.stat_regs
.evergreen
.afmt_status1
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
2919 rdev
->irq
.stat_regs
.evergreen
.afmt_status2
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
2920 rdev
->irq
.stat_regs
.evergreen
.afmt_status3
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
2921 rdev
->irq
.stat_regs
.evergreen
.afmt_status4
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
2922 rdev
->irq
.stat_regs
.evergreen
.afmt_status5
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
2923 rdev
->irq
.stat_regs
.evergreen
.afmt_status6
= RREG32(AFMT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
2925 if (rdev
->irq
.stat_regs
.evergreen
.d1grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2926 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2927 if (rdev
->irq
.stat_regs
.evergreen
.d2grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2928 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2929 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
2930 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
2931 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
)
2932 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
2933 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
2934 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
2935 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
2936 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
2938 if (rdev
->num_crtc
>= 4) {
2939 if (rdev
->irq
.stat_regs
.evergreen
.d3grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2940 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2941 if (rdev
->irq
.stat_regs
.evergreen
.d4grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2942 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2943 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
2944 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
2945 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
2946 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
2947 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
2948 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
2949 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
2950 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
2953 if (rdev
->num_crtc
>= 6) {
2954 if (rdev
->irq
.stat_regs
.evergreen
.d5grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2955 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2956 if (rdev
->irq
.stat_regs
.evergreen
.d6grph_int
& GRPH_PFLIP_INT_OCCURRED
)
2957 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
2958 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
2959 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
2960 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
2961 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
2962 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
2963 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
2964 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
2965 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
2968 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
2969 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
2970 tmp
|= DC_HPDx_INT_ACK
;
2971 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
2973 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
2974 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
2975 tmp
|= DC_HPDx_INT_ACK
;
2976 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
2978 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
2979 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
2980 tmp
|= DC_HPDx_INT_ACK
;
2981 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
2983 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
2984 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
2985 tmp
|= DC_HPDx_INT_ACK
;
2986 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
2988 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
2989 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
2990 tmp
|= DC_HPDx_INT_ACK
;
2991 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
2993 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
2994 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
2995 tmp
|= DC_HPDx_INT_ACK
;
2996 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
2998 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status1
& AFMT_AZ_FORMAT_WTRIG
) {
2999 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
3000 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3001 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, tmp
);
3003 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status2
& AFMT_AZ_FORMAT_WTRIG
) {
3004 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
3005 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3006 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, tmp
);
3008 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status3
& AFMT_AZ_FORMAT_WTRIG
) {
3009 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
3010 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3011 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, tmp
);
3013 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status4
& AFMT_AZ_FORMAT_WTRIG
) {
3014 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
3015 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3016 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, tmp
);
3018 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status5
& AFMT_AZ_FORMAT_WTRIG
) {
3019 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
3020 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3021 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, tmp
);
3023 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status6
& AFMT_AZ_FORMAT_WTRIG
) {
3024 tmp
= RREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
3025 tmp
|= AFMT_AZ_FORMAT_WTRIG_ACK
;
3026 WREG32(AFMT_AUDIO_PACKET_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, tmp
);
3030 static void evergreen_irq_disable(struct radeon_device
*rdev
)
3032 r600_disable_interrupts(rdev
);
3033 /* Wait and acknowledge irq */
3035 evergreen_irq_ack(rdev
);
3036 evergreen_disable_interrupt_state(rdev
);
3039 void evergreen_irq_suspend(struct radeon_device
*rdev
)
3041 evergreen_irq_disable(rdev
);
3042 r600_rlc_stop(rdev
);
3045 static u32
evergreen_get_ih_wptr(struct radeon_device
*rdev
)
3049 if (rdev
->wb
.enabled
)
3050 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
3052 wptr
= RREG32(IH_RB_WPTR
);
3054 if (wptr
& RB_OVERFLOW
) {
3055 /* When a ring buffer overflow happen start parsing interrupt
3056 * from the last not overwritten vector (wptr + 16). Hopefully
3057 * this should allow us to catchup.
3059 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3060 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
3061 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
3062 tmp
= RREG32(IH_RB_CNTL
);
3063 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
3064 WREG32(IH_RB_CNTL
, tmp
);
3066 return (wptr
& rdev
->ih
.ptr_mask
);
3069 int evergreen_irq_process(struct radeon_device
*rdev
)
3073 u32 src_id
, src_data
;
3075 bool queue_hotplug
= false;
3076 bool queue_hdmi
= false;
3078 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
3081 wptr
= evergreen_get_ih_wptr(rdev
);
3084 /* is somebody else already processing irqs? */
3085 if (atomic_xchg(&rdev
->ih
.lock
, 1))
3088 rptr
= rdev
->ih
.rptr
;
3089 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
3091 /* Order reading of wptr vs. reading of IH ring data */
3094 /* display interrupts */
3095 evergreen_irq_ack(rdev
);
3097 while (rptr
!= wptr
) {
3098 /* wptr/rptr are in bytes! */
3099 ring_index
= rptr
/ 4;
3100 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
3101 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
3104 case 1: /* D1 vblank/vline */
3106 case 0: /* D1 vblank */
3107 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
3108 if (rdev
->irq
.crtc_vblank_int
[0]) {
3109 drm_handle_vblank(rdev
->ddev
, 0);
3110 rdev
->pm
.vblank_sync
= true;
3111 wake_up(&rdev
->irq
.vblank_queue
);
3113 if (atomic_read(&rdev
->irq
.pflip
[0]))
3114 radeon_crtc_handle_flip(rdev
, 0);
3115 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
3116 DRM_DEBUG("IH: D1 vblank\n");
3119 case 1: /* D1 vline */
3120 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
3121 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
3122 DRM_DEBUG("IH: D1 vline\n");
3126 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3130 case 2: /* D2 vblank/vline */
3132 case 0: /* D2 vblank */
3133 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
3134 if (rdev
->irq
.crtc_vblank_int
[1]) {
3135 drm_handle_vblank(rdev
->ddev
, 1);
3136 rdev
->pm
.vblank_sync
= true;
3137 wake_up(&rdev
->irq
.vblank_queue
);
3139 if (atomic_read(&rdev
->irq
.pflip
[1]))
3140 radeon_crtc_handle_flip(rdev
, 1);
3141 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
3142 DRM_DEBUG("IH: D2 vblank\n");
3145 case 1: /* D2 vline */
3146 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
3147 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
3148 DRM_DEBUG("IH: D2 vline\n");
3152 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3156 case 3: /* D3 vblank/vline */
3158 case 0: /* D3 vblank */
3159 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
3160 if (rdev
->irq
.crtc_vblank_int
[2]) {
3161 drm_handle_vblank(rdev
->ddev
, 2);
3162 rdev
->pm
.vblank_sync
= true;
3163 wake_up(&rdev
->irq
.vblank_queue
);
3165 if (atomic_read(&rdev
->irq
.pflip
[2]))
3166 radeon_crtc_handle_flip(rdev
, 2);
3167 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
3168 DRM_DEBUG("IH: D3 vblank\n");
3171 case 1: /* D3 vline */
3172 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
3173 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
3174 DRM_DEBUG("IH: D3 vline\n");
3178 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3182 case 4: /* D4 vblank/vline */
3184 case 0: /* D4 vblank */
3185 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
3186 if (rdev
->irq
.crtc_vblank_int
[3]) {
3187 drm_handle_vblank(rdev
->ddev
, 3);
3188 rdev
->pm
.vblank_sync
= true;
3189 wake_up(&rdev
->irq
.vblank_queue
);
3191 if (atomic_read(&rdev
->irq
.pflip
[3]))
3192 radeon_crtc_handle_flip(rdev
, 3);
3193 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
3194 DRM_DEBUG("IH: D4 vblank\n");
3197 case 1: /* D4 vline */
3198 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
3199 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
3200 DRM_DEBUG("IH: D4 vline\n");
3204 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3208 case 5: /* D5 vblank/vline */
3210 case 0: /* D5 vblank */
3211 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
3212 if (rdev
->irq
.crtc_vblank_int
[4]) {
3213 drm_handle_vblank(rdev
->ddev
, 4);
3214 rdev
->pm
.vblank_sync
= true;
3215 wake_up(&rdev
->irq
.vblank_queue
);
3217 if (atomic_read(&rdev
->irq
.pflip
[4]))
3218 radeon_crtc_handle_flip(rdev
, 4);
3219 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
3220 DRM_DEBUG("IH: D5 vblank\n");
3223 case 1: /* D5 vline */
3224 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
3225 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
3226 DRM_DEBUG("IH: D5 vline\n");
3230 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3234 case 6: /* D6 vblank/vline */
3236 case 0: /* D6 vblank */
3237 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
3238 if (rdev
->irq
.crtc_vblank_int
[5]) {
3239 drm_handle_vblank(rdev
->ddev
, 5);
3240 rdev
->pm
.vblank_sync
= true;
3241 wake_up(&rdev
->irq
.vblank_queue
);
3243 if (atomic_read(&rdev
->irq
.pflip
[5]))
3244 radeon_crtc_handle_flip(rdev
, 5);
3245 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
3246 DRM_DEBUG("IH: D6 vblank\n");
3249 case 1: /* D6 vline */
3250 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
3251 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
3252 DRM_DEBUG("IH: D6 vline\n");
3256 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3260 case 42: /* HPD hotplug */
3263 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
3264 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~DC_HPD1_INTERRUPT
;
3265 queue_hotplug
= true;
3266 DRM_DEBUG("IH: HPD1\n");
3270 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
3271 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
3272 queue_hotplug
= true;
3273 DRM_DEBUG("IH: HPD2\n");
3277 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
3278 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
3279 queue_hotplug
= true;
3280 DRM_DEBUG("IH: HPD3\n");
3284 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
3285 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
3286 queue_hotplug
= true;
3287 DRM_DEBUG("IH: HPD4\n");
3291 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
3292 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
3293 queue_hotplug
= true;
3294 DRM_DEBUG("IH: HPD5\n");
3298 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
3299 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
3300 queue_hotplug
= true;
3301 DRM_DEBUG("IH: HPD6\n");
3305 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3312 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status1
& AFMT_AZ_FORMAT_WTRIG
) {
3313 rdev
->irq
.stat_regs
.evergreen
.afmt_status1
&= ~AFMT_AZ_FORMAT_WTRIG
;
3315 DRM_DEBUG("IH: HDMI0\n");
3319 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status2
& AFMT_AZ_FORMAT_WTRIG
) {
3320 rdev
->irq
.stat_regs
.evergreen
.afmt_status2
&= ~AFMT_AZ_FORMAT_WTRIG
;
3322 DRM_DEBUG("IH: HDMI1\n");
3326 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status3
& AFMT_AZ_FORMAT_WTRIG
) {
3327 rdev
->irq
.stat_regs
.evergreen
.afmt_status3
&= ~AFMT_AZ_FORMAT_WTRIG
;
3329 DRM_DEBUG("IH: HDMI2\n");
3333 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status4
& AFMT_AZ_FORMAT_WTRIG
) {
3334 rdev
->irq
.stat_regs
.evergreen
.afmt_status4
&= ~AFMT_AZ_FORMAT_WTRIG
;
3336 DRM_DEBUG("IH: HDMI3\n");
3340 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status5
& AFMT_AZ_FORMAT_WTRIG
) {
3341 rdev
->irq
.stat_regs
.evergreen
.afmt_status5
&= ~AFMT_AZ_FORMAT_WTRIG
;
3343 DRM_DEBUG("IH: HDMI4\n");
3347 if (rdev
->irq
.stat_regs
.evergreen
.afmt_status6
& AFMT_AZ_FORMAT_WTRIG
) {
3348 rdev
->irq
.stat_regs
.evergreen
.afmt_status6
&= ~AFMT_AZ_FORMAT_WTRIG
;
3350 DRM_DEBUG("IH: HDMI5\n");
3354 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3360 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
3361 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3362 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
3363 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3364 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
3365 /* reset addr and status */
3366 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
3368 case 176: /* CP_INT in ring buffer */
3369 case 177: /* CP_INT in IB1 */
3370 case 178: /* CP_INT in IB2 */
3371 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data
);
3372 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
3374 case 181: /* CP EOP event */
3375 DRM_DEBUG("IH: CP EOP\n");
3376 if (rdev
->family
>= CHIP_CAYMAN
) {
3379 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
3382 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
3385 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
3389 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
3391 case 224: /* DMA trap event */
3392 DRM_DEBUG("IH: DMA trap\n");
3393 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
3395 case 233: /* GUI IDLE */
3396 DRM_DEBUG("IH: GUI idle\n");
3398 case 244: /* DMA trap event */
3399 if (rdev
->family
>= CHIP_CAYMAN
) {
3400 DRM_DEBUG("IH: DMA1 trap\n");
3401 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
3405 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
3409 /* wptr/rptr are in bytes! */
3411 rptr
&= rdev
->ih
.ptr_mask
;
3414 schedule_work(&rdev
->hotplug_work
);
3416 schedule_work(&rdev
->audio_work
);
3417 rdev
->ih
.rptr
= rptr
;
3418 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
3419 atomic_set(&rdev
->ih
.lock
, 0);
3421 /* make sure wptr hasn't changed while processing */
3422 wptr
= evergreen_get_ih_wptr(rdev
);
3430 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
3432 * @rdev: radeon_device pointer
3433 * @fence: radeon fence object
3435 * Add a DMA fence packet to the ring to write
3436 * the fence seq number and DMA trap packet to generate
3437 * an interrupt if needed (evergreen-SI).
3439 void evergreen_dma_fence_ring_emit(struct radeon_device
*rdev
,
3440 struct radeon_fence
*fence
)
3442 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3443 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3444 /* write the fence */
3445 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_FENCE
, 0, 0));
3446 radeon_ring_write(ring
, addr
& 0xfffffffc);
3447 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xff));
3448 radeon_ring_write(ring
, fence
->seq
);
3449 /* generate an interrupt */
3450 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_TRAP
, 0, 0));
3452 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0));
3453 radeon_ring_write(ring
, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2));
3454 radeon_ring_write(ring
, 1);
3458 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
3460 * @rdev: radeon_device pointer
3461 * @ib: IB object to schedule
3463 * Schedule an IB in the DMA ring (evergreen).
3465 void evergreen_dma_ring_ib_execute(struct radeon_device
*rdev
,
3466 struct radeon_ib
*ib
)
3468 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3470 if (rdev
->wb
.enabled
) {
3471 u32 next_rptr
= ring
->wptr
+ 4;
3472 while ((next_rptr
& 7) != 5)
3475 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_WRITE
, 0, 1));
3476 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3477 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xff);
3478 radeon_ring_write(ring
, next_rptr
);
3481 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3482 * Pad as necessary with NOPs.
3484 while ((ring
->wptr
& 7) != 5)
3485 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_NOP
, 0, 0));
3486 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER
, 0, 0));
3487 radeon_ring_write(ring
, (ib
->gpu_addr
& 0xFFFFFFE0));
3488 radeon_ring_write(ring
, (ib
->length_dw
<< 12) | (upper_32_bits(ib
->gpu_addr
) & 0xFF));
3493 * evergreen_copy_dma - copy pages using the DMA engine
3495 * @rdev: radeon_device pointer
3496 * @src_offset: src GPU address
3497 * @dst_offset: dst GPU address
3498 * @num_gpu_pages: number of GPU pages to xfer
3499 * @fence: radeon fence object
3501 * Copy GPU paging using the DMA engine (evergreen-cayman).
3502 * Used by the radeon ttm implementation to move pages if
3503 * registered as the asic copy callback.
3505 int evergreen_copy_dma(struct radeon_device
*rdev
,
3506 uint64_t src_offset
, uint64_t dst_offset
,
3507 unsigned num_gpu_pages
,
3508 struct radeon_fence
**fence
)
3510 struct radeon_semaphore
*sem
= NULL
;
3511 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
3512 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
3513 u32 size_in_dw
, cur_size_in_dw
;
3517 r
= radeon_semaphore_create(rdev
, &sem
);
3519 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3523 size_in_dw
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
) / 4;
3524 num_loops
= DIV_ROUND_UP(size_in_dw
, 0xfffff);
3525 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 5 + 11);
3527 DRM_ERROR("radeon: moving bo (%d).\n", r
);
3528 radeon_semaphore_free(rdev
, &sem
, NULL
);
3532 if (radeon_fence_need_sync(*fence
, ring
->idx
)) {
3533 radeon_semaphore_sync_rings(rdev
, sem
, (*fence
)->ring
,
3535 radeon_fence_note_sync(*fence
, ring
->idx
);
3537 radeon_semaphore_free(rdev
, &sem
, NULL
);
3540 for (i
= 0; i
< num_loops
; i
++) {
3541 cur_size_in_dw
= size_in_dw
;
3542 if (cur_size_in_dw
> 0xFFFFF)
3543 cur_size_in_dw
= 0xFFFFF;
3544 size_in_dw
-= cur_size_in_dw
;
3545 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_COPY
, 0, cur_size_in_dw
));
3546 radeon_ring_write(ring
, dst_offset
& 0xfffffffc);
3547 radeon_ring_write(ring
, src_offset
& 0xfffffffc);
3548 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xff);
3549 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xff);
3550 src_offset
+= cur_size_in_dw
* 4;
3551 dst_offset
+= cur_size_in_dw
* 4;
3554 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
3556 radeon_ring_unlock_undo(rdev
, ring
);
3560 radeon_ring_unlock_commit(rdev
, ring
);
3561 radeon_semaphore_free(rdev
, &sem
, *fence
);
3566 static int evergreen_startup(struct radeon_device
*rdev
)
3568 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3571 /* enable pcie gen2 link */
3572 evergreen_pcie_gen2_enable(rdev
);
3574 if (ASIC_IS_DCE5(rdev
)) {
3575 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->rlc_fw
|| !rdev
->mc_fw
) {
3576 r
= ni_init_microcode(rdev
);
3578 DRM_ERROR("Failed to load firmware!\n");
3582 r
= ni_mc_load_microcode(rdev
);
3584 DRM_ERROR("Failed to load MC firmware!\n");
3588 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->rlc_fw
) {
3589 r
= r600_init_microcode(rdev
);
3591 DRM_ERROR("Failed to load firmware!\n");
3597 r
= r600_vram_scratch_init(rdev
);
3601 evergreen_mc_program(rdev
);
3602 if (rdev
->flags
& RADEON_IS_AGP
) {
3603 evergreen_agp_enable(rdev
);
3605 r
= evergreen_pcie_gart_enable(rdev
);
3609 evergreen_gpu_init(rdev
);
3611 r
= evergreen_blit_init(rdev
);
3613 r600_blit_fini(rdev
);
3614 rdev
->asic
->copy
.copy
= NULL
;
3615 dev_warn(rdev
->dev
, "failed blitter (%d) falling back to memcpy\n", r
);
3618 /* allocate wb buffer */
3619 r
= radeon_wb_init(rdev
);
3623 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
3625 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
3629 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
3631 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
3636 r
= r600_irq_init(rdev
);
3638 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
3639 radeon_irq_kms_fini(rdev
);
3642 evergreen_irq_set(rdev
);
3644 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
3645 R600_CP_RB_RPTR
, R600_CP_RB_WPTR
,
3646 0, 0xfffff, RADEON_CP_PACKET2
);
3650 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
3651 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
3652 DMA_RB_RPTR
, DMA_RB_WPTR
,
3653 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP
, 0, 0));
3657 r
= evergreen_cp_load_microcode(rdev
);
3660 r
= evergreen_cp_resume(rdev
);
3663 r
= r600_dma_resume(rdev
);
3667 r
= radeon_ib_pool_init(rdev
);
3669 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
3673 r
= r600_audio_init(rdev
);
3675 DRM_ERROR("radeon: audio init failed\n");
3682 int evergreen_resume(struct radeon_device
*rdev
)
3686 /* reset the asic, the gfx blocks are often in a bad state
3687 * after the driver is unloaded or after a resume
3689 if (radeon_asic_reset(rdev
))
3690 dev_warn(rdev
->dev
, "GPU reset failed !\n");
3691 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3692 * posting will perform necessary task to bring back GPU into good
3696 atom_asic_init(rdev
->mode_info
.atom_context
);
3698 rdev
->accel_working
= true;
3699 r
= evergreen_startup(rdev
);
3701 DRM_ERROR("evergreen startup failed on resume\n");
3702 rdev
->accel_working
= false;
3710 int evergreen_suspend(struct radeon_device
*rdev
)
3712 r600_audio_fini(rdev
);
3714 r600_dma_stop(rdev
);
3715 evergreen_irq_suspend(rdev
);
3716 radeon_wb_disable(rdev
);
3717 evergreen_pcie_gart_disable(rdev
);
3722 /* Plan is to move initialization in that function and use
3723 * helper function so that radeon_device_init pretty much
3724 * do nothing more than calling asic specific function. This
3725 * should also allow to remove a bunch of callback function
3728 int evergreen_init(struct radeon_device
*rdev
)
3733 if (!radeon_get_bios(rdev
)) {
3734 if (ASIC_IS_AVIVO(rdev
))
3737 /* Must be an ATOMBIOS */
3738 if (!rdev
->is_atom_bios
) {
3739 dev_err(rdev
->dev
, "Expecting atombios for evergreen GPU\n");
3742 r
= radeon_atombios_init(rdev
);
3745 /* reset the asic, the gfx blocks are often in a bad state
3746 * after the driver is unloaded or after a resume
3748 if (radeon_asic_reset(rdev
))
3749 dev_warn(rdev
->dev
, "GPU reset failed !\n");
3750 /* Post card if necessary */
3751 if (!radeon_card_posted(rdev
)) {
3753 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
3756 DRM_INFO("GPU not posted. posting now...\n");
3757 atom_asic_init(rdev
->mode_info
.atom_context
);
3759 /* Initialize scratch registers */
3760 r600_scratch_init(rdev
);
3761 /* Initialize surface registers */
3762 radeon_surface_init(rdev
);
3763 /* Initialize clocks */
3764 radeon_get_clock_info(rdev
->ddev
);
3766 r
= radeon_fence_driver_init(rdev
);
3769 /* initialize AGP */
3770 if (rdev
->flags
& RADEON_IS_AGP
) {
3771 r
= radeon_agp_init(rdev
);
3773 radeon_agp_disable(rdev
);
3775 /* initialize memory controller */
3776 r
= evergreen_mc_init(rdev
);
3779 /* Memory manager */
3780 r
= radeon_bo_init(rdev
);
3784 r
= radeon_irq_kms_init(rdev
);
3788 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ring_obj
= NULL
;
3789 r600_ring_init(rdev
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
], 1024 * 1024);
3791 rdev
->ring
[R600_RING_TYPE_DMA_INDEX
].ring_obj
= NULL
;
3792 r600_ring_init(rdev
, &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
], 64 * 1024);
3794 rdev
->ih
.ring_obj
= NULL
;
3795 r600_ih_ring_init(rdev
, 64 * 1024);
3797 r
= r600_pcie_gart_init(rdev
);
3801 rdev
->accel_working
= true;
3802 r
= evergreen_startup(rdev
);
3804 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
3806 r600_dma_fini(rdev
);
3807 r600_irq_fini(rdev
);
3808 radeon_wb_fini(rdev
);
3809 radeon_ib_pool_fini(rdev
);
3810 radeon_irq_kms_fini(rdev
);
3811 evergreen_pcie_gart_fini(rdev
);
3812 rdev
->accel_working
= false;
3815 /* Don't start up if the MC ucode is missing on BTC parts.
3816 * The default clocks and voltages before the MC ucode
3817 * is loaded are not suffient for advanced operations.
3819 if (ASIC_IS_DCE5(rdev
)) {
3820 if (!rdev
->mc_fw
&& !(rdev
->flags
& RADEON_IS_IGP
)) {
3821 DRM_ERROR("radeon: MC ucode required for NI+.\n");
3829 void evergreen_fini(struct radeon_device
*rdev
)
3831 r600_audio_fini(rdev
);
3832 r600_blit_fini(rdev
);
3834 r600_dma_fini(rdev
);
3835 r600_irq_fini(rdev
);
3836 radeon_wb_fini(rdev
);
3837 radeon_ib_pool_fini(rdev
);
3838 radeon_irq_kms_fini(rdev
);
3839 evergreen_pcie_gart_fini(rdev
);
3840 r600_vram_scratch_fini(rdev
);
3841 radeon_gem_fini(rdev
);
3842 radeon_fence_driver_fini(rdev
);
3843 radeon_agp_fini(rdev
);
3844 radeon_bo_fini(rdev
);
3845 radeon_atombios_fini(rdev
);
3850 void evergreen_pcie_gen2_enable(struct radeon_device
*rdev
)
3852 u32 link_width_cntl
, speed_cntl
, mask
;
3855 if (radeon_pcie_gen2
== 0)
3858 if (rdev
->flags
& RADEON_IS_IGP
)
3861 if (!(rdev
->flags
& RADEON_IS_PCIE
))
3864 /* x2 cards have a special sequence */
3865 if (ASIC_IS_X2(rdev
))
3868 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
3872 if (!(mask
& DRM_PCIE_SPEED_50
))
3875 speed_cntl
= RREG32_PCIE_P(PCIE_LC_SPEED_CNTL
);
3876 if (speed_cntl
& LC_CURRENT_DATA_RATE
) {
3877 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
3881 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
3883 if ((speed_cntl
& LC_OTHER_SIDE_EVER_SENT_GEN2
) ||
3884 (speed_cntl
& LC_OTHER_SIDE_SUPPORTS_GEN2
)) {
3886 link_width_cntl
= RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL
);
3887 link_width_cntl
&= ~LC_UPCONFIGURE_DIS
;
3888 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL
, link_width_cntl
);
3890 speed_cntl
= RREG32_PCIE_P(PCIE_LC_SPEED_CNTL
);
3891 speed_cntl
&= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN
;
3892 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL
, speed_cntl
);
3894 speed_cntl
= RREG32_PCIE_P(PCIE_LC_SPEED_CNTL
);
3895 speed_cntl
|= LC_CLR_FAILED_SPD_CHANGE_CNT
;
3896 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL
, speed_cntl
);
3898 speed_cntl
= RREG32_PCIE_P(PCIE_LC_SPEED_CNTL
);
3899 speed_cntl
&= ~LC_CLR_FAILED_SPD_CHANGE_CNT
;
3900 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL
, speed_cntl
);
3902 speed_cntl
= RREG32_PCIE_P(PCIE_LC_SPEED_CNTL
);
3903 speed_cntl
|= LC_GEN2_EN_STRAP
;
3904 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL
, speed_cntl
);
3907 link_width_cntl
= RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL
);
3908 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3910 link_width_cntl
|= LC_UPCONFIGURE_DIS
;
3912 link_width_cntl
&= ~LC_UPCONFIGURE_DIS
;
3913 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL
, link_width_cntl
);