X-Git-Url: http://drtracing.org/?a=blobdiff_plain;f=drivers%2Fgpu%2Fdrm%2Fi915%2Fintel_pm.c;h=79108059b79b5009ac5dea64aa3b1c91e2ccffb8;hb=2aeb7d3a4d425be7e9185e79dd745918f7f72552;hp=40c12295c0bde4648d319f3da48205ab131fc92a;hpb=e9d99a1dec59a9ebf59ddc040ac34a92b1a93c97;p=deliverable%2Flinux.git diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 40c12295c0bd..79108059b79b 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -30,9 +30,6 @@ #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" #include -#include -#include -#include /** * RC6 is a special power stage which allows the GPU to enter an very @@ -66,11 +63,37 @@ * i915.i915_enable_fbc parameter */ +static void gen9_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* + * WaDisableSDEUnitClockGating:skl + * This seems to be a pre-production w/a. + */ + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* + * WaDisableDgMirrorFixInHalfSliceChicken5:skl + * This is a pre-production w/a. + */ + I915_WRITE(GEN9_HALF_SLICE_CHICKEN5, + I915_READ(GEN9_HALF_SLICE_CHICKEN5) & + ~GEN9_DG_MIRROR_FIX_ENABLE); + + /* Wa4x4STCOptimizationDisable:skl */ + I915_WRITE(CACHE_MODE_1, + _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE)); +} + static void i8xx_disable_fbc(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u32 fbc_ctl; + dev_priv->fbc.enabled = false; + /* Disable compression */ fbc_ctl = I915_READ(FBC_CONTROL); if ((fbc_ctl & FBC_CTL_EN) == 0) @@ -99,6 +122,8 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) int i; u32 fbc_ctl; + dev_priv->fbc.enabled = true; + cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE; if (fb->pitches[0] < cfb_pitch) cfb_pitch = fb->pitches[0]; @@ -153,6 +178,8 @@ static void g4x_enable_fbc(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; + dev_priv->fbc.enabled = true; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) dpfc_ctl |= DPFC_CTL_LIMIT_2X; @@ -173,6 +200,8 @@ static void g4x_disable_fbc(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; u32 dpfc_ctl; + dev_priv->fbc.enabled = false; + /* Disable compression */ dpfc_ctl = I915_READ(DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { @@ -224,6 +253,8 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; + dev_priv->fbc.enabled = true; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) dev_priv->fbc.threshold++; @@ -264,6 +295,8 @@ static void ironlake_disable_fbc(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; u32 dpfc_ctl; + dev_priv->fbc.enabled = false; + /* Disable compression */ dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { @@ -290,6 +323,8 @@ static void gen7_enable_fbc(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; + dev_priv->fbc.enabled = true; + dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) dev_priv->fbc.threshold++; @@ -309,6 +344,9 @@ static void gen7_enable_fbc(struct drm_crtc *crtc) dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; + if (dev_priv->fbc.false_color) + dpfc_ctl |= FBC_CTL_FALSE_COLOR; + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); if (IS_IVYBRIDGE(dev)) { @@ -336,10 +374,20 @@ bool intel_fbc_enabled(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (!dev_priv->display.fbc_enabled) - return false; + return dev_priv->fbc.enabled; +} + +void bdw_fbc_sw_flush(struct drm_device *dev, u32 value) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (!IS_GEN8(dev)) + return; + + if (!intel_fbc_enabled(dev)) + return; - return dev_priv->display.fbc_enabled(dev); + I915_WRITE(MSG_FBC_REND_STATE, value); } static void intel_fbc_work_fn(struct work_struct *__work) @@ -578,6 +626,12 @@ void intel_update_fbc(struct drm_device *dev) DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n"); goto out_disable; } + if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && + to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) { + if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) + DRM_DEBUG_KMS("Rotation unsupported, disabling\n"); + goto out_disable; + } /* If the kernel debugger is active, always disable compression */ if (in_dbg_master()) @@ -853,7 +907,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) * A value of 5us seems to be a good balance; safe for very low end * platforms but not overly aggressive on lower latency configs. */ -static const int latency_ns = 5000; +static const int pessimal_latency_ns = 5000; static int i9xx_get_fifo_size(struct drm_device *dev, int plane) { @@ -982,13 +1036,20 @@ static const struct intel_watermark_params i915_wm_info = { .guard_size = 2, .cacheline_size = I915_FIFO_LINE_SIZE, }; -static const struct intel_watermark_params i830_wm_info = { +static const struct intel_watermark_params i830_a_wm_info = { .fifo_size = I855GM_FIFO_SIZE, .max_wm = I915_MAX_WM, .default_wm = 1, .guard_size = 2, .cacheline_size = I830_FIFO_LINE_SIZE, }; +static const struct intel_watermark_params i830_bc_wm_info = { + .fifo_size = I855GM_FIFO_SIZE, + .max_wm = I915_MAX_WM/2, + .default_wm = 1, + .guard_size = 2, + .cacheline_size = I830_FIFO_LINE_SIZE, +}; static const struct intel_watermark_params i845_wm_info = { .fifo_size = I830_FIFO_SIZE, .max_wm = I915_MAX_WM, @@ -1044,6 +1105,17 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz, wm_size = wm->max_wm; if (wm_size <= 0) wm_size = wm->default_wm; + + /* + * Bspec seems to indicate that the value shouldn't be lower than + * 'burst size + 1'. Certainly 830 is quite unhappy with low values. + * Lets go for 8 which is the burst size since certain platforms + * already use a hardcoded 8 (which is what the spec says should be + * done). + */ + if (wm_size <= 8) + wm_size = 8; + return wm_size; } @@ -1268,33 +1340,27 @@ static bool g4x_compute_srwm(struct drm_device *dev, display, cursor); } -static bool vlv_compute_drain_latency(struct drm_device *dev, - int plane, - int *plane_prec_mult, - int *plane_dl, - int *cursor_prec_mult, - int *cursor_dl) +static bool vlv_compute_drain_latency(struct drm_crtc *crtc, + int pixel_size, + int *prec_mult, + int *drain_latency) { - struct drm_crtc *crtc; - int clock, pixel_size; int entries; + int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - crtc = intel_get_crtc_for_plane(dev, plane); - if (!intel_crtc_active(crtc)) + if (WARN(clock == 0, "Pixel clock is zero!\n")) return false; - clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ + if (WARN(pixel_size == 0, "Pixel size is zero!\n")) + return false; - entries = (clock / 1000) * pixel_size; - *plane_prec_mult = (entries > 128) ? - DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32; - *plane_dl = (64 * (*plane_prec_mult) * 4) / entries; + entries = DIV_ROUND_UP(clock, 1000) * pixel_size; + *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 : + DRAIN_LATENCY_PRECISION_32; + *drain_latency = (64 * (*prec_mult) * 4) / entries; - entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */ - *cursor_prec_mult = (entries > 128) ? - DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32; - *cursor_dl = (64 * (*cursor_prec_mult) * 4) / entries; + if (*drain_latency > DRAIN_LATENCY_MASK) + *drain_latency = DRAIN_LATENCY_MASK; return true; } @@ -1307,39 +1373,48 @@ static bool vlv_compute_drain_latency(struct drm_device *dev, * latency value. */ -static void vlv_update_drain_latency(struct drm_device *dev) +static void vlv_update_drain_latency(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; - int planea_prec, planea_dl, planeb_prec, planeb_dl; - int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl; - int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is - either 16 or 32 */ + struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int pixel_size; + int drain_latency; + enum pipe pipe = intel_crtc->pipe; + int plane_prec, prec_mult, plane_dl; + + plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_64 | + DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_64 | + (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT)); - /* For plane A, Cursor A */ - if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl, - &cursor_prec_mult, &cursora_dl)) { - cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_64; - planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_64; + if (!intel_crtc_active(crtc)) { + I915_WRITE(VLV_DDL(pipe), plane_dl); + return; + } - I915_WRITE(VLV_DDL1, cursora_prec | - (cursora_dl << DDL_CURSORA_SHIFT) | - planea_prec | planea_dl); + /* Primary plane Drain Latency */ + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ + if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_PLANE_PRECISION_64 : + DDL_PLANE_PRECISION_32; + plane_dl |= plane_prec | drain_latency; } - /* For plane B, Cursor B */ - if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl, - &cursor_prec_mult, &cursorb_dl)) { - cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_64; - planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_64; + /* Cursor Drain Latency + * BPP is always 4 for cursor + */ + pixel_size = 4; - I915_WRITE(VLV_DDL2, cursorb_prec | - (cursorb_dl << DDL_CURSORB_SHIFT) | - planeb_prec | planeb_dl); + /* Program cursor DL only if it is enabled */ + if (intel_crtc->cursor_base && + vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_CURSOR_PRECISION_64 : + DDL_CURSOR_PRECISION_32; + plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT); } + + I915_WRITE(VLV_DDL(pipe), plane_dl); } #define single_plane_enabled(mask) is_power_of_2(mask) @@ -1355,20 +1430,92 @@ static void valleyview_update_wm(struct drm_crtc *crtc) unsigned int enabled = 0; bool cxsr_enabled; - vlv_update_drain_latency(dev); + vlv_update_drain_latency(crtc); + + if (g4x_compute_wm0(dev, PIPE_A, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planea_wm, &cursora_wm)) + enabled |= 1 << PIPE_A; + + if (g4x_compute_wm0(dev, PIPE_B, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planeb_wm, &cursorb_wm)) + enabled |= 1 << PIPE_B; + + if (single_plane_enabled(enabled) && + g4x_compute_srwm(dev, ffs(enabled) - 1, + sr_latency_ns, + &valleyview_wm_info, + &valleyview_cursor_wm_info, + &plane_sr, &ignore_cursor_sr) && + g4x_compute_srwm(dev, ffs(enabled) - 1, + 2*sr_latency_ns, + &valleyview_wm_info, + &valleyview_cursor_wm_info, + &ignore_plane_sr, &cursor_sr)) { + cxsr_enabled = true; + } else { + cxsr_enabled = false; + intel_set_memory_cxsr(dev_priv, false); + plane_sr = cursor_sr = 0; + } + + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", + planea_wm, cursora_wm, + planeb_wm, cursorb_wm, + plane_sr, cursor_sr); + + I915_WRITE(DSPFW1, + (plane_sr << DSPFW_SR_SHIFT) | + (cursorb_wm << DSPFW_CURSORB_SHIFT) | + (planeb_wm << DSPFW_PLANEB_SHIFT) | + (planea_wm << DSPFW_PLANEA_SHIFT)); + I915_WRITE(DSPFW2, + (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | + (cursora_wm << DSPFW_CURSORA_SHIFT)); + I915_WRITE(DSPFW3, + (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) | + (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); + + if (cxsr_enabled) + intel_set_memory_cxsr(dev_priv, true); +} + +static void cherryview_update_wm(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + static const int sr_latency_ns = 12000; + struct drm_i915_private *dev_priv = dev->dev_private; + int planea_wm, planeb_wm, planec_wm; + int cursora_wm, cursorb_wm, cursorc_wm; + int plane_sr, cursor_sr; + int ignore_plane_sr, ignore_cursor_sr; + unsigned int enabled = 0; + bool cxsr_enabled; + + vlv_update_drain_latency(crtc); if (g4x_compute_wm0(dev, PIPE_A, - &valleyview_wm_info, latency_ns, - &valleyview_cursor_wm_info, latency_ns, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; if (g4x_compute_wm0(dev, PIPE_B, - &valleyview_wm_info, latency_ns, - &valleyview_cursor_wm_info, latency_ns, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; + if (g4x_compute_wm0(dev, PIPE_C, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planec_wm, &cursorc_wm)) + enabled |= 1 << PIPE_C; + if (single_plane_enabled(enabled) && g4x_compute_srwm(dev, ffs(enabled) - 1, sr_latency_ns, @@ -1387,27 +1534,66 @@ static void valleyview_update_wm(struct drm_crtc *crtc) plane_sr = cursor_sr = 0; } - DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, " + "SR: plane=%d, cursor=%d\n", planea_wm, cursora_wm, planeb_wm, cursorb_wm, + planec_wm, cursorc_wm, plane_sr, cursor_sr); I915_WRITE(DSPFW1, (plane_sr << DSPFW_SR_SHIFT) | (cursorb_wm << DSPFW_CURSORB_SHIFT) | (planeb_wm << DSPFW_PLANEB_SHIFT) | - planea_wm); + (planea_wm << DSPFW_PLANEA_SHIFT)); I915_WRITE(DSPFW2, (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | (cursora_wm << DSPFW_CURSORA_SHIFT)); I915_WRITE(DSPFW3, (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) | (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); + I915_WRITE(DSPFW9_CHV, + (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK | + DSPFW_CURSORC_MASK)) | + (planec_wm << DSPFW_PLANEC_SHIFT) | + (cursorc_wm << DSPFW_CURSORC_SHIFT)); if (cxsr_enabled) intel_set_memory_cxsr(dev_priv, true); } +static void valleyview_update_sprite_wm(struct drm_plane *plane, + struct drm_crtc *crtc, + uint32_t sprite_width, + uint32_t sprite_height, + int pixel_size, + bool enabled, bool scaled) +{ + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe = to_intel_plane(plane)->pipe; + int sprite = to_intel_plane(plane)->plane; + int drain_latency; + int plane_prec; + int sprite_dl; + int prec_mult; + + sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_64(sprite) | + (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite))); + + if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, + &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_SPRITE_PRECISION_64(sprite) : + DDL_SPRITE_PRECISION_32(sprite); + sprite_dl |= plane_prec | + (drain_latency << DDL_SPRITE_SHIFT(sprite)); + } + + I915_WRITE(VLV_DDL(pipe), sprite_dl); +} + static void g4x_update_wm(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; @@ -1419,14 +1605,14 @@ static void g4x_update_wm(struct drm_crtc *crtc) bool cxsr_enabled; if (g4x_compute_wm0(dev, PIPE_A, - &g4x_wm_info, latency_ns, - &g4x_cursor_wm_info, latency_ns, + &g4x_wm_info, pessimal_latency_ns, + &g4x_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; if (g4x_compute_wm0(dev, PIPE_B, - &g4x_wm_info, latency_ns, - &g4x_cursor_wm_info, latency_ns, + &g4x_wm_info, pessimal_latency_ns, + &g4x_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; @@ -1443,7 +1629,8 @@ static void g4x_update_wm(struct drm_crtc *crtc) plane_sr = cursor_sr = 0; } - DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", planea_wm, cursora_wm, planeb_wm, cursorb_wm, plane_sr, cursor_sr); @@ -1452,7 +1639,7 @@ static void g4x_update_wm(struct drm_crtc *crtc) (plane_sr << DSPFW_SR_SHIFT) | (cursorb_wm << DSPFW_CURSORB_SHIFT) | (planeb_wm << DSPFW_PLANEB_SHIFT) | - planea_wm); + (planea_wm << DSPFW_PLANEA_SHIFT)); I915_WRITE(DSPFW2, (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | (cursora_wm << DSPFW_CURSORA_SHIFT)); @@ -1526,8 +1713,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) /* 965 has limitations... */ I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) | - (8 << 16) | (8 << 8) | (8 << 0)); - I915_WRITE(DSPFW2, (8 << 8) | (8 << 0)); + (8 << DSPFW_CURSORB_SHIFT) | + (8 << DSPFW_PLANEB_SHIFT) | + (8 << DSPFW_PLANEA_SHIFT)); + I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) | + (8 << DSPFW_PLANEC_SHIFT_OLD)); /* update cursor SR watermark */ I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); @@ -1552,7 +1742,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) else if (!IS_GEN2(dev)) wm_info = &i915_wm_info; else - wm_info = &i830_wm_info; + wm_info = &i830_a_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 0); crtc = intel_get_crtc_for_plane(dev, 0); @@ -1565,10 +1755,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, - latency_ns); + pessimal_latency_ns); enabled = crtc; - } else + } else { planea_wm = fifo_size - wm_info->guard_size; + if (planea_wm > (long)wm_info->max_wm) + planea_wm = wm_info->max_wm; + } + + if (IS_GEN2(dev)) + wm_info = &i830_bc_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 1); crtc = intel_get_crtc_for_plane(dev, 1); @@ -1581,13 +1777,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, - latency_ns); + pessimal_latency_ns); if (enabled == NULL) enabled = crtc; else enabled = NULL; - } else + } else { planeb_wm = fifo_size - wm_info->guard_size; + if (planeb_wm > (long)wm_info->max_wm) + planeb_wm = wm_info->max_wm; + } DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); @@ -1674,7 +1873,7 @@ static void i845_update_wm(struct drm_crtc *unused_crtc) planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, &i845_wm_info, dev_priv->display.get_fifo_size(dev, 0), - 4, latency_ns); + 4, pessimal_latency_ns); fwater_lo = I915_READ(FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; @@ -2121,7 +2320,6 @@ int ilk_wm_max_level(const struct drm_device *dev) else return 2; } - static void intel_print_wm_latency(struct drm_device *dev, const char *name, const uint16_t wm[5]) @@ -2527,7 +2725,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, #define WM_DIRTY_FBC (1 << 24) #define WM_DIRTY_DDB (1 << 25) -static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, +static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, const struct ilk_wm_values *old, const struct ilk_wm_values *new) { @@ -2535,7 +2733,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, enum pipe pipe; int wm_lp; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) { dirty |= WM_DIRTY_LINETIME(pipe); /* Must disable LP1+ watermarks too */ @@ -2621,7 +2819,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, unsigned int dirty; uint32_t val; - dirty = ilk_compute_wm_dirty(dev, previous, results); + dirty = ilk_compute_wm_dirty(dev_priv, previous, results); if (!dirty) return; @@ -3090,6 +3288,9 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) { int new_power; + if (dev_priv->rps.is_bdw_sw_turbo) + return; + new_power = dev_priv->rps.power; switch (dev_priv->rps.power) { case LOW_POWER: @@ -3297,8 +3498,11 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); else if (IS_VALLEYVIEW(dev)) vlv_set_rps_idle(dev_priv); - else + else if (!dev_priv->rps.is_bdw_sw_turbo + || atomic_read(&dev_priv->rps.sw_turbo.flip_received)){ gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); + } + dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3312,8 +3516,11 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv) if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); - else + else if (!dev_priv->rps.is_bdw_sw_turbo + || atomic_read(&dev_priv->rps.sw_turbo.flip_received)){ gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); + } + dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3332,6 +3539,10 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) dev_priv->rps.cur_freq, vlv_gpu_freq(dev_priv, val), val); + if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), + "Odd GPU freq value\n")) + val &= ~1; + if (val != dev_priv->rps.cur_freq) vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); @@ -3344,21 +3555,26 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) static void gen8_disable_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + if (IS_BROADWELL(dev) && dev_priv->rps.is_bdw_sw_turbo){ + if (atomic_read(&dev_priv->rps.sw_turbo.flip_received)) + del_timer(&dev_priv->rps.sw_turbo.flip_timer); + dev_priv-> rps.is_bdw_sw_turbo = false; + } else { + I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); + I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & + ~dev_priv->pm_rps_events); + /* Complete PM interrupt masking here doesn't race with the rps work + * item again unmasking PM interrupts because that is using a different + * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in + * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which + * gen8_enable_rps will clean up. */ - I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); - I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & - ~dev_priv->pm_rps_events); - /* Complete PM interrupt masking here doesn't race with the rps work - * item again unmasking PM interrupts because that is using a different - * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in - * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which - * gen8_enable_rps will clean up. */ - - spin_lock_irq(&dev_priv->irq_lock); - dev_priv->rps.pm_iir = 0; - spin_unlock_irq(&dev_priv->irq_lock); + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->rps.pm_iir = 0; + spin_unlock_irq(&dev_priv->irq_lock); - I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); + I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); + } } static void gen6_disable_rps_interrupts(struct drm_device *dev) @@ -3406,8 +3622,14 @@ static void valleyview_disable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + /* we're doing forcewake before Disabling RC6, + * This what the BIOS expects when going into suspend */ + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + I915_WRITE(GEN6_RC_CONTROL, 0); + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + gen6_disable_rps_interrupts(dev); } @@ -3510,13 +3732,111 @@ static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_c dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; } +static void bdw_sw_calculate_freq(struct drm_device *dev, + struct intel_rps_bdw_cal *c, u32 *cur_time, u32 *c0) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u64 busy = 0; + u32 busyness_pct = 0; + u32 elapsed_time = 0; + u16 new_freq = 0; + + if (!c || !cur_time || !c0) + return; + + if (0 == c->last_c0) + goto out; + + /* Check Evaluation interval */ + elapsed_time = *cur_time - c->last_ts; + if (elapsed_time < c->eval_interval) + return; + + mutex_lock(&dev_priv->rps.hw_lock); + + /* + * c0 unit in 32*1.28 usec, elapsed_time unit in 1 usec. + * Whole busyness_pct calculation should be + * busy = ((u64)(*c0 - c->last_c0) << 5 << 7) / 100; + * busyness_pct = (u32)(busy * 100 / elapsed_time); + * The final formula is to simplify CPU calculation + */ + busy = (u64)(*c0 - c->last_c0) << 12; + do_div(busy, elapsed_time); + busyness_pct = (u32)busy; + + if (c->is_up && busyness_pct >= c->it_threshold_pct) + new_freq = (u16)dev_priv->rps.cur_freq + 3; + if (!c->is_up && busyness_pct <= c->it_threshold_pct) + new_freq = (u16)dev_priv->rps.cur_freq - 1; + + /* Adjust to new frequency busyness and compare with threshold */ + if (0 != new_freq) { + if (new_freq > dev_priv->rps.max_freq_softlimit) + new_freq = dev_priv->rps.max_freq_softlimit; + else if (new_freq < dev_priv->rps.min_freq_softlimit) + new_freq = dev_priv->rps.min_freq_softlimit; + + gen6_set_rps(dev, new_freq); + } + + mutex_unlock(&dev_priv->rps.hw_lock); + +out: + c->last_c0 = *c0; + c->last_ts = *cur_time; +} + +static void gen8_set_frequency_RP0(struct work_struct *work) +{ + struct intel_rps_bdw_turbo *p_bdw_turbo = + container_of(work, struct intel_rps_bdw_turbo, work_max_freq); + struct intel_gen6_power_mgmt *p_power_mgmt = + container_of(p_bdw_turbo, struct intel_gen6_power_mgmt, sw_turbo); + struct drm_i915_private *dev_priv = + container_of(p_power_mgmt, struct drm_i915_private, rps); + + mutex_lock(&dev_priv->rps.hw_lock); + gen6_set_rps(dev_priv->dev, dev_priv->rps.rp0_freq); + mutex_unlock(&dev_priv->rps.hw_lock); +} + +static void flip_active_timeout_handler(unsigned long var) +{ + struct drm_i915_private *dev_priv = (struct drm_i915_private *) var; + + del_timer(&dev_priv->rps.sw_turbo.flip_timer); + atomic_set(&dev_priv->rps.sw_turbo.flip_received, false); + + queue_work(dev_priv->wq, &dev_priv->rps.sw_turbo.work_max_freq); +} + +void bdw_software_turbo(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + u32 current_time = I915_READ(TIMESTAMP_CTR); /* unit in usec */ + u32 current_c0 = I915_READ(MCHBAR_PCU_C0); /* unit in 32*1.28 usec */ + + bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.up, + ¤t_time, ¤t_c0); + bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.down, + ¤t_time, ¤t_c0); +} + static void gen8_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring; uint32_t rc6_mask = 0, rp_state_cap; + uint32_t threshold_up_pct, threshold_down_pct; + uint32_t ei_up, ei_down; /* up and down evaluation interval */ + u32 rp_ctl_flag; int unused; + /* Use software Turbo for BDW */ + dev_priv->rps.is_bdw_sw_turbo = IS_BROADWELL(dev); + /* 1a: Software RC state - RC0 */ I915_WRITE(GEN6_RC_STATE, 0); @@ -3560,35 +3880,74 @@ static void gen8_enable_rps(struct drm_device *dev) HSW_FREQUENCY(dev_priv->rps.rp1_freq)); I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(dev_priv->rps.rp1_freq)); - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ + ei_up = 84480; /* 84.48ms */ + ei_down = 448000; + threshold_up_pct = 90; /* x percent busy */ + threshold_down_pct = 70; + + if (dev_priv->rps.is_bdw_sw_turbo) { + dev_priv->rps.sw_turbo.up.it_threshold_pct = threshold_up_pct; + dev_priv->rps.sw_turbo.up.eval_interval = ei_up; + dev_priv->rps.sw_turbo.up.is_up = true; + dev_priv->rps.sw_turbo.up.last_ts = 0; + dev_priv->rps.sw_turbo.up.last_c0 = 0; + + dev_priv->rps.sw_turbo.down.it_threshold_pct = threshold_down_pct; + dev_priv->rps.sw_turbo.down.eval_interval = ei_down; + dev_priv->rps.sw_turbo.down.is_up = false; + dev_priv->rps.sw_turbo.down.last_ts = 0; + dev_priv->rps.sw_turbo.down.last_c0 = 0; + + /* Start the timer to track if flip comes*/ + dev_priv->rps.sw_turbo.timeout = 200*1000; /* in us */ + + init_timer(&dev_priv->rps.sw_turbo.flip_timer); + dev_priv->rps.sw_turbo.flip_timer.function = flip_active_timeout_handler; + dev_priv->rps.sw_turbo.flip_timer.data = (unsigned long) dev_priv; + dev_priv->rps.sw_turbo.flip_timer.expires = + usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies; + add_timer(&dev_priv->rps.sw_turbo.flip_timer); + INIT_WORK(&dev_priv->rps.sw_turbo.work_max_freq, gen8_set_frequency_RP0); + + atomic_set(&dev_priv->rps.sw_turbo.flip_received, true); + } else { + /* NB: Docs say 1s, and 1000000 - which aren't equivalent + * 1 second timeout*/ + I915_WRITE(GEN6_RP_DOWN_TIMEOUT, FREQ_1_28_US(1000000)); - /* Docs recommend 900MHz, and 300 MHz respectively */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - dev_priv->rps.max_freq_softlimit << 24 | - dev_priv->rps.min_freq_softlimit << 16); + /* Docs recommend 900MHz, and 300 MHz respectively */ + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + dev_priv->rps.max_freq_softlimit << 24 | + dev_priv->rps.min_freq_softlimit << 16); - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ + I915_WRITE(GEN6_RP_UP_THRESHOLD, + FREQ_1_28_US(ei_up * threshold_up_pct / 100)); + I915_WRITE(GEN6_RP_DOWN_THRESHOLD, + FREQ_1_28_US(ei_down * threshold_down_pct / 100)); + I915_WRITE(GEN6_RP_UP_EI, + FREQ_1_28_US(ei_up)); + I915_WRITE(GEN6_RP_DOWN_EI, + FREQ_1_28_US(ei_down)); - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); + I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); + } /* 5: Enable RPS */ - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_TURBO | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); - - /* 6: Ring frequency + overclocking (our driver does this later */ - + rp_ctl_flag = GEN6_RP_MEDIA_TURBO | + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_AVG; + if (!dev_priv->rps.is_bdw_sw_turbo) + rp_ctl_flag |= GEN6_RP_ENABLE; + + I915_WRITE(GEN6_RP_CONTROL, rp_ctl_flag); + + /* 6: Ring frequency + overclocking + * (our driver does this later */ gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8); - - gen8_enable_rps_interrupts(dev); + if (!dev_priv->rps.is_bdw_sw_turbo) + gen8_enable_rps_interrupts(dev); gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } @@ -3598,7 +3957,6 @@ static void gen6_enable_rps(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring; u32 rp_state_cap; - u32 gt_perf_status; u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; int rc6_mode; @@ -3623,7 +3981,6 @@ static void gen6_enable_rps(struct drm_device *dev) gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); - gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); parse_rp_state_cap(dev_priv, rp_state_cap); @@ -3965,11 +4322,27 @@ static void valleyview_cleanup_pctx(struct drm_device *dev) static void valleyview_init_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; valleyview_setup_pctx(dev); mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); + switch ((val >> 6) & 3) { + case 0: + case 1: + dev_priv->mem_freq = 800; + break; + case 2: + dev_priv->mem_freq = 1066; + break; + case 3: + dev_priv->mem_freq = 1333; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); + dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", @@ -4004,11 +4377,38 @@ static void valleyview_init_gt_powersave(struct drm_device *dev) static void cherryview_init_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; cherryview_setup_pctx(dev); mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, CCK_FUSE_REG); + switch ((val >> 2) & 0x7) { + case 0: + case 1: + dev_priv->rps.cz_freq = 200; + dev_priv->mem_freq = 1600; + break; + case 2: + dev_priv->rps.cz_freq = 267; + dev_priv->mem_freq = 1600; + break; + case 3: + dev_priv->rps.cz_freq = 333; + dev_priv->mem_freq = 2000; + break; + case 4: + dev_priv->rps.cz_freq = 320; + dev_priv->mem_freq = 1600; + break; + case 5: + dev_priv->rps.cz_freq = 400; + dev_priv->mem_freq = 1600; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); + dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", @@ -4030,6 +4430,12 @@ static void cherryview_init_gt_powersave(struct drm_device *dev) vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), dev_priv->rps.min_freq); + WARN_ONCE((dev_priv->rps.max_freq | + dev_priv->rps.efficient_freq | + dev_priv->rps.rp1_freq | + dev_priv->rps.min_freq) & 1, + "Odd GPU freq values\n"); + /* Preserve min/max settings in case of re-init */ if (dev_priv->rps.max_freq_softlimit == 0) dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; @@ -5015,6 +5421,8 @@ static void intel_gen6_powersave_work(struct work_struct *work) rps.delayed_resume_work.work); struct drm_device *dev = dev_priv->dev; + dev_priv->rps.is_bdw_sw_turbo = false; + mutex_lock(&dev_priv->rps.hw_lock); if (IS_CHERRYVIEW(dev)) { @@ -5088,7 +5496,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(DSPCNTR(pipe), I915_READ(DSPCNTR(pipe)) | DISPPLANE_TRICKLE_FEED_DISABLE); @@ -5203,7 +5611,7 @@ static void cpt_init_clock_gating(struct drm_device *dev) /* The below fixes the weird display corruption, a few pixels shifted * downward, on (only) LVDS of some HP laptops with IVY. */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { val = I915_READ(TRANS_CHICKEN2(pipe)); val |= TRANS_CHICKEN2_TIMING_OVERRIDE; val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; @@ -5215,7 +5623,7 @@ static void cpt_init_clock_gating(struct drm_device *dev) I915_WRITE(TRANS_CHICKEN2(pipe), val); } /* WADP0ClockGatingDisable */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(TRANS_CHICKEN1(pipe), TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } @@ -5383,7 +5791,7 @@ static void lpt_suspend_hw(struct drm_device *dev) } } -static void gen8_init_clock_gating(struct drm_device *dev) +static void broadwell_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; @@ -5395,77 +5803,33 @@ static void gen8_init_clock_gating(struct drm_device *dev) /* FIXME(BDW): Check all the w/a, some might only apply to * pre-production hw. */ - /* WaDisablePartialInstShootdown:bdw */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); - /* WaDisableThreadStallDopClockGating:bdw */ - /* FIXME: Unclear whether we really need this on production bdw. */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); - - /* - * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for - * pre-production hardware - */ - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS)); - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE)); I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2))); - I915_WRITE(COMMON_SLICE_CHICKEN2, - _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE)); - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, - _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE)); - - /* WaDisableDopClockGating:bdw May not be needed for production */ - I915_WRITE(GEN7_ROW_CHICKEN2, - _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - - /* WaSwitchSolVfFArbitrationPriority:bdw */ - I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); + /* WaSwitchSolVfFArbitrationPriority:bdw */ + I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); /* WaPsrDPAMaskVBlankInSRD:bdw */ I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(CHICKEN_PIPESL_1(pipe), I915_READ(CHICKEN_PIPESL_1(pipe)) | BDW_DPRS_MASK_VBLANK_SRD); } - /* Use Force Non-Coherent whenever executing a 3D context. This is a - * workaround for for a possible hang in the unlikely event a TLB - * invalidation occurs during a PSD flush. - */ - I915_WRITE(HDC_CHICKEN0, - I915_READ(HDC_CHICKEN0) | - _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT)); - /* WaVSRefCountFullforceMissDisable:bdw */ /* WaDSRefCountFullforceMissDisable:bdw */ I915_WRITE(GEN7_FF_THREAD_MODE, I915_READ(GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN7_GT_MODE, - GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); - I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); @@ -5473,9 +5837,7 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - /* Wa4x4STCOptimizationDisable:bdw */ - I915_WRITE(CACHE_MODE_1, - _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE)); + lpt_init_clock_gating(dev); } static void haswell_init_clock_gating(struct drm_device *dev) @@ -5631,24 +5993,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) static void valleyview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - - mutex_lock(&dev_priv->rps.hw_lock); - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - mutex_unlock(&dev_priv->rps.hw_lock); - switch ((val >> 6) & 3) { - case 0: - case 1: - dev_priv->mem_freq = 800; - break; - case 2: - dev_priv->mem_freq = 1066; - break; - case 3: - dev_priv->mem_freq = 1333; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); @@ -5724,48 +6068,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev) static void cherryview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - - mutex_lock(&dev_priv->rps.hw_lock); - val = vlv_punit_read(dev_priv, CCK_FUSE_REG); - mutex_unlock(&dev_priv->rps.hw_lock); - switch ((val >> 2) & 0x7) { - case 0: - case 1: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_200; - dev_priv->mem_freq = 1600; - break; - case 2: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_267; - dev_priv->mem_freq = 1600; - break; - case 3: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_333; - dev_priv->mem_freq = 2000; - break; - case 4: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_320; - dev_priv->mem_freq = 1600; - break; - case 5: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_400; - dev_priv->mem_freq = 1600; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); - /* WaDisablePartialInstShootdown:chv */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); - - /* WaDisableThreadStallDopClockGating:chv */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); - /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ I915_WRITE(GEN7_FF_THREAD_MODE, @@ -5784,10 +6091,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - /* WaDisableSamplerPowerBypass:chv (pre-production hw) */ - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); - /* WaDisableGunitClockGating:chv (pre-production hw) */ I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) | GINT_DIS); @@ -5797,8 +6100,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE)); /* WaDisableDopClockGating:chv (pre-production hw) */ - I915_WRITE(GEN7_ROW_CHICKEN2, - _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); } @@ -5883,6 +6184,9 @@ static void gen3_init_clock_gating(struct drm_device *dev) /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); + + I915_WRITE(MI_ARB_STATE, + _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } static void i85x_init_clock_gating(struct drm_device *dev) @@ -5894,6 +6198,9 @@ static void i85x_init_clock_gating(struct drm_device *dev) /* interrupts should cause a wake up from C3 */ I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); + + I915_WRITE(MEM_MODE, + _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); } static void i830_init_clock_gating(struct drm_device *dev) @@ -5901,6 +6208,10 @@ static void i830_init_clock_gating(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); + + I915_WRITE(MEM_MODE, + _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | + _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); } void intel_init_clock_gating(struct drm_device *dev) @@ -5916,863 +6227,35 @@ void intel_suspend_hw(struct drm_device *dev) lpt_suspend_hw(dev); } -#define for_each_power_well(i, power_well, domain_mask, power_domains) \ - for (i = 0; \ - i < (power_domains)->power_well_count && \ - ((power_well) = &(power_domains)->power_wells[i]); \ - i++) \ - if ((power_well)->domains & (domain_mask)) - -#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \ - for (i = (power_domains)->power_well_count - 1; \ - i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\ - i--) \ - if ((power_well)->domains & (domain_mask)) - -/** - * We should only use the power well if we explicitly asked the hardware to - * enable it, so check if it's enabled and also check if we've requested it to - * be enabled. - */ -static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - return I915_READ(HSW_PWR_WELL_DRIVER) == - (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); -} - -bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) -{ - struct i915_power_domains *power_domains; - struct i915_power_well *power_well; - bool is_enabled; - int i; - - if (dev_priv->pm.suspended) - return false; - - power_domains = &dev_priv->power_domains; - - is_enabled = true; - - for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { - if (power_well->always_on) - continue; - - if (!power_well->hw_enabled) { - is_enabled = false; - break; - } - } - - return is_enabled; -} - -bool intel_display_power_enabled(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) -{ - struct i915_power_domains *power_domains; - bool ret; - - power_domains = &dev_priv->power_domains; - - mutex_lock(&power_domains->lock); - ret = intel_display_power_enabled_unlocked(dev_priv, domain); - mutex_unlock(&power_domains->lock); - - return ret; -} - -/* - * Starting with Haswell, we have a "Power Down Well" that can be turned off - * when not needed anymore. We have 4 registers that can request the power well - * to be enabled, and it will only be disabled if none of the registers is - * requesting it to be enabled. - */ -static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - - /* - * After we re-enable the power well, if we touch VGA register 0x3d5 - * we'll get unclaimed register interrupts. This stops after we write - * anything to the VGA MSR register. The vgacon module uses this - * register all the time, so if we unbind our driver and, as a - * consequence, bind vgacon, we'll get stuck in an infinite loop at - * console_unlock(). So make here we touch the VGA MSR register, making - * sure vgacon can keep working normally without triggering interrupts - * and error messages. - */ - vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); - outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); - vga_put(dev->pdev, VGA_RSRC_LEGACY_IO); - - if (IS_BROADWELL(dev)) - gen8_irq_power_well_post_enable(dev_priv); -} - -static void hsw_set_power_well(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well, bool enable) -{ - bool is_enabled, enable_requested; - uint32_t tmp; - - tmp = I915_READ(HSW_PWR_WELL_DRIVER); - is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED; - enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST; - - if (enable) { - if (!enable_requested) - I915_WRITE(HSW_PWR_WELL_DRIVER, - HSW_PWR_WELL_ENABLE_REQUEST); - - if (!is_enabled) { - DRM_DEBUG_KMS("Enabling power well\n"); - if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) & - HSW_PWR_WELL_STATE_ENABLED), 20)) - DRM_ERROR("Timeout enabling power well\n"); - } - - hsw_power_well_post_enable(dev_priv); - } else { - if (enable_requested) { - I915_WRITE(HSW_PWR_WELL_DRIVER, 0); - POSTING_READ(HSW_PWR_WELL_DRIVER); - DRM_DEBUG_KMS("Requesting to disable the power well\n"); - } - } -} - -static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - hsw_set_power_well(dev_priv, power_well, power_well->count > 0); - - /* - * We're taking over the BIOS, so clear any requests made by it since - * the driver is in charge now. - */ - if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) - I915_WRITE(HSW_PWR_WELL_BIOS, 0); -} - -static void hsw_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - hsw_set_power_well(dev_priv, power_well, true); -} - -static void hsw_power_well_disable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - hsw_set_power_well(dev_priv, power_well, false); -} - -static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ -} - -static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) +static void intel_init_fbc(struct drm_i915_private *dev_priv) { - return true; -} - -static void vlv_set_power_well(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well, bool enable) -{ - enum punit_power_well power_well_id = power_well->data; - u32 mask; - u32 state; - u32 ctrl; - - mask = PUNIT_PWRGT_MASK(power_well_id); - state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) : - PUNIT_PWRGT_PWR_GATE(power_well_id); - - mutex_lock(&dev_priv->rps.hw_lock); - -#define COND \ - ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state) - - if (COND) - goto out; - - ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL); - ctrl &= ~mask; - ctrl |= state; - vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl); - - if (wait_for(COND, 100)) - DRM_ERROR("timout setting power well state %08x (%08x)\n", - state, - vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL)); - -#undef COND - -out: - mutex_unlock(&dev_priv->rps.hw_lock); -} - -static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - vlv_set_power_well(dev_priv, power_well, power_well->count > 0); -} - -static void vlv_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - vlv_set_power_well(dev_priv, power_well, true); -} - -static void vlv_power_well_disable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - vlv_set_power_well(dev_priv, power_well, false); -} - -static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - int power_well_id = power_well->data; - bool enabled = false; - u32 mask; - u32 state; - u32 ctrl; - - mask = PUNIT_PWRGT_MASK(power_well_id); - ctrl = PUNIT_PWRGT_PWR_ON(power_well_id); - - mutex_lock(&dev_priv->rps.hw_lock); - - state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask; - /* - * We only ever set the power-on and power-gate states, anything - * else is unexpected. - */ - WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) && - state != PUNIT_PWRGT_PWR_GATE(power_well_id)); - if (state == ctrl) - enabled = true; - - /* - * A transient state at this point would mean some unexpected party - * is poking at the power controls too. - */ - ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask; - WARN_ON(ctrl != state); - - mutex_unlock(&dev_priv->rps.hw_lock); - - return enabled; -} - -static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); - - vlv_set_power_well(dev_priv, power_well, true); - - spin_lock_irq(&dev_priv->irq_lock); - valleyview_enable_display_irqs(dev_priv); - spin_unlock_irq(&dev_priv->irq_lock); - - /* - * During driver initialization/resume we can avoid restoring the - * part of the HW/SW state that will be inited anyway explicitly. - */ - if (dev_priv->power_domains.initializing) - return; - - intel_hpd_init(dev_priv->dev); - - i915_redisable_vga_power_on(dev_priv->dev); -} - -static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); - - spin_lock_irq(&dev_priv->irq_lock); - valleyview_disable_display_irqs(dev_priv); - spin_unlock_irq(&dev_priv->irq_lock); - - vlv_set_power_well(dev_priv, power_well, false); -} - -static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); - - /* - * Enable the CRI clock source so we can get at the - * display and the reference clock for VGA - * hotplug / manual detection. - */ - I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); - udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ - - vlv_set_power_well(dev_priv, power_well, true); - - /* - * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx - - * 6. De-assert cmn_reset/side_reset. Same as VLV X0. - * a. GUnit 0x2110 bit[0] set to 1 (def 0) - * b. The other bits such as sfr settings / modesel may all - * be set to 0. - * - * This should only be done on init and resume from S3 with - * both PLLs disabled, or we risk losing DPIO and PLL - * synchronization. - */ - I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST); -} - -static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - struct drm_device *dev = dev_priv->dev; - enum pipe pipe; - - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); - - for_each_pipe(pipe) - assert_pll_disabled(dev_priv, pipe); - - /* Assert common reset */ - I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST); - - vlv_set_power_well(dev_priv, power_well, false); -} - -static void check_power_well_state(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - bool enabled = power_well->ops->is_enabled(dev_priv, power_well); - - if (power_well->always_on || !i915.disable_power_well) { - if (!enabled) - goto mismatch; - + if (!HAS_FBC(dev_priv)) { + dev_priv->fbc.enabled = false; return; } - if (enabled != (power_well->count > 0)) - goto mismatch; - - return; - -mismatch: - WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n", - power_well->name, power_well->always_on, enabled, - power_well->count, i915.disable_power_well); -} - -void intel_display_power_get(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) -{ - struct i915_power_domains *power_domains; - struct i915_power_well *power_well; - int i; - - intel_runtime_pm_get(dev_priv); - - power_domains = &dev_priv->power_domains; - - mutex_lock(&power_domains->lock); - - for_each_power_well(i, power_well, BIT(domain), power_domains) { - if (!power_well->count++) { - DRM_DEBUG_KMS("enabling %s\n", power_well->name); - power_well->ops->enable(dev_priv, power_well); - power_well->hw_enabled = true; - } - - check_power_well_state(dev_priv, power_well); - } - - power_domains->domain_use_count[domain]++; - - mutex_unlock(&power_domains->lock); -} - -void intel_display_power_put(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) -{ - struct i915_power_domains *power_domains; - struct i915_power_well *power_well; - int i; - - power_domains = &dev_priv->power_domains; - - mutex_lock(&power_domains->lock); - - WARN_ON(!power_domains->domain_use_count[domain]); - power_domains->domain_use_count[domain]--; - - for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { - WARN_ON(!power_well->count); - - if (!--power_well->count && i915.disable_power_well) { - DRM_DEBUG_KMS("disabling %s\n", power_well->name); - power_well->hw_enabled = false; - power_well->ops->disable(dev_priv, power_well); - } - - check_power_well_state(dev_priv, power_well); - } - - mutex_unlock(&power_domains->lock); - - intel_runtime_pm_put(dev_priv); -} - -static struct i915_power_domains *hsw_pwr; - -/* Display audio driver power well request */ -int i915_request_power_well(void) -{ - struct drm_i915_private *dev_priv; - - if (!hsw_pwr) - return -ENODEV; - - dev_priv = container_of(hsw_pwr, struct drm_i915_private, - power_domains); - intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); - return 0; -} -EXPORT_SYMBOL_GPL(i915_request_power_well); - -/* Display audio driver power well release */ -int i915_release_power_well(void) -{ - struct drm_i915_private *dev_priv; - - if (!hsw_pwr) - return -ENODEV; - - dev_priv = container_of(hsw_pwr, struct drm_i915_private, - power_domains); - intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); - return 0; -} -EXPORT_SYMBOL_GPL(i915_release_power_well); - -/* - * Private interface for the audio driver to get CDCLK in kHz. - * - * Caller must request power well using i915_request_power_well() prior to - * making the call. - */ -int i915_get_cdclk_freq(void) -{ - struct drm_i915_private *dev_priv; - - if (!hsw_pwr) - return -ENODEV; - - dev_priv = container_of(hsw_pwr, struct drm_i915_private, - power_domains); - - return intel_ddi_get_cdclk_freq(dev_priv); -} -EXPORT_SYMBOL_GPL(i915_get_cdclk_freq); - - -#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) - -#define HSW_ALWAYS_ON_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PIPE_A) | \ - BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_CRT) | \ - BIT(POWER_DOMAIN_PLLS) | \ - BIT(POWER_DOMAIN_INIT)) -#define HSW_DISPLAY_POWER_DOMAINS ( \ - (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \ - BIT(POWER_DOMAIN_INIT)) - -#define BDW_ALWAYS_ON_POWER_DOMAINS ( \ - HSW_ALWAYS_ON_POWER_DOMAINS | \ - BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER)) -#define BDW_DISPLAY_POWER_DOMAINS ( \ - (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \ - BIT(POWER_DOMAIN_INIT)) - -#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT) -#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK - -#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_CRT) | \ - BIT(POWER_DOMAIN_INIT)) - -#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_INIT)) - -#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_INIT)) - -#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_INIT)) - -#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_INIT)) - -static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { - .sync_hw = i9xx_always_on_power_well_noop, - .enable = i9xx_always_on_power_well_noop, - .disable = i9xx_always_on_power_well_noop, - .is_enabled = i9xx_always_on_power_well_enabled, -}; - -static struct i915_power_well i9xx_always_on_power_well[] = { - { - .name = "always-on", - .always_on = 1, - .domains = POWER_DOMAIN_MASK, - .ops = &i9xx_always_on_power_well_ops, - }, -}; - -static const struct i915_power_well_ops hsw_power_well_ops = { - .sync_hw = hsw_power_well_sync_hw, - .enable = hsw_power_well_enable, - .disable = hsw_power_well_disable, - .is_enabled = hsw_power_well_enabled, -}; - -static struct i915_power_well hsw_power_wells[] = { - { - .name = "always-on", - .always_on = 1, - .domains = HSW_ALWAYS_ON_POWER_DOMAINS, - .ops = &i9xx_always_on_power_well_ops, - }, - { - .name = "display", - .domains = HSW_DISPLAY_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - }, -}; - -static struct i915_power_well bdw_power_wells[] = { - { - .name = "always-on", - .always_on = 1, - .domains = BDW_ALWAYS_ON_POWER_DOMAINS, - .ops = &i9xx_always_on_power_well_ops, - }, - { - .name = "display", - .domains = BDW_DISPLAY_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - }, -}; - -static const struct i915_power_well_ops vlv_display_power_well_ops = { - .sync_hw = vlv_power_well_sync_hw, - .enable = vlv_display_power_well_enable, - .disable = vlv_display_power_well_disable, - .is_enabled = vlv_power_well_enabled, -}; - -static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = { - .sync_hw = vlv_power_well_sync_hw, - .enable = vlv_dpio_cmn_power_well_enable, - .disable = vlv_dpio_cmn_power_well_disable, - .is_enabled = vlv_power_well_enabled, -}; - -static const struct i915_power_well_ops vlv_dpio_power_well_ops = { - .sync_hw = vlv_power_well_sync_hw, - .enable = vlv_power_well_enable, - .disable = vlv_power_well_disable, - .is_enabled = vlv_power_well_enabled, -}; - -static struct i915_power_well vlv_power_wells[] = { - { - .name = "always-on", - .always_on = 1, - .domains = VLV_ALWAYS_ON_POWER_DOMAINS, - .ops = &i9xx_always_on_power_well_ops, - }, - { - .name = "display", - .domains = VLV_DISPLAY_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DISP2D, - .ops = &vlv_display_power_well_ops, - }, - { - .name = "dpio-tx-b-01", - .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, - .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, - }, - { - .name = "dpio-tx-b-23", - .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, - .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, - }, - { - .name = "dpio-tx-c-01", - .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, - .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, - }, - { - .name = "dpio-tx-c-23", - .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | - VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, - .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, - }, - { - .name = "dpio-common", - .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_BC, - .ops = &vlv_dpio_cmn_power_well_ops, - }, -}; - -static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv, - enum punit_power_well power_well_id) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *power_well; - int i; - - for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { - if (power_well->data == power_well_id) - return power_well; - } - - return NULL; -} - -#define set_power_wells(power_domains, __power_wells) ({ \ - (power_domains)->power_wells = (__power_wells); \ - (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ -}) - -int intel_power_domains_init(struct drm_i915_private *dev_priv) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - - mutex_init(&power_domains->lock); - - /* - * The enabling order will be from lower to higher indexed wells, - * the disabling order is reversed. - */ - if (IS_HASWELL(dev_priv->dev)) { - set_power_wells(power_domains, hsw_power_wells); - hsw_pwr = power_domains; - } else if (IS_BROADWELL(dev_priv->dev)) { - set_power_wells(power_domains, bdw_power_wells); - hsw_pwr = power_domains; - } else if (IS_VALLEYVIEW(dev_priv->dev)) { - set_power_wells(power_domains, vlv_power_wells); + if (INTEL_INFO(dev_priv)->gen >= 7) { + dev_priv->display.fbc_enabled = ironlake_fbc_enabled; + dev_priv->display.enable_fbc = gen7_enable_fbc; + dev_priv->display.disable_fbc = ironlake_disable_fbc; + } else if (INTEL_INFO(dev_priv)->gen >= 5) { + dev_priv->display.fbc_enabled = ironlake_fbc_enabled; + dev_priv->display.enable_fbc = ironlake_enable_fbc; + dev_priv->display.disable_fbc = ironlake_disable_fbc; + } else if (IS_GM45(dev_priv)) { + dev_priv->display.fbc_enabled = g4x_fbc_enabled; + dev_priv->display.enable_fbc = g4x_enable_fbc; + dev_priv->display.disable_fbc = g4x_disable_fbc; } else { - set_power_wells(power_domains, i9xx_always_on_power_well); - } - - return 0; -} - -void intel_power_domains_remove(struct drm_i915_private *dev_priv) -{ - hsw_pwr = NULL; -} - -static void intel_power_domains_resume(struct drm_i915_private *dev_priv) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *power_well; - int i; - - mutex_lock(&power_domains->lock); - for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { - power_well->ops->sync_hw(dev_priv, power_well); - power_well->hw_enabled = power_well->ops->is_enabled(dev_priv, - power_well); - } - mutex_unlock(&power_domains->lock); -} - -static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv) -{ - struct i915_power_well *cmn = - lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC); - struct i915_power_well *disp2d = - lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D); - - /* nothing to do if common lane is already off */ - if (!cmn->ops->is_enabled(dev_priv, cmn)) - return; - - /* If the display might be already active skip this */ - if (disp2d->ops->is_enabled(dev_priv, disp2d) && - I915_READ(DPIO_CTL) & DPIO_CMNRST) - return; - - DRM_DEBUG_KMS("toggling display PHY side reset\n"); - - /* cmnlane needs DPLL registers */ - disp2d->ops->enable(dev_priv, disp2d); - - /* - * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx: - * Need to assert and de-assert PHY SB reset by gating the - * common lane power, then un-gating it. - * Simply ungating isn't enough to reset the PHY enough to get - * ports and lanes running. - */ - cmn->ops->disable(dev_priv, cmn); -} - -void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct i915_power_domains *power_domains = &dev_priv->power_domains; - - power_domains->initializing = true; - - if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) { - mutex_lock(&power_domains->lock); - vlv_cmnlane_wa(dev_priv); - mutex_unlock(&power_domains->lock); - } - - /* For now, we need the power well to be always enabled. */ - intel_display_set_init_power(dev_priv, true); - intel_power_domains_resume(dev_priv); - power_domains->initializing = false; -} - -void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv) -{ - intel_runtime_pm_get(dev_priv); -} - -void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) -{ - intel_runtime_pm_put(dev_priv); -} - -void intel_runtime_pm_get(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; - - if (!HAS_RUNTIME_PM(dev)) - return; - - pm_runtime_get_sync(device); - WARN(dev_priv->pm.suspended, "Device still suspended.\n"); -} - -void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; + dev_priv->display.fbc_enabled = i8xx_fbc_enabled; + dev_priv->display.enable_fbc = i8xx_enable_fbc; + dev_priv->display.disable_fbc = i8xx_disable_fbc; - if (!HAS_RUNTIME_PM(dev)) - return; - - WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n"); - pm_runtime_get_noresume(device); -} - -void intel_runtime_pm_put(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; - - if (!HAS_RUNTIME_PM(dev)) - return; - - pm_runtime_mark_last_busy(device); - pm_runtime_put_autosuspend(device); -} - -void intel_init_runtime_pm(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; - - if (!HAS_RUNTIME_PM(dev)) - return; - - pm_runtime_set_active(device); - - /* - * RPM depends on RC6 to save restore the GT HW context, so make RC6 a - * requirement. - */ - if (!intel_enable_rc6(dev)) { - DRM_INFO("RC6 disabled, disabling runtime PM support\n"); - return; + /* This value was pulled out of someone's hat */ + I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); } - pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ - pm_runtime_mark_last_busy(device); - pm_runtime_use_autosuspend(device); - - pm_runtime_put_autosuspend(device); -} - -void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; - - if (!HAS_RUNTIME_PM(dev)) - return; - - if (!intel_enable_rc6(dev)) - return; - - /* Make sure we're not suspended first. */ - pm_runtime_get_sync(device); - pm_runtime_disable(device); + dev_priv->fbc.enabled = dev_priv->display.fbc_enabled(dev_priv->dev); } /* Set up chip specific power management-related functions */ @@ -6780,28 +6263,7 @@ void intel_init_pm(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (HAS_FBC(dev)) { - if (INTEL_INFO(dev)->gen >= 7) { - dev_priv->display.fbc_enabled = ironlake_fbc_enabled; - dev_priv->display.enable_fbc = gen7_enable_fbc; - dev_priv->display.disable_fbc = ironlake_disable_fbc; - } else if (INTEL_INFO(dev)->gen >= 5) { - dev_priv->display.fbc_enabled = ironlake_fbc_enabled; - dev_priv->display.enable_fbc = ironlake_enable_fbc; - dev_priv->display.disable_fbc = ironlake_disable_fbc; - } else if (IS_GM45(dev)) { - dev_priv->display.fbc_enabled = g4x_fbc_enabled; - dev_priv->display.enable_fbc = g4x_enable_fbc; - dev_priv->display.disable_fbc = g4x_disable_fbc; - } else { - dev_priv->display.fbc_enabled = i8xx_fbc_enabled; - dev_priv->display.enable_fbc = i8xx_enable_fbc; - dev_priv->display.disable_fbc = i8xx_disable_fbc; - - /* This value was pulled out of someone's hat */ - I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); - } - } + intel_init_fbc(dev_priv); /* For cxsr */ if (IS_PINEVIEW(dev)) @@ -6810,7 +6272,9 @@ void intel_init_pm(struct drm_device *dev) i915_ironlake_get_mem_freq(dev); /* For FIFO watermark updates */ - if (HAS_PCH_SPLIT(dev)) { + if (IS_GEN9(dev)) { + dev_priv->display.init_clock_gating = gen9_init_clock_gating; + } else if (HAS_PCH_SPLIT(dev)) { ilk_setup_wm_latency(dev); if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && @@ -6833,13 +6297,15 @@ void intel_init_pm(struct drm_device *dev) else if (IS_HASWELL(dev)) dev_priv->display.init_clock_gating = haswell_init_clock_gating; else if (INTEL_INFO(dev)->gen == 8) - dev_priv->display.init_clock_gating = gen8_init_clock_gating; + dev_priv->display.init_clock_gating = broadwell_init_clock_gating; } else if (IS_CHERRYVIEW(dev)) { - dev_priv->display.update_wm = valleyview_update_wm; + dev_priv->display.update_wm = cherryview_update_wm; + dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; dev_priv->display.init_clock_gating = cherryview_init_clock_gating; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.update_wm = valleyview_update_wm; + dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; dev_priv->display.init_clock_gating = valleyview_init_clock_gating; } else if (IS_PINEVIEW(dev)) { @@ -7025,6 +6491,7 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) return -1; } + /* CHV needs even values */ opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2); return opcode; @@ -7064,5 +6531,4 @@ void intel_pm_setup(struct drm_device *dev) intel_gen6_powersave_work); dev_priv->pm.suspended = false; - dev_priv->pm._irqs_disabled = false; }