2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/cpufreq.h>
30 #include "intel_drv.h"
31 #include "../../../platform/x86/intel_ips.h"
32 #include <linux/module.h>
34 #define FORCEWAKE_ACK_TIMEOUT_MS 2
36 /* FBC, or Frame Buffer Compression, is a technique employed to compress the
37 * framebuffer contents in-memory, aiming at reducing the required bandwidth
38 * during in-memory transfers and, therefore, reduce the power packet.
40 * The benefits of FBC are mostly visible with solid backgrounds and
41 * variation-less patterns.
43 * FBC-related functionality can be enabled by the means of the
44 * i915.i915_enable_fbc parameter
47 static bool intel_crtc_active(struct drm_crtc
*crtc
)
49 /* Be paranoid as we can arrive here with only partial
50 * state retrieved from the hardware during setup.
52 return to_intel_crtc(crtc
)->active
&& crtc
->fb
&& crtc
->mode
.clock
;
55 static void i8xx_disable_fbc(struct drm_device
*dev
)
57 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
60 /* Disable compression */
61 fbc_ctl
= I915_READ(FBC_CONTROL
);
62 if ((fbc_ctl
& FBC_CTL_EN
) == 0)
65 fbc_ctl
&= ~FBC_CTL_EN
;
66 I915_WRITE(FBC_CONTROL
, fbc_ctl
);
68 /* Wait for compressing bit to clear */
69 if (wait_for((I915_READ(FBC_STATUS
) & FBC_STAT_COMPRESSING
) == 0, 10)) {
70 DRM_DEBUG_KMS("FBC idle timed out\n");
74 DRM_DEBUG_KMS("disabled FBC\n");
77 static void i8xx_enable_fbc(struct drm_crtc
*crtc
, unsigned long interval
)
79 struct drm_device
*dev
= crtc
->dev
;
80 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
81 struct drm_framebuffer
*fb
= crtc
->fb
;
82 struct intel_framebuffer
*intel_fb
= to_intel_framebuffer(fb
);
83 struct drm_i915_gem_object
*obj
= intel_fb
->obj
;
84 struct intel_crtc
*intel_crtc
= to_intel_crtc(crtc
);
87 u32 fbc_ctl
, fbc_ctl2
;
89 cfb_pitch
= dev_priv
->cfb_size
/ FBC_LL_SIZE
;
90 if (fb
->pitches
[0] < cfb_pitch
)
91 cfb_pitch
= fb
->pitches
[0];
93 /* FBC_CTL wants 64B units */
94 cfb_pitch
= (cfb_pitch
/ 64) - 1;
95 plane
= intel_crtc
->plane
== 0 ? FBC_CTL_PLANEA
: FBC_CTL_PLANEB
;
98 for (i
= 0; i
< (FBC_LL_SIZE
/ 32) + 1; i
++)
99 I915_WRITE(FBC_TAG
+ (i
* 4), 0);
102 fbc_ctl2
= FBC_CTL_FENCE_DBL
| FBC_CTL_IDLE_IMM
| FBC_CTL_CPU_FENCE
;
104 I915_WRITE(FBC_CONTROL2
, fbc_ctl2
);
105 I915_WRITE(FBC_FENCE_OFF
, crtc
->y
);
108 fbc_ctl
= FBC_CTL_EN
| FBC_CTL_PERIODIC
;
110 fbc_ctl
|= FBC_CTL_C3_IDLE
; /* 945 needs special SR handling */
111 fbc_ctl
|= (cfb_pitch
& 0xff) << FBC_CTL_STRIDE_SHIFT
;
112 fbc_ctl
|= (interval
& 0x2fff) << FBC_CTL_INTERVAL_SHIFT
;
113 fbc_ctl
|= obj
->fence_reg
;
114 I915_WRITE(FBC_CONTROL
, fbc_ctl
);
116 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
117 cfb_pitch
, crtc
->y
, plane_name(intel_crtc
->plane
));
120 static bool i8xx_fbc_enabled(struct drm_device
*dev
)
122 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
124 return I915_READ(FBC_CONTROL
) & FBC_CTL_EN
;
127 static void g4x_enable_fbc(struct drm_crtc
*crtc
, unsigned long interval
)
129 struct drm_device
*dev
= crtc
->dev
;
130 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
131 struct drm_framebuffer
*fb
= crtc
->fb
;
132 struct intel_framebuffer
*intel_fb
= to_intel_framebuffer(fb
);
133 struct drm_i915_gem_object
*obj
= intel_fb
->obj
;
134 struct intel_crtc
*intel_crtc
= to_intel_crtc(crtc
);
135 int plane
= intel_crtc
->plane
== 0 ? DPFC_CTL_PLANEA
: DPFC_CTL_PLANEB
;
136 unsigned long stall_watermark
= 200;
139 dpfc_ctl
= plane
| DPFC_SR_EN
| DPFC_CTL_LIMIT_1X
;
140 dpfc_ctl
|= DPFC_CTL_FENCE_EN
| obj
->fence_reg
;
141 I915_WRITE(DPFC_CHICKEN
, DPFC_HT_MODIFY
);
143 I915_WRITE(DPFC_RECOMP_CTL
, DPFC_RECOMP_STALL_EN
|
144 (stall_watermark
<< DPFC_RECOMP_STALL_WM_SHIFT
) |
145 (interval
<< DPFC_RECOMP_TIMER_COUNT_SHIFT
));
146 I915_WRITE(DPFC_FENCE_YOFF
, crtc
->y
);
149 I915_WRITE(DPFC_CONTROL
, I915_READ(DPFC_CONTROL
) | DPFC_CTL_EN
);
151 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc
->plane
));
154 static void g4x_disable_fbc(struct drm_device
*dev
)
156 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
159 /* Disable compression */
160 dpfc_ctl
= I915_READ(DPFC_CONTROL
);
161 if (dpfc_ctl
& DPFC_CTL_EN
) {
162 dpfc_ctl
&= ~DPFC_CTL_EN
;
163 I915_WRITE(DPFC_CONTROL
, dpfc_ctl
);
165 DRM_DEBUG_KMS("disabled FBC\n");
169 static bool g4x_fbc_enabled(struct drm_device
*dev
)
171 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
173 return I915_READ(DPFC_CONTROL
) & DPFC_CTL_EN
;
176 static void sandybridge_blit_fbc_update(struct drm_device
*dev
)
178 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
181 /* Make sure blitter notifies FBC of writes */
182 gen6_gt_force_wake_get(dev_priv
);
183 blt_ecoskpd
= I915_READ(GEN6_BLITTER_ECOSKPD
);
184 blt_ecoskpd
|= GEN6_BLITTER_FBC_NOTIFY
<<
185 GEN6_BLITTER_LOCK_SHIFT
;
186 I915_WRITE(GEN6_BLITTER_ECOSKPD
, blt_ecoskpd
);
187 blt_ecoskpd
|= GEN6_BLITTER_FBC_NOTIFY
;
188 I915_WRITE(GEN6_BLITTER_ECOSKPD
, blt_ecoskpd
);
189 blt_ecoskpd
&= ~(GEN6_BLITTER_FBC_NOTIFY
<<
190 GEN6_BLITTER_LOCK_SHIFT
);
191 I915_WRITE(GEN6_BLITTER_ECOSKPD
, blt_ecoskpd
);
192 POSTING_READ(GEN6_BLITTER_ECOSKPD
);
193 gen6_gt_force_wake_put(dev_priv
);
196 static void ironlake_enable_fbc(struct drm_crtc
*crtc
, unsigned long interval
)
198 struct drm_device
*dev
= crtc
->dev
;
199 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
200 struct drm_framebuffer
*fb
= crtc
->fb
;
201 struct intel_framebuffer
*intel_fb
= to_intel_framebuffer(fb
);
202 struct drm_i915_gem_object
*obj
= intel_fb
->obj
;
203 struct intel_crtc
*intel_crtc
= to_intel_crtc(crtc
);
204 int plane
= intel_crtc
->plane
== 0 ? DPFC_CTL_PLANEA
: DPFC_CTL_PLANEB
;
205 unsigned long stall_watermark
= 200;
208 dpfc_ctl
= I915_READ(ILK_DPFC_CONTROL
);
209 dpfc_ctl
&= DPFC_RESERVED
;
210 dpfc_ctl
|= (plane
| DPFC_CTL_LIMIT_1X
);
211 /* Set persistent mode for front-buffer rendering, ala X. */
212 dpfc_ctl
|= DPFC_CTL_PERSISTENT_MODE
;
213 dpfc_ctl
|= (DPFC_CTL_FENCE_EN
| obj
->fence_reg
);
214 I915_WRITE(ILK_DPFC_CHICKEN
, DPFC_HT_MODIFY
);
216 I915_WRITE(ILK_DPFC_RECOMP_CTL
, DPFC_RECOMP_STALL_EN
|
217 (stall_watermark
<< DPFC_RECOMP_STALL_WM_SHIFT
) |
218 (interval
<< DPFC_RECOMP_TIMER_COUNT_SHIFT
));
219 I915_WRITE(ILK_DPFC_FENCE_YOFF
, crtc
->y
);
220 I915_WRITE(ILK_FBC_RT_BASE
, obj
->gtt_offset
| ILK_FBC_RT_VALID
);
222 I915_WRITE(ILK_DPFC_CONTROL
, dpfc_ctl
| DPFC_CTL_EN
);
225 I915_WRITE(SNB_DPFC_CTL_SA
,
226 SNB_CPU_FENCE_ENABLE
| obj
->fence_reg
);
227 I915_WRITE(DPFC_CPU_FENCE_OFFSET
, crtc
->y
);
228 sandybridge_blit_fbc_update(dev
);
231 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc
->plane
));
234 static void ironlake_disable_fbc(struct drm_device
*dev
)
236 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
239 /* Disable compression */
240 dpfc_ctl
= I915_READ(ILK_DPFC_CONTROL
);
241 if (dpfc_ctl
& DPFC_CTL_EN
) {
242 dpfc_ctl
&= ~DPFC_CTL_EN
;
243 I915_WRITE(ILK_DPFC_CONTROL
, dpfc_ctl
);
245 DRM_DEBUG_KMS("disabled FBC\n");
249 static bool ironlake_fbc_enabled(struct drm_device
*dev
)
251 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
253 return I915_READ(ILK_DPFC_CONTROL
) & DPFC_CTL_EN
;
256 bool intel_fbc_enabled(struct drm_device
*dev
)
258 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
260 if (!dev_priv
->display
.fbc_enabled
)
263 return dev_priv
->display
.fbc_enabled(dev
);
266 static void intel_fbc_work_fn(struct work_struct
*__work
)
268 struct intel_fbc_work
*work
=
269 container_of(to_delayed_work(__work
),
270 struct intel_fbc_work
, work
);
271 struct drm_device
*dev
= work
->crtc
->dev
;
272 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
274 mutex_lock(&dev
->struct_mutex
);
275 if (work
== dev_priv
->fbc_work
) {
276 /* Double check that we haven't switched fb without cancelling
279 if (work
->crtc
->fb
== work
->fb
) {
280 dev_priv
->display
.enable_fbc(work
->crtc
,
283 dev_priv
->cfb_plane
= to_intel_crtc(work
->crtc
)->plane
;
284 dev_priv
->cfb_fb
= work
->crtc
->fb
->base
.id
;
285 dev_priv
->cfb_y
= work
->crtc
->y
;
288 dev_priv
->fbc_work
= NULL
;
290 mutex_unlock(&dev
->struct_mutex
);
295 static void intel_cancel_fbc_work(struct drm_i915_private
*dev_priv
)
297 if (dev_priv
->fbc_work
== NULL
)
300 DRM_DEBUG_KMS("cancelling pending FBC enable\n");
302 /* Synchronisation is provided by struct_mutex and checking of
303 * dev_priv->fbc_work, so we can perform the cancellation
304 * entirely asynchronously.
306 if (cancel_delayed_work(&dev_priv
->fbc_work
->work
))
307 /* tasklet was killed before being run, clean up */
308 kfree(dev_priv
->fbc_work
);
310 /* Mark the work as no longer wanted so that if it does
311 * wake-up (because the work was already running and waiting
312 * for our mutex), it will discover that is no longer
315 dev_priv
->fbc_work
= NULL
;
318 void intel_enable_fbc(struct drm_crtc
*crtc
, unsigned long interval
)
320 struct intel_fbc_work
*work
;
321 struct drm_device
*dev
= crtc
->dev
;
322 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
324 if (!dev_priv
->display
.enable_fbc
)
327 intel_cancel_fbc_work(dev_priv
);
329 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
331 dev_priv
->display
.enable_fbc(crtc
, interval
);
337 work
->interval
= interval
;
338 INIT_DELAYED_WORK(&work
->work
, intel_fbc_work_fn
);
340 dev_priv
->fbc_work
= work
;
342 DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
344 /* Delay the actual enabling to let pageflipping cease and the
345 * display to settle before starting the compression. Note that
346 * this delay also serves a second purpose: it allows for a
347 * vblank to pass after disabling the FBC before we attempt
348 * to modify the control registers.
350 * A more complicated solution would involve tracking vblanks
351 * following the termination of the page-flipping sequence
352 * and indeed performing the enable as a co-routine and not
353 * waiting synchronously upon the vblank.
355 schedule_delayed_work(&work
->work
, msecs_to_jiffies(50));
358 void intel_disable_fbc(struct drm_device
*dev
)
360 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
362 intel_cancel_fbc_work(dev_priv
);
364 if (!dev_priv
->display
.disable_fbc
)
367 dev_priv
->display
.disable_fbc(dev
);
368 dev_priv
->cfb_plane
= -1;
372 * intel_update_fbc - enable/disable FBC as needed
373 * @dev: the drm_device
375 * Set up the framebuffer compression hardware at mode set time. We
376 * enable it if possible:
377 * - plane A only (on pre-965)
378 * - no pixel mulitply/line duplication
379 * - no alpha buffer discard
381 * - framebuffer <= 2048 in width, 1536 in height
383 * We can't assume that any compression will take place (worst case),
384 * so the compressed buffer has to be the same size as the uncompressed
385 * one. It also must reside (along with the line length buffer) in
388 * We need to enable/disable FBC on a global basis.
390 void intel_update_fbc(struct drm_device
*dev
)
392 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
393 struct drm_crtc
*crtc
= NULL
, *tmp_crtc
;
394 struct intel_crtc
*intel_crtc
;
395 struct drm_framebuffer
*fb
;
396 struct intel_framebuffer
*intel_fb
;
397 struct drm_i915_gem_object
*obj
;
403 if (!I915_HAS_FBC(dev
))
407 * If FBC is already on, we just have to verify that we can
408 * keep it that way...
409 * Need to disable if:
410 * - more than one pipe is active
411 * - changing FBC params (stride, fence, mode)
412 * - new fb is too large to fit in compressed buffer
413 * - going to an unsupported config (interlace, pixel multiply, etc.)
415 list_for_each_entry(tmp_crtc
, &dev
->mode_config
.crtc_list
, head
) {
416 if (intel_crtc_active(tmp_crtc
) &&
417 !to_intel_crtc(tmp_crtc
)->primary_disabled
) {
419 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
420 dev_priv
->no_fbc_reason
= FBC_MULTIPLE_PIPES
;
427 if (!crtc
|| crtc
->fb
== NULL
) {
428 DRM_DEBUG_KMS("no output, disabling\n");
429 dev_priv
->no_fbc_reason
= FBC_NO_OUTPUT
;
433 intel_crtc
= to_intel_crtc(crtc
);
435 intel_fb
= to_intel_framebuffer(fb
);
438 enable_fbc
= i915_enable_fbc
;
439 if (enable_fbc
< 0) {
440 DRM_DEBUG_KMS("fbc set to per-chip default\n");
442 if (INTEL_INFO(dev
)->gen
<= 6)
446 DRM_DEBUG_KMS("fbc disabled per module param\n");
447 dev_priv
->no_fbc_reason
= FBC_MODULE_PARAM
;
450 if ((crtc
->mode
.flags
& DRM_MODE_FLAG_INTERLACE
) ||
451 (crtc
->mode
.flags
& DRM_MODE_FLAG_DBLSCAN
)) {
452 DRM_DEBUG_KMS("mode incompatible with compression, "
454 dev_priv
->no_fbc_reason
= FBC_UNSUPPORTED_MODE
;
457 if ((crtc
->mode
.hdisplay
> 2048) ||
458 (crtc
->mode
.vdisplay
> 1536)) {
459 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
460 dev_priv
->no_fbc_reason
= FBC_MODE_TOO_LARGE
;
463 if ((IS_I915GM(dev
) || IS_I945GM(dev
)) && intel_crtc
->plane
!= 0) {
464 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
465 dev_priv
->no_fbc_reason
= FBC_BAD_PLANE
;
469 /* The use of a CPU fence is mandatory in order to detect writes
470 * by the CPU to the scanout and trigger updates to the FBC.
472 if (obj
->tiling_mode
!= I915_TILING_X
||
473 obj
->fence_reg
== I915_FENCE_REG_NONE
) {
474 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
475 dev_priv
->no_fbc_reason
= FBC_NOT_TILED
;
479 /* If the kernel debugger is active, always disable compression */
483 if (i915_gem_stolen_setup_compression(dev
, intel_fb
->obj
->base
.size
)) {
484 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
485 dev_priv
->no_fbc_reason
= FBC_STOLEN_TOO_SMALL
;
489 /* If the scanout has not changed, don't modify the FBC settings.
490 * Note that we make the fundamental assumption that the fb->obj
491 * cannot be unpinned (and have its GTT offset and fence revoked)
492 * without first being decoupled from the scanout and FBC disabled.
494 if (dev_priv
->cfb_plane
== intel_crtc
->plane
&&
495 dev_priv
->cfb_fb
== fb
->base
.id
&&
496 dev_priv
->cfb_y
== crtc
->y
)
499 if (intel_fbc_enabled(dev
)) {
500 /* We update FBC along two paths, after changing fb/crtc
501 * configuration (modeswitching) and after page-flipping
502 * finishes. For the latter, we know that not only did
503 * we disable the FBC at the start of the page-flip
504 * sequence, but also more than one vblank has passed.
506 * For the former case of modeswitching, it is possible
507 * to switch between two FBC valid configurations
508 * instantaneously so we do need to disable the FBC
509 * before we can modify its control registers. We also
510 * have to wait for the next vblank for that to take
511 * effect. However, since we delay enabling FBC we can
512 * assume that a vblank has passed since disabling and
513 * that we can safely alter the registers in the deferred
516 * In the scenario that we go from a valid to invalid
517 * and then back to valid FBC configuration we have
518 * no strict enforcement that a vblank occurred since
519 * disabling the FBC. However, along all current pipe
520 * disabling paths we do need to wait for a vblank at
521 * some point. And we wait before enabling FBC anyway.
523 DRM_DEBUG_KMS("disabling active FBC for update\n");
524 intel_disable_fbc(dev
);
527 intel_enable_fbc(crtc
, 500);
531 /* Multiple disables should be harmless */
532 if (intel_fbc_enabled(dev
)) {
533 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
534 intel_disable_fbc(dev
);
536 i915_gem_stolen_cleanup_compression(dev
);
539 static void i915_pineview_get_mem_freq(struct drm_device
*dev
)
541 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
544 tmp
= I915_READ(CLKCFG
);
546 switch (tmp
& CLKCFG_FSB_MASK
) {
548 dev_priv
->fsb_freq
= 533; /* 133*4 */
551 dev_priv
->fsb_freq
= 800; /* 200*4 */
554 dev_priv
->fsb_freq
= 667; /* 167*4 */
557 dev_priv
->fsb_freq
= 400; /* 100*4 */
561 switch (tmp
& CLKCFG_MEM_MASK
) {
563 dev_priv
->mem_freq
= 533;
566 dev_priv
->mem_freq
= 667;
569 dev_priv
->mem_freq
= 800;
573 /* detect pineview DDR3 setting */
574 tmp
= I915_READ(CSHRDDR3CTL
);
575 dev_priv
->is_ddr3
= (tmp
& CSHRDDR3CTL_DDR3
) ? 1 : 0;
578 static void i915_ironlake_get_mem_freq(struct drm_device
*dev
)
580 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
583 ddrpll
= I915_READ16(DDRMPLL1
);
584 csipll
= I915_READ16(CSIPLL0
);
586 switch (ddrpll
& 0xff) {
588 dev_priv
->mem_freq
= 800;
591 dev_priv
->mem_freq
= 1066;
594 dev_priv
->mem_freq
= 1333;
597 dev_priv
->mem_freq
= 1600;
600 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
602 dev_priv
->mem_freq
= 0;
606 dev_priv
->ips
.r_t
= dev_priv
->mem_freq
;
608 switch (csipll
& 0x3ff) {
610 dev_priv
->fsb_freq
= 3200;
613 dev_priv
->fsb_freq
= 3733;
616 dev_priv
->fsb_freq
= 4266;
619 dev_priv
->fsb_freq
= 4800;
622 dev_priv
->fsb_freq
= 5333;
625 dev_priv
->fsb_freq
= 5866;
628 dev_priv
->fsb_freq
= 6400;
631 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
633 dev_priv
->fsb_freq
= 0;
637 if (dev_priv
->fsb_freq
== 3200) {
638 dev_priv
->ips
.c_m
= 0;
639 } else if (dev_priv
->fsb_freq
> 3200 && dev_priv
->fsb_freq
<= 4800) {
640 dev_priv
->ips
.c_m
= 1;
642 dev_priv
->ips
.c_m
= 2;
646 static const struct cxsr_latency cxsr_latency_table
[] = {
647 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
648 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
649 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
650 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
651 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
653 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
654 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
655 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
656 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
657 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
659 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
660 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
661 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
662 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
663 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
665 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
666 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
667 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
668 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
669 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
671 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
672 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
673 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
674 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
675 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
677 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
678 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
679 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
680 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
681 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
684 static const struct cxsr_latency
*intel_get_cxsr_latency(int is_desktop
,
689 const struct cxsr_latency
*latency
;
692 if (fsb
== 0 || mem
== 0)
695 for (i
= 0; i
< ARRAY_SIZE(cxsr_latency_table
); i
++) {
696 latency
= &cxsr_latency_table
[i
];
697 if (is_desktop
== latency
->is_desktop
&&
698 is_ddr3
== latency
->is_ddr3
&&
699 fsb
== latency
->fsb_freq
&& mem
== latency
->mem_freq
)
703 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
708 static void pineview_disable_cxsr(struct drm_device
*dev
)
710 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
712 /* deactivate cxsr */
713 I915_WRITE(DSPFW3
, I915_READ(DSPFW3
) & ~PINEVIEW_SELF_REFRESH_EN
);
717 * Latency for FIFO fetches is dependent on several factors:
718 * - memory configuration (speed, channels)
720 * - current MCH state
721 * It can be fairly high in some situations, so here we assume a fairly
722 * pessimal value. It's a tradeoff between extra memory fetches (if we
723 * set this value too high, the FIFO will fetch frequently to stay full)
724 * and power consumption (set it too low to save power and we might see
725 * FIFO underruns and display "flicker").
727 * A value of 5us seems to be a good balance; safe for very low end
728 * platforms but not overly aggressive on lower latency configs.
730 static const int latency_ns
= 5000;
732 static int i9xx_get_fifo_size(struct drm_device
*dev
, int plane
)
734 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
735 uint32_t dsparb
= I915_READ(DSPARB
);
738 size
= dsparb
& 0x7f;
740 size
= ((dsparb
>> DSPARB_CSTART_SHIFT
) & 0x7f) - size
;
742 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb
,
743 plane
? "B" : "A", size
);
748 static int i85x_get_fifo_size(struct drm_device
*dev
, int plane
)
750 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
751 uint32_t dsparb
= I915_READ(DSPARB
);
754 size
= dsparb
& 0x1ff;
756 size
= ((dsparb
>> DSPARB_BEND_SHIFT
) & 0x1ff) - size
;
757 size
>>= 1; /* Convert to cachelines */
759 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb
,
760 plane
? "B" : "A", size
);
765 static int i845_get_fifo_size(struct drm_device
*dev
, int plane
)
767 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
768 uint32_t dsparb
= I915_READ(DSPARB
);
771 size
= dsparb
& 0x7f;
772 size
>>= 2; /* Convert to cachelines */
774 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb
,
781 static int i830_get_fifo_size(struct drm_device
*dev
, int plane
)
783 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
784 uint32_t dsparb
= I915_READ(DSPARB
);
787 size
= dsparb
& 0x7f;
788 size
>>= 1; /* Convert to cachelines */
790 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb
,
791 plane
? "B" : "A", size
);
796 /* Pineview has different values for various configs */
797 static const struct intel_watermark_params pineview_display_wm
= {
798 PINEVIEW_DISPLAY_FIFO
,
802 PINEVIEW_FIFO_LINE_SIZE
804 static const struct intel_watermark_params pineview_display_hplloff_wm
= {
805 PINEVIEW_DISPLAY_FIFO
,
807 PINEVIEW_DFT_HPLLOFF_WM
,
809 PINEVIEW_FIFO_LINE_SIZE
811 static const struct intel_watermark_params pineview_cursor_wm
= {
812 PINEVIEW_CURSOR_FIFO
,
813 PINEVIEW_CURSOR_MAX_WM
,
814 PINEVIEW_CURSOR_DFT_WM
,
815 PINEVIEW_CURSOR_GUARD_WM
,
816 PINEVIEW_FIFO_LINE_SIZE
,
818 static const struct intel_watermark_params pineview_cursor_hplloff_wm
= {
819 PINEVIEW_CURSOR_FIFO
,
820 PINEVIEW_CURSOR_MAX_WM
,
821 PINEVIEW_CURSOR_DFT_WM
,
822 PINEVIEW_CURSOR_GUARD_WM
,
823 PINEVIEW_FIFO_LINE_SIZE
825 static const struct intel_watermark_params g4x_wm_info
= {
832 static const struct intel_watermark_params g4x_cursor_wm_info
= {
839 static const struct intel_watermark_params valleyview_wm_info
= {
840 VALLEYVIEW_FIFO_SIZE
,
846 static const struct intel_watermark_params valleyview_cursor_wm_info
= {
848 VALLEYVIEW_CURSOR_MAX_WM
,
853 static const struct intel_watermark_params i965_cursor_wm_info
= {
860 static const struct intel_watermark_params i945_wm_info
= {
867 static const struct intel_watermark_params i915_wm_info
= {
874 static const struct intel_watermark_params i855_wm_info
= {
881 static const struct intel_watermark_params i830_wm_info
= {
889 static const struct intel_watermark_params ironlake_display_wm_info
= {
896 static const struct intel_watermark_params ironlake_cursor_wm_info
= {
903 static const struct intel_watermark_params ironlake_display_srwm_info
= {
905 ILK_DISPLAY_MAX_SRWM
,
906 ILK_DISPLAY_DFT_SRWM
,
910 static const struct intel_watermark_params ironlake_cursor_srwm_info
= {
918 static const struct intel_watermark_params sandybridge_display_wm_info
= {
925 static const struct intel_watermark_params sandybridge_cursor_wm_info
= {
932 static const struct intel_watermark_params sandybridge_display_srwm_info
= {
934 SNB_DISPLAY_MAX_SRWM
,
935 SNB_DISPLAY_DFT_SRWM
,
939 static const struct intel_watermark_params sandybridge_cursor_srwm_info
= {
949 * intel_calculate_wm - calculate watermark level
950 * @clock_in_khz: pixel clock
951 * @wm: chip FIFO params
952 * @pixel_size: display pixel size
953 * @latency_ns: memory latency for the platform
955 * Calculate the watermark level (the level at which the display plane will
956 * start fetching from memory again). Each chip has a different display
957 * FIFO size and allocation, so the caller needs to figure that out and pass
958 * in the correct intel_watermark_params structure.
960 * As the pixel clock runs, the FIFO will be drained at a rate that depends
961 * on the pixel size. When it reaches the watermark level, it'll start
962 * fetching FIFO line sized based chunks from memory until the FIFO fills
963 * past the watermark point. If the FIFO drains completely, a FIFO underrun
964 * will occur, and a display engine hang could result.
966 static unsigned long intel_calculate_wm(unsigned long clock_in_khz
,
967 const struct intel_watermark_params
*wm
,
970 unsigned long latency_ns
)
972 long entries_required
, wm_size
;
975 * Note: we need to make sure we don't overflow for various clock &
977 * clocks go from a few thousand to several hundred thousand.
978 * latency is usually a few thousand
980 entries_required
= ((clock_in_khz
/ 1000) * pixel_size
* latency_ns
) /
982 entries_required
= DIV_ROUND_UP(entries_required
, wm
->cacheline_size
);
984 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required
);
986 wm_size
= fifo_size
- (entries_required
+ wm
->guard_size
);
988 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size
);
990 /* Don't promote wm_size to unsigned... */
991 if (wm_size
> (long)wm
->max_wm
)
992 wm_size
= wm
->max_wm
;
994 wm_size
= wm
->default_wm
;
998 static struct drm_crtc
*single_enabled_crtc(struct drm_device
*dev
)
1000 struct drm_crtc
*crtc
, *enabled
= NULL
;
1002 list_for_each_entry(crtc
, &dev
->mode_config
.crtc_list
, head
) {
1003 if (intel_crtc_active(crtc
)) {
1013 static void pineview_update_wm(struct drm_device
*dev
)
1015 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1016 struct drm_crtc
*crtc
;
1017 const struct cxsr_latency
*latency
;
1021 latency
= intel_get_cxsr_latency(IS_PINEVIEW_G(dev
), dev_priv
->is_ddr3
,
1022 dev_priv
->fsb_freq
, dev_priv
->mem_freq
);
1024 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
1025 pineview_disable_cxsr(dev
);
1029 crtc
= single_enabled_crtc(dev
);
1031 int clock
= crtc
->mode
.clock
;
1032 int pixel_size
= crtc
->fb
->bits_per_pixel
/ 8;
1035 wm
= intel_calculate_wm(clock
, &pineview_display_wm
,
1036 pineview_display_wm
.fifo_size
,
1037 pixel_size
, latency
->display_sr
);
1038 reg
= I915_READ(DSPFW1
);
1039 reg
&= ~DSPFW_SR_MASK
;
1040 reg
|= wm
<< DSPFW_SR_SHIFT
;
1041 I915_WRITE(DSPFW1
, reg
);
1042 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg
);
1045 wm
= intel_calculate_wm(clock
, &pineview_cursor_wm
,
1046 pineview_display_wm
.fifo_size
,
1047 pixel_size
, latency
->cursor_sr
);
1048 reg
= I915_READ(DSPFW3
);
1049 reg
&= ~DSPFW_CURSOR_SR_MASK
;
1050 reg
|= (wm
& 0x3f) << DSPFW_CURSOR_SR_SHIFT
;
1051 I915_WRITE(DSPFW3
, reg
);
1053 /* Display HPLL off SR */
1054 wm
= intel_calculate_wm(clock
, &pineview_display_hplloff_wm
,
1055 pineview_display_hplloff_wm
.fifo_size
,
1056 pixel_size
, latency
->display_hpll_disable
);
1057 reg
= I915_READ(DSPFW3
);
1058 reg
&= ~DSPFW_HPLL_SR_MASK
;
1059 reg
|= wm
& DSPFW_HPLL_SR_MASK
;
1060 I915_WRITE(DSPFW3
, reg
);
1062 /* cursor HPLL off SR */
1063 wm
= intel_calculate_wm(clock
, &pineview_cursor_hplloff_wm
,
1064 pineview_display_hplloff_wm
.fifo_size
,
1065 pixel_size
, latency
->cursor_hpll_disable
);
1066 reg
= I915_READ(DSPFW3
);
1067 reg
&= ~DSPFW_HPLL_CURSOR_MASK
;
1068 reg
|= (wm
& 0x3f) << DSPFW_HPLL_CURSOR_SHIFT
;
1069 I915_WRITE(DSPFW3
, reg
);
1070 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg
);
1074 I915_READ(DSPFW3
) | PINEVIEW_SELF_REFRESH_EN
);
1075 DRM_DEBUG_KMS("Self-refresh is enabled\n");
1077 pineview_disable_cxsr(dev
);
1078 DRM_DEBUG_KMS("Self-refresh is disabled\n");
1082 static bool g4x_compute_wm0(struct drm_device
*dev
,
1084 const struct intel_watermark_params
*display
,
1085 int display_latency_ns
,
1086 const struct intel_watermark_params
*cursor
,
1087 int cursor_latency_ns
,
1091 struct drm_crtc
*crtc
;
1092 int htotal
, hdisplay
, clock
, pixel_size
;
1093 int line_time_us
, line_count
;
1094 int entries
, tlb_miss
;
1096 crtc
= intel_get_crtc_for_plane(dev
, plane
);
1097 if (!intel_crtc_active(crtc
)) {
1098 *cursor_wm
= cursor
->guard_size
;
1099 *plane_wm
= display
->guard_size
;
1103 htotal
= crtc
->mode
.htotal
;
1104 hdisplay
= crtc
->mode
.hdisplay
;
1105 clock
= crtc
->mode
.clock
;
1106 pixel_size
= crtc
->fb
->bits_per_pixel
/ 8;
1108 /* Use the small buffer method to calculate plane watermark */
1109 entries
= ((clock
* pixel_size
/ 1000) * display_latency_ns
) / 1000;
1110 tlb_miss
= display
->fifo_size
*display
->cacheline_size
- hdisplay
* 8;
1112 entries
+= tlb_miss
;
1113 entries
= DIV_ROUND_UP(entries
, display
->cacheline_size
);
1114 *plane_wm
= entries
+ display
->guard_size
;
1115 if (*plane_wm
> (int)display
->max_wm
)
1116 *plane_wm
= display
->max_wm
;
1118 /* Use the large buffer method to calculate cursor watermark */
1119 line_time_us
= ((htotal
* 1000) / clock
);
1120 line_count
= (cursor_latency_ns
/ line_time_us
+ 1000) / 1000;
1121 entries
= line_count
* 64 * pixel_size
;
1122 tlb_miss
= cursor
->fifo_size
*cursor
->cacheline_size
- hdisplay
* 8;
1124 entries
+= tlb_miss
;
1125 entries
= DIV_ROUND_UP(entries
, cursor
->cacheline_size
);
1126 *cursor_wm
= entries
+ cursor
->guard_size
;
1127 if (*cursor_wm
> (int)cursor
->max_wm
)
1128 *cursor_wm
= (int)cursor
->max_wm
;
1134 * Check the wm result.
1136 * If any calculated watermark values is larger than the maximum value that
1137 * can be programmed into the associated watermark register, that watermark
1140 static bool g4x_check_srwm(struct drm_device
*dev
,
1141 int display_wm
, int cursor_wm
,
1142 const struct intel_watermark_params
*display
,
1143 const struct intel_watermark_params
*cursor
)
1145 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1146 display_wm
, cursor_wm
);
1148 if (display_wm
> display
->max_wm
) {
1149 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1150 display_wm
, display
->max_wm
);
1154 if (cursor_wm
> cursor
->max_wm
) {
1155 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1156 cursor_wm
, cursor
->max_wm
);
1160 if (!(display_wm
|| cursor_wm
)) {
1161 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1168 static bool g4x_compute_srwm(struct drm_device
*dev
,
1171 const struct intel_watermark_params
*display
,
1172 const struct intel_watermark_params
*cursor
,
1173 int *display_wm
, int *cursor_wm
)
1175 struct drm_crtc
*crtc
;
1176 int hdisplay
, htotal
, pixel_size
, clock
;
1177 unsigned long line_time_us
;
1178 int line_count
, line_size
;
1183 *display_wm
= *cursor_wm
= 0;
1187 crtc
= intel_get_crtc_for_plane(dev
, plane
);
1188 hdisplay
= crtc
->mode
.hdisplay
;
1189 htotal
= crtc
->mode
.htotal
;
1190 clock
= crtc
->mode
.clock
;
1191 pixel_size
= crtc
->fb
->bits_per_pixel
/ 8;
1193 line_time_us
= (htotal
* 1000) / clock
;
1194 line_count
= (latency_ns
/ line_time_us
+ 1000) / 1000;
1195 line_size
= hdisplay
* pixel_size
;
1197 /* Use the minimum of the small and large buffer method for primary */
1198 small
= ((clock
* pixel_size
/ 1000) * latency_ns
) / 1000;
1199 large
= line_count
* line_size
;
1201 entries
= DIV_ROUND_UP(min(small
, large
), display
->cacheline_size
);
1202 *display_wm
= entries
+ display
->guard_size
;
1204 /* calculate the self-refresh watermark for display cursor */
1205 entries
= line_count
* pixel_size
* 64;
1206 entries
= DIV_ROUND_UP(entries
, cursor
->cacheline_size
);
1207 *cursor_wm
= entries
+ cursor
->guard_size
;
1209 return g4x_check_srwm(dev
,
1210 *display_wm
, *cursor_wm
,
1214 static bool vlv_compute_drain_latency(struct drm_device
*dev
,
1216 int *plane_prec_mult
,
1218 int *cursor_prec_mult
,
1221 struct drm_crtc
*crtc
;
1222 int clock
, pixel_size
;
1225 crtc
= intel_get_crtc_for_plane(dev
, plane
);
1226 if (!intel_crtc_active(crtc
))
1229 clock
= crtc
->mode
.clock
; /* VESA DOT Clock */
1230 pixel_size
= crtc
->fb
->bits_per_pixel
/ 8; /* BPP */
1232 entries
= (clock
/ 1000) * pixel_size
;
1233 *plane_prec_mult
= (entries
> 256) ?
1234 DRAIN_LATENCY_PRECISION_32
: DRAIN_LATENCY_PRECISION_16
;
1235 *plane_dl
= (64 * (*plane_prec_mult
) * 4) / ((clock
/ 1000) *
1238 entries
= (clock
/ 1000) * 4; /* BPP is always 4 for cursor */
1239 *cursor_prec_mult
= (entries
> 256) ?
1240 DRAIN_LATENCY_PRECISION_32
: DRAIN_LATENCY_PRECISION_16
;
1241 *cursor_dl
= (64 * (*cursor_prec_mult
) * 4) / ((clock
/ 1000) * 4);
1247 * Update drain latency registers of memory arbiter
1249 * Valleyview SoC has a new memory arbiter and needs drain latency registers
1250 * to be programmed. Each plane has a drain latency multiplier and a drain
1254 static void vlv_update_drain_latency(struct drm_device
*dev
)
1256 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1257 int planea_prec
, planea_dl
, planeb_prec
, planeb_dl
;
1258 int cursora_prec
, cursora_dl
, cursorb_prec
, cursorb_dl
;
1259 int plane_prec_mult
, cursor_prec_mult
; /* Precision multiplier is
1262 /* For plane A, Cursor A */
1263 if (vlv_compute_drain_latency(dev
, 0, &plane_prec_mult
, &planea_dl
,
1264 &cursor_prec_mult
, &cursora_dl
)) {
1265 cursora_prec
= (cursor_prec_mult
== DRAIN_LATENCY_PRECISION_32
) ?
1266 DDL_CURSORA_PRECISION_32
: DDL_CURSORA_PRECISION_16
;
1267 planea_prec
= (plane_prec_mult
== DRAIN_LATENCY_PRECISION_32
) ?
1268 DDL_PLANEA_PRECISION_32
: DDL_PLANEA_PRECISION_16
;
1270 I915_WRITE(VLV_DDL1
, cursora_prec
|
1271 (cursora_dl
<< DDL_CURSORA_SHIFT
) |
1272 planea_prec
| planea_dl
);
1275 /* For plane B, Cursor B */
1276 if (vlv_compute_drain_latency(dev
, 1, &plane_prec_mult
, &planeb_dl
,
1277 &cursor_prec_mult
, &cursorb_dl
)) {
1278 cursorb_prec
= (cursor_prec_mult
== DRAIN_LATENCY_PRECISION_32
) ?
1279 DDL_CURSORB_PRECISION_32
: DDL_CURSORB_PRECISION_16
;
1280 planeb_prec
= (plane_prec_mult
== DRAIN_LATENCY_PRECISION_32
) ?
1281 DDL_PLANEB_PRECISION_32
: DDL_PLANEB_PRECISION_16
;
1283 I915_WRITE(VLV_DDL2
, cursorb_prec
|
1284 (cursorb_dl
<< DDL_CURSORB_SHIFT
) |
1285 planeb_prec
| planeb_dl
);
1289 #define single_plane_enabled(mask) is_power_of_2(mask)
1291 static void valleyview_update_wm(struct drm_device
*dev
)
1293 static const int sr_latency_ns
= 12000;
1294 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1295 int planea_wm
, planeb_wm
, cursora_wm
, cursorb_wm
;
1296 int plane_sr
, cursor_sr
;
1297 int ignore_plane_sr
, ignore_cursor_sr
;
1298 unsigned int enabled
= 0;
1300 vlv_update_drain_latency(dev
);
1302 if (g4x_compute_wm0(dev
, 0,
1303 &valleyview_wm_info
, latency_ns
,
1304 &valleyview_cursor_wm_info
, latency_ns
,
1305 &planea_wm
, &cursora_wm
))
1308 if (g4x_compute_wm0(dev
, 1,
1309 &valleyview_wm_info
, latency_ns
,
1310 &valleyview_cursor_wm_info
, latency_ns
,
1311 &planeb_wm
, &cursorb_wm
))
1314 if (single_plane_enabled(enabled
) &&
1315 g4x_compute_srwm(dev
, ffs(enabled
) - 1,
1317 &valleyview_wm_info
,
1318 &valleyview_cursor_wm_info
,
1319 &plane_sr
, &ignore_cursor_sr
) &&
1320 g4x_compute_srwm(dev
, ffs(enabled
) - 1,
1322 &valleyview_wm_info
,
1323 &valleyview_cursor_wm_info
,
1324 &ignore_plane_sr
, &cursor_sr
)) {
1325 I915_WRITE(FW_BLC_SELF_VLV
, FW_CSPWRDWNEN
);
1327 I915_WRITE(FW_BLC_SELF_VLV
,
1328 I915_READ(FW_BLC_SELF_VLV
) & ~FW_CSPWRDWNEN
);
1329 plane_sr
= cursor_sr
= 0;
1332 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1333 planea_wm
, cursora_wm
,
1334 planeb_wm
, cursorb_wm
,
1335 plane_sr
, cursor_sr
);
1338 (plane_sr
<< DSPFW_SR_SHIFT
) |
1339 (cursorb_wm
<< DSPFW_CURSORB_SHIFT
) |
1340 (planeb_wm
<< DSPFW_PLANEB_SHIFT
) |
1343 (I915_READ(DSPFW2
) & ~DSPFW_CURSORA_MASK
) |
1344 (cursora_wm
<< DSPFW_CURSORA_SHIFT
));
1346 (I915_READ(DSPFW3
) & ~DSPFW_CURSOR_SR_MASK
) |
1347 (cursor_sr
<< DSPFW_CURSOR_SR_SHIFT
));
1350 static void g4x_update_wm(struct drm_device
*dev
)
1352 static const int sr_latency_ns
= 12000;
1353 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1354 int planea_wm
, planeb_wm
, cursora_wm
, cursorb_wm
;
1355 int plane_sr
, cursor_sr
;
1356 unsigned int enabled
= 0;
1358 if (g4x_compute_wm0(dev
, 0,
1359 &g4x_wm_info
, latency_ns
,
1360 &g4x_cursor_wm_info
, latency_ns
,
1361 &planea_wm
, &cursora_wm
))
1364 if (g4x_compute_wm0(dev
, 1,
1365 &g4x_wm_info
, latency_ns
,
1366 &g4x_cursor_wm_info
, latency_ns
,
1367 &planeb_wm
, &cursorb_wm
))
1370 if (single_plane_enabled(enabled
) &&
1371 g4x_compute_srwm(dev
, ffs(enabled
) - 1,
1374 &g4x_cursor_wm_info
,
1375 &plane_sr
, &cursor_sr
)) {
1376 I915_WRITE(FW_BLC_SELF
, FW_BLC_SELF_EN
);
1378 I915_WRITE(FW_BLC_SELF
,
1379 I915_READ(FW_BLC_SELF
) & ~FW_BLC_SELF_EN
);
1380 plane_sr
= cursor_sr
= 0;
1383 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1384 planea_wm
, cursora_wm
,
1385 planeb_wm
, cursorb_wm
,
1386 plane_sr
, cursor_sr
);
1389 (plane_sr
<< DSPFW_SR_SHIFT
) |
1390 (cursorb_wm
<< DSPFW_CURSORB_SHIFT
) |
1391 (planeb_wm
<< DSPFW_PLANEB_SHIFT
) |
1394 (I915_READ(DSPFW2
) & ~DSPFW_CURSORA_MASK
) |
1395 (cursora_wm
<< DSPFW_CURSORA_SHIFT
));
1396 /* HPLL off in SR has some issues on G4x... disable it */
1398 (I915_READ(DSPFW3
) & ~(DSPFW_HPLL_SR_EN
| DSPFW_CURSOR_SR_MASK
)) |
1399 (cursor_sr
<< DSPFW_CURSOR_SR_SHIFT
));
1402 static void i965_update_wm(struct drm_device
*dev
)
1404 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1405 struct drm_crtc
*crtc
;
1409 /* Calc sr entries for one plane configs */
1410 crtc
= single_enabled_crtc(dev
);
1412 /* self-refresh has much higher latency */
1413 static const int sr_latency_ns
= 12000;
1414 int clock
= crtc
->mode
.clock
;
1415 int htotal
= crtc
->mode
.htotal
;
1416 int hdisplay
= crtc
->mode
.hdisplay
;
1417 int pixel_size
= crtc
->fb
->bits_per_pixel
/ 8;
1418 unsigned long line_time_us
;
1421 line_time_us
= ((htotal
* 1000) / clock
);
1423 /* Use ns/us then divide to preserve precision */
1424 entries
= (((sr_latency_ns
/ line_time_us
) + 1000) / 1000) *
1425 pixel_size
* hdisplay
;
1426 entries
= DIV_ROUND_UP(entries
, I915_FIFO_LINE_SIZE
);
1427 srwm
= I965_FIFO_SIZE
- entries
;
1431 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1434 entries
= (((sr_latency_ns
/ line_time_us
) + 1000) / 1000) *
1436 entries
= DIV_ROUND_UP(entries
,
1437 i965_cursor_wm_info
.cacheline_size
);
1438 cursor_sr
= i965_cursor_wm_info
.fifo_size
-
1439 (entries
+ i965_cursor_wm_info
.guard_size
);
1441 if (cursor_sr
> i965_cursor_wm_info
.max_wm
)
1442 cursor_sr
= i965_cursor_wm_info
.max_wm
;
1444 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1445 "cursor %d\n", srwm
, cursor_sr
);
1447 if (IS_CRESTLINE(dev
))
1448 I915_WRITE(FW_BLC_SELF
, FW_BLC_SELF_EN
);
1450 /* Turn off self refresh if both pipes are enabled */
1451 if (IS_CRESTLINE(dev
))
1452 I915_WRITE(FW_BLC_SELF
, I915_READ(FW_BLC_SELF
)
1456 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1459 /* 965 has limitations... */
1460 I915_WRITE(DSPFW1
, (srwm
<< DSPFW_SR_SHIFT
) |
1461 (8 << 16) | (8 << 8) | (8 << 0));
1462 I915_WRITE(DSPFW2
, (8 << 8) | (8 << 0));
1463 /* update cursor SR watermark */
1464 I915_WRITE(DSPFW3
, (cursor_sr
<< DSPFW_CURSOR_SR_SHIFT
));
1467 static void i9xx_update_wm(struct drm_device
*dev
)
1469 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1470 const struct intel_watermark_params
*wm_info
;
1475 int planea_wm
, planeb_wm
;
1476 struct drm_crtc
*crtc
, *enabled
= NULL
;
1479 wm_info
= &i945_wm_info
;
1480 else if (!IS_GEN2(dev
))
1481 wm_info
= &i915_wm_info
;
1483 wm_info
= &i855_wm_info
;
1485 fifo_size
= dev_priv
->display
.get_fifo_size(dev
, 0);
1486 crtc
= intel_get_crtc_for_plane(dev
, 0);
1487 if (intel_crtc_active(crtc
)) {
1488 int cpp
= crtc
->fb
->bits_per_pixel
/ 8;
1492 planea_wm
= intel_calculate_wm(crtc
->mode
.clock
,
1493 wm_info
, fifo_size
, cpp
,
1497 planea_wm
= fifo_size
- wm_info
->guard_size
;
1499 fifo_size
= dev_priv
->display
.get_fifo_size(dev
, 1);
1500 crtc
= intel_get_crtc_for_plane(dev
, 1);
1501 if (intel_crtc_active(crtc
)) {
1502 int cpp
= crtc
->fb
->bits_per_pixel
/ 8;
1506 planeb_wm
= intel_calculate_wm(crtc
->mode
.clock
,
1507 wm_info
, fifo_size
, cpp
,
1509 if (enabled
== NULL
)
1514 planeb_wm
= fifo_size
- wm_info
->guard_size
;
1516 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm
, planeb_wm
);
1519 * Overlay gets an aggressive default since video jitter is bad.
1523 /* Play safe and disable self-refresh before adjusting watermarks. */
1524 if (IS_I945G(dev
) || IS_I945GM(dev
))
1525 I915_WRITE(FW_BLC_SELF
, FW_BLC_SELF_EN_MASK
| 0);
1526 else if (IS_I915GM(dev
))
1527 I915_WRITE(INSTPM
, I915_READ(INSTPM
) & ~INSTPM_SELF_EN
);
1529 /* Calc sr entries for one plane configs */
1530 if (HAS_FW_BLC(dev
) && enabled
) {
1531 /* self-refresh has much higher latency */
1532 static const int sr_latency_ns
= 6000;
1533 int clock
= enabled
->mode
.clock
;
1534 int htotal
= enabled
->mode
.htotal
;
1535 int hdisplay
= enabled
->mode
.hdisplay
;
1536 int pixel_size
= enabled
->fb
->bits_per_pixel
/ 8;
1537 unsigned long line_time_us
;
1540 line_time_us
= (htotal
* 1000) / clock
;
1542 /* Use ns/us then divide to preserve precision */
1543 entries
= (((sr_latency_ns
/ line_time_us
) + 1000) / 1000) *
1544 pixel_size
* hdisplay
;
1545 entries
= DIV_ROUND_UP(entries
, wm_info
->cacheline_size
);
1546 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries
);
1547 srwm
= wm_info
->fifo_size
- entries
;
1551 if (IS_I945G(dev
) || IS_I945GM(dev
))
1552 I915_WRITE(FW_BLC_SELF
,
1553 FW_BLC_SELF_FIFO_MASK
| (srwm
& 0xff));
1554 else if (IS_I915GM(dev
))
1555 I915_WRITE(FW_BLC_SELF
, srwm
& 0x3f);
1558 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1559 planea_wm
, planeb_wm
, cwm
, srwm
);
1561 fwater_lo
= ((planeb_wm
& 0x3f) << 16) | (planea_wm
& 0x3f);
1562 fwater_hi
= (cwm
& 0x1f);
1564 /* Set request length to 8 cachelines per fetch */
1565 fwater_lo
= fwater_lo
| (1 << 24) | (1 << 8);
1566 fwater_hi
= fwater_hi
| (1 << 8);
1568 I915_WRITE(FW_BLC
, fwater_lo
);
1569 I915_WRITE(FW_BLC2
, fwater_hi
);
1571 if (HAS_FW_BLC(dev
)) {
1573 if (IS_I945G(dev
) || IS_I945GM(dev
))
1574 I915_WRITE(FW_BLC_SELF
,
1575 FW_BLC_SELF_EN_MASK
| FW_BLC_SELF_EN
);
1576 else if (IS_I915GM(dev
))
1577 I915_WRITE(INSTPM
, I915_READ(INSTPM
) | INSTPM_SELF_EN
);
1578 DRM_DEBUG_KMS("memory self refresh enabled\n");
1580 DRM_DEBUG_KMS("memory self refresh disabled\n");
1584 static void i830_update_wm(struct drm_device
*dev
)
1586 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1587 struct drm_crtc
*crtc
;
1591 crtc
= single_enabled_crtc(dev
);
1595 planea_wm
= intel_calculate_wm(crtc
->mode
.clock
, &i830_wm_info
,
1596 dev_priv
->display
.get_fifo_size(dev
, 0),
1598 fwater_lo
= I915_READ(FW_BLC
) & ~0xfff;
1599 fwater_lo
|= (3<<8) | planea_wm
;
1601 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm
);
1603 I915_WRITE(FW_BLC
, fwater_lo
);
1606 #define ILK_LP0_PLANE_LATENCY 700
1607 #define ILK_LP0_CURSOR_LATENCY 1300
1610 * Check the wm result.
1612 * If any calculated watermark values is larger than the maximum value that
1613 * can be programmed into the associated watermark register, that watermark
1616 static bool ironlake_check_srwm(struct drm_device
*dev
, int level
,
1617 int fbc_wm
, int display_wm
, int cursor_wm
,
1618 const struct intel_watermark_params
*display
,
1619 const struct intel_watermark_params
*cursor
)
1621 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1623 DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
1624 " cursor %d\n", level
, display_wm
, fbc_wm
, cursor_wm
);
1626 if (fbc_wm
> SNB_FBC_MAX_SRWM
) {
1627 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
1628 fbc_wm
, SNB_FBC_MAX_SRWM
, level
);
1630 /* fbc has it's own way to disable FBC WM */
1631 I915_WRITE(DISP_ARB_CTL
,
1632 I915_READ(DISP_ARB_CTL
) | DISP_FBC_WM_DIS
);
1636 if (display_wm
> display
->max_wm
) {
1637 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
1638 display_wm
, SNB_DISPLAY_MAX_SRWM
, level
);
1642 if (cursor_wm
> cursor
->max_wm
) {
1643 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
1644 cursor_wm
, SNB_CURSOR_MAX_SRWM
, level
);
1648 if (!(fbc_wm
|| display_wm
|| cursor_wm
)) {
1649 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level
, level
);
1657 * Compute watermark values of WM[1-3],
1659 static bool ironlake_compute_srwm(struct drm_device
*dev
, int level
, int plane
,
1661 const struct intel_watermark_params
*display
,
1662 const struct intel_watermark_params
*cursor
,
1663 int *fbc_wm
, int *display_wm
, int *cursor_wm
)
1665 struct drm_crtc
*crtc
;
1666 unsigned long line_time_us
;
1667 int hdisplay
, htotal
, pixel_size
, clock
;
1668 int line_count
, line_size
;
1673 *fbc_wm
= *display_wm
= *cursor_wm
= 0;
1677 crtc
= intel_get_crtc_for_plane(dev
, plane
);
1678 hdisplay
= crtc
->mode
.hdisplay
;
1679 htotal
= crtc
->mode
.htotal
;
1680 clock
= crtc
->mode
.clock
;
1681 pixel_size
= crtc
->fb
->bits_per_pixel
/ 8;
1683 line_time_us
= (htotal
* 1000) / clock
;
1684 line_count
= (latency_ns
/ line_time_us
+ 1000) / 1000;
1685 line_size
= hdisplay
* pixel_size
;
1687 /* Use the minimum of the small and large buffer method for primary */
1688 small
= ((clock
* pixel_size
/ 1000) * latency_ns
) / 1000;
1689 large
= line_count
* line_size
;
1691 entries
= DIV_ROUND_UP(min(small
, large
), display
->cacheline_size
);
1692 *display_wm
= entries
+ display
->guard_size
;
1696 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
1698 *fbc_wm
= DIV_ROUND_UP(*display_wm
* 64, line_size
) + 2;
1700 /* calculate the self-refresh watermark for display cursor */
1701 entries
= line_count
* pixel_size
* 64;
1702 entries
= DIV_ROUND_UP(entries
, cursor
->cacheline_size
);
1703 *cursor_wm
= entries
+ cursor
->guard_size
;
1705 return ironlake_check_srwm(dev
, level
,
1706 *fbc_wm
, *display_wm
, *cursor_wm
,
1710 static void ironlake_update_wm(struct drm_device
*dev
)
1712 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1713 int fbc_wm
, plane_wm
, cursor_wm
;
1714 unsigned int enabled
;
1717 if (g4x_compute_wm0(dev
, 0,
1718 &ironlake_display_wm_info
,
1719 ILK_LP0_PLANE_LATENCY
,
1720 &ironlake_cursor_wm_info
,
1721 ILK_LP0_CURSOR_LATENCY
,
1722 &plane_wm
, &cursor_wm
)) {
1723 I915_WRITE(WM0_PIPEA_ILK
,
1724 (plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
);
1725 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1726 " plane %d, " "cursor: %d\n",
1727 plane_wm
, cursor_wm
);
1731 if (g4x_compute_wm0(dev
, 1,
1732 &ironlake_display_wm_info
,
1733 ILK_LP0_PLANE_LATENCY
,
1734 &ironlake_cursor_wm_info
,
1735 ILK_LP0_CURSOR_LATENCY
,
1736 &plane_wm
, &cursor_wm
)) {
1737 I915_WRITE(WM0_PIPEB_ILK
,
1738 (plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
);
1739 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1740 " plane %d, cursor: %d\n",
1741 plane_wm
, cursor_wm
);
1746 * Calculate and update the self-refresh watermark only when one
1747 * display plane is used.
1749 I915_WRITE(WM3_LP_ILK
, 0);
1750 I915_WRITE(WM2_LP_ILK
, 0);
1751 I915_WRITE(WM1_LP_ILK
, 0);
1753 if (!single_plane_enabled(enabled
))
1755 enabled
= ffs(enabled
) - 1;
1758 if (!ironlake_compute_srwm(dev
, 1, enabled
,
1759 ILK_READ_WM1_LATENCY() * 500,
1760 &ironlake_display_srwm_info
,
1761 &ironlake_cursor_srwm_info
,
1762 &fbc_wm
, &plane_wm
, &cursor_wm
))
1765 I915_WRITE(WM1_LP_ILK
,
1767 (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1768 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1769 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1773 if (!ironlake_compute_srwm(dev
, 2, enabled
,
1774 ILK_READ_WM2_LATENCY() * 500,
1775 &ironlake_display_srwm_info
,
1776 &ironlake_cursor_srwm_info
,
1777 &fbc_wm
, &plane_wm
, &cursor_wm
))
1780 I915_WRITE(WM2_LP_ILK
,
1782 (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1783 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1784 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1788 * WM3 is unsupported on ILK, probably because we don't have latency
1789 * data for that power state
1793 static void sandybridge_update_wm(struct drm_device
*dev
)
1795 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1796 int latency
= SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
1798 int fbc_wm
, plane_wm
, cursor_wm
;
1799 unsigned int enabled
;
1802 if (g4x_compute_wm0(dev
, 0,
1803 &sandybridge_display_wm_info
, latency
,
1804 &sandybridge_cursor_wm_info
, latency
,
1805 &plane_wm
, &cursor_wm
)) {
1806 val
= I915_READ(WM0_PIPEA_ILK
);
1807 val
&= ~(WM0_PIPE_PLANE_MASK
| WM0_PIPE_CURSOR_MASK
);
1808 I915_WRITE(WM0_PIPEA_ILK
, val
|
1809 ((plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
));
1810 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1811 " plane %d, " "cursor: %d\n",
1812 plane_wm
, cursor_wm
);
1816 if (g4x_compute_wm0(dev
, 1,
1817 &sandybridge_display_wm_info
, latency
,
1818 &sandybridge_cursor_wm_info
, latency
,
1819 &plane_wm
, &cursor_wm
)) {
1820 val
= I915_READ(WM0_PIPEB_ILK
);
1821 val
&= ~(WM0_PIPE_PLANE_MASK
| WM0_PIPE_CURSOR_MASK
);
1822 I915_WRITE(WM0_PIPEB_ILK
, val
|
1823 ((plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
));
1824 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1825 " plane %d, cursor: %d\n",
1826 plane_wm
, cursor_wm
);
1831 * Calculate and update the self-refresh watermark only when one
1832 * display plane is used.
1834 * SNB support 3 levels of watermark.
1836 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
1837 * and disabled in the descending order
1840 I915_WRITE(WM3_LP_ILK
, 0);
1841 I915_WRITE(WM2_LP_ILK
, 0);
1842 I915_WRITE(WM1_LP_ILK
, 0);
1844 if (!single_plane_enabled(enabled
) ||
1845 dev_priv
->sprite_scaling_enabled
)
1847 enabled
= ffs(enabled
) - 1;
1850 if (!ironlake_compute_srwm(dev
, 1, enabled
,
1851 SNB_READ_WM1_LATENCY() * 500,
1852 &sandybridge_display_srwm_info
,
1853 &sandybridge_cursor_srwm_info
,
1854 &fbc_wm
, &plane_wm
, &cursor_wm
))
1857 I915_WRITE(WM1_LP_ILK
,
1859 (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1860 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1861 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1865 if (!ironlake_compute_srwm(dev
, 2, enabled
,
1866 SNB_READ_WM2_LATENCY() * 500,
1867 &sandybridge_display_srwm_info
,
1868 &sandybridge_cursor_srwm_info
,
1869 &fbc_wm
, &plane_wm
, &cursor_wm
))
1872 I915_WRITE(WM2_LP_ILK
,
1874 (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1875 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1876 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1880 if (!ironlake_compute_srwm(dev
, 3, enabled
,
1881 SNB_READ_WM3_LATENCY() * 500,
1882 &sandybridge_display_srwm_info
,
1883 &sandybridge_cursor_srwm_info
,
1884 &fbc_wm
, &plane_wm
, &cursor_wm
))
1887 I915_WRITE(WM3_LP_ILK
,
1889 (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1890 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1891 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1895 static void ivybridge_update_wm(struct drm_device
*dev
)
1897 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1898 int latency
= SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
1900 int fbc_wm
, plane_wm
, cursor_wm
;
1901 int ignore_fbc_wm
, ignore_plane_wm
, ignore_cursor_wm
;
1902 unsigned int enabled
;
1905 if (g4x_compute_wm0(dev
, 0,
1906 &sandybridge_display_wm_info
, latency
,
1907 &sandybridge_cursor_wm_info
, latency
,
1908 &plane_wm
, &cursor_wm
)) {
1909 val
= I915_READ(WM0_PIPEA_ILK
);
1910 val
&= ~(WM0_PIPE_PLANE_MASK
| WM0_PIPE_CURSOR_MASK
);
1911 I915_WRITE(WM0_PIPEA_ILK
, val
|
1912 ((plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
));
1913 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1914 " plane %d, " "cursor: %d\n",
1915 plane_wm
, cursor_wm
);
1919 if (g4x_compute_wm0(dev
, 1,
1920 &sandybridge_display_wm_info
, latency
,
1921 &sandybridge_cursor_wm_info
, latency
,
1922 &plane_wm
, &cursor_wm
)) {
1923 val
= I915_READ(WM0_PIPEB_ILK
);
1924 val
&= ~(WM0_PIPE_PLANE_MASK
| WM0_PIPE_CURSOR_MASK
);
1925 I915_WRITE(WM0_PIPEB_ILK
, val
|
1926 ((plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
));
1927 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1928 " plane %d, cursor: %d\n",
1929 plane_wm
, cursor_wm
);
1933 if (g4x_compute_wm0(dev
, 2,
1934 &sandybridge_display_wm_info
, latency
,
1935 &sandybridge_cursor_wm_info
, latency
,
1936 &plane_wm
, &cursor_wm
)) {
1937 val
= I915_READ(WM0_PIPEC_IVB
);
1938 val
&= ~(WM0_PIPE_PLANE_MASK
| WM0_PIPE_CURSOR_MASK
);
1939 I915_WRITE(WM0_PIPEC_IVB
, val
|
1940 ((plane_wm
<< WM0_PIPE_PLANE_SHIFT
) | cursor_wm
));
1941 DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
1942 " plane %d, cursor: %d\n",
1943 plane_wm
, cursor_wm
);
1948 * Calculate and update the self-refresh watermark only when one
1949 * display plane is used.
1951 * SNB support 3 levels of watermark.
1953 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
1954 * and disabled in the descending order
1957 I915_WRITE(WM3_LP_ILK
, 0);
1958 I915_WRITE(WM2_LP_ILK
, 0);
1959 I915_WRITE(WM1_LP_ILK
, 0);
1961 if (!single_plane_enabled(enabled
) ||
1962 dev_priv
->sprite_scaling_enabled
)
1964 enabled
= ffs(enabled
) - 1;
1967 if (!ironlake_compute_srwm(dev
, 1, enabled
,
1968 SNB_READ_WM1_LATENCY() * 500,
1969 &sandybridge_display_srwm_info
,
1970 &sandybridge_cursor_srwm_info
,
1971 &fbc_wm
, &plane_wm
, &cursor_wm
))
1974 I915_WRITE(WM1_LP_ILK
,
1976 (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1977 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1978 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1982 if (!ironlake_compute_srwm(dev
, 2, enabled
,
1983 SNB_READ_WM2_LATENCY() * 500,
1984 &sandybridge_display_srwm_info
,
1985 &sandybridge_cursor_srwm_info
,
1986 &fbc_wm
, &plane_wm
, &cursor_wm
))
1989 I915_WRITE(WM2_LP_ILK
,
1991 (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
1992 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
1993 (plane_wm
<< WM1_LP_SR_SHIFT
) |
1996 /* WM3, note we have to correct the cursor latency */
1997 if (!ironlake_compute_srwm(dev
, 3, enabled
,
1998 SNB_READ_WM3_LATENCY() * 500,
1999 &sandybridge_display_srwm_info
,
2000 &sandybridge_cursor_srwm_info
,
2001 &fbc_wm
, &plane_wm
, &ignore_cursor_wm
) ||
2002 !ironlake_compute_srwm(dev
, 3, enabled
,
2003 2 * SNB_READ_WM3_LATENCY() * 500,
2004 &sandybridge_display_srwm_info
,
2005 &sandybridge_cursor_srwm_info
,
2006 &ignore_fbc_wm
, &ignore_plane_wm
, &cursor_wm
))
2009 I915_WRITE(WM3_LP_ILK
,
2011 (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT
) |
2012 (fbc_wm
<< WM1_LP_FBC_SHIFT
) |
2013 (plane_wm
<< WM1_LP_SR_SHIFT
) |
2018 haswell_update_linetime_wm(struct drm_device
*dev
, int pipe
,
2019 struct drm_display_mode
*mode
)
2021 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2024 temp
= I915_READ(PIPE_WM_LINETIME(pipe
));
2025 temp
&= ~PIPE_WM_LINETIME_MASK
;
2027 /* The WM are computed with base on how long it takes to fill a single
2028 * row at the given clock rate, multiplied by 8.
2030 temp
|= PIPE_WM_LINETIME_TIME(
2031 ((mode
->crtc_hdisplay
* 1000) / mode
->clock
) * 8);
2033 /* IPS watermarks are only used by pipe A, and are ignored by
2034 * pipes B and C. They are calculated similarly to the common
2035 * linetime values, except that we are using CD clock frequency
2036 * in MHz instead of pixel rate for the division.
2038 * This is a placeholder for the IPS watermark calculation code.
2041 I915_WRITE(PIPE_WM_LINETIME(pipe
), temp
);
2045 sandybridge_compute_sprite_wm(struct drm_device
*dev
, int plane
,
2046 uint32_t sprite_width
, int pixel_size
,
2047 const struct intel_watermark_params
*display
,
2048 int display_latency_ns
, int *sprite_wm
)
2050 struct drm_crtc
*crtc
;
2052 int entries
, tlb_miss
;
2054 crtc
= intel_get_crtc_for_plane(dev
, plane
);
2055 if (!intel_crtc_active(crtc
)) {
2056 *sprite_wm
= display
->guard_size
;
2060 clock
= crtc
->mode
.clock
;
2062 /* Use the small buffer method to calculate the sprite watermark */
2063 entries
= ((clock
* pixel_size
/ 1000) * display_latency_ns
) / 1000;
2064 tlb_miss
= display
->fifo_size
*display
->cacheline_size
-
2067 entries
+= tlb_miss
;
2068 entries
= DIV_ROUND_UP(entries
, display
->cacheline_size
);
2069 *sprite_wm
= entries
+ display
->guard_size
;
2070 if (*sprite_wm
> (int)display
->max_wm
)
2071 *sprite_wm
= display
->max_wm
;
2077 sandybridge_compute_sprite_srwm(struct drm_device
*dev
, int plane
,
2078 uint32_t sprite_width
, int pixel_size
,
2079 const struct intel_watermark_params
*display
,
2080 int latency_ns
, int *sprite_wm
)
2082 struct drm_crtc
*crtc
;
2083 unsigned long line_time_us
;
2085 int line_count
, line_size
;
2094 crtc
= intel_get_crtc_for_plane(dev
, plane
);
2095 clock
= crtc
->mode
.clock
;
2101 line_time_us
= (sprite_width
* 1000) / clock
;
2102 if (!line_time_us
) {
2107 line_count
= (latency_ns
/ line_time_us
+ 1000) / 1000;
2108 line_size
= sprite_width
* pixel_size
;
2110 /* Use the minimum of the small and large buffer method for primary */
2111 small
= ((clock
* pixel_size
/ 1000) * latency_ns
) / 1000;
2112 large
= line_count
* line_size
;
2114 entries
= DIV_ROUND_UP(min(small
, large
), display
->cacheline_size
);
2115 *sprite_wm
= entries
+ display
->guard_size
;
2117 return *sprite_wm
> 0x3ff ? false : true;
2120 static void sandybridge_update_sprite_wm(struct drm_device
*dev
, int pipe
,
2121 uint32_t sprite_width
, int pixel_size
)
2123 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2124 int latency
= SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
2131 reg
= WM0_PIPEA_ILK
;
2134 reg
= WM0_PIPEB_ILK
;
2137 reg
= WM0_PIPEC_IVB
;
2140 return; /* bad pipe */
2143 ret
= sandybridge_compute_sprite_wm(dev
, pipe
, sprite_width
, pixel_size
,
2144 &sandybridge_display_wm_info
,
2145 latency
, &sprite_wm
);
2147 DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
2152 val
= I915_READ(reg
);
2153 val
&= ~WM0_PIPE_SPRITE_MASK
;
2154 I915_WRITE(reg
, val
| (sprite_wm
<< WM0_PIPE_SPRITE_SHIFT
));
2155 DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe
), sprite_wm
);
2158 ret
= sandybridge_compute_sprite_srwm(dev
, pipe
, sprite_width
,
2160 &sandybridge_display_srwm_info
,
2161 SNB_READ_WM1_LATENCY() * 500,
2164 DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
2168 I915_WRITE(WM1S_LP_ILK
, sprite_wm
);
2170 /* Only IVB has two more LP watermarks for sprite */
2171 if (!IS_IVYBRIDGE(dev
))
2174 ret
= sandybridge_compute_sprite_srwm(dev
, pipe
, sprite_width
,
2176 &sandybridge_display_srwm_info
,
2177 SNB_READ_WM2_LATENCY() * 500,
2180 DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
2184 I915_WRITE(WM2S_LP_IVB
, sprite_wm
);
2186 ret
= sandybridge_compute_sprite_srwm(dev
, pipe
, sprite_width
,
2188 &sandybridge_display_srwm_info
,
2189 SNB_READ_WM3_LATENCY() * 500,
2192 DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
2196 I915_WRITE(WM3S_LP_IVB
, sprite_wm
);
2200 * intel_update_watermarks - update FIFO watermark values based on current modes
2202 * Calculate watermark values for the various WM regs based on current mode
2203 * and plane configuration.
2205 * There are several cases to deal with here:
2206 * - normal (i.e. non-self-refresh)
2207 * - self-refresh (SR) mode
2208 * - lines are large relative to FIFO size (buffer can hold up to 2)
2209 * - lines are small relative to FIFO size (buffer can hold more than 2
2210 * lines), so need to account for TLB latency
2212 * The normal calculation is:
2213 * watermark = dotclock * bytes per pixel * latency
2214 * where latency is platform & configuration dependent (we assume pessimal
2217 * The SR calculation is:
2218 * watermark = (trunc(latency/line time)+1) * surface width *
2221 * line time = htotal / dotclock
2222 * surface width = hdisplay for normal plane and 64 for cursor
2223 * and latency is assumed to be high, as above.
2225 * The final value programmed to the register should always be rounded up,
2226 * and include an extra 2 entries to account for clock crossings.
2228 * We don't use the sprite, so we can ignore that. And on Crestline we have
2229 * to set the non-SR watermarks to 8.
2231 void intel_update_watermarks(struct drm_device
*dev
)
2233 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2235 if (dev_priv
->display
.update_wm
)
2236 dev_priv
->display
.update_wm(dev
);
2239 void intel_update_linetime_watermarks(struct drm_device
*dev
,
2240 int pipe
, struct drm_display_mode
*mode
)
2242 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2244 if (dev_priv
->display
.update_linetime_wm
)
2245 dev_priv
->display
.update_linetime_wm(dev
, pipe
, mode
);
2248 void intel_update_sprite_watermarks(struct drm_device
*dev
, int pipe
,
2249 uint32_t sprite_width
, int pixel_size
)
2251 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2253 if (dev_priv
->display
.update_sprite_wm
)
2254 dev_priv
->display
.update_sprite_wm(dev
, pipe
, sprite_width
,
2258 static struct drm_i915_gem_object
*
2259 intel_alloc_context_page(struct drm_device
*dev
)
2261 struct drm_i915_gem_object
*ctx
;
2264 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
2266 ctx
= i915_gem_alloc_object(dev
, 4096);
2268 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
2272 ret
= i915_gem_object_pin(ctx
, 4096, true, false);
2274 DRM_ERROR("failed to pin power context: %d\n", ret
);
2278 ret
= i915_gem_object_set_to_gtt_domain(ctx
, 1);
2280 DRM_ERROR("failed to set-domain on power context: %d\n", ret
);
2287 i915_gem_object_unpin(ctx
);
2289 drm_gem_object_unreference(&ctx
->base
);
2294 * Lock protecting IPS related data structures
2296 DEFINE_SPINLOCK(mchdev_lock
);
2298 /* Global for IPS driver to get at the current i915 device. Protected by
2300 static struct drm_i915_private
*i915_mch_dev
;
2302 bool ironlake_set_drps(struct drm_device
*dev
, u8 val
)
2304 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2307 assert_spin_locked(&mchdev_lock
);
2309 rgvswctl
= I915_READ16(MEMSWCTL
);
2310 if (rgvswctl
& MEMCTL_CMD_STS
) {
2311 DRM_DEBUG("gpu busy, RCS change rejected\n");
2312 return false; /* still busy with another command */
2315 rgvswctl
= (MEMCTL_CMD_CHFREQ
<< MEMCTL_CMD_SHIFT
) |
2316 (val
<< MEMCTL_FREQ_SHIFT
) | MEMCTL_SFCAVM
;
2317 I915_WRITE16(MEMSWCTL
, rgvswctl
);
2318 POSTING_READ16(MEMSWCTL
);
2320 rgvswctl
|= MEMCTL_CMD_STS
;
2321 I915_WRITE16(MEMSWCTL
, rgvswctl
);
2326 static void ironlake_enable_drps(struct drm_device
*dev
)
2328 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2329 u32 rgvmodectl
= I915_READ(MEMMODECTL
);
2330 u8 fmax
, fmin
, fstart
, vstart
;
2332 spin_lock_irq(&mchdev_lock
);
2334 /* Enable temp reporting */
2335 I915_WRITE16(PMMISC
, I915_READ(PMMISC
) | MCPPCE_EN
);
2336 I915_WRITE16(TSC1
, I915_READ(TSC1
) | TSE
);
2338 /* 100ms RC evaluation intervals */
2339 I915_WRITE(RCUPEI
, 100000);
2340 I915_WRITE(RCDNEI
, 100000);
2342 /* Set max/min thresholds to 90ms and 80ms respectively */
2343 I915_WRITE(RCBMAXAVG
, 90000);
2344 I915_WRITE(RCBMINAVG
, 80000);
2346 I915_WRITE(MEMIHYST
, 1);
2348 /* Set up min, max, and cur for interrupt handling */
2349 fmax
= (rgvmodectl
& MEMMODE_FMAX_MASK
) >> MEMMODE_FMAX_SHIFT
;
2350 fmin
= (rgvmodectl
& MEMMODE_FMIN_MASK
);
2351 fstart
= (rgvmodectl
& MEMMODE_FSTART_MASK
) >>
2352 MEMMODE_FSTART_SHIFT
;
2354 vstart
= (I915_READ(PXVFREQ_BASE
+ (fstart
* 4)) & PXVFREQ_PX_MASK
) >>
2357 dev_priv
->ips
.fmax
= fmax
; /* IPS callback will increase this */
2358 dev_priv
->ips
.fstart
= fstart
;
2360 dev_priv
->ips
.max_delay
= fstart
;
2361 dev_priv
->ips
.min_delay
= fmin
;
2362 dev_priv
->ips
.cur_delay
= fstart
;
2364 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
2365 fmax
, fmin
, fstart
);
2367 I915_WRITE(MEMINTREN
, MEMINT_CX_SUPR_EN
| MEMINT_EVAL_CHG_EN
);
2370 * Interrupts will be enabled in ironlake_irq_postinstall
2373 I915_WRITE(VIDSTART
, vstart
);
2374 POSTING_READ(VIDSTART
);
2376 rgvmodectl
|= MEMMODE_SWMODE_EN
;
2377 I915_WRITE(MEMMODECTL
, rgvmodectl
);
2379 if (wait_for_atomic((I915_READ(MEMSWCTL
) & MEMCTL_CMD_STS
) == 0, 10))
2380 DRM_ERROR("stuck trying to change perf mode\n");
2383 ironlake_set_drps(dev
, fstart
);
2385 dev_priv
->ips
.last_count1
= I915_READ(0x112e4) + I915_READ(0x112e8) +
2387 dev_priv
->ips
.last_time1
= jiffies_to_msecs(jiffies
);
2388 dev_priv
->ips
.last_count2
= I915_READ(0x112f4);
2389 getrawmonotonic(&dev_priv
->ips
.last_time2
);
2391 spin_unlock_irq(&mchdev_lock
);
2394 static void ironlake_disable_drps(struct drm_device
*dev
)
2396 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2399 spin_lock_irq(&mchdev_lock
);
2401 rgvswctl
= I915_READ16(MEMSWCTL
);
2403 /* Ack interrupts, disable EFC interrupt */
2404 I915_WRITE(MEMINTREN
, I915_READ(MEMINTREN
) & ~MEMINT_EVAL_CHG_EN
);
2405 I915_WRITE(MEMINTRSTS
, MEMINT_EVAL_CHG
);
2406 I915_WRITE(DEIER
, I915_READ(DEIER
) & ~DE_PCU_EVENT
);
2407 I915_WRITE(DEIIR
, DE_PCU_EVENT
);
2408 I915_WRITE(DEIMR
, I915_READ(DEIMR
) | DE_PCU_EVENT
);
2410 /* Go back to the starting frequency */
2411 ironlake_set_drps(dev
, dev_priv
->ips
.fstart
);
2413 rgvswctl
|= MEMCTL_CMD_STS
;
2414 I915_WRITE(MEMSWCTL
, rgvswctl
);
2417 spin_unlock_irq(&mchdev_lock
);
2420 /* There's a funny hw issue where the hw returns all 0 when reading from
2421 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
2422 * ourselves, instead of doing a rmw cycle (which might result in us clearing
2423 * all limits and the gpu stuck at whatever frequency it is at atm).
2425 static u32
gen6_rps_limits(struct drm_i915_private
*dev_priv
, u8
*val
)
2431 if (*val
>= dev_priv
->rps
.max_delay
)
2432 *val
= dev_priv
->rps
.max_delay
;
2433 limits
|= dev_priv
->rps
.max_delay
<< 24;
2435 /* Only set the down limit when we've reached the lowest level to avoid
2436 * getting more interrupts, otherwise leave this clear. This prevents a
2437 * race in the hw when coming out of rc6: There's a tiny window where
2438 * the hw runs at the minimal clock before selecting the desired
2439 * frequency, if the down threshold expires in that window we will not
2440 * receive a down interrupt. */
2441 if (*val
<= dev_priv
->rps
.min_delay
) {
2442 *val
= dev_priv
->rps
.min_delay
;
2443 limits
|= dev_priv
->rps
.min_delay
<< 16;
2449 void gen6_set_rps(struct drm_device
*dev
, u8 val
)
2451 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2452 u32 limits
= gen6_rps_limits(dev_priv
, &val
);
2454 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
2455 WARN_ON(val
> dev_priv
->rps
.max_delay
);
2456 WARN_ON(val
< dev_priv
->rps
.min_delay
);
2458 if (val
== dev_priv
->rps
.cur_delay
)
2461 if (IS_HASWELL(dev
))
2462 I915_WRITE(GEN6_RPNSWREQ
,
2463 HSW_FREQUENCY(val
));
2465 I915_WRITE(GEN6_RPNSWREQ
,
2466 GEN6_FREQUENCY(val
) |
2468 GEN6_AGGRESSIVE_TURBO
);
2470 /* Make sure we continue to get interrupts
2471 * until we hit the minimum or maximum frequencies.
2473 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS
, limits
);
2475 POSTING_READ(GEN6_RPNSWREQ
);
2477 dev_priv
->rps
.cur_delay
= val
;
2479 trace_intel_gpu_freq_change(val
* 50);
2482 void valleyview_set_rps(struct drm_device
*dev
, u8 val
)
2484 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2485 unsigned long timeout
= jiffies
+ msecs_to_jiffies(10);
2486 u32 limits
= gen6_rps_limits(dev_priv
, &val
);
2489 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
2490 WARN_ON(val
> dev_priv
->rps
.max_delay
);
2491 WARN_ON(val
< dev_priv
->rps
.min_delay
);
2493 DRM_DEBUG_DRIVER("gpu freq request from %d to %d\n",
2494 vlv_gpu_freq(dev_priv
->mem_freq
,
2495 dev_priv
->rps
.cur_delay
),
2496 vlv_gpu_freq(dev_priv
->mem_freq
, val
));
2498 if (val
== dev_priv
->rps
.cur_delay
)
2501 valleyview_punit_write(dev_priv
, PUNIT_REG_GPU_FREQ_REQ
, val
);
2504 valleyview_punit_read(dev_priv
, PUNIT_REG_GPU_FREQ_STS
, &pval
);
2505 if (time_after(jiffies
, timeout
)) {
2506 DRM_DEBUG_DRIVER("timed out waiting for Punit\n");
2512 valleyview_punit_read(dev_priv
, PUNIT_REG_GPU_FREQ_STS
, &pval
);
2513 if ((pval
>> 8) != val
)
2514 DRM_DEBUG_DRIVER("punit overrode freq: %d requested, but got %d\n",
2517 /* Make sure we continue to get interrupts
2518 * until we hit the minimum or maximum frequencies.
2520 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS
, limits
);
2522 dev_priv
->rps
.cur_delay
= pval
>> 8;
2524 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv
->mem_freq
, val
));
2528 static void gen6_disable_rps(struct drm_device
*dev
)
2530 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2532 I915_WRITE(GEN6_RC_CONTROL
, 0);
2533 I915_WRITE(GEN6_RPNSWREQ
, 1 << 31);
2534 I915_WRITE(GEN6_PMINTRMSK
, 0xffffffff);
2535 I915_WRITE(GEN6_PMIER
, 0);
2536 /* Complete PM interrupt masking here doesn't race with the rps work
2537 * item again unmasking PM interrupts because that is using a different
2538 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
2539 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
2541 spin_lock_irq(&dev_priv
->rps
.lock
);
2542 dev_priv
->rps
.pm_iir
= 0;
2543 spin_unlock_irq(&dev_priv
->rps
.lock
);
2545 I915_WRITE(GEN6_PMIIR
, I915_READ(GEN6_PMIIR
));
2548 static void valleyview_disable_rps(struct drm_device
*dev
)
2550 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2552 I915_WRITE(GEN6_RC_CONTROL
, 0);
2553 I915_WRITE(GEN6_PMINTRMSK
, 0xffffffff);
2554 I915_WRITE(GEN6_PMIER
, 0);
2555 /* Complete PM interrupt masking here doesn't race with the rps work
2556 * item again unmasking PM interrupts because that is using a different
2557 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
2558 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
2560 spin_lock_irq(&dev_priv
->rps
.lock
);
2561 dev_priv
->rps
.pm_iir
= 0;
2562 spin_unlock_irq(&dev_priv
->rps
.lock
);
2564 I915_WRITE(GEN6_PMIIR
, I915_READ(GEN6_PMIIR
));
2567 int intel_enable_rc6(const struct drm_device
*dev
)
2569 /* Respect the kernel parameter if it is set */
2570 if (i915_enable_rc6
>= 0)
2571 return i915_enable_rc6
;
2573 /* Disable RC6 on Ironlake */
2574 if (INTEL_INFO(dev
)->gen
== 5)
2577 if (IS_HASWELL(dev
)) {
2578 DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
2579 return INTEL_RC6_ENABLE
;
2582 /* snb/ivb have more than one rc6 state. */
2583 if (INTEL_INFO(dev
)->gen
== 6) {
2584 DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
2585 return INTEL_RC6_ENABLE
;
2588 DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
2589 return (INTEL_RC6_ENABLE
| INTEL_RC6p_ENABLE
);
2592 static void gen6_enable_rps(struct drm_device
*dev
)
2594 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2595 struct intel_ring_buffer
*ring
;
2598 u32 rc6vids
, pcu_mbox
, rc6_mask
= 0;
2603 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
2605 /* Here begins a magic sequence of register writes to enable
2606 * auto-downclocking.
2608 * Perhaps there might be some value in exposing these to
2611 I915_WRITE(GEN6_RC_STATE
, 0);
2613 /* Clear the DBG now so we don't confuse earlier errors */
2614 if ((gtfifodbg
= I915_READ(GTFIFODBG
))) {
2615 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg
);
2616 I915_WRITE(GTFIFODBG
, gtfifodbg
);
2619 gen6_gt_force_wake_get(dev_priv
);
2621 rp_state_cap
= I915_READ(GEN6_RP_STATE_CAP
);
2622 gt_perf_status
= I915_READ(GEN6_GT_PERF_STATUS
);
2624 /* In units of 50MHz */
2625 dev_priv
->rps
.hw_max
= dev_priv
->rps
.max_delay
= rp_state_cap
& 0xff;
2626 dev_priv
->rps
.min_delay
= (rp_state_cap
& 0xff0000) >> 16;
2627 dev_priv
->rps
.cur_delay
= 0;
2629 /* disable the counters and set deterministic thresholds */
2630 I915_WRITE(GEN6_RC_CONTROL
, 0);
2632 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT
, 1000 << 16);
2633 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT
, 40 << 16 | 30);
2634 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT
, 30);
2635 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL
, 125000);
2636 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS
, 25);
2638 for_each_ring(ring
, dev_priv
, i
)
2639 I915_WRITE(RING_MAX_IDLE(ring
->mmio_base
), 10);
2641 I915_WRITE(GEN6_RC_SLEEP
, 0);
2642 I915_WRITE(GEN6_RC1e_THRESHOLD
, 1000);
2643 I915_WRITE(GEN6_RC6_THRESHOLD
, 50000);
2644 I915_WRITE(GEN6_RC6p_THRESHOLD
, 150000);
2645 I915_WRITE(GEN6_RC6pp_THRESHOLD
, 64000); /* unused */
2647 /* Check if we are enabling RC6 */
2648 rc6_mode
= intel_enable_rc6(dev_priv
->dev
);
2649 if (rc6_mode
& INTEL_RC6_ENABLE
)
2650 rc6_mask
|= GEN6_RC_CTL_RC6_ENABLE
;
2652 /* We don't use those on Haswell */
2653 if (!IS_HASWELL(dev
)) {
2654 if (rc6_mode
& INTEL_RC6p_ENABLE
)
2655 rc6_mask
|= GEN6_RC_CTL_RC6p_ENABLE
;
2657 if (rc6_mode
& INTEL_RC6pp_ENABLE
)
2658 rc6_mask
|= GEN6_RC_CTL_RC6pp_ENABLE
;
2661 DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
2662 (rc6_mask
& GEN6_RC_CTL_RC6_ENABLE
) ? "on" : "off",
2663 (rc6_mask
& GEN6_RC_CTL_RC6p_ENABLE
) ? "on" : "off",
2664 (rc6_mask
& GEN6_RC_CTL_RC6pp_ENABLE
) ? "on" : "off");
2666 I915_WRITE(GEN6_RC_CONTROL
,
2668 GEN6_RC_CTL_EI_MODE(1) |
2669 GEN6_RC_CTL_HW_ENABLE
);
2671 if (IS_HASWELL(dev
)) {
2672 I915_WRITE(GEN6_RPNSWREQ
,
2674 I915_WRITE(GEN6_RC_VIDEO_FREQ
,
2677 I915_WRITE(GEN6_RPNSWREQ
,
2678 GEN6_FREQUENCY(10) |
2680 GEN6_AGGRESSIVE_TURBO
);
2681 I915_WRITE(GEN6_RC_VIDEO_FREQ
,
2682 GEN6_FREQUENCY(12));
2685 I915_WRITE(GEN6_RP_DOWN_TIMEOUT
, 1000000);
2686 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS
,
2687 dev_priv
->rps
.max_delay
<< 24 |
2688 dev_priv
->rps
.min_delay
<< 16);
2690 I915_WRITE(GEN6_RP_UP_THRESHOLD
, 59400);
2691 I915_WRITE(GEN6_RP_DOWN_THRESHOLD
, 245000);
2692 I915_WRITE(GEN6_RP_UP_EI
, 66000);
2693 I915_WRITE(GEN6_RP_DOWN_EI
, 350000);
2695 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS
, 10);
2696 I915_WRITE(GEN6_RP_CONTROL
,
2697 GEN6_RP_MEDIA_TURBO
|
2698 GEN6_RP_MEDIA_HW_NORMAL_MODE
|
2699 GEN6_RP_MEDIA_IS_GFX
|
2701 GEN6_RP_UP_BUSY_AVG
|
2702 (IS_HASWELL(dev
) ? GEN7_RP_DOWN_IDLE_AVG
: GEN6_RP_DOWN_IDLE_CONT
));
2704 ret
= sandybridge_pcode_write(dev_priv
, GEN6_PCODE_WRITE_MIN_FREQ_TABLE
, 0);
2705 if (!ret
&& (IS_GEN6(dev
) || IS_IVYBRIDGE(dev
))) {
2707 ret
= sandybridge_pcode_read(dev_priv
, GEN6_READ_OC_PARAMS
, &pcu_mbox
);
2708 if (!ret
&& (pcu_mbox
& (1<<31))) { /* OC supported */
2709 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
2710 (dev_priv
->rps
.max_delay
& 0xff) * 50,
2711 (pcu_mbox
& 0xff) * 50);
2712 dev_priv
->rps
.hw_max
= pcu_mbox
& 0xff;
2715 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
2718 gen6_set_rps(dev_priv
->dev
, (gt_perf_status
& 0xff00) >> 8);
2720 /* requires MSI enabled */
2721 I915_WRITE(GEN6_PMIER
, GEN6_PM_DEFERRED_EVENTS
);
2722 spin_lock_irq(&dev_priv
->rps
.lock
);
2723 WARN_ON(dev_priv
->rps
.pm_iir
!= 0);
2724 I915_WRITE(GEN6_PMIMR
, 0);
2725 spin_unlock_irq(&dev_priv
->rps
.lock
);
2726 /* enable all PM interrupts */
2727 I915_WRITE(GEN6_PMINTRMSK
, 0);
2730 ret
= sandybridge_pcode_read(dev_priv
, GEN6_PCODE_READ_RC6VIDS
, &rc6vids
);
2731 if (IS_GEN6(dev
) && ret
) {
2732 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
2733 } else if (IS_GEN6(dev
) && (GEN6_DECODE_RC6_VID(rc6vids
& 0xff) < 450)) {
2734 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
2735 GEN6_DECODE_RC6_VID(rc6vids
& 0xff), 450);
2736 rc6vids
&= 0xffff00;
2737 rc6vids
|= GEN6_ENCODE_RC6_VID(450);
2738 ret
= sandybridge_pcode_write(dev_priv
, GEN6_PCODE_WRITE_RC6VIDS
, rc6vids
);
2740 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
2743 gen6_gt_force_wake_put(dev_priv
);
2746 static void gen6_update_ring_freq(struct drm_device
*dev
)
2748 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2750 unsigned int gpu_freq
;
2751 unsigned int max_ia_freq
, min_ring_freq
;
2752 int scaling_factor
= 180;
2754 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
2756 max_ia_freq
= cpufreq_quick_get_max(0);
2758 * Default to measured freq if none found, PCU will ensure we don't go
2762 max_ia_freq
= tsc_khz
;
2764 /* Convert from kHz to MHz */
2765 max_ia_freq
/= 1000;
2767 min_ring_freq
= I915_READ(MCHBAR_MIRROR_BASE_SNB
+ DCLK
);
2768 /* convert DDR frequency from units of 133.3MHz to bandwidth */
2769 min_ring_freq
= (2 * 4 * min_ring_freq
+ 2) / 3;
2772 * For each potential GPU frequency, load a ring frequency we'd like
2773 * to use for memory access. We do this by specifying the IA frequency
2774 * the PCU should use as a reference to determine the ring frequency.
2776 for (gpu_freq
= dev_priv
->rps
.max_delay
; gpu_freq
>= dev_priv
->rps
.min_delay
;
2778 int diff
= dev_priv
->rps
.max_delay
- gpu_freq
;
2779 unsigned int ia_freq
= 0, ring_freq
= 0;
2781 if (IS_HASWELL(dev
)) {
2782 ring_freq
= (gpu_freq
* 5 + 3) / 4;
2783 ring_freq
= max(min_ring_freq
, ring_freq
);
2784 /* leave ia_freq as the default, chosen by cpufreq */
2786 /* On older processors, there is no separate ring
2787 * clock domain, so in order to boost the bandwidth
2788 * of the ring, we need to upclock the CPU (ia_freq).
2790 * For GPU frequencies less than 750MHz,
2791 * just use the lowest ring freq.
2793 if (gpu_freq
< min_freq
)
2796 ia_freq
= max_ia_freq
- ((diff
* scaling_factor
) / 2);
2797 ia_freq
= DIV_ROUND_CLOSEST(ia_freq
, 100);
2800 sandybridge_pcode_write(dev_priv
,
2801 GEN6_PCODE_WRITE_MIN_FREQ_TABLE
,
2802 ia_freq
<< GEN6_PCODE_FREQ_IA_RATIO_SHIFT
|
2803 ring_freq
<< GEN6_PCODE_FREQ_RING_RATIO_SHIFT
|
2808 int valleyview_rps_max_freq(struct drm_i915_private
*dev_priv
)
2812 valleyview_nc_read(dev_priv
, IOSF_NC_FB_GFX_FREQ_FUSE
, &val
);
2814 rp0
= (val
& FB_GFX_MAX_FREQ_FUSE_MASK
) >> FB_GFX_MAX_FREQ_FUSE_SHIFT
;
2816 rp0
= min_t(u32
, rp0
, 0xea);
2821 static int valleyview_rps_rpe_freq(struct drm_i915_private
*dev_priv
)
2825 valleyview_nc_read(dev_priv
, IOSF_NC_FB_GFX_FMAX_FUSE_LO
, &val
);
2826 rpe
= (val
& FB_FMAX_VMIN_FREQ_LO_MASK
) >> FB_FMAX_VMIN_FREQ_LO_SHIFT
;
2827 valleyview_nc_read(dev_priv
, IOSF_NC_FB_GFX_FMAX_FUSE_HI
, &val
);
2828 rpe
|= (val
& FB_FMAX_VMIN_FREQ_HI_MASK
) << 5;
2833 int valleyview_rps_min_freq(struct drm_i915_private
*dev_priv
)
2837 valleyview_punit_read(dev_priv
, PUNIT_REG_GPU_LFM
, &val
);
2842 static void vlv_rps_timer_work(struct work_struct
*work
)
2844 drm_i915_private_t
*dev_priv
= container_of(work
, drm_i915_private_t
,
2848 * Timer fired, we must be idle. Drop to min voltage state.
2849 * Note: we use RPe here since it should match the
2850 * Vmin we were shooting for. That should give us better
2851 * perf when we come back out of RC6 than if we used the
2852 * min freq available.
2854 mutex_lock(&dev_priv
->rps
.hw_lock
);
2855 valleyview_set_rps(dev_priv
->dev
, dev_priv
->rps
.rpe_delay
);
2856 mutex_unlock(&dev_priv
->rps
.hw_lock
);
2859 static void valleyview_enable_rps(struct drm_device
*dev
)
2861 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2862 struct intel_ring_buffer
*ring
;
2863 u32 gtfifodbg
, val
, rpe
;
2866 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
2868 if ((gtfifodbg
= I915_READ(GTFIFODBG
))) {
2869 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg
);
2870 I915_WRITE(GTFIFODBG
, gtfifodbg
);
2873 gen6_gt_force_wake_get(dev_priv
);
2875 I915_WRITE(GEN6_RP_UP_THRESHOLD
, 59400);
2876 I915_WRITE(GEN6_RP_DOWN_THRESHOLD
, 245000);
2877 I915_WRITE(GEN6_RP_UP_EI
, 66000);
2878 I915_WRITE(GEN6_RP_DOWN_EI
, 350000);
2880 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS
, 10);
2882 I915_WRITE(GEN6_RP_CONTROL
,
2883 GEN6_RP_MEDIA_TURBO
|
2884 GEN6_RP_MEDIA_HW_NORMAL_MODE
|
2885 GEN6_RP_MEDIA_IS_GFX
|
2887 GEN6_RP_UP_BUSY_AVG
|
2888 GEN6_RP_DOWN_IDLE_CONT
);
2890 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT
, 0x00280000);
2891 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL
, 125000);
2892 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS
, 25);
2894 for_each_ring(ring
, dev_priv
, i
)
2895 I915_WRITE(RING_MAX_IDLE(ring
->mmio_base
), 10);
2897 I915_WRITE(GEN6_RC6_THRESHOLD
, 0xc350);
2899 /* allows RC6 residency counter to work */
2900 I915_WRITE(0x138104, _MASKED_BIT_ENABLE(0x3));
2901 I915_WRITE(GEN6_RC_CONTROL
,
2902 GEN7_RC_CTL_TO_MODE
);
2904 valleyview_punit_read(dev_priv
, PUNIT_REG_GPU_FREQ_STS
, &val
);
2905 switch ((val
>> 6) & 3) {
2908 dev_priv
->mem_freq
= 800;
2911 dev_priv
->mem_freq
= 1066;
2914 dev_priv
->mem_freq
= 1333;
2917 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv
->mem_freq
);
2919 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val
& 0x10 ? "yes" : "no");
2920 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val
);
2922 DRM_DEBUG_DRIVER("current GPU freq: %d\n",
2923 vlv_gpu_freq(dev_priv
->mem_freq
, (val
>> 8) & 0xff));
2924 dev_priv
->rps
.cur_delay
= (val
>> 8) & 0xff;
2926 dev_priv
->rps
.max_delay
= valleyview_rps_max_freq(dev_priv
);
2927 dev_priv
->rps
.hw_max
= dev_priv
->rps
.max_delay
;
2928 DRM_DEBUG_DRIVER("max GPU freq: %d\n", vlv_gpu_freq(dev_priv
->mem_freq
,
2929 dev_priv
->rps
.max_delay
));
2931 rpe
= valleyview_rps_rpe_freq(dev_priv
);
2932 DRM_DEBUG_DRIVER("RPe GPU freq: %d\n",
2933 vlv_gpu_freq(dev_priv
->mem_freq
, rpe
));
2934 dev_priv
->rps
.rpe_delay
= rpe
;
2936 val
= valleyview_rps_min_freq(dev_priv
);
2937 DRM_DEBUG_DRIVER("min GPU freq: %d\n", vlv_gpu_freq(dev_priv
->mem_freq
,
2939 dev_priv
->rps
.min_delay
= val
;
2941 DRM_DEBUG_DRIVER("setting GPU freq to %d\n",
2942 vlv_gpu_freq(dev_priv
->mem_freq
, rpe
));
2944 INIT_DELAYED_WORK(&dev_priv
->rps
.vlv_work
, vlv_rps_timer_work
);
2946 valleyview_set_rps(dev_priv
->dev
, rpe
);
2948 /* requires MSI enabled */
2949 I915_WRITE(GEN6_PMIER
, GEN6_PM_DEFERRED_EVENTS
);
2950 spin_lock_irq(&dev_priv
->rps
.lock
);
2951 WARN_ON(dev_priv
->rps
.pm_iir
!= 0);
2952 I915_WRITE(GEN6_PMIMR
, 0);
2953 spin_unlock_irq(&dev_priv
->rps
.lock
);
2954 /* enable all PM interrupts */
2955 I915_WRITE(GEN6_PMINTRMSK
, 0);
2957 gen6_gt_force_wake_put(dev_priv
);
2960 void ironlake_teardown_rc6(struct drm_device
*dev
)
2962 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2964 if (dev_priv
->ips
.renderctx
) {
2965 i915_gem_object_unpin(dev_priv
->ips
.renderctx
);
2966 drm_gem_object_unreference(&dev_priv
->ips
.renderctx
->base
);
2967 dev_priv
->ips
.renderctx
= NULL
;
2970 if (dev_priv
->ips
.pwrctx
) {
2971 i915_gem_object_unpin(dev_priv
->ips
.pwrctx
);
2972 drm_gem_object_unreference(&dev_priv
->ips
.pwrctx
->base
);
2973 dev_priv
->ips
.pwrctx
= NULL
;
2977 static void ironlake_disable_rc6(struct drm_device
*dev
)
2979 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2981 if (I915_READ(PWRCTXA
)) {
2982 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
2983 I915_WRITE(RSTDBYCTL
, I915_READ(RSTDBYCTL
) | RCX_SW_EXIT
);
2984 wait_for(((I915_READ(RSTDBYCTL
) & RSX_STATUS_MASK
) == RSX_STATUS_ON
),
2987 I915_WRITE(PWRCTXA
, 0);
2988 POSTING_READ(PWRCTXA
);
2990 I915_WRITE(RSTDBYCTL
, I915_READ(RSTDBYCTL
) & ~RCX_SW_EXIT
);
2991 POSTING_READ(RSTDBYCTL
);
2995 static int ironlake_setup_rc6(struct drm_device
*dev
)
2997 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2999 if (dev_priv
->ips
.renderctx
== NULL
)
3000 dev_priv
->ips
.renderctx
= intel_alloc_context_page(dev
);
3001 if (!dev_priv
->ips
.renderctx
)
3004 if (dev_priv
->ips
.pwrctx
== NULL
)
3005 dev_priv
->ips
.pwrctx
= intel_alloc_context_page(dev
);
3006 if (!dev_priv
->ips
.pwrctx
) {
3007 ironlake_teardown_rc6(dev
);
3014 static void ironlake_enable_rc6(struct drm_device
*dev
)
3016 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3017 struct intel_ring_buffer
*ring
= &dev_priv
->ring
[RCS
];
3018 bool was_interruptible
;
3021 /* rc6 disabled by default due to repeated reports of hanging during
3024 if (!intel_enable_rc6(dev
))
3027 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
3029 ret
= ironlake_setup_rc6(dev
);
3033 was_interruptible
= dev_priv
->mm
.interruptible
;
3034 dev_priv
->mm
.interruptible
= false;
3037 * GPU can automatically power down the render unit if given a page
3040 ret
= intel_ring_begin(ring
, 6);
3042 ironlake_teardown_rc6(dev
);
3043 dev_priv
->mm
.interruptible
= was_interruptible
;
3047 intel_ring_emit(ring
, MI_SUSPEND_FLUSH
| MI_SUSPEND_FLUSH_EN
);
3048 intel_ring_emit(ring
, MI_SET_CONTEXT
);
3049 intel_ring_emit(ring
, dev_priv
->ips
.renderctx
->gtt_offset
|
3051 MI_SAVE_EXT_STATE_EN
|
3052 MI_RESTORE_EXT_STATE_EN
|
3053 MI_RESTORE_INHIBIT
);
3054 intel_ring_emit(ring
, MI_SUSPEND_FLUSH
);
3055 intel_ring_emit(ring
, MI_NOOP
);
3056 intel_ring_emit(ring
, MI_FLUSH
);
3057 intel_ring_advance(ring
);
3060 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
3061 * does an implicit flush, combined with MI_FLUSH above, it should be
3062 * safe to assume that renderctx is valid
3064 ret
= intel_ring_idle(ring
);
3065 dev_priv
->mm
.interruptible
= was_interruptible
;
3067 DRM_ERROR("failed to enable ironlake power savings\n");
3068 ironlake_teardown_rc6(dev
);
3072 I915_WRITE(PWRCTXA
, dev_priv
->ips
.pwrctx
->gtt_offset
| PWRCTX_EN
);
3073 I915_WRITE(RSTDBYCTL
, I915_READ(RSTDBYCTL
) & ~RCX_SW_EXIT
);
3076 static unsigned long intel_pxfreq(u32 vidfreq
)
3079 int div
= (vidfreq
& 0x3f0000) >> 16;
3080 int post
= (vidfreq
& 0x3000) >> 12;
3081 int pre
= (vidfreq
& 0x7);
3086 freq
= ((div
* 133333) / ((1<<post
) * pre
));
3091 static const struct cparams
{
3097 { 1, 1333, 301, 28664 },
3098 { 1, 1066, 294, 24460 },
3099 { 1, 800, 294, 25192 },
3100 { 0, 1333, 276, 27605 },
3101 { 0, 1066, 276, 27605 },
3102 { 0, 800, 231, 23784 },
3105 static unsigned long __i915_chipset_val(struct drm_i915_private
*dev_priv
)
3107 u64 total_count
, diff
, ret
;
3108 u32 count1
, count2
, count3
, m
= 0, c
= 0;
3109 unsigned long now
= jiffies_to_msecs(jiffies
), diff1
;
3112 assert_spin_locked(&mchdev_lock
);
3114 diff1
= now
- dev_priv
->ips
.last_time1
;
3116 /* Prevent division-by-zero if we are asking too fast.
3117 * Also, we don't get interesting results if we are polling
3118 * faster than once in 10ms, so just return the saved value
3122 return dev_priv
->ips
.chipset_power
;
3124 count1
= I915_READ(DMIEC
);
3125 count2
= I915_READ(DDREC
);
3126 count3
= I915_READ(CSIEC
);
3128 total_count
= count1
+ count2
+ count3
;
3130 /* FIXME: handle per-counter overflow */
3131 if (total_count
< dev_priv
->ips
.last_count1
) {
3132 diff
= ~0UL - dev_priv
->ips
.last_count1
;
3133 diff
+= total_count
;
3135 diff
= total_count
- dev_priv
->ips
.last_count1
;
3138 for (i
= 0; i
< ARRAY_SIZE(cparams
); i
++) {
3139 if (cparams
[i
].i
== dev_priv
->ips
.c_m
&&
3140 cparams
[i
].t
== dev_priv
->ips
.r_t
) {
3147 diff
= div_u64(diff
, diff1
);
3148 ret
= ((m
* diff
) + c
);
3149 ret
= div_u64(ret
, 10);
3151 dev_priv
->ips
.last_count1
= total_count
;
3152 dev_priv
->ips
.last_time1
= now
;
3154 dev_priv
->ips
.chipset_power
= ret
;
3159 unsigned long i915_chipset_val(struct drm_i915_private
*dev_priv
)
3163 if (dev_priv
->info
->gen
!= 5)
3166 spin_lock_irq(&mchdev_lock
);
3168 val
= __i915_chipset_val(dev_priv
);
3170 spin_unlock_irq(&mchdev_lock
);
3175 unsigned long i915_mch_val(struct drm_i915_private
*dev_priv
)
3177 unsigned long m
, x
, b
;
3180 tsfs
= I915_READ(TSFS
);
3182 m
= ((tsfs
& TSFS_SLOPE_MASK
) >> TSFS_SLOPE_SHIFT
);
3183 x
= I915_READ8(TR1
);
3185 b
= tsfs
& TSFS_INTR_MASK
;
3187 return ((m
* x
) / 127) - b
;
3190 static u16
pvid_to_extvid(struct drm_i915_private
*dev_priv
, u8 pxvid
)
3192 static const struct v_table
{
3193 u16 vd
; /* in .1 mil */
3194 u16 vm
; /* in .1 mil */
3325 if (dev_priv
->info
->is_mobile
)
3326 return v_table
[pxvid
].vm
;
3328 return v_table
[pxvid
].vd
;
3331 static void __i915_update_gfx_val(struct drm_i915_private
*dev_priv
)
3333 struct timespec now
, diff1
;
3335 unsigned long diffms
;
3338 assert_spin_locked(&mchdev_lock
);
3340 getrawmonotonic(&now
);
3341 diff1
= timespec_sub(now
, dev_priv
->ips
.last_time2
);
3343 /* Don't divide by 0 */
3344 diffms
= diff1
.tv_sec
* 1000 + diff1
.tv_nsec
/ 1000000;
3348 count
= I915_READ(GFXEC
);
3350 if (count
< dev_priv
->ips
.last_count2
) {
3351 diff
= ~0UL - dev_priv
->ips
.last_count2
;
3354 diff
= count
- dev_priv
->ips
.last_count2
;
3357 dev_priv
->ips
.last_count2
= count
;
3358 dev_priv
->ips
.last_time2
= now
;
3360 /* More magic constants... */
3362 diff
= div_u64(diff
, diffms
* 10);
3363 dev_priv
->ips
.gfx_power
= diff
;
3366 void i915_update_gfx_val(struct drm_i915_private
*dev_priv
)
3368 if (dev_priv
->info
->gen
!= 5)
3371 spin_lock_irq(&mchdev_lock
);
3373 __i915_update_gfx_val(dev_priv
);
3375 spin_unlock_irq(&mchdev_lock
);
3378 static unsigned long __i915_gfx_val(struct drm_i915_private
*dev_priv
)
3380 unsigned long t
, corr
, state1
, corr2
, state2
;
3383 assert_spin_locked(&mchdev_lock
);
3385 pxvid
= I915_READ(PXVFREQ_BASE
+ (dev_priv
->rps
.cur_delay
* 4));
3386 pxvid
= (pxvid
>> 24) & 0x7f;
3387 ext_v
= pvid_to_extvid(dev_priv
, pxvid
);
3391 t
= i915_mch_val(dev_priv
);
3393 /* Revel in the empirically derived constants */
3395 /* Correction factor in 1/100000 units */
3397 corr
= ((t
* 2349) + 135940);
3399 corr
= ((t
* 964) + 29317);
3401 corr
= ((t
* 301) + 1004);
3403 corr
= corr
* ((150142 * state1
) / 10000 - 78642);
3405 corr2
= (corr
* dev_priv
->ips
.corr
);
3407 state2
= (corr2
* state1
) / 10000;
3408 state2
/= 100; /* convert to mW */
3410 __i915_update_gfx_val(dev_priv
);
3412 return dev_priv
->ips
.gfx_power
+ state2
;
3415 unsigned long i915_gfx_val(struct drm_i915_private
*dev_priv
)
3419 if (dev_priv
->info
->gen
!= 5)
3422 spin_lock_irq(&mchdev_lock
);
3424 val
= __i915_gfx_val(dev_priv
);
3426 spin_unlock_irq(&mchdev_lock
);
3432 * i915_read_mch_val - return value for IPS use
3434 * Calculate and return a value for the IPS driver to use when deciding whether
3435 * we have thermal and power headroom to increase CPU or GPU power budget.
3437 unsigned long i915_read_mch_val(void)
3439 struct drm_i915_private
*dev_priv
;
3440 unsigned long chipset_val
, graphics_val
, ret
= 0;
3442 spin_lock_irq(&mchdev_lock
);
3445 dev_priv
= i915_mch_dev
;
3447 chipset_val
= __i915_chipset_val(dev_priv
);
3448 graphics_val
= __i915_gfx_val(dev_priv
);
3450 ret
= chipset_val
+ graphics_val
;
3453 spin_unlock_irq(&mchdev_lock
);
3457 EXPORT_SYMBOL_GPL(i915_read_mch_val
);
3460 * i915_gpu_raise - raise GPU frequency limit
3462 * Raise the limit; IPS indicates we have thermal headroom.
3464 bool i915_gpu_raise(void)
3466 struct drm_i915_private
*dev_priv
;
3469 spin_lock_irq(&mchdev_lock
);
3470 if (!i915_mch_dev
) {
3474 dev_priv
= i915_mch_dev
;
3476 if (dev_priv
->ips
.max_delay
> dev_priv
->ips
.fmax
)
3477 dev_priv
->ips
.max_delay
--;
3480 spin_unlock_irq(&mchdev_lock
);
3484 EXPORT_SYMBOL_GPL(i915_gpu_raise
);
3487 * i915_gpu_lower - lower GPU frequency limit
3489 * IPS indicates we're close to a thermal limit, so throttle back the GPU
3490 * frequency maximum.
3492 bool i915_gpu_lower(void)
3494 struct drm_i915_private
*dev_priv
;
3497 spin_lock_irq(&mchdev_lock
);
3498 if (!i915_mch_dev
) {
3502 dev_priv
= i915_mch_dev
;
3504 if (dev_priv
->ips
.max_delay
< dev_priv
->ips
.min_delay
)
3505 dev_priv
->ips
.max_delay
++;
3508 spin_unlock_irq(&mchdev_lock
);
3512 EXPORT_SYMBOL_GPL(i915_gpu_lower
);
3515 * i915_gpu_busy - indicate GPU business to IPS
3517 * Tell the IPS driver whether or not the GPU is busy.
3519 bool i915_gpu_busy(void)
3521 struct drm_i915_private
*dev_priv
;
3522 struct intel_ring_buffer
*ring
;
3526 spin_lock_irq(&mchdev_lock
);
3529 dev_priv
= i915_mch_dev
;
3531 for_each_ring(ring
, dev_priv
, i
)
3532 ret
|= !list_empty(&ring
->request_list
);
3535 spin_unlock_irq(&mchdev_lock
);
3539 EXPORT_SYMBOL_GPL(i915_gpu_busy
);
3542 * i915_gpu_turbo_disable - disable graphics turbo
3544 * Disable graphics turbo by resetting the max frequency and setting the
3545 * current frequency to the default.
3547 bool i915_gpu_turbo_disable(void)
3549 struct drm_i915_private
*dev_priv
;
3552 spin_lock_irq(&mchdev_lock
);
3553 if (!i915_mch_dev
) {
3557 dev_priv
= i915_mch_dev
;
3559 dev_priv
->ips
.max_delay
= dev_priv
->ips
.fstart
;
3561 if (!ironlake_set_drps(dev_priv
->dev
, dev_priv
->ips
.fstart
))
3565 spin_unlock_irq(&mchdev_lock
);
3569 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable
);
3572 * Tells the intel_ips driver that the i915 driver is now loaded, if
3573 * IPS got loaded first.
3575 * This awkward dance is so that neither module has to depend on the
3576 * other in order for IPS to do the appropriate communication of
3577 * GPU turbo limits to i915.
3580 ips_ping_for_i915_load(void)
3584 link
= symbol_get(ips_link_to_i915_driver
);
3587 symbol_put(ips_link_to_i915_driver
);
3591 void intel_gpu_ips_init(struct drm_i915_private
*dev_priv
)
3593 /* We only register the i915 ips part with intel-ips once everything is
3594 * set up, to avoid intel-ips sneaking in and reading bogus values. */
3595 spin_lock_irq(&mchdev_lock
);
3596 i915_mch_dev
= dev_priv
;
3597 spin_unlock_irq(&mchdev_lock
);
3599 ips_ping_for_i915_load();
3602 void intel_gpu_ips_teardown(void)
3604 spin_lock_irq(&mchdev_lock
);
3605 i915_mch_dev
= NULL
;
3606 spin_unlock_irq(&mchdev_lock
);
3608 static void intel_init_emon(struct drm_device
*dev
)
3610 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3615 /* Disable to program */
3619 /* Program energy weights for various events */
3620 I915_WRITE(SDEW
, 0x15040d00);
3621 I915_WRITE(CSIEW0
, 0x007f0000);
3622 I915_WRITE(CSIEW1
, 0x1e220004);
3623 I915_WRITE(CSIEW2
, 0x04000004);
3625 for (i
= 0; i
< 5; i
++)
3626 I915_WRITE(PEW
+ (i
* 4), 0);
3627 for (i
= 0; i
< 3; i
++)
3628 I915_WRITE(DEW
+ (i
* 4), 0);
3630 /* Program P-state weights to account for frequency power adjustment */
3631 for (i
= 0; i
< 16; i
++) {
3632 u32 pxvidfreq
= I915_READ(PXVFREQ_BASE
+ (i
* 4));
3633 unsigned long freq
= intel_pxfreq(pxvidfreq
);
3634 unsigned long vid
= (pxvidfreq
& PXVFREQ_PX_MASK
) >>
3639 val
*= (freq
/ 1000);
3641 val
/= (127*127*900);
3643 DRM_ERROR("bad pxval: %ld\n", val
);
3646 /* Render standby states get 0 weight */
3650 for (i
= 0; i
< 4; i
++) {
3651 u32 val
= (pxw
[i
*4] << 24) | (pxw
[(i
*4)+1] << 16) |
3652 (pxw
[(i
*4)+2] << 8) | (pxw
[(i
*4)+3]);
3653 I915_WRITE(PXW
+ (i
* 4), val
);
3656 /* Adjust magic regs to magic values (more experimental results) */
3657 I915_WRITE(OGW0
, 0);
3658 I915_WRITE(OGW1
, 0);
3659 I915_WRITE(EG0
, 0x00007f00);
3660 I915_WRITE(EG1
, 0x0000000e);
3661 I915_WRITE(EG2
, 0x000e0000);
3662 I915_WRITE(EG3
, 0x68000300);
3663 I915_WRITE(EG4
, 0x42000000);
3664 I915_WRITE(EG5
, 0x00140031);
3668 for (i
= 0; i
< 8; i
++)
3669 I915_WRITE(PXWL
+ (i
* 4), 0);
3671 /* Enable PMON + select events */
3672 I915_WRITE(ECR
, 0x80000019);
3674 lcfuse
= I915_READ(LCFUSE02
);
3676 dev_priv
->ips
.corr
= (lcfuse
& LCFUSE_HIV_MASK
);
3679 void intel_disable_gt_powersave(struct drm_device
*dev
)
3681 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3683 /* Interrupts should be disabled already to avoid re-arming. */
3684 WARN_ON(dev
->irq_enabled
);
3686 if (IS_IRONLAKE_M(dev
)) {
3687 ironlake_disable_drps(dev
);
3688 ironlake_disable_rc6(dev
);
3689 } else if (INTEL_INFO(dev
)->gen
>= 6) {
3690 cancel_delayed_work_sync(&dev_priv
->rps
.delayed_resume_work
);
3691 cancel_work_sync(&dev_priv
->rps
.work
);
3692 if (IS_VALLEYVIEW(dev
))
3693 cancel_delayed_work_sync(&dev_priv
->rps
.vlv_work
);
3694 mutex_lock(&dev_priv
->rps
.hw_lock
);
3695 if (IS_VALLEYVIEW(dev
))
3696 valleyview_disable_rps(dev
);
3698 gen6_disable_rps(dev
);
3699 mutex_unlock(&dev_priv
->rps
.hw_lock
);
3703 static void intel_gen6_powersave_work(struct work_struct
*work
)
3705 struct drm_i915_private
*dev_priv
=
3706 container_of(work
, struct drm_i915_private
,
3707 rps
.delayed_resume_work
.work
);
3708 struct drm_device
*dev
= dev_priv
->dev
;
3710 mutex_lock(&dev_priv
->rps
.hw_lock
);
3712 if (IS_VALLEYVIEW(dev
)) {
3713 valleyview_enable_rps(dev
);
3715 gen6_enable_rps(dev
);
3716 gen6_update_ring_freq(dev
);
3718 mutex_unlock(&dev_priv
->rps
.hw_lock
);
3721 void intel_enable_gt_powersave(struct drm_device
*dev
)
3723 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3725 if (IS_IRONLAKE_M(dev
)) {
3726 ironlake_enable_drps(dev
);
3727 ironlake_enable_rc6(dev
);
3728 intel_init_emon(dev
);
3729 } else if (IS_GEN6(dev
) || IS_GEN7(dev
)) {
3731 * PCU communication is slow and this doesn't need to be
3732 * done at any specific time, so do this out of our fast path
3733 * to make resume and init faster.
3735 schedule_delayed_work(&dev_priv
->rps
.delayed_resume_work
,
3736 round_jiffies_up_relative(HZ
));
3740 static void ibx_init_clock_gating(struct drm_device
*dev
)
3742 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3745 * On Ibex Peak and Cougar Point, we need to disable clock
3746 * gating for the panel power sequencer or it will fail to
3747 * start up when no ports are active.
3749 I915_WRITE(SOUTH_DSPCLK_GATE_D
, PCH_DPLSUNIT_CLOCK_GATE_DISABLE
);
3752 static void ironlake_init_clock_gating(struct drm_device
*dev
)
3754 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3755 uint32_t dspclk_gate
= ILK_VRHUNIT_CLOCK_GATE_DISABLE
;
3757 /* Required for FBC */
3758 dspclk_gate
|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE
|
3759 ILK_DPFCUNIT_CLOCK_GATE_DISABLE
|
3760 ILK_DPFDUNIT_CLOCK_GATE_ENABLE
;
3762 I915_WRITE(PCH_3DCGDIS0
,
3763 MARIUNIT_CLOCK_GATE_DISABLE
|
3764 SVSMUNIT_CLOCK_GATE_DISABLE
);
3765 I915_WRITE(PCH_3DCGDIS1
,
3766 VFMUNIT_CLOCK_GATE_DISABLE
);
3769 * According to the spec the following bits should be set in
3770 * order to enable memory self-refresh
3771 * The bit 22/21 of 0x42004
3772 * The bit 5 of 0x42020
3773 * The bit 15 of 0x45000
3775 I915_WRITE(ILK_DISPLAY_CHICKEN2
,
3776 (I915_READ(ILK_DISPLAY_CHICKEN2
) |
3777 ILK_DPARB_GATE
| ILK_VSDPFD_FULL
));
3778 dspclk_gate
|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE
;
3779 I915_WRITE(DISP_ARB_CTL
,
3780 (I915_READ(DISP_ARB_CTL
) |
3782 I915_WRITE(WM3_LP_ILK
, 0);
3783 I915_WRITE(WM2_LP_ILK
, 0);
3784 I915_WRITE(WM1_LP_ILK
, 0);
3787 * Based on the document from hardware guys the following bits
3788 * should be set unconditionally in order to enable FBC.
3789 * The bit 22 of 0x42000
3790 * The bit 22 of 0x42004
3791 * The bit 7,8,9 of 0x42020.
3793 if (IS_IRONLAKE_M(dev
)) {
3794 I915_WRITE(ILK_DISPLAY_CHICKEN1
,
3795 I915_READ(ILK_DISPLAY_CHICKEN1
) |
3797 I915_WRITE(ILK_DISPLAY_CHICKEN2
,
3798 I915_READ(ILK_DISPLAY_CHICKEN2
) |
3802 I915_WRITE(ILK_DSPCLK_GATE_D
, dspclk_gate
);
3804 I915_WRITE(ILK_DISPLAY_CHICKEN2
,
3805 I915_READ(ILK_DISPLAY_CHICKEN2
) |
3806 ILK_ELPIN_409_SELECT
);
3807 I915_WRITE(_3D_CHICKEN2
,
3808 _3D_CHICKEN2_WM_READ_PIPELINED
<< 16 |
3809 _3D_CHICKEN2_WM_READ_PIPELINED
);
3811 /* WaDisableRenderCachePipelinedFlush:ilk */
3812 I915_WRITE(CACHE_MODE_0
,
3813 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE
));
3815 ibx_init_clock_gating(dev
);
3818 static void cpt_init_clock_gating(struct drm_device
*dev
)
3820 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3825 * On Ibex Peak and Cougar Point, we need to disable clock
3826 * gating for the panel power sequencer or it will fail to
3827 * start up when no ports are active.
3829 I915_WRITE(SOUTH_DSPCLK_GATE_D
, PCH_DPLSUNIT_CLOCK_GATE_DISABLE
);
3830 I915_WRITE(SOUTH_CHICKEN2
, I915_READ(SOUTH_CHICKEN2
) |
3831 DPLS_EDP_PPS_FIX_DIS
);
3832 /* The below fixes the weird display corruption, a few pixels shifted
3833 * downward, on (only) LVDS of some HP laptops with IVY.
3835 for_each_pipe(pipe
) {
3836 val
= I915_READ(TRANS_CHICKEN2(pipe
));
3837 val
|= TRANS_CHICKEN2_TIMING_OVERRIDE
;
3838 val
&= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED
;
3839 if (dev_priv
->fdi_rx_polarity_inverted
)
3840 val
|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED
;
3841 val
&= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK
;
3842 val
&= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER
;
3843 val
&= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH
;
3844 I915_WRITE(TRANS_CHICKEN2(pipe
), val
);
3846 /* WADP0ClockGatingDisable */
3847 for_each_pipe(pipe
) {
3848 I915_WRITE(TRANS_CHICKEN1(pipe
),
3849 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE
);
3853 static void gen6_check_mch_setup(struct drm_device
*dev
)
3855 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3858 tmp
= I915_READ(MCH_SSKPD
);
3859 if ((tmp
& MCH_SSKPD_WM0_MASK
) != MCH_SSKPD_WM0_VAL
) {
3860 DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp
);
3861 DRM_INFO("This can cause pipe underruns and display issues.\n");
3862 DRM_INFO("Please upgrade your BIOS to fix this.\n");
3866 static void gen6_init_clock_gating(struct drm_device
*dev
)
3868 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3870 uint32_t dspclk_gate
= ILK_VRHUNIT_CLOCK_GATE_DISABLE
;
3872 I915_WRITE(ILK_DSPCLK_GATE_D
, dspclk_gate
);
3874 I915_WRITE(ILK_DISPLAY_CHICKEN2
,
3875 I915_READ(ILK_DISPLAY_CHICKEN2
) |
3876 ILK_ELPIN_409_SELECT
);
3878 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3879 I915_WRITE(_3D_CHICKEN
,
3880 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB
));
3882 /* WaSetupGtModeTdRowDispatch:snb */
3883 if (IS_SNB_GT1(dev
))
3884 I915_WRITE(GEN6_GT_MODE
,
3885 _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE
));
3887 I915_WRITE(WM3_LP_ILK
, 0);
3888 I915_WRITE(WM2_LP_ILK
, 0);
3889 I915_WRITE(WM1_LP_ILK
, 0);
3891 I915_WRITE(CACHE_MODE_0
,
3892 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB
));
3894 I915_WRITE(GEN6_UCGCTL1
,
3895 I915_READ(GEN6_UCGCTL1
) |
3896 GEN6_BLBUNIT_CLOCK_GATE_DISABLE
|
3897 GEN6_CSUNIT_CLOCK_GATE_DISABLE
);
3899 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
3900 * gating disable must be set. Failure to set it results in
3901 * flickering pixels due to Z write ordering failures after
3902 * some amount of runtime in the Mesa "fire" demo, and Unigine
3903 * Sanctuary and Tropics, and apparently anything else with
3904 * alpha test or pixel discard.
3906 * According to the spec, bit 11 (RCCUNIT) must also be set,
3907 * but we didn't debug actual testcases to find it out.
3909 * Also apply WaDisableVDSUnitClockGating:snb and
3910 * WaDisableRCPBUnitClockGating:snb.
3912 I915_WRITE(GEN6_UCGCTL2
,
3913 GEN7_VDSUNIT_CLOCK_GATE_DISABLE
|
3914 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE
|
3915 GEN6_RCCUNIT_CLOCK_GATE_DISABLE
);
3917 /* Bspec says we need to always set all mask bits. */
3918 I915_WRITE(_3D_CHICKEN3
, (0xFFFF << 16) |
3919 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL
);
3922 * According to the spec the following bits should be
3923 * set in order to enable memory self-refresh and fbc:
3924 * The bit21 and bit22 of 0x42000
3925 * The bit21 and bit22 of 0x42004
3926 * The bit5 and bit7 of 0x42020
3927 * The bit14 of 0x70180
3928 * The bit14 of 0x71180
3930 I915_WRITE(ILK_DISPLAY_CHICKEN1
,
3931 I915_READ(ILK_DISPLAY_CHICKEN1
) |
3932 ILK_FBCQ_DIS
| ILK_PABSTRETCH_DIS
);
3933 I915_WRITE(ILK_DISPLAY_CHICKEN2
,
3934 I915_READ(ILK_DISPLAY_CHICKEN2
) |
3935 ILK_DPARB_GATE
| ILK_VSDPFD_FULL
);
3936 I915_WRITE(ILK_DSPCLK_GATE_D
,
3937 I915_READ(ILK_DSPCLK_GATE_D
) |
3938 ILK_DPARBUNIT_CLOCK_GATE_ENABLE
|
3939 ILK_DPFDUNIT_CLOCK_GATE_ENABLE
);
3941 /* WaMbcDriverBootEnable:snb */
3942 I915_WRITE(GEN6_MBCTL
, I915_READ(GEN6_MBCTL
) |
3943 GEN6_MBCTL_ENABLE_BOOT_FETCH
);
3945 for_each_pipe(pipe
) {
3946 I915_WRITE(DSPCNTR(pipe
),
3947 I915_READ(DSPCNTR(pipe
)) |
3948 DISPPLANE_TRICKLE_FEED_DISABLE
);
3949 intel_flush_display_plane(dev_priv
, pipe
);
3952 /* The default value should be 0x200 according to docs, but the two
3953 * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
3954 I915_WRITE(GEN6_GT_MODE
, _MASKED_BIT_DISABLE(0xffff));
3955 I915_WRITE(GEN6_GT_MODE
, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI
));
3957 cpt_init_clock_gating(dev
);
3959 gen6_check_mch_setup(dev
);
3962 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private
*dev_priv
)
3964 uint32_t reg
= I915_READ(GEN7_FF_THREAD_MODE
);
3966 reg
&= ~GEN7_FF_SCHED_MASK
;
3967 reg
|= GEN7_FF_TS_SCHED_HW
;
3968 reg
|= GEN7_FF_VS_SCHED_HW
;
3969 reg
|= GEN7_FF_DS_SCHED_HW
;
3971 if (IS_HASWELL(dev_priv
->dev
))
3972 reg
&= ~GEN7_FF_VS_REF_CNT_FFME
;
3974 I915_WRITE(GEN7_FF_THREAD_MODE
, reg
);
3977 static void lpt_init_clock_gating(struct drm_device
*dev
)
3979 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3982 * TODO: this bit should only be enabled when really needed, then
3983 * disabled when not needed anymore in order to save power.
3985 if (dev_priv
->pch_id
== INTEL_PCH_LPT_LP_DEVICE_ID_TYPE
)
3986 I915_WRITE(SOUTH_DSPCLK_GATE_D
,
3987 I915_READ(SOUTH_DSPCLK_GATE_D
) |
3988 PCH_LP_PARTITION_LEVEL_DISABLE
);
3991 static void haswell_init_clock_gating(struct drm_device
*dev
)
3993 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3996 I915_WRITE(WM3_LP_ILK
, 0);
3997 I915_WRITE(WM2_LP_ILK
, 0);
3998 I915_WRITE(WM1_LP_ILK
, 0);
4000 /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4001 * This implements the WaDisableRCZUnitClockGating:hsw workaround.
4003 I915_WRITE(GEN6_UCGCTL2
, GEN6_RCZUNIT_CLOCK_GATE_DISABLE
);
4005 /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
4006 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1
,
4007 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC
);
4009 /* WaApplyL3ControlAndL3ChickenMode:hsw */
4010 I915_WRITE(GEN7_L3CNTLREG1
,
4011 GEN7_WA_FOR_GEN7_L3_CONTROL
);
4012 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER
,
4013 GEN7_WA_L3_CHICKEN_MODE
);
4015 /* This is required by WaCatErrorRejectionIssue:hsw */
4016 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
,
4017 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
) |
4018 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB
);
4020 for_each_pipe(pipe
) {
4021 I915_WRITE(DSPCNTR(pipe
),
4022 I915_READ(DSPCNTR(pipe
)) |
4023 DISPPLANE_TRICKLE_FEED_DISABLE
);
4024 intel_flush_display_plane(dev_priv
, pipe
);
4027 /* WaVSRefCountFullforceMissDisable:hsw */
4028 gen7_setup_fixed_func_scheduler(dev_priv
);
4030 /* WaDisable4x2SubspanOptimization:hsw */
4031 I915_WRITE(CACHE_MODE_1
,
4032 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE
));
4034 /* WaMbcDriverBootEnable:hsw */
4035 I915_WRITE(GEN6_MBCTL
, I915_READ(GEN6_MBCTL
) |
4036 GEN6_MBCTL_ENABLE_BOOT_FETCH
);
4038 /* WaSwitchSolVfFArbitrationPriority:hsw */
4039 I915_WRITE(GAM_ECOCHK
, I915_READ(GAM_ECOCHK
) | HSW_ECOCHK_ARB_PRIO_SOL
);
4041 /* XXX: This is a workaround for early silicon revisions and should be
4046 WM_DBG_DISALLOW_MULTIPLE_LP
|
4047 WM_DBG_DISALLOW_SPRITE
|
4048 WM_DBG_DISALLOW_MAXFIFO
);
4050 lpt_init_clock_gating(dev
);
4053 static void ivybridge_init_clock_gating(struct drm_device
*dev
)
4055 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4059 I915_WRITE(WM3_LP_ILK
, 0);
4060 I915_WRITE(WM2_LP_ILK
, 0);
4061 I915_WRITE(WM1_LP_ILK
, 0);
4063 I915_WRITE(ILK_DSPCLK_GATE_D
, ILK_VRHUNIT_CLOCK_GATE_DISABLE
);
4065 /* WaDisableEarlyCull:ivb */
4066 I915_WRITE(_3D_CHICKEN3
,
4067 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL
));
4069 /* WaDisableBackToBackFlipFix:ivb */
4070 I915_WRITE(IVB_CHICKEN3
,
4071 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE
|
4072 CHICKEN3_DGMG_DONE_FIX_DISABLE
);
4074 /* WaDisablePSDDualDispatchEnable:ivb */
4075 if (IS_IVB_GT1(dev
))
4076 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1
,
4077 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE
));
4079 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2
,
4080 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE
));
4082 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
4083 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1
,
4084 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC
);
4086 /* WaApplyL3ControlAndL3ChickenMode:ivb */
4087 I915_WRITE(GEN7_L3CNTLREG1
,
4088 GEN7_WA_FOR_GEN7_L3_CONTROL
);
4089 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER
,
4090 GEN7_WA_L3_CHICKEN_MODE
);
4091 if (IS_IVB_GT1(dev
))
4092 I915_WRITE(GEN7_ROW_CHICKEN2
,
4093 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE
));
4095 I915_WRITE(GEN7_ROW_CHICKEN2_GT2
,
4096 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE
));
4099 /* WaForceL3Serialization:ivb */
4100 I915_WRITE(GEN7_L3SQCREG4
, I915_READ(GEN7_L3SQCREG4
) &
4101 ~L3SQ_URB_READ_CAM_MATCH_DISABLE
);
4103 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4104 * gating disable must be set. Failure to set it results in
4105 * flickering pixels due to Z write ordering failures after
4106 * some amount of runtime in the Mesa "fire" demo, and Unigine
4107 * Sanctuary and Tropics, and apparently anything else with
4108 * alpha test or pixel discard.
4110 * According to the spec, bit 11 (RCCUNIT) must also be set,
4111 * but we didn't debug actual testcases to find it out.
4113 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4114 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4116 I915_WRITE(GEN6_UCGCTL2
,
4117 GEN6_RCZUNIT_CLOCK_GATE_DISABLE
|
4118 GEN6_RCCUNIT_CLOCK_GATE_DISABLE
);
4120 /* This is required by WaCatErrorRejectionIssue:ivb */
4121 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
,
4122 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
) |
4123 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB
);
4125 for_each_pipe(pipe
) {
4126 I915_WRITE(DSPCNTR(pipe
),
4127 I915_READ(DSPCNTR(pipe
)) |
4128 DISPPLANE_TRICKLE_FEED_DISABLE
);
4129 intel_flush_display_plane(dev_priv
, pipe
);
4132 /* WaMbcDriverBootEnable:ivb */
4133 I915_WRITE(GEN6_MBCTL
, I915_READ(GEN6_MBCTL
) |
4134 GEN6_MBCTL_ENABLE_BOOT_FETCH
);
4136 /* WaVSRefCountFullforceMissDisable:ivb */
4137 gen7_setup_fixed_func_scheduler(dev_priv
);
4139 /* WaDisable4x2SubspanOptimization:ivb */
4140 I915_WRITE(CACHE_MODE_1
,
4141 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE
));
4143 snpcr
= I915_READ(GEN6_MBCUNIT_SNPCR
);
4144 snpcr
&= ~GEN6_MBC_SNPCR_MASK
;
4145 snpcr
|= GEN6_MBC_SNPCR_MED
;
4146 I915_WRITE(GEN6_MBCUNIT_SNPCR
, snpcr
);
4148 if (!HAS_PCH_NOP(dev
))
4149 cpt_init_clock_gating(dev
);
4151 gen6_check_mch_setup(dev
);
4154 static void valleyview_init_clock_gating(struct drm_device
*dev
)
4156 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4159 I915_WRITE(WM3_LP_ILK
, 0);
4160 I915_WRITE(WM2_LP_ILK
, 0);
4161 I915_WRITE(WM1_LP_ILK
, 0);
4163 I915_WRITE(ILK_DSPCLK_GATE_D
, ILK_VRHUNIT_CLOCK_GATE_DISABLE
);
4165 /* WaDisableEarlyCull:vlv */
4166 I915_WRITE(_3D_CHICKEN3
,
4167 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL
));
4169 /* WaDisableBackToBackFlipFix:vlv */
4170 I915_WRITE(IVB_CHICKEN3
,
4171 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE
|
4172 CHICKEN3_DGMG_DONE_FIX_DISABLE
);
4174 /* WaDisablePSDDualDispatchEnable:vlv */
4175 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1
,
4176 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP
|
4177 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE
));
4179 /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
4180 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1
,
4181 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC
);
4183 /* WaApplyL3ControlAndL3ChickenMode:vlv */
4184 I915_WRITE(GEN7_L3CNTLREG1
, I915_READ(GEN7_L3CNTLREG1
) | GEN7_L3AGDIS
);
4185 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER
, GEN7_WA_L3_CHICKEN_MODE
);
4187 /* WaForceL3Serialization:vlv */
4188 I915_WRITE(GEN7_L3SQCREG4
, I915_READ(GEN7_L3SQCREG4
) &
4189 ~L3SQ_URB_READ_CAM_MATCH_DISABLE
);
4191 /* WaDisableDopClockGating:vlv */
4192 I915_WRITE(GEN7_ROW_CHICKEN2
,
4193 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE
));
4195 /* WaForceL3Serialization:vlv */
4196 I915_WRITE(GEN7_L3SQCREG4
, I915_READ(GEN7_L3SQCREG4
) &
4197 ~L3SQ_URB_READ_CAM_MATCH_DISABLE
);
4199 /* This is required by WaCatErrorRejectionIssue:vlv */
4200 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
,
4201 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG
) |
4202 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB
);
4204 /* WaMbcDriverBootEnable:vlv */
4205 I915_WRITE(GEN6_MBCTL
, I915_READ(GEN6_MBCTL
) |
4206 GEN6_MBCTL_ENABLE_BOOT_FETCH
);
4209 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4210 * gating disable must be set. Failure to set it results in
4211 * flickering pixels due to Z write ordering failures after
4212 * some amount of runtime in the Mesa "fire" demo, and Unigine
4213 * Sanctuary and Tropics, and apparently anything else with
4214 * alpha test or pixel discard.
4216 * According to the spec, bit 11 (RCCUNIT) must also be set,
4217 * but we didn't debug actual testcases to find it out.
4219 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4220 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4222 * Also apply WaDisableVDSUnitClockGating:vlv and
4223 * WaDisableRCPBUnitClockGating:vlv.
4225 I915_WRITE(GEN6_UCGCTL2
,
4226 GEN7_VDSUNIT_CLOCK_GATE_DISABLE
|
4227 GEN7_TDLUNIT_CLOCK_GATE_DISABLE
|
4228 GEN6_RCZUNIT_CLOCK_GATE_DISABLE
|
4229 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE
|
4230 GEN6_RCCUNIT_CLOCK_GATE_DISABLE
);
4232 I915_WRITE(GEN7_UCGCTL4
, GEN7_L3BANK2X_CLOCK_GATE_DISABLE
);
4234 for_each_pipe(pipe
) {
4235 I915_WRITE(DSPCNTR(pipe
),
4236 I915_READ(DSPCNTR(pipe
)) |
4237 DISPPLANE_TRICKLE_FEED_DISABLE
);
4238 intel_flush_display_plane(dev_priv
, pipe
);
4241 I915_WRITE(CACHE_MODE_1
,
4242 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE
));
4245 * WaDisableVLVClockGating_VBIIssue:vlv
4246 * Disable clock gating on th GCFG unit to prevent a delay
4247 * in the reporting of vblank events.
4249 I915_WRITE(VLV_GUNIT_CLOCK_GATE
, 0xffffffff);
4251 /* Conservative clock gating settings for now */
4252 I915_WRITE(0x9400, 0xffffffff);
4253 I915_WRITE(0x9404, 0xffffffff);
4254 I915_WRITE(0x9408, 0xffffffff);
4255 I915_WRITE(0x940c, 0xffffffff);
4256 I915_WRITE(0x9410, 0xffffffff);
4257 I915_WRITE(0x9414, 0xffffffff);
4258 I915_WRITE(0x9418, 0xffffffff);
4261 static void g4x_init_clock_gating(struct drm_device
*dev
)
4263 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4264 uint32_t dspclk_gate
;
4266 I915_WRITE(RENCLK_GATE_D1
, 0);
4267 I915_WRITE(RENCLK_GATE_D2
, VF_UNIT_CLOCK_GATE_DISABLE
|
4268 GS_UNIT_CLOCK_GATE_DISABLE
|
4269 CL_UNIT_CLOCK_GATE_DISABLE
);
4270 I915_WRITE(RAMCLK_GATE_D
, 0);
4271 dspclk_gate
= VRHUNIT_CLOCK_GATE_DISABLE
|
4272 OVRUNIT_CLOCK_GATE_DISABLE
|
4273 OVCUNIT_CLOCK_GATE_DISABLE
;
4275 dspclk_gate
|= DSSUNIT_CLOCK_GATE_DISABLE
;
4276 I915_WRITE(DSPCLK_GATE_D
, dspclk_gate
);
4278 /* WaDisableRenderCachePipelinedFlush */
4279 I915_WRITE(CACHE_MODE_0
,
4280 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE
));
4283 static void crestline_init_clock_gating(struct drm_device
*dev
)
4285 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4287 I915_WRITE(RENCLK_GATE_D1
, I965_RCC_CLOCK_GATE_DISABLE
);
4288 I915_WRITE(RENCLK_GATE_D2
, 0);
4289 I915_WRITE(DSPCLK_GATE_D
, 0);
4290 I915_WRITE(RAMCLK_GATE_D
, 0);
4291 I915_WRITE16(DEUC
, 0);
4294 static void broadwater_init_clock_gating(struct drm_device
*dev
)
4296 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4298 I915_WRITE(RENCLK_GATE_D1
, I965_RCZ_CLOCK_GATE_DISABLE
|
4299 I965_RCC_CLOCK_GATE_DISABLE
|
4300 I965_RCPB_CLOCK_GATE_DISABLE
|
4301 I965_ISC_CLOCK_GATE_DISABLE
|
4302 I965_FBC_CLOCK_GATE_DISABLE
);
4303 I915_WRITE(RENCLK_GATE_D2
, 0);
4306 static void gen3_init_clock_gating(struct drm_device
*dev
)
4308 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4309 u32 dstate
= I915_READ(D_STATE
);
4311 dstate
|= DSTATE_PLL_D3_OFF
| DSTATE_GFX_CLOCK_GATING
|
4312 DSTATE_DOT_CLOCK_GATING
;
4313 I915_WRITE(D_STATE
, dstate
);
4315 if (IS_PINEVIEW(dev
))
4316 I915_WRITE(ECOSKPD
, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY
));
4318 /* IIR "flip pending" means done if this bit is set */
4319 I915_WRITE(ECOSKPD
, _MASKED_BIT_DISABLE(ECO_FLIP_DONE
));
4322 static void i85x_init_clock_gating(struct drm_device
*dev
)
4324 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4326 I915_WRITE(RENCLK_GATE_D1
, SV_CLOCK_GATE_DISABLE
);
4329 static void i830_init_clock_gating(struct drm_device
*dev
)
4331 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4333 I915_WRITE(DSPCLK_GATE_D
, OVRUNIT_CLOCK_GATE_DISABLE
);
4336 void intel_init_clock_gating(struct drm_device
*dev
)
4338 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4340 dev_priv
->display
.init_clock_gating(dev
);
4344 * We should only use the power well if we explicitly asked the hardware to
4345 * enable it, so check if it's enabled and also check if we've requested it to
4348 bool intel_display_power_enabled(struct drm_device
*dev
,
4349 enum intel_display_power_domain domain
)
4351 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4353 if (!HAS_POWER_WELL(dev
))
4357 case POWER_DOMAIN_PIPE_A
:
4358 case POWER_DOMAIN_TRANSCODER_EDP
:
4360 case POWER_DOMAIN_PIPE_B
:
4361 case POWER_DOMAIN_PIPE_C
:
4362 case POWER_DOMAIN_PIPE_A_PANEL_FITTER
:
4363 case POWER_DOMAIN_PIPE_B_PANEL_FITTER
:
4364 case POWER_DOMAIN_PIPE_C_PANEL_FITTER
:
4365 case POWER_DOMAIN_TRANSCODER_A
:
4366 case POWER_DOMAIN_TRANSCODER_B
:
4367 case POWER_DOMAIN_TRANSCODER_C
:
4368 return I915_READ(HSW_PWR_WELL_DRIVER
) ==
4369 (HSW_PWR_WELL_ENABLE
| HSW_PWR_WELL_STATE
);
4375 void intel_set_power_well(struct drm_device
*dev
, bool enable
)
4377 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4378 bool is_enabled
, enable_requested
;
4381 if (!HAS_POWER_WELL(dev
))
4384 if (!i915_disable_power_well
&& !enable
)
4387 tmp
= I915_READ(HSW_PWR_WELL_DRIVER
);
4388 is_enabled
= tmp
& HSW_PWR_WELL_STATE
;
4389 enable_requested
= tmp
& HSW_PWR_WELL_ENABLE
;
4392 if (!enable_requested
)
4393 I915_WRITE(HSW_PWR_WELL_DRIVER
, HSW_PWR_WELL_ENABLE
);
4396 DRM_DEBUG_KMS("Enabling power well\n");
4397 if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER
) &
4398 HSW_PWR_WELL_STATE
), 20))
4399 DRM_ERROR("Timeout enabling power well\n");
4402 if (enable_requested
) {
4403 I915_WRITE(HSW_PWR_WELL_DRIVER
, 0);
4404 DRM_DEBUG_KMS("Requesting to disable the power well\n");
4410 * Starting with Haswell, we have a "Power Down Well" that can be turned off
4411 * when not needed anymore. We have 4 registers that can request the power well
4412 * to be enabled, and it will only be disabled if none of the registers is
4413 * requesting it to be enabled.
4415 void intel_init_power_well(struct drm_device
*dev
)
4417 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4419 if (!HAS_POWER_WELL(dev
))
4422 /* For now, we need the power well to be always enabled. */
4423 intel_set_power_well(dev
, true);
4425 /* We're taking over the BIOS, so clear any requests made by it since
4426 * the driver is in charge now. */
4427 if (I915_READ(HSW_PWR_WELL_BIOS
) & HSW_PWR_WELL_ENABLE
)
4428 I915_WRITE(HSW_PWR_WELL_BIOS
, 0);
4431 /* Set up chip specific power management-related functions */
4432 void intel_init_pm(struct drm_device
*dev
)
4434 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4436 if (I915_HAS_FBC(dev
)) {
4437 if (HAS_PCH_SPLIT(dev
)) {
4438 dev_priv
->display
.fbc_enabled
= ironlake_fbc_enabled
;
4439 dev_priv
->display
.enable_fbc
= ironlake_enable_fbc
;
4440 dev_priv
->display
.disable_fbc
= ironlake_disable_fbc
;
4441 } else if (IS_GM45(dev
)) {
4442 dev_priv
->display
.fbc_enabled
= g4x_fbc_enabled
;
4443 dev_priv
->display
.enable_fbc
= g4x_enable_fbc
;
4444 dev_priv
->display
.disable_fbc
= g4x_disable_fbc
;
4445 } else if (IS_CRESTLINE(dev
)) {
4446 dev_priv
->display
.fbc_enabled
= i8xx_fbc_enabled
;
4447 dev_priv
->display
.enable_fbc
= i8xx_enable_fbc
;
4448 dev_priv
->display
.disable_fbc
= i8xx_disable_fbc
;
4450 /* 855GM needs testing */
4454 if (IS_PINEVIEW(dev
))
4455 i915_pineview_get_mem_freq(dev
);
4456 else if (IS_GEN5(dev
))
4457 i915_ironlake_get_mem_freq(dev
);
4459 /* For FIFO watermark updates */
4460 if (HAS_PCH_SPLIT(dev
)) {
4462 if (I915_READ(MLTR_ILK
) & ILK_SRLT_MASK
)
4463 dev_priv
->display
.update_wm
= ironlake_update_wm
;
4465 DRM_DEBUG_KMS("Failed to get proper latency. "
4467 dev_priv
->display
.update_wm
= NULL
;
4469 dev_priv
->display
.init_clock_gating
= ironlake_init_clock_gating
;
4470 } else if (IS_GEN6(dev
)) {
4471 if (SNB_READ_WM0_LATENCY()) {
4472 dev_priv
->display
.update_wm
= sandybridge_update_wm
;
4473 dev_priv
->display
.update_sprite_wm
= sandybridge_update_sprite_wm
;
4475 DRM_DEBUG_KMS("Failed to read display plane latency. "
4477 dev_priv
->display
.update_wm
= NULL
;
4479 dev_priv
->display
.init_clock_gating
= gen6_init_clock_gating
;
4480 } else if (IS_IVYBRIDGE(dev
)) {
4481 if (SNB_READ_WM0_LATENCY()) {
4482 dev_priv
->display
.update_wm
= ivybridge_update_wm
;
4483 dev_priv
->display
.update_sprite_wm
= sandybridge_update_sprite_wm
;
4485 DRM_DEBUG_KMS("Failed to read display plane latency. "
4487 dev_priv
->display
.update_wm
= NULL
;
4489 dev_priv
->display
.init_clock_gating
= ivybridge_init_clock_gating
;
4490 } else if (IS_HASWELL(dev
)) {
4491 if (SNB_READ_WM0_LATENCY()) {
4492 dev_priv
->display
.update_wm
= sandybridge_update_wm
;
4493 dev_priv
->display
.update_sprite_wm
= sandybridge_update_sprite_wm
;
4494 dev_priv
->display
.update_linetime_wm
= haswell_update_linetime_wm
;
4496 DRM_DEBUG_KMS("Failed to read display plane latency. "
4498 dev_priv
->display
.update_wm
= NULL
;
4500 dev_priv
->display
.init_clock_gating
= haswell_init_clock_gating
;
4502 dev_priv
->display
.update_wm
= NULL
;
4503 } else if (IS_VALLEYVIEW(dev
)) {
4504 dev_priv
->display
.update_wm
= valleyview_update_wm
;
4505 dev_priv
->display
.init_clock_gating
=
4506 valleyview_init_clock_gating
;
4507 } else if (IS_PINEVIEW(dev
)) {
4508 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev
),
4511 dev_priv
->mem_freq
)) {
4512 DRM_INFO("failed to find known CxSR latency "
4513 "(found ddr%s fsb freq %d, mem freq %d), "
4515 (dev_priv
->is_ddr3
== 1) ? "3" : "2",
4516 dev_priv
->fsb_freq
, dev_priv
->mem_freq
);
4517 /* Disable CxSR and never update its watermark again */
4518 pineview_disable_cxsr(dev
);
4519 dev_priv
->display
.update_wm
= NULL
;
4521 dev_priv
->display
.update_wm
= pineview_update_wm
;
4522 dev_priv
->display
.init_clock_gating
= gen3_init_clock_gating
;
4523 } else if (IS_G4X(dev
)) {
4524 dev_priv
->display
.update_wm
= g4x_update_wm
;
4525 dev_priv
->display
.init_clock_gating
= g4x_init_clock_gating
;
4526 } else if (IS_GEN4(dev
)) {
4527 dev_priv
->display
.update_wm
= i965_update_wm
;
4528 if (IS_CRESTLINE(dev
))
4529 dev_priv
->display
.init_clock_gating
= crestline_init_clock_gating
;
4530 else if (IS_BROADWATER(dev
))
4531 dev_priv
->display
.init_clock_gating
= broadwater_init_clock_gating
;
4532 } else if (IS_GEN3(dev
)) {
4533 dev_priv
->display
.update_wm
= i9xx_update_wm
;
4534 dev_priv
->display
.get_fifo_size
= i9xx_get_fifo_size
;
4535 dev_priv
->display
.init_clock_gating
= gen3_init_clock_gating
;
4536 } else if (IS_I865G(dev
)) {
4537 dev_priv
->display
.update_wm
= i830_update_wm
;
4538 dev_priv
->display
.init_clock_gating
= i85x_init_clock_gating
;
4539 dev_priv
->display
.get_fifo_size
= i830_get_fifo_size
;
4540 } else if (IS_I85X(dev
)) {
4541 dev_priv
->display
.update_wm
= i9xx_update_wm
;
4542 dev_priv
->display
.get_fifo_size
= i85x_get_fifo_size
;
4543 dev_priv
->display
.init_clock_gating
= i85x_init_clock_gating
;
4545 dev_priv
->display
.update_wm
= i830_update_wm
;
4546 dev_priv
->display
.init_clock_gating
= i830_init_clock_gating
;
4548 dev_priv
->display
.get_fifo_size
= i845_get_fifo_size
;
4550 dev_priv
->display
.get_fifo_size
= i830_get_fifo_size
;
4554 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private
*dev_priv
)
4556 u32 gt_thread_status_mask
;
4558 if (IS_HASWELL(dev_priv
->dev
))
4559 gt_thread_status_mask
= GEN6_GT_THREAD_STATUS_CORE_MASK_HSW
;
4561 gt_thread_status_mask
= GEN6_GT_THREAD_STATUS_CORE_MASK
;
4563 /* w/a for a sporadic read returning 0 by waiting for the GT
4564 * thread to wake up.
4566 if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG
) & gt_thread_status_mask
) == 0, 500))
4567 DRM_ERROR("GT thread status wait timed out\n");
4570 static void __gen6_gt_force_wake_reset(struct drm_i915_private
*dev_priv
)
4572 I915_WRITE_NOTRACE(FORCEWAKE
, 0);
4573 POSTING_READ(ECOBUS
); /* something from same cacheline, but !FORCEWAKE */
4576 static void __gen6_gt_force_wake_get(struct drm_i915_private
*dev_priv
)
4578 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK
) & 1) == 0,
4579 FORCEWAKE_ACK_TIMEOUT_MS
))
4580 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
4582 I915_WRITE_NOTRACE(FORCEWAKE
, 1);
4583 POSTING_READ(ECOBUS
); /* something from same cacheline, but !FORCEWAKE */
4585 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK
) & 1),
4586 FORCEWAKE_ACK_TIMEOUT_MS
))
4587 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
4589 __gen6_gt_wait_for_thread_c0(dev_priv
);
4592 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private
*dev_priv
)
4594 I915_WRITE_NOTRACE(FORCEWAKE_MT
, _MASKED_BIT_DISABLE(0xffff));
4595 /* something from same cacheline, but !FORCEWAKE_MT */
4596 POSTING_READ(ECOBUS
);
4599 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private
*dev_priv
)
4603 if (IS_HASWELL(dev_priv
->dev
))
4604 forcewake_ack
= FORCEWAKE_ACK_HSW
;
4606 forcewake_ack
= FORCEWAKE_MT_ACK
;
4608 if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack
) & FORCEWAKE_KERNEL
) == 0,
4609 FORCEWAKE_ACK_TIMEOUT_MS
))
4610 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
4612 I915_WRITE_NOTRACE(FORCEWAKE_MT
, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL
));
4613 /* something from same cacheline, but !FORCEWAKE_MT */
4614 POSTING_READ(ECOBUS
);
4616 if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack
) & FORCEWAKE_KERNEL
),
4617 FORCEWAKE_ACK_TIMEOUT_MS
))
4618 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
4620 __gen6_gt_wait_for_thread_c0(dev_priv
);
4624 * Generally this is called implicitly by the register read function. However,
4625 * if some sequence requires the GT to not power down then this function should
4626 * be called at the beginning of the sequence followed by a call to
4627 * gen6_gt_force_wake_put() at the end of the sequence.
4629 void gen6_gt_force_wake_get(struct drm_i915_private
*dev_priv
)
4631 unsigned long irqflags
;
4633 spin_lock_irqsave(&dev_priv
->gt_lock
, irqflags
);
4634 if (dev_priv
->forcewake_count
++ == 0)
4635 dev_priv
->gt
.force_wake_get(dev_priv
);
4636 spin_unlock_irqrestore(&dev_priv
->gt_lock
, irqflags
);
4639 void gen6_gt_check_fifodbg(struct drm_i915_private
*dev_priv
)
4642 gtfifodbg
= I915_READ_NOTRACE(GTFIFODBG
);
4643 if (WARN(gtfifodbg
& GT_FIFO_CPU_ERROR_MASK
,
4644 "MMIO read or write has been dropped %x\n", gtfifodbg
))
4645 I915_WRITE_NOTRACE(GTFIFODBG
, GT_FIFO_CPU_ERROR_MASK
);
4648 static void __gen6_gt_force_wake_put(struct drm_i915_private
*dev_priv
)
4650 I915_WRITE_NOTRACE(FORCEWAKE
, 0);
4651 /* something from same cacheline, but !FORCEWAKE */
4652 POSTING_READ(ECOBUS
);
4653 gen6_gt_check_fifodbg(dev_priv
);
4656 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private
*dev_priv
)
4658 I915_WRITE_NOTRACE(FORCEWAKE_MT
, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL
));
4659 /* something from same cacheline, but !FORCEWAKE_MT */
4660 POSTING_READ(ECOBUS
);
4661 gen6_gt_check_fifodbg(dev_priv
);
4665 * see gen6_gt_force_wake_get()
4667 void gen6_gt_force_wake_put(struct drm_i915_private
*dev_priv
)
4669 unsigned long irqflags
;
4671 spin_lock_irqsave(&dev_priv
->gt_lock
, irqflags
);
4672 if (--dev_priv
->forcewake_count
== 0)
4673 dev_priv
->gt
.force_wake_put(dev_priv
);
4674 spin_unlock_irqrestore(&dev_priv
->gt_lock
, irqflags
);
4677 int __gen6_gt_wait_for_fifo(struct drm_i915_private
*dev_priv
)
4681 if (dev_priv
->gt_fifo_count
< GT_FIFO_NUM_RESERVED_ENTRIES
) {
4683 u32 fifo
= I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES
);
4684 while (fifo
<= GT_FIFO_NUM_RESERVED_ENTRIES
&& loop
--) {
4686 fifo
= I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES
);
4688 if (WARN_ON(loop
< 0 && fifo
<= GT_FIFO_NUM_RESERVED_ENTRIES
))
4690 dev_priv
->gt_fifo_count
= fifo
;
4692 dev_priv
->gt_fifo_count
--;
4697 static void vlv_force_wake_reset(struct drm_i915_private
*dev_priv
)
4699 I915_WRITE_NOTRACE(FORCEWAKE_VLV
, _MASKED_BIT_DISABLE(0xffff));
4700 /* something from same cacheline, but !FORCEWAKE_VLV */
4701 POSTING_READ(FORCEWAKE_ACK_VLV
);
4704 static void vlv_force_wake_get(struct drm_i915_private
*dev_priv
)
4706 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV
) & FORCEWAKE_KERNEL
) == 0,
4707 FORCEWAKE_ACK_TIMEOUT_MS
))
4708 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
4710 I915_WRITE_NOTRACE(FORCEWAKE_VLV
, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL
));
4711 I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV
,
4712 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL
));
4714 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV
) & FORCEWAKE_KERNEL
),
4715 FORCEWAKE_ACK_TIMEOUT_MS
))
4716 DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n");
4718 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_MEDIA_VLV
) &
4720 FORCEWAKE_ACK_TIMEOUT_MS
))
4721 DRM_ERROR("Timed out waiting for media to ack forcewake request.\n");
4723 __gen6_gt_wait_for_thread_c0(dev_priv
);
4726 static void vlv_force_wake_put(struct drm_i915_private
*dev_priv
)
4728 I915_WRITE_NOTRACE(FORCEWAKE_VLV
, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL
));
4729 I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV
,
4730 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL
));
4731 /* The below doubles as a POSTING_READ */
4732 gen6_gt_check_fifodbg(dev_priv
);
4735 void intel_gt_reset(struct drm_device
*dev
)
4737 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4739 if (IS_VALLEYVIEW(dev
)) {
4740 vlv_force_wake_reset(dev_priv
);
4741 } else if (INTEL_INFO(dev
)->gen
>= 6) {
4742 __gen6_gt_force_wake_reset(dev_priv
);
4743 if (IS_IVYBRIDGE(dev
) || IS_HASWELL(dev
))
4744 __gen6_gt_force_wake_mt_reset(dev_priv
);
4748 void intel_gt_init(struct drm_device
*dev
)
4750 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
4752 spin_lock_init(&dev_priv
->gt_lock
);
4754 intel_gt_reset(dev
);
4756 if (IS_VALLEYVIEW(dev
)) {
4757 dev_priv
->gt
.force_wake_get
= vlv_force_wake_get
;
4758 dev_priv
->gt
.force_wake_put
= vlv_force_wake_put
;
4759 } else if (IS_IVYBRIDGE(dev
) || IS_HASWELL(dev
)) {
4760 dev_priv
->gt
.force_wake_get
= __gen6_gt_force_wake_mt_get
;
4761 dev_priv
->gt
.force_wake_put
= __gen6_gt_force_wake_mt_put
;
4762 } else if (IS_GEN6(dev
)) {
4763 dev_priv
->gt
.force_wake_get
= __gen6_gt_force_wake_get
;
4764 dev_priv
->gt
.force_wake_put
= __gen6_gt_force_wake_put
;
4766 INIT_DELAYED_WORK(&dev_priv
->rps
.delayed_resume_work
,
4767 intel_gen6_powersave_work
);
4770 int sandybridge_pcode_read(struct drm_i915_private
*dev_priv
, u8 mbox
, u32
*val
)
4772 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
4774 if (I915_READ(GEN6_PCODE_MAILBOX
) & GEN6_PCODE_READY
) {
4775 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
4779 I915_WRITE(GEN6_PCODE_DATA
, *val
);
4780 I915_WRITE(GEN6_PCODE_MAILBOX
, GEN6_PCODE_READY
| mbox
);
4782 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX
) & GEN6_PCODE_READY
) == 0,
4784 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox
);
4788 *val
= I915_READ(GEN6_PCODE_DATA
);
4789 I915_WRITE(GEN6_PCODE_DATA
, 0);
4794 int sandybridge_pcode_write(struct drm_i915_private
*dev_priv
, u8 mbox
, u32 val
)
4796 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
4798 if (I915_READ(GEN6_PCODE_MAILBOX
) & GEN6_PCODE_READY
) {
4799 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
4803 I915_WRITE(GEN6_PCODE_DATA
, val
);
4804 I915_WRITE(GEN6_PCODE_MAILBOX
, GEN6_PCODE_READY
| mbox
);
4806 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX
) & GEN6_PCODE_READY
) == 0,
4808 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox
);
4812 I915_WRITE(GEN6_PCODE_DATA
, 0);
4817 static int vlv_punit_rw(struct drm_i915_private
*dev_priv
, u32 port
, u8 opcode
,
4820 u32 cmd
, devfn
, be
, bar
;
4824 devfn
= PCI_DEVFN(2, 0);
4826 cmd
= (devfn
<< IOSF_DEVFN_SHIFT
) | (opcode
<< IOSF_OPCODE_SHIFT
) |
4827 (port
<< IOSF_PORT_SHIFT
) | (be
<< IOSF_BYTE_ENABLES_SHIFT
) |
4828 (bar
<< IOSF_BAR_SHIFT
);
4830 WARN_ON(!mutex_is_locked(&dev_priv
->rps
.hw_lock
));
4832 if (I915_READ(VLV_IOSF_DOORBELL_REQ
) & IOSF_SB_BUSY
) {
4833 DRM_DEBUG_DRIVER("warning: pcode (%s) mailbox access failed\n",
4834 opcode
== PUNIT_OPCODE_REG_READ
?
4839 I915_WRITE(VLV_IOSF_ADDR
, addr
);
4840 if (opcode
== PUNIT_OPCODE_REG_WRITE
)
4841 I915_WRITE(VLV_IOSF_DATA
, *val
);
4842 I915_WRITE(VLV_IOSF_DOORBELL_REQ
, cmd
);
4844 if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ
) & IOSF_SB_BUSY
) == 0,
4846 DRM_ERROR("timeout waiting for pcode %s (%d) to finish\n",
4847 opcode
== PUNIT_OPCODE_REG_READ
? "read" : "write",
4852 if (opcode
== PUNIT_OPCODE_REG_READ
)
4853 *val
= I915_READ(VLV_IOSF_DATA
);
4854 I915_WRITE(VLV_IOSF_DATA
, 0);
4859 int valleyview_punit_read(struct drm_i915_private
*dev_priv
, u8 addr
, u32
*val
)
4861 return vlv_punit_rw(dev_priv
, IOSF_PORT_PUNIT
, PUNIT_OPCODE_REG_READ
,
4865 int valleyview_punit_write(struct drm_i915_private
*dev_priv
, u8 addr
, u32 val
)
4867 return vlv_punit_rw(dev_priv
, IOSF_PORT_PUNIT
, PUNIT_OPCODE_REG_WRITE
,
4871 int valleyview_nc_read(struct drm_i915_private
*dev_priv
, u8 addr
, u32
*val
)
4873 return vlv_punit_rw(dev_priv
, IOSF_PORT_NC
, PUNIT_OPCODE_REG_READ
,
4877 int vlv_gpu_freq(int ddr_freq
, int val
)
4898 return ((val
- 0xbd) * mult
) + base
;
4901 int vlv_freq_opcode(int ddr_freq
, int val
)