projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: Update DRIVER_DATE to 20150911
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_irq.c
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3 /*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/sysrq.h>
32 #include <linux/slab.h>
33 #include <linux/circ_buf.h>
34 #include <drm/drmP.h>
35 #include <drm/i915_drm.h>
36 #include "i915_drv.h"
37 #include "i915_trace.h"
38 #include "intel_drv.h"
39
40 /**
41 * DOC: interrupt handling
42 *
43 * These functions provide the basic support for enabling and disabling the
44 * interrupt handling support. There's a lot more functionality in i915_irq.c
45 * and related files, but that will be described in separate chapters.
46 */
47
48 static const u32 hpd_ilk[HPD_NUM_PINS] = {
49 [HPD_PORT_A] = DE_DP_A_HOTPLUG,
50 };
51
52 static const u32 hpd_ivb[HPD_NUM_PINS] = {
53 [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
54 };
55
56 static const u32 hpd_bdw[HPD_NUM_PINS] = {
57 [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
58 };
59
60 static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 [HPD_CRT] = SDE_CRT_HOTPLUG,
62 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
63 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
64 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
65 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
66 };
67
68 static const u32 hpd_cpt[HPD_NUM_PINS] = {
69 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
72 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
73 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
74 };
75
76 static const u32 hpd_spt[HPD_NUM_PINS] = {
77 [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
78 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
79 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
80 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
81 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
82 };
83
84 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
86 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
87 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
88 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
89 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
90 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
91 };
92
93 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
95 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
96 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
97 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
98 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
99 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
100 };
101
102 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
104 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
105 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
106 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
107 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
108 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
109 };
110
111 /* BXT hpd list */
112 static const u32 hpd_bxt[HPD_NUM_PINS] = {
113 [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
115 [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
116 };
117
118 /* IIR can theoretically queue up two events. Be paranoid. */
119 #define GEN8_IRQ_RESET_NDX(type, which) do { \
120 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
121 POSTING_READ(GEN8_##type##_IMR(which)); \
122 I915_WRITE(GEN8_##type##_IER(which), 0); \
123 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
124 POSTING_READ(GEN8_##type##_IIR(which)); \
125 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
126 POSTING_READ(GEN8_##type##_IIR(which)); \
127 } while (0)
128
129 #define GEN5_IRQ_RESET(type) do { \
130 I915_WRITE(type##IMR, 0xffffffff); \
131 POSTING_READ(type##IMR); \
132 I915_WRITE(type##IER, 0); \
133 I915_WRITE(type##IIR, 0xffffffff); \
134 POSTING_READ(type##IIR); \
135 I915_WRITE(type##IIR, 0xffffffff); \
136 POSTING_READ(type##IIR); \
137 } while (0)
138
139 /*
140 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
141 */
142 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
143 u32 val = I915_READ(reg); \
144 if (val) { \
145 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
146 (reg), val); \
147 I915_WRITE((reg), 0xffffffff); \
148 POSTING_READ(reg); \
149 I915_WRITE((reg), 0xffffffff); \
150 POSTING_READ(reg); \
151 } \
152 } while (0)
153
154 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
155 GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
156 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
157 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
158 POSTING_READ(GEN8_##type##_IMR(which)); \
159 } while (0)
160
161 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
162 GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
163 I915_WRITE(type##IER, (ier_val)); \
164 I915_WRITE(type##IMR, (imr_val)); \
165 POSTING_READ(type##IMR); \
166 } while (0)
167
168 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
169
170 /**
171 * ilk_update_display_irq - update DEIMR
172 * @dev_priv: driver private
173 * @interrupt_mask: mask of interrupt bits to update
174 * @enabled_irq_mask: mask of interrupt bits to enable
175 */
176 static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
177 uint32_t interrupt_mask,
178 uint32_t enabled_irq_mask)
179 {
180 uint32_t new_val;
181
182 assert_spin_locked(&dev_priv->irq_lock);
183
184 WARN_ON(enabled_irq_mask & ~interrupt_mask);
185
186 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
187 return;
188
189 new_val = dev_priv->irq_mask;
190 new_val &= ~interrupt_mask;
191 new_val |= (~enabled_irq_mask & interrupt_mask);
192
193 if (new_val != dev_priv->irq_mask) {
194 dev_priv->irq_mask = new_val;
195 I915_WRITE(DEIMR, dev_priv->irq_mask);
196 POSTING_READ(DEIMR);
197 }
198 }
199
200 void
201 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
202 {
203 ilk_update_display_irq(dev_priv, mask, mask);
204 }
205
206 void
207 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
208 {
209 ilk_update_display_irq(dev_priv, mask, 0);
210 }
211
212 /**
213 * ilk_update_gt_irq - update GTIMR
214 * @dev_priv: driver private
215 * @interrupt_mask: mask of interrupt bits to update
216 * @enabled_irq_mask: mask of interrupt bits to enable
217 */
218 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
219 uint32_t interrupt_mask,
220 uint32_t enabled_irq_mask)
221 {
222 assert_spin_locked(&dev_priv->irq_lock);
223
224 WARN_ON(enabled_irq_mask & ~interrupt_mask);
225
226 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
227 return;
228
229 dev_priv->gt_irq_mask &= ~interrupt_mask;
230 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
231 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
232 POSTING_READ(GTIMR);
233 }
234
235 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
236 {
237 ilk_update_gt_irq(dev_priv, mask, mask);
238 }
239
240 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
241 {
242 ilk_update_gt_irq(dev_priv, mask, 0);
243 }
244
245 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
246 {
247 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
248 }
249
250 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
251 {
252 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
253 }
254
255 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
256 {
257 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
258 }
259
260 /**
261 * snb_update_pm_irq - update GEN6_PMIMR
262 * @dev_priv: driver private
263 * @interrupt_mask: mask of interrupt bits to update
264 * @enabled_irq_mask: mask of interrupt bits to enable
265 */
266 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
267 uint32_t interrupt_mask,
268 uint32_t enabled_irq_mask)
269 {
270 uint32_t new_val;
271
272 WARN_ON(enabled_irq_mask & ~interrupt_mask);
273
274 assert_spin_locked(&dev_priv->irq_lock);
275
276 new_val = dev_priv->pm_irq_mask;
277 new_val &= ~interrupt_mask;
278 new_val |= (~enabled_irq_mask & interrupt_mask);
279
280 if (new_val != dev_priv->pm_irq_mask) {
281 dev_priv->pm_irq_mask = new_val;
282 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
283 POSTING_READ(gen6_pm_imr(dev_priv));
284 }
285 }
286
287 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
288 {
289 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
290 return;
291
292 snb_update_pm_irq(dev_priv, mask, mask);
293 }
294
295 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
296 uint32_t mask)
297 {
298 snb_update_pm_irq(dev_priv, mask, 0);
299 }
300
301 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
302 {
303 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
304 return;
305
306 __gen6_disable_pm_irq(dev_priv, mask);
307 }
308
309 void gen6_reset_rps_interrupts(struct drm_device *dev)
310 {
311 struct drm_i915_private *dev_priv = dev->dev_private;
312 uint32_t reg = gen6_pm_iir(dev_priv);
313
314 spin_lock_irq(&dev_priv->irq_lock);
315 I915_WRITE(reg, dev_priv->pm_rps_events);
316 I915_WRITE(reg, dev_priv->pm_rps_events);
317 POSTING_READ(reg);
318 dev_priv->rps.pm_iir = 0;
319 spin_unlock_irq(&dev_priv->irq_lock);
320 }
321
322 void gen6_enable_rps_interrupts(struct drm_device *dev)
323 {
324 struct drm_i915_private *dev_priv = dev->dev_private;
325
326 spin_lock_irq(&dev_priv->irq_lock);
327
328 WARN_ON(dev_priv->rps.pm_iir);
329 WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
330 dev_priv->rps.interrupts_enabled = true;
331 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
332 dev_priv->pm_rps_events);
333 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
334
335 spin_unlock_irq(&dev_priv->irq_lock);
336 }
337
338 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
339 {
340 /*
341 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
342 * if GEN6_PM_UP_EI_EXPIRED is masked.
343 *
344 * TODO: verify if this can be reproduced on VLV,CHV.
345 */
346 if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
347 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
348
349 if (INTEL_INFO(dev_priv)->gen >= 8)
350 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
351
352 return mask;
353 }
354
355 void gen6_disable_rps_interrupts(struct drm_device *dev)
356 {
357 struct drm_i915_private *dev_priv = dev->dev_private;
358
359 spin_lock_irq(&dev_priv->irq_lock);
360 dev_priv->rps.interrupts_enabled = false;
361 spin_unlock_irq(&dev_priv->irq_lock);
362
363 cancel_work_sync(&dev_priv->rps.work);
364
365 spin_lock_irq(&dev_priv->irq_lock);
366
367 I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
368
369 __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
370 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
371 ~dev_priv->pm_rps_events);
372
373 spin_unlock_irq(&dev_priv->irq_lock);
374
375 synchronize_irq(dev->irq);
376 }
377
378 /**
379 * bdw_update_port_irq - update DE port interrupt
380 * @dev_priv: driver private
381 * @interrupt_mask: mask of interrupt bits to update
382 * @enabled_irq_mask: mask of interrupt bits to enable
383 */
384 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
385 uint32_t interrupt_mask,
386 uint32_t enabled_irq_mask)
387 {
388 uint32_t new_val;
389 uint32_t old_val;
390
391 assert_spin_locked(&dev_priv->irq_lock);
392
393 WARN_ON(enabled_irq_mask & ~interrupt_mask);
394
395 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
396 return;
397
398 old_val = I915_READ(GEN8_DE_PORT_IMR);
399
400 new_val = old_val;
401 new_val &= ~interrupt_mask;
402 new_val |= (~enabled_irq_mask & interrupt_mask);
403
404 if (new_val != old_val) {
405 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
406 POSTING_READ(GEN8_DE_PORT_IMR);
407 }
408 }
409
410 /**
411 * ibx_display_interrupt_update - update SDEIMR
412 * @dev_priv: driver private
413 * @interrupt_mask: mask of interrupt bits to update
414 * @enabled_irq_mask: mask of interrupt bits to enable
415 */
416 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
417 uint32_t interrupt_mask,
418 uint32_t enabled_irq_mask)
419 {
420 uint32_t sdeimr = I915_READ(SDEIMR);
421 sdeimr &= ~interrupt_mask;
422 sdeimr |= (~enabled_irq_mask & interrupt_mask);
423
424 WARN_ON(enabled_irq_mask & ~interrupt_mask);
425
426 assert_spin_locked(&dev_priv->irq_lock);
427
428 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
429 return;
430
431 I915_WRITE(SDEIMR, sdeimr);
432 POSTING_READ(SDEIMR);
433 }
434
435 static void
436 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
437 u32 enable_mask, u32 status_mask)
438 {
439 u32 reg = PIPESTAT(pipe);
440 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
441
442 assert_spin_locked(&dev_priv->irq_lock);
443 WARN_ON(!intel_irqs_enabled(dev_priv));
444
445 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
446 status_mask & ~PIPESTAT_INT_STATUS_MASK,
447 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
448 pipe_name(pipe), enable_mask, status_mask))
449 return;
450
451 if ((pipestat & enable_mask) == enable_mask)
452 return;
453
454 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
455
456 /* Enable the interrupt, clear any pending status */
457 pipestat |= enable_mask | status_mask;
458 I915_WRITE(reg, pipestat);
459 POSTING_READ(reg);
460 }
461
462 static void
463 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
464 u32 enable_mask, u32 status_mask)
465 {
466 u32 reg = PIPESTAT(pipe);
467 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
468
469 assert_spin_locked(&dev_priv->irq_lock);
470 WARN_ON(!intel_irqs_enabled(dev_priv));
471
472 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
473 status_mask & ~PIPESTAT_INT_STATUS_MASK,
474 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
475 pipe_name(pipe), enable_mask, status_mask))
476 return;
477
478 if ((pipestat & enable_mask) == 0)
479 return;
480
481 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
482
483 pipestat &= ~enable_mask;
484 I915_WRITE(reg, pipestat);
485 POSTING_READ(reg);
486 }
487
488 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
489 {
490 u32 enable_mask = status_mask << 16;
491
492 /*
493 * On pipe A we don't support the PSR interrupt yet,
494 * on pipe B and C the same bit MBZ.
495 */
496 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
497 return 0;
498 /*
499 * On pipe B and C we don't support the PSR interrupt yet, on pipe
500 * A the same bit is for perf counters which we don't use either.
501 */
502 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
503 return 0;
504
505 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
506 SPRITE0_FLIP_DONE_INT_EN_VLV |
507 SPRITE1_FLIP_DONE_INT_EN_VLV);
508 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
509 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
510 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
511 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
512
513 return enable_mask;
514 }
515
516 void
517 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
518 u32 status_mask)
519 {
520 u32 enable_mask;
521
522 if (IS_VALLEYVIEW(dev_priv->dev))
523 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
524 status_mask);
525 else
526 enable_mask = status_mask << 16;
527 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
528 }
529
530 void
531 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
532 u32 status_mask)
533 {
534 u32 enable_mask;
535
536 if (IS_VALLEYVIEW(dev_priv->dev))
537 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
538 status_mask);
539 else
540 enable_mask = status_mask << 16;
541 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
542 }
543
544 /**
545 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
546 */
547 static void i915_enable_asle_pipestat(struct drm_device *dev)
548 {
549 struct drm_i915_private *dev_priv = dev->dev_private;
550
551 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
552 return;
553
554 spin_lock_irq(&dev_priv->irq_lock);
555
556 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
557 if (INTEL_INFO(dev)->gen >= 4)
558 i915_enable_pipestat(dev_priv, PIPE_A,
559 PIPE_LEGACY_BLC_EVENT_STATUS);
560
561 spin_unlock_irq(&dev_priv->irq_lock);
562 }
563
564 /*
565 * This timing diagram depicts the video signal in and
566 * around the vertical blanking period.
567 *
568 * Assumptions about the fictitious mode used in this example:
569 * vblank_start >= 3
570 * vsync_start = vblank_start + 1
571 * vsync_end = vblank_start + 2
572 * vtotal = vblank_start + 3
573 *
574 * start of vblank:
575 * latch double buffered registers
576 * increment frame counter (ctg+)
577 * generate start of vblank interrupt (gen4+)
578 * |
579 * | frame start:
580 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
581 * | may be shifted forward 1-3 extra lines via PIPECONF
582 * | |
583 * | | start of vsync:
584 * | | generate vsync interrupt
585 * | | |
586 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
587 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
588 * ----va---> <-----------------vb--------------------> <--------va-------------
589 * | | <----vs-----> |
590 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
591 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
592 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
593 * | | |
594 * last visible pixel first visible pixel
595 * | increment frame counter (gen3/4)
596 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
597 *
598 * x = horizontal active
599 * _ = horizontal blanking
600 * hs = horizontal sync
601 * va = vertical active
602 * vb = vertical blanking
603 * vs = vertical sync
604 * vbs = vblank_start (number)
605 *
606 * Summary:
607 * - most events happen at the start of horizontal sync
608 * - frame start happens at the start of horizontal blank, 1-4 lines
609 * (depending on PIPECONF settings) after the start of vblank
610 * - gen3/4 pixel and frame counter are synchronized with the start
611 * of horizontal active on the first line of vertical active
612 */
613
614 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
615 {
616 /* Gen2 doesn't have a hardware frame counter */
617 return 0;
618 }
619
620 /* Called from drm generic code, passed a 'crtc', which
621 * we use as a pipe index
622 */
623 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
624 {
625 struct drm_i915_private *dev_priv = dev->dev_private;
626 unsigned long high_frame;
627 unsigned long low_frame;
628 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
629 struct intel_crtc *intel_crtc =
630 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
631 const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
632
633 htotal = mode->crtc_htotal;
634 hsync_start = mode->crtc_hsync_start;
635 vbl_start = mode->crtc_vblank_start;
636 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
637 vbl_start = DIV_ROUND_UP(vbl_start, 2);
638
639 /* Convert to pixel count */
640 vbl_start *= htotal;
641
642 /* Start of vblank event occurs at start of hsync */
643 vbl_start -= htotal - hsync_start;
644
645 high_frame = PIPEFRAME(pipe);
646 low_frame = PIPEFRAMEPIXEL(pipe);
647
648 /*
649 * High & low register fields aren't synchronized, so make sure
650 * we get a low value that's stable across two reads of the high
651 * register.
652 */
653 do {
654 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
655 low = I915_READ(low_frame);
656 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
657 } while (high1 != high2);
658
659 high1 >>= PIPE_FRAME_HIGH_SHIFT;
660 pixel = low & PIPE_PIXEL_MASK;
661 low >>= PIPE_FRAME_LOW_SHIFT;
662
663 /*
664 * The frame counter increments at beginning of active.
665 * Cook up a vblank counter by also checking the pixel
666 * counter against vblank start.
667 */
668 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
669 }
670
671 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
672 {
673 struct drm_i915_private *dev_priv = dev->dev_private;
674 int reg = PIPE_FRMCOUNT_GM45(pipe);
675
676 return I915_READ(reg);
677 }
678
679 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
680 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
681
682 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
683 {
684 struct drm_device *dev = crtc->base.dev;
685 struct drm_i915_private *dev_priv = dev->dev_private;
686 const struct drm_display_mode *mode = &crtc->base.hwmode;
687 enum pipe pipe = crtc->pipe;
688 int position, vtotal;
689
690 vtotal = mode->crtc_vtotal;
691 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
692 vtotal /= 2;
693
694 if (IS_GEN2(dev))
695 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
696 else
697 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
698
699 /*
700 * See update_scanline_offset() for the details on the
701 * scanline_offset adjustment.
702 */
703 return (position + crtc->scanline_offset) % vtotal;
704 }
705
706 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
707 unsigned int flags, int *vpos, int *hpos,
708 ktime_t *stime, ktime_t *etime)
709 {
710 struct drm_i915_private *dev_priv = dev->dev_private;
711 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
712 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
713 const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
714 int position;
715 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
716 bool in_vbl = true;
717 int ret = 0;
718 unsigned long irqflags;
719
720 if (WARN_ON(!mode->crtc_clock)) {
721 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
722 "pipe %c\n", pipe_name(pipe));
723 return 0;
724 }
725
726 htotal = mode->crtc_htotal;
727 hsync_start = mode->crtc_hsync_start;
728 vtotal = mode->crtc_vtotal;
729 vbl_start = mode->crtc_vblank_start;
730 vbl_end = mode->crtc_vblank_end;
731
732 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
733 vbl_start = DIV_ROUND_UP(vbl_start, 2);
734 vbl_end /= 2;
735 vtotal /= 2;
736 }
737
738 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
739
740 /*
741 * Lock uncore.lock, as we will do multiple timing critical raw
742 * register reads, potentially with preemption disabled, so the
743 * following code must not block on uncore.lock.
744 */
745 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
746
747 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
748
749 /* Get optional system timestamp before query. */
750 if (stime)
751 *stime = ktime_get();
752
753 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
754 /* No obvious pixelcount register. Only query vertical
755 * scanout position from Display scan line register.
756 */
757 position = __intel_get_crtc_scanline(intel_crtc);
758 } else {
759 /* Have access to pixelcount since start of frame.
760 * We can split this into vertical and horizontal
761 * scanout position.
762 */
763 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
764
765 /* convert to pixel counts */
766 vbl_start *= htotal;
767 vbl_end *= htotal;
768 vtotal *= htotal;
769
770 /*
771 * In interlaced modes, the pixel counter counts all pixels,
772 * so one field will have htotal more pixels. In order to avoid
773 * the reported position from jumping backwards when the pixel
774 * counter is beyond the length of the shorter field, just
775 * clamp the position the length of the shorter field. This
776 * matches how the scanline counter based position works since
777 * the scanline counter doesn't count the two half lines.
778 */
779 if (position >= vtotal)
780 position = vtotal - 1;
781
782 /*
783 * Start of vblank interrupt is triggered at start of hsync,
784 * just prior to the first active line of vblank. However we
785 * consider lines to start at the leading edge of horizontal
786 * active. So, should we get here before we've crossed into
787 * the horizontal active of the first line in vblank, we would
788 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
789 * always add htotal-hsync_start to the current pixel position.
790 */
791 position = (position + htotal - hsync_start) % vtotal;
792 }
793
794 /* Get optional system timestamp after query. */
795 if (etime)
796 *etime = ktime_get();
797
798 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
799
800 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
801
802 in_vbl = position >= vbl_start && position < vbl_end;
803
804 /*
805 * While in vblank, position will be negative
806 * counting up towards 0 at vbl_end. And outside
807 * vblank, position will be positive counting
808 * up since vbl_end.
809 */
810 if (position >= vbl_start)
811 position -= vbl_end;
812 else
813 position += vtotal - vbl_end;
814
815 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
816 *vpos = position;
817 *hpos = 0;
818 } else {
819 *vpos = position / htotal;
820 *hpos = position - (*vpos * htotal);
821 }
822
823 /* In vblank? */
824 if (in_vbl)
825 ret |= DRM_SCANOUTPOS_IN_VBLANK;
826
827 return ret;
828 }
829
830 int intel_get_crtc_scanline(struct intel_crtc *crtc)
831 {
832 struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
833 unsigned long irqflags;
834 int position;
835
836 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
837 position = __intel_get_crtc_scanline(crtc);
838 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
839
840 return position;
841 }
842
843 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
844 int *max_error,
845 struct timeval *vblank_time,
846 unsigned flags)
847 {
848 struct drm_crtc *crtc;
849
850 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
851 DRM_ERROR("Invalid crtc %d\n", pipe);
852 return -EINVAL;
853 }
854
855 /* Get drm_crtc to timestamp: */
856 crtc = intel_get_crtc_for_pipe(dev, pipe);
857 if (crtc == NULL) {
858 DRM_ERROR("Invalid crtc %d\n", pipe);
859 return -EINVAL;
860 }
861
862 if (!crtc->hwmode.crtc_clock) {
863 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
864 return -EBUSY;
865 }
866
867 /* Helper routine in DRM core does all the work: */
868 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
869 vblank_time, flags,
870 crtc,
871 &crtc->hwmode);
872 }
873
874 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
875 {
876 struct drm_i915_private *dev_priv = dev->dev_private;
877 u32 busy_up, busy_down, max_avg, min_avg;
878 u8 new_delay;
879
880 spin_lock(&mchdev_lock);
881
882 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
883
884 new_delay = dev_priv->ips.cur_delay;
885
886 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
887 busy_up = I915_READ(RCPREVBSYTUPAVG);
888 busy_down = I915_READ(RCPREVBSYTDNAVG);
889 max_avg = I915_READ(RCBMAXAVG);
890 min_avg = I915_READ(RCBMINAVG);
891
892 /* Handle RCS change request from hw */
893 if (busy_up > max_avg) {
894 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
895 new_delay = dev_priv->ips.cur_delay - 1;
896 if (new_delay < dev_priv->ips.max_delay)
897 new_delay = dev_priv->ips.max_delay;
898 } else if (busy_down < min_avg) {
899 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
900 new_delay = dev_priv->ips.cur_delay + 1;
901 if (new_delay > dev_priv->ips.min_delay)
902 new_delay = dev_priv->ips.min_delay;
903 }
904
905 if (ironlake_set_drps(dev, new_delay))
906 dev_priv->ips.cur_delay = new_delay;
907
908 spin_unlock(&mchdev_lock);
909
910 return;
911 }
912
913 static void notify_ring(struct intel_engine_cs *ring)
914 {
915 if (!intel_ring_initialized(ring))
916 return;
917
918 trace_i915_gem_request_notify(ring);
919
920 wake_up_all(&ring->irq_queue);
921 }
922
923 static void vlv_c0_read(struct drm_i915_private *dev_priv,
924 struct intel_rps_ei *ei)
925 {
926 ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
927 ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
928 ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
929 }
930
931 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
932 const struct intel_rps_ei *old,
933 const struct intel_rps_ei *now,
934 int threshold)
935 {
936 u64 time, c0;
937
938 if (old->cz_clock == 0)
939 return false;
940
941 time = now->cz_clock - old->cz_clock;
942 time *= threshold * dev_priv->mem_freq;
943
944 /* Workload can be split between render + media, e.g. SwapBuffers
945 * being blitted in X after being rendered in mesa. To account for
946 * this we need to combine both engines into our activity counter.
947 */
948 c0 = now->render_c0 - old->render_c0;
949 c0 += now->media_c0 - old->media_c0;
950 c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
951
952 return c0 >= time;
953 }
954
955 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
956 {
957 vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
958 dev_priv->rps.up_ei = dev_priv->rps.down_ei;
959 }
960
961 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
962 {
963 struct intel_rps_ei now;
964 u32 events = 0;
965
966 if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
967 return 0;
968
969 vlv_c0_read(dev_priv, &now);
970 if (now.cz_clock == 0)
971 return 0;
972
973 if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
974 if (!vlv_c0_above(dev_priv,
975 &dev_priv->rps.down_ei, &now,
976 dev_priv->rps.down_threshold))
977 events |= GEN6_PM_RP_DOWN_THRESHOLD;
978 dev_priv->rps.down_ei = now;
979 }
980
981 if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
982 if (vlv_c0_above(dev_priv,
983 &dev_priv->rps.up_ei, &now,
984 dev_priv->rps.up_threshold))
985 events |= GEN6_PM_RP_UP_THRESHOLD;
986 dev_priv->rps.up_ei = now;
987 }
988
989 return events;
990 }
991
992 static bool any_waiters(struct drm_i915_private *dev_priv)
993 {
994 struct intel_engine_cs *ring;
995 int i;
996
997 for_each_ring(ring, dev_priv, i)
998 if (ring->irq_refcount)
999 return true;
1000
1001 return false;
1002 }
1003
1004 static void gen6_pm_rps_work(struct work_struct *work)
1005 {
1006 struct drm_i915_private *dev_priv =
1007 container_of(work, struct drm_i915_private, rps.work);
1008 bool client_boost;
1009 int new_delay, adj, min, max;
1010 u32 pm_iir;
1011
1012 spin_lock_irq(&dev_priv->irq_lock);
1013 /* Speed up work cancelation during disabling rps interrupts. */
1014 if (!dev_priv->rps.interrupts_enabled) {
1015 spin_unlock_irq(&dev_priv->irq_lock);
1016 return;
1017 }
1018 pm_iir = dev_priv->rps.pm_iir;
1019 dev_priv->rps.pm_iir = 0;
1020 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1021 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1022 client_boost = dev_priv->rps.client_boost;
1023 dev_priv->rps.client_boost = false;
1024 spin_unlock_irq(&dev_priv->irq_lock);
1025
1026 /* Make sure we didn't queue anything we're not going to process. */
1027 WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1028
1029 if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1030 return;
1031
1032 mutex_lock(&dev_priv->rps.hw_lock);
1033
1034 pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1035
1036 adj = dev_priv->rps.last_adj;
1037 new_delay = dev_priv->rps.cur_freq;
1038 min = dev_priv->rps.min_freq_softlimit;
1039 max = dev_priv->rps.max_freq_softlimit;
1040
1041 if (client_boost) {
1042 new_delay = dev_priv->rps.max_freq_softlimit;
1043 adj = 0;
1044 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1045 if (adj > 0)
1046 adj *= 2;
1047 else /* CHV needs even encode values */
1048 adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1049 /*
1050 * For better performance, jump directly
1051 * to RPe if we're below it.
1052 */
1053 if (new_delay < dev_priv->rps.efficient_freq - adj) {
1054 new_delay = dev_priv->rps.efficient_freq;
1055 adj = 0;
1056 }
1057 } else if (any_waiters(dev_priv)) {
1058 adj = 0;
1059 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1060 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1061 new_delay = dev_priv->rps.efficient_freq;
1062 else
1063 new_delay = dev_priv->rps.min_freq_softlimit;
1064 adj = 0;
1065 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1066 if (adj < 0)
1067 adj *= 2;
1068 else /* CHV needs even encode values */
1069 adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1070 } else { /* unknown event */
1071 adj = 0;
1072 }
1073
1074 dev_priv->rps.last_adj = adj;
1075
1076 /* sysfs frequency interfaces may have snuck in while servicing the
1077 * interrupt
1078 */
1079 new_delay += adj;
1080 new_delay = clamp_t(int, new_delay, min, max);
1081
1082 intel_set_rps(dev_priv->dev, new_delay);
1083
1084 mutex_unlock(&dev_priv->rps.hw_lock);
1085 }
1086
1087
1088 /**
1089 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1090 * occurred.
1091 * @work: workqueue struct
1092 *
1093 * Doesn't actually do anything except notify userspace. As a consequence of
1094 * this event, userspace should try to remap the bad rows since statistically
1095 * it is likely the same row is more likely to go bad again.
1096 */
1097 static void ivybridge_parity_work(struct work_struct *work)
1098 {
1099 struct drm_i915_private *dev_priv =
1100 container_of(work, struct drm_i915_private, l3_parity.error_work);
1101 u32 error_status, row, bank, subbank;
1102 char *parity_event[6];
1103 uint32_t misccpctl;
1104 uint8_t slice = 0;
1105
1106 /* We must turn off DOP level clock gating to access the L3 registers.
1107 * In order to prevent a get/put style interface, acquire struct mutex
1108 * any time we access those registers.
1109 */
1110 mutex_lock(&dev_priv->dev->struct_mutex);
1111
1112 /* If we've screwed up tracking, just let the interrupt fire again */
1113 if (WARN_ON(!dev_priv->l3_parity.which_slice))
1114 goto out;
1115
1116 misccpctl = I915_READ(GEN7_MISCCPCTL);
1117 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1118 POSTING_READ(GEN7_MISCCPCTL);
1119
1120 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1121 u32 reg;
1122
1123 slice--;
1124 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1125 break;
1126
1127 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1128
1129 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1130
1131 error_status = I915_READ(reg);
1132 row = GEN7_PARITY_ERROR_ROW(error_status);
1133 bank = GEN7_PARITY_ERROR_BANK(error_status);
1134 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1135
1136 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1137 POSTING_READ(reg);
1138
1139 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1140 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1141 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1142 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1143 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1144 parity_event[5] = NULL;
1145
1146 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1147 KOBJ_CHANGE, parity_event);
1148
1149 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1150 slice, row, bank, subbank);
1151
1152 kfree(parity_event[4]);
1153 kfree(parity_event[3]);
1154 kfree(parity_event[2]);
1155 kfree(parity_event[1]);
1156 }
1157
1158 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1159
1160 out:
1161 WARN_ON(dev_priv->l3_parity.which_slice);
1162 spin_lock_irq(&dev_priv->irq_lock);
1163 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1164 spin_unlock_irq(&dev_priv->irq_lock);
1165
1166 mutex_unlock(&dev_priv->dev->struct_mutex);
1167 }
1168
1169 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1170 {
1171 struct drm_i915_private *dev_priv = dev->dev_private;
1172
1173 if (!HAS_L3_DPF(dev))
1174 return;
1175
1176 spin_lock(&dev_priv->irq_lock);
1177 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1178 spin_unlock(&dev_priv->irq_lock);
1179
1180 iir &= GT_PARITY_ERROR(dev);
1181 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1182 dev_priv->l3_parity.which_slice |= 1 << 1;
1183
1184 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1185 dev_priv->l3_parity.which_slice |= 1 << 0;
1186
1187 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1188 }
1189
1190 static void ilk_gt_irq_handler(struct drm_device *dev,
1191 struct drm_i915_private *dev_priv,
1192 u32 gt_iir)
1193 {
1194 if (gt_iir &
1195 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1196 notify_ring(&dev_priv->ring[RCS]);
1197 if (gt_iir & ILK_BSD_USER_INTERRUPT)
1198 notify_ring(&dev_priv->ring[VCS]);
1199 }
1200
1201 static void snb_gt_irq_handler(struct drm_device *dev,
1202 struct drm_i915_private *dev_priv,
1203 u32 gt_iir)
1204 {
1205
1206 if (gt_iir &
1207 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1208 notify_ring(&dev_priv->ring[RCS]);
1209 if (gt_iir & GT_BSD_USER_INTERRUPT)
1210 notify_ring(&dev_priv->ring[VCS]);
1211 if (gt_iir & GT_BLT_USER_INTERRUPT)
1212 notify_ring(&dev_priv->ring[BCS]);
1213
1214 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1215 GT_BSD_CS_ERROR_INTERRUPT |
1216 GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1217 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1218
1219 if (gt_iir & GT_PARITY_ERROR(dev))
1220 ivybridge_parity_error_irq_handler(dev, gt_iir);
1221 }
1222
1223 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1224 u32 master_ctl)
1225 {
1226 irqreturn_t ret = IRQ_NONE;
1227
1228 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1229 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1230 if (tmp) {
1231 I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1232 ret = IRQ_HANDLED;
1233
1234 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1235 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1236 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1237 notify_ring(&dev_priv->ring[RCS]);
1238
1239 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1240 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1241 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1242 notify_ring(&dev_priv->ring[BCS]);
1243 } else
1244 DRM_ERROR("The master control interrupt lied (GT0)!\n");
1245 }
1246
1247 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1248 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1249 if (tmp) {
1250 I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1251 ret = IRQ_HANDLED;
1252
1253 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1254 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1255 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1256 notify_ring(&dev_priv->ring[VCS]);
1257
1258 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1259 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1260 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1261 notify_ring(&dev_priv->ring[VCS2]);
1262 } else
1263 DRM_ERROR("The master control interrupt lied (GT1)!\n");
1264 }
1265
1266 if (master_ctl & GEN8_GT_VECS_IRQ) {
1267 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1268 if (tmp) {
1269 I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1270 ret = IRQ_HANDLED;
1271
1272 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1273 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1274 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1275 notify_ring(&dev_priv->ring[VECS]);
1276 } else
1277 DRM_ERROR("The master control interrupt lied (GT3)!\n");
1278 }
1279
1280 if (master_ctl & GEN8_GT_PM_IRQ) {
1281 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1282 if (tmp & dev_priv->pm_rps_events) {
1283 I915_WRITE_FW(GEN8_GT_IIR(2),
1284 tmp & dev_priv->pm_rps_events);
1285 ret = IRQ_HANDLED;
1286 gen6_rps_irq_handler(dev_priv, tmp);
1287 } else
1288 DRM_ERROR("The master control interrupt lied (PM)!\n");
1289 }
1290
1291 return ret;
1292 }
1293
1294 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1295 {
1296 switch (port) {
1297 case PORT_A:
1298 return val & PORTA_HOTPLUG_LONG_DETECT;
1299 case PORT_B:
1300 return val & PORTB_HOTPLUG_LONG_DETECT;
1301 case PORT_C:
1302 return val & PORTC_HOTPLUG_LONG_DETECT;
1303 default:
1304 return false;
1305 }
1306 }
1307
1308 static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1309 {
1310 switch (port) {
1311 case PORT_E:
1312 return val & PORTE_HOTPLUG_LONG_DETECT;
1313 default:
1314 return false;
1315 }
1316 }
1317
1318 static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1319 {
1320 switch (port) {
1321 case PORT_A:
1322 return val & PORTA_HOTPLUG_LONG_DETECT;
1323 case PORT_B:
1324 return val & PORTB_HOTPLUG_LONG_DETECT;
1325 case PORT_C:
1326 return val & PORTC_HOTPLUG_LONG_DETECT;
1327 case PORT_D:
1328 return val & PORTD_HOTPLUG_LONG_DETECT;
1329 default:
1330 return false;
1331 }
1332 }
1333
1334 static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1335 {
1336 switch (port) {
1337 case PORT_A:
1338 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1339 default:
1340 return false;
1341 }
1342 }
1343
1344 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1345 {
1346 switch (port) {
1347 case PORT_B:
1348 return val & PORTB_HOTPLUG_LONG_DETECT;
1349 case PORT_C:
1350 return val & PORTC_HOTPLUG_LONG_DETECT;
1351 case PORT_D:
1352 return val & PORTD_HOTPLUG_LONG_DETECT;
1353 default:
1354 return false;
1355 }
1356 }
1357
1358 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1359 {
1360 switch (port) {
1361 case PORT_B:
1362 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1363 case PORT_C:
1364 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1365 case PORT_D:
1366 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1367 default:
1368 return false;
1369 }
1370 }
1371
1372 /*
1373 * Get a bit mask of pins that have triggered, and which ones may be long.
1374 * This can be called multiple times with the same masks to accumulate
1375 * hotplug detection results from several registers.
1376 *
1377 * Note that the caller is expected to zero out the masks initially.
1378 */
1379 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1380 u32 hotplug_trigger, u32 dig_hotplug_reg,
1381 const u32 hpd[HPD_NUM_PINS],
1382 bool long_pulse_detect(enum port port, u32 val))
1383 {
1384 enum port port;
1385 int i;
1386
1387 for_each_hpd_pin(i) {
1388 if ((hpd[i] & hotplug_trigger) == 0)
1389 continue;
1390
1391 *pin_mask |= BIT(i);
1392
1393 if (!intel_hpd_pin_to_port(i, &port))
1394 continue;
1395
1396 if (long_pulse_detect(port, dig_hotplug_reg))
1397 *long_mask |= BIT(i);
1398 }
1399
1400 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1401 hotplug_trigger, dig_hotplug_reg, *pin_mask);
1402
1403 }
1404
1405 static void gmbus_irq_handler(struct drm_device *dev)
1406 {
1407 struct drm_i915_private *dev_priv = dev->dev_private;
1408
1409 wake_up_all(&dev_priv->gmbus_wait_queue);
1410 }
1411
1412 static void dp_aux_irq_handler(struct drm_device *dev)
1413 {
1414 struct drm_i915_private *dev_priv = dev->dev_private;
1415
1416 wake_up_all(&dev_priv->gmbus_wait_queue);
1417 }
1418
1419 #if defined(CONFIG_DEBUG_FS)
1420 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1421 uint32_t crc0, uint32_t crc1,
1422 uint32_t crc2, uint32_t crc3,
1423 uint32_t crc4)
1424 {
1425 struct drm_i915_private *dev_priv = dev->dev_private;
1426 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1427 struct intel_pipe_crc_entry *entry;
1428 int head, tail;
1429
1430 spin_lock(&pipe_crc->lock);
1431
1432 if (!pipe_crc->entries) {
1433 spin_unlock(&pipe_crc->lock);
1434 DRM_DEBUG_KMS("spurious interrupt\n");
1435 return;
1436 }
1437
1438 head = pipe_crc->head;
1439 tail = pipe_crc->tail;
1440
1441 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1442 spin_unlock(&pipe_crc->lock);
1443 DRM_ERROR("CRC buffer overflowing\n");
1444 return;
1445 }
1446
1447 entry = &pipe_crc->entries[head];
1448
1449 entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1450 entry->crc[0] = crc0;
1451 entry->crc[1] = crc1;
1452 entry->crc[2] = crc2;
1453 entry->crc[3] = crc3;
1454 entry->crc[4] = crc4;
1455
1456 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1457 pipe_crc->head = head;
1458
1459 spin_unlock(&pipe_crc->lock);
1460
1461 wake_up_interruptible(&pipe_crc->wq);
1462 }
1463 #else
1464 static inline void
1465 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1466 uint32_t crc0, uint32_t crc1,
1467 uint32_t crc2, uint32_t crc3,
1468 uint32_t crc4) {}
1469 #endif
1470
1471
1472 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1473 {
1474 struct drm_i915_private *dev_priv = dev->dev_private;
1475
1476 display_pipe_crc_irq_handler(dev, pipe,
1477 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1478 0, 0, 0, 0);
1479 }
1480
1481 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1482 {
1483 struct drm_i915_private *dev_priv = dev->dev_private;
1484
1485 display_pipe_crc_irq_handler(dev, pipe,
1486 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1487 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1488 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1489 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1490 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1491 }
1492
1493 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1494 {
1495 struct drm_i915_private *dev_priv = dev->dev_private;
1496 uint32_t res1, res2;
1497
1498 if (INTEL_INFO(dev)->gen >= 3)
1499 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1500 else
1501 res1 = 0;
1502
1503 if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1504 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1505 else
1506 res2 = 0;
1507
1508 display_pipe_crc_irq_handler(dev, pipe,
1509 I915_READ(PIPE_CRC_RES_RED(pipe)),
1510 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1511 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1512 res1, res2);
1513 }
1514
1515 /* The RPS events need forcewake, so we add them to a work queue and mask their
1516 * IMR bits until the work is done. Other interrupts can be processed without
1517 * the work queue. */
1518 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1519 {
1520 if (pm_iir & dev_priv->pm_rps_events) {
1521 spin_lock(&dev_priv->irq_lock);
1522 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1523 if (dev_priv->rps.interrupts_enabled) {
1524 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1525 queue_work(dev_priv->wq, &dev_priv->rps.work);
1526 }
1527 spin_unlock(&dev_priv->irq_lock);
1528 }
1529
1530 if (INTEL_INFO(dev_priv)->gen >= 8)
1531 return;
1532
1533 if (HAS_VEBOX(dev_priv->dev)) {
1534 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1535 notify_ring(&dev_priv->ring[VECS]);
1536
1537 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1538 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1539 }
1540 }
1541
1542 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1543 {
1544 if (!drm_handle_vblank(dev, pipe))
1545 return false;
1546
1547 return true;
1548 }
1549
1550 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1551 {
1552 struct drm_i915_private *dev_priv = dev->dev_private;
1553 u32 pipe_stats[I915_MAX_PIPES] = { };
1554 int pipe;
1555
1556 spin_lock(&dev_priv->irq_lock);
1557 for_each_pipe(dev_priv, pipe) {
1558 int reg;
1559 u32 mask, iir_bit = 0;
1560
1561 /*
1562 * PIPESTAT bits get signalled even when the interrupt is
1563 * disabled with the mask bits, and some of the status bits do
1564 * not generate interrupts at all (like the underrun bit). Hence
1565 * we need to be careful that we only handle what we want to
1566 * handle.
1567 */
1568
1569 /* fifo underruns are filterered in the underrun handler. */
1570 mask = PIPE_FIFO_UNDERRUN_STATUS;
1571
1572 switch (pipe) {
1573 case PIPE_A:
1574 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1575 break;
1576 case PIPE_B:
1577 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1578 break;
1579 case PIPE_C:
1580 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1581 break;
1582 }
1583 if (iir & iir_bit)
1584 mask |= dev_priv->pipestat_irq_mask[pipe];
1585
1586 if (!mask)
1587 continue;
1588
1589 reg = PIPESTAT(pipe);
1590 mask |= PIPESTAT_INT_ENABLE_MASK;
1591 pipe_stats[pipe] = I915_READ(reg) & mask;
1592
1593 /*
1594 * Clear the PIPE*STAT regs before the IIR
1595 */
1596 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1597 PIPESTAT_INT_STATUS_MASK))
1598 I915_WRITE(reg, pipe_stats[pipe]);
1599 }
1600 spin_unlock(&dev_priv->irq_lock);
1601
1602 for_each_pipe(dev_priv, pipe) {
1603 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1604 intel_pipe_handle_vblank(dev, pipe))
1605 intel_check_page_flip(dev, pipe);
1606
1607 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1608 intel_prepare_page_flip(dev, pipe);
1609 intel_finish_page_flip(dev, pipe);
1610 }
1611
1612 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1613 i9xx_pipe_crc_irq_handler(dev, pipe);
1614
1615 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1616 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1617 }
1618
1619 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1620 gmbus_irq_handler(dev);
1621 }
1622
1623 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1624 {
1625 struct drm_i915_private *dev_priv = dev->dev_private;
1626 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1627 u32 pin_mask = 0, long_mask = 0;
1628
1629 if (!hotplug_status)
1630 return;
1631
1632 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1633 /*
1634 * Make sure hotplug status is cleared before we clear IIR, or else we
1635 * may miss hotplug events.
1636 */
1637 POSTING_READ(PORT_HOTPLUG_STAT);
1638
1639 if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1640 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1641
1642 if (hotplug_trigger) {
1643 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1644 hotplug_trigger, hpd_status_g4x,
1645 i9xx_port_hotplug_long_detect);
1646
1647 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1648 }
1649
1650 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1651 dp_aux_irq_handler(dev);
1652 } else {
1653 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1654
1655 if (hotplug_trigger) {
1656 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1657 hotplug_trigger, hpd_status_g4x,
1658 i9xx_port_hotplug_long_detect);
1659
1660 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1661 }
1662 }
1663 }
1664
1665 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1666 {
1667 struct drm_device *dev = arg;
1668 struct drm_i915_private *dev_priv = dev->dev_private;
1669 u32 iir, gt_iir, pm_iir;
1670 irqreturn_t ret = IRQ_NONE;
1671
1672 if (!intel_irqs_enabled(dev_priv))
1673 return IRQ_NONE;
1674
1675 while (true) {
1676 /* Find, clear, then process each source of interrupt */
1677
1678 gt_iir = I915_READ(GTIIR);
1679 if (gt_iir)
1680 I915_WRITE(GTIIR, gt_iir);
1681
1682 pm_iir = I915_READ(GEN6_PMIIR);
1683 if (pm_iir)
1684 I915_WRITE(GEN6_PMIIR, pm_iir);
1685
1686 iir = I915_READ(VLV_IIR);
1687 if (iir) {
1688 /* Consume port before clearing IIR or we'll miss events */
1689 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1690 i9xx_hpd_irq_handler(dev);
1691 I915_WRITE(VLV_IIR, iir);
1692 }
1693
1694 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1695 goto out;
1696
1697 ret = IRQ_HANDLED;
1698
1699 if (gt_iir)
1700 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1701 if (pm_iir)
1702 gen6_rps_irq_handler(dev_priv, pm_iir);
1703 /* Call regardless, as some status bits might not be
1704 * signalled in iir */
1705 valleyview_pipestat_irq_handler(dev, iir);
1706 }
1707
1708 out:
1709 return ret;
1710 }
1711
1712 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1713 {
1714 struct drm_device *dev = arg;
1715 struct drm_i915_private *dev_priv = dev->dev_private;
1716 u32 master_ctl, iir;
1717 irqreturn_t ret = IRQ_NONE;
1718
1719 if (!intel_irqs_enabled(dev_priv))
1720 return IRQ_NONE;
1721
1722 for (;;) {
1723 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1724 iir = I915_READ(VLV_IIR);
1725
1726 if (master_ctl == 0 && iir == 0)
1727 break;
1728
1729 ret = IRQ_HANDLED;
1730
1731 I915_WRITE(GEN8_MASTER_IRQ, 0);
1732
1733 /* Find, clear, then process each source of interrupt */
1734
1735 if (iir) {
1736 /* Consume port before clearing IIR or we'll miss events */
1737 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1738 i9xx_hpd_irq_handler(dev);
1739 I915_WRITE(VLV_IIR, iir);
1740 }
1741
1742 gen8_gt_irq_handler(dev_priv, master_ctl);
1743
1744 /* Call regardless, as some status bits might not be
1745 * signalled in iir */
1746 valleyview_pipestat_irq_handler(dev, iir);
1747
1748 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1749 POSTING_READ(GEN8_MASTER_IRQ);
1750 }
1751
1752 return ret;
1753 }
1754
1755 static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
1756 const u32 hpd[HPD_NUM_PINS])
1757 {
1758 struct drm_i915_private *dev_priv = to_i915(dev);
1759 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1760
1761 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1762 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1763
1764 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1765 dig_hotplug_reg, hpd,
1766 pch_port_hotplug_long_detect);
1767
1768 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1769 }
1770
1771 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1772 {
1773 struct drm_i915_private *dev_priv = dev->dev_private;
1774 int pipe;
1775 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1776
1777 if (hotplug_trigger)
1778 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1779
1780 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1781 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1782 SDE_AUDIO_POWER_SHIFT);
1783 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1784 port_name(port));
1785 }
1786
1787 if (pch_iir & SDE_AUX_MASK)
1788 dp_aux_irq_handler(dev);
1789
1790 if (pch_iir & SDE_GMBUS)
1791 gmbus_irq_handler(dev);
1792
1793 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1794 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1795
1796 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1797 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1798
1799 if (pch_iir & SDE_POISON)
1800 DRM_ERROR("PCH poison interrupt\n");
1801
1802 if (pch_iir & SDE_FDI_MASK)
1803 for_each_pipe(dev_priv, pipe)
1804 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1805 pipe_name(pipe),
1806 I915_READ(FDI_RX_IIR(pipe)));
1807
1808 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1809 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1810
1811 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1812 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1813
1814 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1815 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1816
1817 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1818 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1819 }
1820
1821 static void ivb_err_int_handler(struct drm_device *dev)
1822 {
1823 struct drm_i915_private *dev_priv = dev->dev_private;
1824 u32 err_int = I915_READ(GEN7_ERR_INT);
1825 enum pipe pipe;
1826
1827 if (err_int & ERR_INT_POISON)
1828 DRM_ERROR("Poison interrupt\n");
1829
1830 for_each_pipe(dev_priv, pipe) {
1831 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1832 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1833
1834 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1835 if (IS_IVYBRIDGE(dev))
1836 ivb_pipe_crc_irq_handler(dev, pipe);
1837 else
1838 hsw_pipe_crc_irq_handler(dev, pipe);
1839 }
1840 }
1841
1842 I915_WRITE(GEN7_ERR_INT, err_int);
1843 }
1844
1845 static void cpt_serr_int_handler(struct drm_device *dev)
1846 {
1847 struct drm_i915_private *dev_priv = dev->dev_private;
1848 u32 serr_int = I915_READ(SERR_INT);
1849
1850 if (serr_int & SERR_INT_POISON)
1851 DRM_ERROR("PCH poison interrupt\n");
1852
1853 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1854 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1855
1856 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1857 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1858
1859 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1860 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1861
1862 I915_WRITE(SERR_INT, serr_int);
1863 }
1864
1865 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1866 {
1867 struct drm_i915_private *dev_priv = dev->dev_private;
1868 int pipe;
1869 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1870
1871 if (hotplug_trigger)
1872 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1873
1874 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1875 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1876 SDE_AUDIO_POWER_SHIFT_CPT);
1877 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1878 port_name(port));
1879 }
1880
1881 if (pch_iir & SDE_AUX_MASK_CPT)
1882 dp_aux_irq_handler(dev);
1883
1884 if (pch_iir & SDE_GMBUS_CPT)
1885 gmbus_irq_handler(dev);
1886
1887 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1888 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1889
1890 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1891 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1892
1893 if (pch_iir & SDE_FDI_MASK_CPT)
1894 for_each_pipe(dev_priv, pipe)
1895 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1896 pipe_name(pipe),
1897 I915_READ(FDI_RX_IIR(pipe)));
1898
1899 if (pch_iir & SDE_ERROR_CPT)
1900 cpt_serr_int_handler(dev);
1901 }
1902
1903 static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
1904 {
1905 struct drm_i915_private *dev_priv = dev->dev_private;
1906 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1907 ~SDE_PORTE_HOTPLUG_SPT;
1908 u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1909 u32 pin_mask = 0, long_mask = 0;
1910
1911 if (hotplug_trigger) {
1912 u32 dig_hotplug_reg;
1913
1914 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1915 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1916
1917 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1918 dig_hotplug_reg, hpd_spt,
1919 spt_port_hotplug_long_detect);
1920 }
1921
1922 if (hotplug2_trigger) {
1923 u32 dig_hotplug_reg;
1924
1925 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1926 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1927
1928 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
1929 dig_hotplug_reg, hpd_spt,
1930 spt_port_hotplug2_long_detect);
1931 }
1932
1933 if (pin_mask)
1934 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1935
1936 if (pch_iir & SDE_GMBUS_CPT)
1937 gmbus_irq_handler(dev);
1938 }
1939
1940 static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
1941 const u32 hpd[HPD_NUM_PINS])
1942 {
1943 struct drm_i915_private *dev_priv = to_i915(dev);
1944 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1945
1946 dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
1947 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
1948
1949 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1950 dig_hotplug_reg, hpd,
1951 ilk_port_hotplug_long_detect);
1952
1953 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1954 }
1955
1956 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1957 {
1958 struct drm_i915_private *dev_priv = dev->dev_private;
1959 enum pipe pipe;
1960 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
1961
1962 if (hotplug_trigger)
1963 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
1964
1965 if (de_iir & DE_AUX_CHANNEL_A)
1966 dp_aux_irq_handler(dev);
1967
1968 if (de_iir & DE_GSE)
1969 intel_opregion_asle_intr(dev);
1970
1971 if (de_iir & DE_POISON)
1972 DRM_ERROR("Poison interrupt\n");
1973
1974 for_each_pipe(dev_priv, pipe) {
1975 if (de_iir & DE_PIPE_VBLANK(pipe) &&
1976 intel_pipe_handle_vblank(dev, pipe))
1977 intel_check_page_flip(dev, pipe);
1978
1979 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1980 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1981
1982 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1983 i9xx_pipe_crc_irq_handler(dev, pipe);
1984
1985 /* plane/pipes map 1:1 on ilk+ */
1986 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1987 intel_prepare_page_flip(dev, pipe);
1988 intel_finish_page_flip_plane(dev, pipe);
1989 }
1990 }
1991
1992 /* check event from PCH */
1993 if (de_iir & DE_PCH_EVENT) {
1994 u32 pch_iir = I915_READ(SDEIIR);
1995
1996 if (HAS_PCH_CPT(dev))
1997 cpt_irq_handler(dev, pch_iir);
1998 else
1999 ibx_irq_handler(dev, pch_iir);
2000
2001 /* should clear PCH hotplug event before clear CPU irq */
2002 I915_WRITE(SDEIIR, pch_iir);
2003 }
2004
2005 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2006 ironlake_rps_change_irq_handler(dev);
2007 }
2008
2009 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
2010 {
2011 struct drm_i915_private *dev_priv = dev->dev_private;
2012 enum pipe pipe;
2013 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2014
2015 if (hotplug_trigger)
2016 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2017
2018 if (de_iir & DE_ERR_INT_IVB)
2019 ivb_err_int_handler(dev);
2020
2021 if (de_iir & DE_AUX_CHANNEL_A_IVB)
2022 dp_aux_irq_handler(dev);
2023
2024 if (de_iir & DE_GSE_IVB)
2025 intel_opregion_asle_intr(dev);
2026
2027 for_each_pipe(dev_priv, pipe) {
2028 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
2029 intel_pipe_handle_vblank(dev, pipe))
2030 intel_check_page_flip(dev, pipe);
2031
2032 /* plane/pipes map 1:1 on ilk+ */
2033 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
2034 intel_prepare_page_flip(dev, pipe);
2035 intel_finish_page_flip_plane(dev, pipe);
2036 }
2037 }
2038
2039 /* check event from PCH */
2040 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
2041 u32 pch_iir = I915_READ(SDEIIR);
2042
2043 cpt_irq_handler(dev, pch_iir);
2044
2045 /* clear PCH hotplug event before clear CPU irq */
2046 I915_WRITE(SDEIIR, pch_iir);
2047 }
2048 }
2049
2050 /*
2051 * To handle irqs with the minimum potential races with fresh interrupts, we:
2052 * 1 - Disable Master Interrupt Control.
2053 * 2 - Find the source(s) of the interrupt.
2054 * 3 - Clear the Interrupt Identity bits (IIR).
2055 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2056 * 5 - Re-enable Master Interrupt Control.
2057 */
2058 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2059 {
2060 struct drm_device *dev = arg;
2061 struct drm_i915_private *dev_priv = dev->dev_private;
2062 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2063 irqreturn_t ret = IRQ_NONE;
2064
2065 if (!intel_irqs_enabled(dev_priv))
2066 return IRQ_NONE;
2067
2068 /* We get interrupts on unclaimed registers, so check for this before we
2069 * do any I915_{READ,WRITE}. */
2070 intel_uncore_check_errors(dev);
2071
2072 /* disable master interrupt before clearing iir */
2073 de_ier = I915_READ(DEIER);
2074 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2075 POSTING_READ(DEIER);
2076
2077 /* Disable south interrupts. We'll only write to SDEIIR once, so further
2078 * interrupts will will be stored on its back queue, and then we'll be
2079 * able to process them after we restore SDEIER (as soon as we restore
2080 * it, we'll get an interrupt if SDEIIR still has something to process
2081 * due to its back queue). */
2082 if (!HAS_PCH_NOP(dev)) {
2083 sde_ier = I915_READ(SDEIER);
2084 I915_WRITE(SDEIER, 0);
2085 POSTING_READ(SDEIER);
2086 }
2087
2088 /* Find, clear, then process each source of interrupt */
2089
2090 gt_iir = I915_READ(GTIIR);
2091 if (gt_iir) {
2092 I915_WRITE(GTIIR, gt_iir);
2093 ret = IRQ_HANDLED;
2094 if (INTEL_INFO(dev)->gen >= 6)
2095 snb_gt_irq_handler(dev, dev_priv, gt_iir);
2096 else
2097 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2098 }
2099
2100 de_iir = I915_READ(DEIIR);
2101 if (de_iir) {
2102 I915_WRITE(DEIIR, de_iir);
2103 ret = IRQ_HANDLED;
2104 if (INTEL_INFO(dev)->gen >= 7)
2105 ivb_display_irq_handler(dev, de_iir);
2106 else
2107 ilk_display_irq_handler(dev, de_iir);
2108 }
2109
2110 if (INTEL_INFO(dev)->gen >= 6) {
2111 u32 pm_iir = I915_READ(GEN6_PMIIR);
2112 if (pm_iir) {
2113 I915_WRITE(GEN6_PMIIR, pm_iir);
2114 ret = IRQ_HANDLED;
2115 gen6_rps_irq_handler(dev_priv, pm_iir);
2116 }
2117 }
2118
2119 I915_WRITE(DEIER, de_ier);
2120 POSTING_READ(DEIER);
2121 if (!HAS_PCH_NOP(dev)) {
2122 I915_WRITE(SDEIER, sde_ier);
2123 POSTING_READ(SDEIER);
2124 }
2125
2126 return ret;
2127 }
2128
2129 static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2130 const u32 hpd[HPD_NUM_PINS])
2131 {
2132 struct drm_i915_private *dev_priv = to_i915(dev);
2133 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2134
2135 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2136 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2137
2138 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2139 dig_hotplug_reg, hpd,
2140 bxt_port_hotplug_long_detect);
2141
2142 intel_hpd_irq_handler(dev, pin_mask, long_mask);
2143 }
2144
2145 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2146 {
2147 struct drm_device *dev = arg;
2148 struct drm_i915_private *dev_priv = dev->dev_private;
2149 u32 master_ctl;
2150 irqreturn_t ret = IRQ_NONE;
2151 uint32_t tmp = 0;
2152 enum pipe pipe;
2153 u32 aux_mask = GEN8_AUX_CHANNEL_A;
2154
2155 if (!intel_irqs_enabled(dev_priv))
2156 return IRQ_NONE;
2157
2158 if (INTEL_INFO(dev_priv)->gen >= 9)
2159 aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2160 GEN9_AUX_CHANNEL_D;
2161
2162 master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2163 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2164 if (!master_ctl)
2165 return IRQ_NONE;
2166
2167 I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2168
2169 /* Find, clear, then process each source of interrupt */
2170
2171 ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2172
2173 if (master_ctl & GEN8_DE_MISC_IRQ) {
2174 tmp = I915_READ(GEN8_DE_MISC_IIR);
2175 if (tmp) {
2176 I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2177 ret = IRQ_HANDLED;
2178 if (tmp & GEN8_DE_MISC_GSE)
2179 intel_opregion_asle_intr(dev);
2180 else
2181 DRM_ERROR("Unexpected DE Misc interrupt\n");
2182 }
2183 else
2184 DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2185 }
2186
2187 if (master_ctl & GEN8_DE_PORT_IRQ) {
2188 tmp = I915_READ(GEN8_DE_PORT_IIR);
2189 if (tmp) {
2190 bool found = false;
2191 u32 hotplug_trigger = 0;
2192
2193 if (IS_BROXTON(dev_priv))
2194 hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
2195 else if (IS_BROADWELL(dev_priv))
2196 hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2197
2198 I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2199 ret = IRQ_HANDLED;
2200
2201 if (tmp & aux_mask) {
2202 dp_aux_irq_handler(dev);
2203 found = true;
2204 }
2205
2206 if (hotplug_trigger) {
2207 if (IS_BROXTON(dev))
2208 bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
2209 else
2210 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2211 found = true;
2212 }
2213
2214 if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2215 gmbus_irq_handler(dev);
2216 found = true;
2217 }
2218
2219 if (!found)
2220 DRM_ERROR("Unexpected DE Port interrupt\n");
2221 }
2222 else
2223 DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2224 }
2225
2226 for_each_pipe(dev_priv, pipe) {
2227 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2228
2229 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2230 continue;
2231
2232 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2233 if (pipe_iir) {
2234 ret = IRQ_HANDLED;
2235 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2236
2237 if (pipe_iir & GEN8_PIPE_VBLANK &&
2238 intel_pipe_handle_vblank(dev, pipe))
2239 intel_check_page_flip(dev, pipe);
2240
2241 if (INTEL_INFO(dev_priv)->gen >= 9)
2242 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2243 else
2244 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2245
2246 if (flip_done) {
2247 intel_prepare_page_flip(dev, pipe);
2248 intel_finish_page_flip_plane(dev, pipe);
2249 }
2250
2251 if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2252 hsw_pipe_crc_irq_handler(dev, pipe);
2253
2254 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2255 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2256 pipe);
2257
2258
2259 if (INTEL_INFO(dev_priv)->gen >= 9)
2260 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2261 else
2262 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2263
2264 if (fault_errors)
2265 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2266 pipe_name(pipe),
2267 pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2268 } else
2269 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2270 }
2271
2272 if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2273 master_ctl & GEN8_DE_PCH_IRQ) {
2274 /*
2275 * FIXME(BDW): Assume for now that the new interrupt handling
2276 * scheme also closed the SDE interrupt handling race we've seen
2277 * on older pch-split platforms. But this needs testing.
2278 */
2279 u32 pch_iir = I915_READ(SDEIIR);
2280 if (pch_iir) {
2281 I915_WRITE(SDEIIR, pch_iir);
2282 ret = IRQ_HANDLED;
2283
2284 if (HAS_PCH_SPT(dev_priv))
2285 spt_irq_handler(dev, pch_iir);
2286 else
2287 cpt_irq_handler(dev, pch_iir);
2288 } else
2289 DRM_ERROR("The master control interrupt lied (SDE)!\n");
2290
2291 }
2292
2293 I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2294 POSTING_READ_FW(GEN8_MASTER_IRQ);
2295
2296 return ret;
2297 }
2298
2299 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2300 bool reset_completed)
2301 {
2302 struct intel_engine_cs *ring;
2303 int i;
2304
2305 /*
2306 * Notify all waiters for GPU completion events that reset state has
2307 * been changed, and that they need to restart their wait after
2308 * checking for potential errors (and bail out to drop locks if there is
2309 * a gpu reset pending so that i915_error_work_func can acquire them).
2310 */
2311
2312 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2313 for_each_ring(ring, dev_priv, i)
2314 wake_up_all(&ring->irq_queue);
2315
2316 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2317 wake_up_all(&dev_priv->pending_flip_queue);
2318
2319 /*
2320 * Signal tasks blocked in i915_gem_wait_for_error that the pending
2321 * reset state is cleared.
2322 */
2323 if (reset_completed)
2324 wake_up_all(&dev_priv->gpu_error.reset_queue);
2325 }
2326
2327 /**
2328 * i915_reset_and_wakeup - do process context error handling work
2329 *
2330 * Fire an error uevent so userspace can see that a hang or error
2331 * was detected.
2332 */
2333 static void i915_reset_and_wakeup(struct drm_device *dev)
2334 {
2335 struct drm_i915_private *dev_priv = to_i915(dev);
2336 struct i915_gpu_error *error = &dev_priv->gpu_error;
2337 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2338 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2339 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2340 int ret;
2341
2342 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2343
2344 /*
2345 * Note that there's only one work item which does gpu resets, so we
2346 * need not worry about concurrent gpu resets potentially incrementing
2347 * error->reset_counter twice. We only need to take care of another
2348 * racing irq/hangcheck declaring the gpu dead for a second time. A
2349 * quick check for that is good enough: schedule_work ensures the
2350 * correct ordering between hang detection and this work item, and since
2351 * the reset in-progress bit is only ever set by code outside of this
2352 * work we don't need to worry about any other races.
2353 */
2354 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2355 DRM_DEBUG_DRIVER("resetting chip\n");
2356 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2357 reset_event);
2358
2359 /*
2360 * In most cases it's guaranteed that we get here with an RPM
2361 * reference held, for example because there is a pending GPU
2362 * request that won't finish until the reset is done. This
2363 * isn't the case at least when we get here by doing a
2364 * simulated reset via debugs, so get an RPM reference.
2365 */
2366 intel_runtime_pm_get(dev_priv);
2367
2368 intel_prepare_reset(dev);
2369
2370 /*
2371 * All state reset _must_ be completed before we update the
2372 * reset counter, for otherwise waiters might miss the reset
2373 * pending state and not properly drop locks, resulting in
2374 * deadlocks with the reset work.
2375 */
2376 ret = i915_reset(dev);
2377
2378 intel_finish_reset(dev);
2379
2380 intel_runtime_pm_put(dev_priv);
2381
2382 if (ret == 0) {
2383 /*
2384 * After all the gem state is reset, increment the reset
2385 * counter and wake up everyone waiting for the reset to
2386 * complete.
2387 *
2388 * Since unlock operations are a one-sided barrier only,
2389 * we need to insert a barrier here to order any seqno
2390 * updates before
2391 * the counter increment.
2392 */
2393 smp_mb__before_atomic();
2394 atomic_inc(&dev_priv->gpu_error.reset_counter);
2395
2396 kobject_uevent_env(&dev->primary->kdev->kobj,
2397 KOBJ_CHANGE, reset_done_event);
2398 } else {
2399 atomic_set_mask(I915_WEDGED, &error->reset_counter);
2400 }
2401
2402 /*
2403 * Note: The wake_up also serves as a memory barrier so that
2404 * waiters see the update value of the reset counter atomic_t.
2405 */
2406 i915_error_wake_up(dev_priv, true);
2407 }
2408 }
2409
2410 static void i915_report_and_clear_eir(struct drm_device *dev)
2411 {
2412 struct drm_i915_private *dev_priv = dev->dev_private;
2413 uint32_t instdone[I915_NUM_INSTDONE_REG];
2414 u32 eir = I915_READ(EIR);
2415 int pipe, i;
2416
2417 if (!eir)
2418 return;
2419
2420 pr_err("render error detected, EIR: 0x%08x\n", eir);
2421
2422 i915_get_extra_instdone(dev, instdone);
2423
2424 if (IS_G4X(dev)) {
2425 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2426 u32 ipeir = I915_READ(IPEIR_I965);
2427
2428 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2429 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2430 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2431 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2432 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
2433 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2434 I915_WRITE(IPEIR_I965, ipeir);
2435 POSTING_READ(IPEIR_I965);
2436 }
2437 if (eir & GM45_ERROR_PAGE_TABLE) {
2438 u32 pgtbl_err = I915_READ(PGTBL_ER);
2439 pr_err("page table error\n");
2440 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
2441 I915_WRITE(PGTBL_ER, pgtbl_err);
2442 POSTING_READ(PGTBL_ER);
2443 }
2444 }
2445
2446 if (!IS_GEN2(dev)) {
2447 if (eir & I915_ERROR_PAGE_TABLE) {
2448 u32 pgtbl_err = I915_READ(PGTBL_ER);
2449 pr_err("page table error\n");
2450 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
2451 I915_WRITE(PGTBL_ER, pgtbl_err);
2452 POSTING_READ(PGTBL_ER);
2453 }
2454 }
2455
2456 if (eir & I915_ERROR_MEMORY_REFRESH) {
2457 pr_err("memory refresh error:\n");
2458 for_each_pipe(dev_priv, pipe)
2459 pr_err("pipe %c stat: 0x%08x\n",
2460 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2461 /* pipestat has already been acked */
2462 }
2463 if (eir & I915_ERROR_INSTRUCTION) {
2464 pr_err("instruction error\n");
2465 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
2466 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2467 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2468 if (INTEL_INFO(dev)->gen < 4) {
2469 u32 ipeir = I915_READ(IPEIR);
2470
2471 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
2472 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
2473 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
2474 I915_WRITE(IPEIR, ipeir);
2475 POSTING_READ(IPEIR);
2476 } else {
2477 u32 ipeir = I915_READ(IPEIR_I965);
2478
2479 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2480 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2481 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
2482 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2483 I915_WRITE(IPEIR_I965, ipeir);
2484 POSTING_READ(IPEIR_I965);
2485 }
2486 }
2487
2488 I915_WRITE(EIR, eir);
2489 POSTING_READ(EIR);
2490 eir = I915_READ(EIR);
2491 if (eir) {
2492 /*
2493 * some errors might have become stuck,
2494 * mask them.
2495 */
2496 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2497 I915_WRITE(EMR, I915_READ(EMR) | eir);
2498 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2499 }
2500 }
2501
2502 /**
2503 * i915_handle_error - handle a gpu error
2504 * @dev: drm device
2505 *
2506 * Do some basic checking of regsiter state at error time and
2507 * dump it to the syslog. Also call i915_capture_error_state() to make
2508 * sure we get a record and make it available in debugfs. Fire a uevent
2509 * so userspace knows something bad happened (should trigger collection
2510 * of a ring dump etc.).
2511 */
2512 void i915_handle_error(struct drm_device *dev, bool wedged,
2513 const char *fmt, ...)
2514 {
2515 struct drm_i915_private *dev_priv = dev->dev_private;
2516 va_list args;
2517 char error_msg[80];
2518
2519 va_start(args, fmt);
2520 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2521 va_end(args);
2522
2523 i915_capture_error_state(dev, wedged, error_msg);
2524 i915_report_and_clear_eir(dev);
2525
2526 if (wedged) {
2527 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2528 &dev_priv->gpu_error.reset_counter);
2529
2530 /*
2531 * Wakeup waiting processes so that the reset function
2532 * i915_reset_and_wakeup doesn't deadlock trying to grab
2533 * various locks. By bumping the reset counter first, the woken
2534 * processes will see a reset in progress and back off,
2535 * releasing their locks and then wait for the reset completion.
2536 * We must do this for _all_ gpu waiters that might hold locks
2537 * that the reset work needs to acquire.
2538 *
2539 * Note: The wake_up serves as the required memory barrier to
2540 * ensure that the waiters see the updated value of the reset
2541 * counter atomic_t.
2542 */
2543 i915_error_wake_up(dev_priv, false);
2544 }
2545
2546 i915_reset_and_wakeup(dev);
2547 }
2548
2549 /* Called from drm generic code, passed 'crtc' which
2550 * we use as a pipe index
2551 */
2552 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2553 {
2554 struct drm_i915_private *dev_priv = dev->dev_private;
2555 unsigned long irqflags;
2556
2557 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2558 if (INTEL_INFO(dev)->gen >= 4)
2559 i915_enable_pipestat(dev_priv, pipe,
2560 PIPE_START_VBLANK_INTERRUPT_STATUS);
2561 else
2562 i915_enable_pipestat(dev_priv, pipe,
2563 PIPE_VBLANK_INTERRUPT_STATUS);
2564 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2565
2566 return 0;
2567 }
2568
2569 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2570 {
2571 struct drm_i915_private *dev_priv = dev->dev_private;
2572 unsigned long irqflags;
2573 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2574 DE_PIPE_VBLANK(pipe);
2575
2576 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2577 ironlake_enable_display_irq(dev_priv, bit);
2578 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2579
2580 return 0;
2581 }
2582
2583 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2584 {
2585 struct drm_i915_private *dev_priv = dev->dev_private;
2586 unsigned long irqflags;
2587
2588 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2589 i915_enable_pipestat(dev_priv, pipe,
2590 PIPE_START_VBLANK_INTERRUPT_STATUS);
2591 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2592
2593 return 0;
2594 }
2595
2596 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2597 {
2598 struct drm_i915_private *dev_priv = dev->dev_private;
2599 unsigned long irqflags;
2600
2601 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2602 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2603 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2604 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2605 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2606 return 0;
2607 }
2608
2609 /* Called from drm generic code, passed 'crtc' which
2610 * we use as a pipe index
2611 */
2612 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2613 {
2614 struct drm_i915_private *dev_priv = dev->dev_private;
2615 unsigned long irqflags;
2616
2617 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2618 i915_disable_pipestat(dev_priv, pipe,
2619 PIPE_VBLANK_INTERRUPT_STATUS |
2620 PIPE_START_VBLANK_INTERRUPT_STATUS);
2621 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2622 }
2623
2624 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2625 {
2626 struct drm_i915_private *dev_priv = dev->dev_private;
2627 unsigned long irqflags;
2628 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2629 DE_PIPE_VBLANK(pipe);
2630
2631 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2632 ironlake_disable_display_irq(dev_priv, bit);
2633 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2634 }
2635
2636 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2637 {
2638 struct drm_i915_private *dev_priv = dev->dev_private;
2639 unsigned long irqflags;
2640
2641 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2642 i915_disable_pipestat(dev_priv, pipe,
2643 PIPE_START_VBLANK_INTERRUPT_STATUS);
2644 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2645 }
2646
2647 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2648 {
2649 struct drm_i915_private *dev_priv = dev->dev_private;
2650 unsigned long irqflags;
2651
2652 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2653 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2654 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2655 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2656 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2657 }
2658
2659 static bool
2660 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2661 {
2662 return (list_empty(&ring->request_list) ||
2663 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2664 }
2665
2666 static bool
2667 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2668 {
2669 if (INTEL_INFO(dev)->gen >= 8) {
2670 return (ipehr >> 23) == 0x1c;
2671 } else {
2672 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2673 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2674 MI_SEMAPHORE_REGISTER);
2675 }
2676 }
2677
2678 static struct intel_engine_cs *
2679 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2680 {
2681 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2682 struct intel_engine_cs *signaller;
2683 int i;
2684
2685 if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2686 for_each_ring(signaller, dev_priv, i) {
2687 if (ring == signaller)
2688 continue;
2689
2690 if (offset == signaller->semaphore.signal_ggtt[ring->id])
2691 return signaller;
2692 }
2693 } else {
2694 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2695
2696 for_each_ring(signaller, dev_priv, i) {
2697 if(ring == signaller)
2698 continue;
2699
2700 if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2701 return signaller;
2702 }
2703 }
2704
2705 DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2706 ring->id, ipehr, offset);
2707
2708 return NULL;
2709 }
2710
2711 static struct intel_engine_cs *
2712 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2713 {
2714 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2715 u32 cmd, ipehr, head;
2716 u64 offset = 0;
2717 int i, backwards;
2718
2719 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2720 if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2721 return NULL;
2722
2723 /*
2724 * HEAD is likely pointing to the dword after the actual command,
2725 * so scan backwards until we find the MBOX. But limit it to just 3
2726 * or 4 dwords depending on the semaphore wait command size.
2727 * Note that we don't care about ACTHD here since that might
2728 * point at at batch, and semaphores are always emitted into the
2729 * ringbuffer itself.
2730 */
2731 head = I915_READ_HEAD(ring) & HEAD_ADDR;
2732 backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2733
2734 for (i = backwards; i; --i) {
2735 /*
2736 * Be paranoid and presume the hw has gone off into the wild -
2737 * our ring is smaller than what the hardware (and hence
2738 * HEAD_ADDR) allows. Also handles wrap-around.
2739 */
2740 head &= ring->buffer->size - 1;
2741
2742 /* This here seems to blow up */
2743 cmd = ioread32(ring->buffer->virtual_start + head);
2744 if (cmd == ipehr)
2745 break;
2746
2747 head -= 4;
2748 }
2749
2750 if (!i)
2751 return NULL;
2752
2753 *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2754 if (INTEL_INFO(ring->dev)->gen >= 8) {
2755 offset = ioread32(ring->buffer->virtual_start + head + 12);
2756 offset <<= 32;
2757 offset = ioread32(ring->buffer->virtual_start + head + 8);
2758 }
2759 return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2760 }
2761
2762 static int semaphore_passed(struct intel_engine_cs *ring)
2763 {
2764 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2765 struct intel_engine_cs *signaller;
2766 u32 seqno;
2767
2768 ring->hangcheck.deadlock++;
2769
2770 signaller = semaphore_waits_for(ring, &seqno);
2771 if (signaller == NULL)
2772 return -1;
2773
2774 /* Prevent pathological recursion due to driver bugs */
2775 if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2776 return -1;
2777
2778 if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2779 return 1;
2780
2781 /* cursory check for an unkickable deadlock */
2782 if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2783 semaphore_passed(signaller) < 0)
2784 return -1;
2785
2786 return 0;
2787 }
2788
2789 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2790 {
2791 struct intel_engine_cs *ring;
2792 int i;
2793
2794 for_each_ring(ring, dev_priv, i)
2795 ring->hangcheck.deadlock = 0;
2796 }
2797
2798 static enum intel_ring_hangcheck_action
2799 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2800 {
2801 struct drm_device *dev = ring->dev;
2802 struct drm_i915_private *dev_priv = dev->dev_private;
2803 u32 tmp;
2804
2805 if (acthd != ring->hangcheck.acthd) {
2806 if (acthd > ring->hangcheck.max_acthd) {
2807 ring->hangcheck.max_acthd = acthd;
2808 return HANGCHECK_ACTIVE;
2809 }
2810
2811 return HANGCHECK_ACTIVE_LOOP;
2812 }
2813
2814 if (IS_GEN2(dev))
2815 return HANGCHECK_HUNG;
2816
2817 /* Is the chip hanging on a WAIT_FOR_EVENT?
2818 * If so we can simply poke the RB_WAIT bit
2819 * and break the hang. This should work on
2820 * all but the second generation chipsets.
2821 */
2822 tmp = I915_READ_CTL(ring);
2823 if (tmp & RING_WAIT) {
2824 i915_handle_error(dev, false,
2825 "Kicking stuck wait on %s",
2826 ring->name);
2827 I915_WRITE_CTL(ring, tmp);
2828 return HANGCHECK_KICK;
2829 }
2830
2831 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2832 switch (semaphore_passed(ring)) {
2833 default:
2834 return HANGCHECK_HUNG;
2835 case 1:
2836 i915_handle_error(dev, false,
2837 "Kicking stuck semaphore on %s",
2838 ring->name);
2839 I915_WRITE_CTL(ring, tmp);
2840 return HANGCHECK_KICK;
2841 case 0:
2842 return HANGCHECK_WAIT;
2843 }
2844 }
2845
2846 return HANGCHECK_HUNG;
2847 }
2848
2849 /*
2850 * This is called when the chip hasn't reported back with completed
2851 * batchbuffers in a long time. We keep track per ring seqno progress and
2852 * if there are no progress, hangcheck score for that ring is increased.
2853 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2854 * we kick the ring. If we see no progress on three subsequent calls
2855 * we assume chip is wedged and try to fix it by resetting the chip.
2856 */
2857 static void i915_hangcheck_elapsed(struct work_struct *work)
2858 {
2859 struct drm_i915_private *dev_priv =
2860 container_of(work, typeof(*dev_priv),
2861 gpu_error.hangcheck_work.work);
2862 struct drm_device *dev = dev_priv->dev;
2863 struct intel_engine_cs *ring;
2864 int i;
2865 int busy_count = 0, rings_hung = 0;
2866 bool stuck[I915_NUM_RINGS] = { 0 };
2867 #define BUSY 1
2868 #define KICK 5
2869 #define HUNG 20
2870
2871 if (!i915.enable_hangcheck)
2872 return;
2873
2874 for_each_ring(ring, dev_priv, i) {
2875 u64 acthd;
2876 u32 seqno;
2877 bool busy = true;
2878
2879 semaphore_clear_deadlocks(dev_priv);
2880
2881 seqno = ring->get_seqno(ring, false);
2882 acthd = intel_ring_get_active_head(ring);
2883
2884 if (ring->hangcheck.seqno == seqno) {
2885 if (ring_idle(ring, seqno)) {
2886 ring->hangcheck.action = HANGCHECK_IDLE;
2887
2888 if (waitqueue_active(&ring->irq_queue)) {
2889 /* Issue a wake-up to catch stuck h/w. */
2890 if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2891 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2892 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2893 ring->name);
2894 else
2895 DRM_INFO("Fake missed irq on %s\n",
2896 ring->name);
2897 wake_up_all(&ring->irq_queue);
2898 }
2899 /* Safeguard against driver failure */
2900 ring->hangcheck.score += BUSY;
2901 } else
2902 busy = false;
2903 } else {
2904 /* We always increment the hangcheck score
2905 * if the ring is busy and still processing
2906 * the same request, so that no single request
2907 * can run indefinitely (such as a chain of
2908 * batches). The only time we do not increment
2909 * the hangcheck score on this ring, if this
2910 * ring is in a legitimate wait for another
2911 * ring. In that case the waiting ring is a
2912 * victim and we want to be sure we catch the
2913 * right culprit. Then every time we do kick
2914 * the ring, add a small increment to the
2915 * score so that we can catch a batch that is
2916 * being repeatedly kicked and so responsible
2917 * for stalling the machine.
2918 */
2919 ring->hangcheck.action = ring_stuck(ring,
2920 acthd);
2921
2922 switch (ring->hangcheck.action) {
2923 case HANGCHECK_IDLE:
2924 case HANGCHECK_WAIT:
2925 case HANGCHECK_ACTIVE:
2926 break;
2927 case HANGCHECK_ACTIVE_LOOP:
2928 ring->hangcheck.score += BUSY;
2929 break;
2930 case HANGCHECK_KICK:
2931 ring->hangcheck.score += KICK;
2932 break;
2933 case HANGCHECK_HUNG:
2934 ring->hangcheck.score += HUNG;
2935 stuck[i] = true;
2936 break;
2937 }
2938 }
2939 } else {
2940 ring->hangcheck.action = HANGCHECK_ACTIVE;
2941
2942 /* Gradually reduce the count so that we catch DoS
2943 * attempts across multiple batches.
2944 */
2945 if (ring->hangcheck.score > 0)
2946 ring->hangcheck.score--;
2947
2948 ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2949 }
2950
2951 ring->hangcheck.seqno = seqno;
2952 ring->hangcheck.acthd = acthd;
2953 busy_count += busy;
2954 }
2955
2956 for_each_ring(ring, dev_priv, i) {
2957 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2958 DRM_INFO("%s on %s\n",
2959 stuck[i] ? "stuck" : "no progress",
2960 ring->name);
2961 rings_hung++;
2962 }
2963 }
2964
2965 if (rings_hung)
2966 return i915_handle_error(dev, true, "Ring hung");
2967
2968 if (busy_count)
2969 /* Reset timer case chip hangs without another request
2970 * being added */
2971 i915_queue_hangcheck(dev);
2972 }
2973
2974 void i915_queue_hangcheck(struct drm_device *dev)
2975 {
2976 struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
2977
2978 if (!i915.enable_hangcheck)
2979 return;
2980
2981 /* Don't continually defer the hangcheck so that it is always run at
2982 * least once after work has been scheduled on any ring. Otherwise,
2983 * we will ignore a hung ring if a second ring is kept busy.
2984 */
2985
2986 queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
2987 round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
2988 }
2989
2990 static void ibx_irq_reset(struct drm_device *dev)
2991 {
2992 struct drm_i915_private *dev_priv = dev->dev_private;
2993
2994 if (HAS_PCH_NOP(dev))
2995 return;
2996
2997 GEN5_IRQ_RESET(SDE);
2998
2999 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3000 I915_WRITE(SERR_INT, 0xffffffff);
3001 }
3002
3003 /*
3004 * SDEIER is also touched by the interrupt handler to work around missed PCH
3005 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3006 * instead we unconditionally enable all PCH interrupt sources here, but then
3007 * only unmask them as needed with SDEIMR.
3008 *
3009 * This function needs to be called before interrupts are enabled.
3010 */
3011 static void ibx_irq_pre_postinstall(struct drm_device *dev)
3012 {
3013 struct drm_i915_private *dev_priv = dev->dev_private;
3014
3015 if (HAS_PCH_NOP(dev))
3016 return;
3017
3018 WARN_ON(I915_READ(SDEIER) != 0);
3019 I915_WRITE(SDEIER, 0xffffffff);
3020 POSTING_READ(SDEIER);
3021 }
3022
3023 static void gen5_gt_irq_reset(struct drm_device *dev)
3024 {
3025 struct drm_i915_private *dev_priv = dev->dev_private;
3026
3027 GEN5_IRQ_RESET(GT);
3028 if (INTEL_INFO(dev)->gen >= 6)
3029 GEN5_IRQ_RESET(GEN6_PM);
3030 }
3031
3032 /* drm_dma.h hooks
3033 */
3034 static void ironlake_irq_reset(struct drm_device *dev)
3035 {
3036 struct drm_i915_private *dev_priv = dev->dev_private;
3037
3038 I915_WRITE(HWSTAM, 0xffffffff);
3039
3040 GEN5_IRQ_RESET(DE);
3041 if (IS_GEN7(dev))
3042 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3043
3044 gen5_gt_irq_reset(dev);
3045
3046 ibx_irq_reset(dev);
3047 }
3048
3049 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3050 {
3051 enum pipe pipe;
3052
3053 I915_WRITE(PORT_HOTPLUG_EN, 0);
3054 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3055
3056 for_each_pipe(dev_priv, pipe)
3057 I915_WRITE(PIPESTAT(pipe), 0xffff);
3058
3059 GEN5_IRQ_RESET(VLV_);
3060 }
3061
3062 static void valleyview_irq_preinstall(struct drm_device *dev)
3063 {
3064 struct drm_i915_private *dev_priv = dev->dev_private;
3065
3066 /* VLV magic */
3067 I915_WRITE(VLV_IMR, 0);
3068 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
3069 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
3070 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
3071
3072 gen5_gt_irq_reset(dev);
3073
3074 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3075
3076 vlv_display_irq_reset(dev_priv);
3077 }
3078
3079 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3080 {
3081 GEN8_IRQ_RESET_NDX(GT, 0);
3082 GEN8_IRQ_RESET_NDX(GT, 1);
3083 GEN8_IRQ_RESET_NDX(GT, 2);
3084 GEN8_IRQ_RESET_NDX(GT, 3);
3085 }
3086
3087 static void gen8_irq_reset(struct drm_device *dev)
3088 {
3089 struct drm_i915_private *dev_priv = dev->dev_private;
3090 int pipe;
3091
3092 I915_WRITE(GEN8_MASTER_IRQ, 0);
3093 POSTING_READ(GEN8_MASTER_IRQ);
3094
3095 gen8_gt_irq_reset(dev_priv);
3096
3097 for_each_pipe(dev_priv, pipe)
3098 if (intel_display_power_is_enabled(dev_priv,
3099 POWER_DOMAIN_PIPE(pipe)))
3100 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3101
3102 GEN5_IRQ_RESET(GEN8_DE_PORT_);
3103 GEN5_IRQ_RESET(GEN8_DE_MISC_);
3104 GEN5_IRQ_RESET(GEN8_PCU_);
3105
3106 if (HAS_PCH_SPLIT(dev))
3107 ibx_irq_reset(dev);
3108 }
3109
3110 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3111 unsigned int pipe_mask)
3112 {
3113 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3114
3115 spin_lock_irq(&dev_priv->irq_lock);
3116 if (pipe_mask & 1 << PIPE_A)
3117 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
3118 dev_priv->de_irq_mask[PIPE_A],
3119 ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3120 if (pipe_mask & 1 << PIPE_B)
3121 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
3122 dev_priv->de_irq_mask[PIPE_B],
3123 ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3124 if (pipe_mask & 1 << PIPE_C)
3125 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
3126 dev_priv->de_irq_mask[PIPE_C],
3127 ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3128 spin_unlock_irq(&dev_priv->irq_lock);
3129 }
3130
3131 static void cherryview_irq_preinstall(struct drm_device *dev)
3132 {
3133 struct drm_i915_private *dev_priv = dev->dev_private;
3134
3135 I915_WRITE(GEN8_MASTER_IRQ, 0);
3136 POSTING_READ(GEN8_MASTER_IRQ);
3137
3138 gen8_gt_irq_reset(dev_priv);
3139
3140 GEN5_IRQ_RESET(GEN8_PCU_);
3141
3142 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3143
3144 vlv_display_irq_reset(dev_priv);
3145 }
3146
3147 static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
3148 const u32 hpd[HPD_NUM_PINS])
3149 {
3150 struct drm_i915_private *dev_priv = to_i915(dev);
3151 struct intel_encoder *encoder;
3152 u32 enabled_irqs = 0;
3153
3154 for_each_intel_encoder(dev, encoder)
3155 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3156 enabled_irqs |= hpd[encoder->hpd_pin];
3157
3158 return enabled_irqs;
3159 }
3160
3161 static void ibx_hpd_irq_setup(struct drm_device *dev)
3162 {
3163 struct drm_i915_private *dev_priv = dev->dev_private;
3164 u32 hotplug_irqs, hotplug, enabled_irqs;
3165
3166 if (HAS_PCH_IBX(dev)) {
3167 hotplug_irqs = SDE_HOTPLUG_MASK;
3168 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3169 } else {
3170 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3171 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3172 }
3173
3174 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3175
3176 /*
3177 * Enable digital hotplug on the PCH, and configure the DP short pulse
3178 * duration to 2ms (which is the minimum in the Display Port spec).
3179 * The pulse duration bits are reserved on LPT+.
3180 */
3181 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3182 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3183 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3184 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3185 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3186 /*
3187 * When CPU and PCH are on the same package, port A
3188 * HPD must be enabled in both north and south.
3189 */
3190 if (HAS_PCH_LPT_LP(dev))
3191 hotplug |= PORTA_HOTPLUG_ENABLE;
3192 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3193 }
3194
3195 static void spt_hpd_irq_setup(struct drm_device *dev)
3196 {
3197 struct drm_i915_private *dev_priv = dev->dev_private;
3198 u32 hotplug_irqs, hotplug, enabled_irqs;
3199
3200 hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3201 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
3202
3203 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3204
3205 /* Enable digital hotplug on the PCH */
3206 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3207 hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3208 PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3209 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3210
3211 hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3212 hotplug |= PORTE_HOTPLUG_ENABLE;
3213 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3214 }
3215
3216 static void ilk_hpd_irq_setup(struct drm_device *dev)
3217 {
3218 struct drm_i915_private *dev_priv = dev->dev_private;
3219 u32 hotplug_irqs, hotplug, enabled_irqs;
3220
3221 if (INTEL_INFO(dev)->gen >= 8) {
3222 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3223 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
3224
3225 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3226 } else if (INTEL_INFO(dev)->gen >= 7) {
3227 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3228 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3229
3230 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3231 } else {
3232 hotplug_irqs = DE_DP_A_HOTPLUG;
3233 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3234
3235 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3236 }
3237
3238 /*
3239 * Enable digital hotplug on the CPU, and configure the DP short pulse
3240 * duration to 2ms (which is the minimum in the Display Port spec)
3241 * The pulse duration bits are reserved on HSW+.
3242 */
3243 hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3244 hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3245 hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
3246 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3247
3248 ibx_hpd_irq_setup(dev);
3249 }
3250
3251 static void bxt_hpd_irq_setup(struct drm_device *dev)
3252 {
3253 struct drm_i915_private *dev_priv = dev->dev_private;
3254 u32 hotplug_irqs, hotplug, enabled_irqs;
3255
3256 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
3257 hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3258
3259 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3260
3261 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3262 hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
3263 PORTA_HOTPLUG_ENABLE;
3264 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3265 }
3266
3267 static void ibx_irq_postinstall(struct drm_device *dev)
3268 {
3269 struct drm_i915_private *dev_priv = dev->dev_private;
3270 u32 mask;
3271
3272 if (HAS_PCH_NOP(dev))
3273 return;
3274
3275 if (HAS_PCH_IBX(dev))
3276 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3277 else
3278 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3279
3280 GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3281 I915_WRITE(SDEIMR, ~mask);
3282 }
3283
3284 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3285 {
3286 struct drm_i915_private *dev_priv = dev->dev_private;
3287 u32 pm_irqs, gt_irqs;
3288
3289 pm_irqs = gt_irqs = 0;
3290
3291 dev_priv->gt_irq_mask = ~0;
3292 if (HAS_L3_DPF(dev)) {
3293 /* L3 parity interrupt is always unmasked. */
3294 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3295 gt_irqs |= GT_PARITY_ERROR(dev);
3296 }
3297
3298 gt_irqs |= GT_RENDER_USER_INTERRUPT;
3299 if (IS_GEN5(dev)) {
3300 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3301 ILK_BSD_USER_INTERRUPT;
3302 } else {
3303 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3304 }
3305
3306 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3307
3308 if (INTEL_INFO(dev)->gen >= 6) {
3309 /*
3310 * RPS interrupts will get enabled/disabled on demand when RPS
3311 * itself is enabled/disabled.
3312 */
3313 if (HAS_VEBOX(dev))
3314 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3315
3316 dev_priv->pm_irq_mask = 0xffffffff;
3317 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3318 }
3319 }
3320
3321 static int ironlake_irq_postinstall(struct drm_device *dev)
3322 {
3323 struct drm_i915_private *dev_priv = dev->dev_private;
3324 u32 display_mask, extra_mask;
3325
3326 if (INTEL_INFO(dev)->gen >= 7) {
3327 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3328 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3329 DE_PLANEB_FLIP_DONE_IVB |
3330 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3331 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3332 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3333 DE_DP_A_HOTPLUG_IVB);
3334 } else {
3335 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3336 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3337 DE_AUX_CHANNEL_A |
3338 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3339 DE_POISON);
3340 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3341 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3342 DE_DP_A_HOTPLUG);
3343 }
3344
3345 dev_priv->irq_mask = ~display_mask;
3346
3347 I915_WRITE(HWSTAM, 0xeffe);
3348
3349 ibx_irq_pre_postinstall(dev);
3350
3351 GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3352
3353 gen5_gt_irq_postinstall(dev);
3354
3355 ibx_irq_postinstall(dev);
3356
3357 if (IS_IRONLAKE_M(dev)) {
3358 /* Enable PCU event interrupts
3359 *
3360 * spinlocking not required here for correctness since interrupt
3361 * setup is guaranteed to run in single-threaded context. But we
3362 * need it to make the assert_spin_locked happy. */
3363 spin_lock_irq(&dev_priv->irq_lock);
3364 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3365 spin_unlock_irq(&dev_priv->irq_lock);
3366 }
3367
3368 return 0;
3369 }
3370
3371 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3372 {
3373 u32 pipestat_mask;
3374 u32 iir_mask;
3375 enum pipe pipe;
3376
3377 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3378 PIPE_FIFO_UNDERRUN_STATUS;
3379
3380 for_each_pipe(dev_priv, pipe)
3381 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3382 POSTING_READ(PIPESTAT(PIPE_A));
3383
3384 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3385 PIPE_CRC_DONE_INTERRUPT_STATUS;
3386
3387 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3388 for_each_pipe(dev_priv, pipe)
3389 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3390
3391 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3392 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3393 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3394 if (IS_CHERRYVIEW(dev_priv))
3395 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3396 dev_priv->irq_mask &= ~iir_mask;
3397
3398 I915_WRITE(VLV_IIR, iir_mask);
3399 I915_WRITE(VLV_IIR, iir_mask);
3400 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3401 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3402 POSTING_READ(VLV_IMR);
3403 }
3404
3405 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3406 {
3407 u32 pipestat_mask;
3408 u32 iir_mask;
3409 enum pipe pipe;
3410
3411 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3412 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3413 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3414 if (IS_CHERRYVIEW(dev_priv))
3415 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3416
3417 dev_priv->irq_mask |= iir_mask;
3418 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3419 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3420 I915_WRITE(VLV_IIR, iir_mask);
3421 I915_WRITE(VLV_IIR, iir_mask);
3422 POSTING_READ(VLV_IIR);
3423
3424 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3425 PIPE_CRC_DONE_INTERRUPT_STATUS;
3426
3427 i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3428 for_each_pipe(dev_priv, pipe)
3429 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3430
3431 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3432 PIPE_FIFO_UNDERRUN_STATUS;
3433
3434 for_each_pipe(dev_priv, pipe)
3435 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3436 POSTING_READ(PIPESTAT(PIPE_A));
3437 }
3438
3439 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3440 {
3441 assert_spin_locked(&dev_priv->irq_lock);
3442
3443 if (dev_priv->display_irqs_enabled)
3444 return;
3445
3446 dev_priv->display_irqs_enabled = true;
3447
3448 if (intel_irqs_enabled(dev_priv))
3449 valleyview_display_irqs_install(dev_priv);
3450 }
3451
3452 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3453 {
3454 assert_spin_locked(&dev_priv->irq_lock);
3455
3456 if (!dev_priv->display_irqs_enabled)
3457 return;
3458
3459 dev_priv->display_irqs_enabled = false;
3460
3461 if (intel_irqs_enabled(dev_priv))
3462 valleyview_display_irqs_uninstall(dev_priv);
3463 }
3464
3465 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3466 {
3467 dev_priv->irq_mask = ~0;
3468
3469 I915_WRITE(PORT_HOTPLUG_EN, 0);
3470 POSTING_READ(PORT_HOTPLUG_EN);
3471
3472 I915_WRITE(VLV_IIR, 0xffffffff);
3473 I915_WRITE(VLV_IIR, 0xffffffff);
3474 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3475 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3476 POSTING_READ(VLV_IMR);
3477
3478 /* Interrupt setup is already guaranteed to be single-threaded, this is
3479 * just to make the assert_spin_locked check happy. */
3480 spin_lock_irq(&dev_priv->irq_lock);
3481 if (dev_priv->display_irqs_enabled)
3482 valleyview_display_irqs_install(dev_priv);
3483 spin_unlock_irq(&dev_priv->irq_lock);
3484 }
3485
3486 static int valleyview_irq_postinstall(struct drm_device *dev)
3487 {
3488 struct drm_i915_private *dev_priv = dev->dev_private;
3489
3490 vlv_display_irq_postinstall(dev_priv);
3491
3492 gen5_gt_irq_postinstall(dev);
3493
3494 /* ack & enable invalid PTE error interrupts */
3495 #if 0 /* FIXME: add support to irq handler for checking these bits */
3496 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3497 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3498 #endif
3499
3500 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3501
3502 return 0;
3503 }
3504
3505 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3506 {
3507 /* These are interrupts we'll toggle with the ring mask register */
3508 uint32_t gt_interrupts[] = {
3509 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3510 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3511 GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3512 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3513 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3514 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3515 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3516 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3517 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3518 0,
3519 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3520 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3521 };
3522
3523 dev_priv->pm_irq_mask = 0xffffffff;
3524 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3525 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3526 /*
3527 * RPS interrupts will get enabled/disabled on demand when RPS itself
3528 * is enabled/disabled.
3529 */
3530 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3531 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3532 }
3533
3534 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3535 {
3536 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3537 uint32_t de_pipe_enables;
3538 u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3539 u32 de_port_enables;
3540 enum pipe pipe;
3541
3542 if (INTEL_INFO(dev_priv)->gen >= 9) {
3543 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3544 GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3545 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3546 GEN9_AUX_CHANNEL_D;
3547 if (IS_BROXTON(dev_priv))
3548 de_port_masked |= BXT_DE_PORT_GMBUS;
3549 } else {
3550 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3551 GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3552 }
3553
3554 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3555 GEN8_PIPE_FIFO_UNDERRUN;
3556
3557 de_port_enables = de_port_masked;
3558 if (IS_BROXTON(dev_priv))
3559 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3560 else if (IS_BROADWELL(dev_priv))
3561 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3562
3563 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3564 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3565 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3566
3567 for_each_pipe(dev_priv, pipe)
3568 if (intel_display_power_is_enabled(dev_priv,
3569 POWER_DOMAIN_PIPE(pipe)))
3570 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3571 dev_priv->de_irq_mask[pipe],
3572 de_pipe_enables);
3573
3574 GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3575 }
3576
3577 static int gen8_irq_postinstall(struct drm_device *dev)
3578 {
3579 struct drm_i915_private *dev_priv = dev->dev_private;
3580
3581 if (HAS_PCH_SPLIT(dev))
3582 ibx_irq_pre_postinstall(dev);
3583
3584 gen8_gt_irq_postinstall(dev_priv);
3585 gen8_de_irq_postinstall(dev_priv);
3586
3587 if (HAS_PCH_SPLIT(dev))
3588 ibx_irq_postinstall(dev);
3589
3590 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3591 POSTING_READ(GEN8_MASTER_IRQ);
3592
3593 return 0;
3594 }
3595
3596 static int cherryview_irq_postinstall(struct drm_device *dev)
3597 {
3598 struct drm_i915_private *dev_priv = dev->dev_private;
3599
3600 vlv_display_irq_postinstall(dev_priv);
3601
3602 gen8_gt_irq_postinstall(dev_priv);
3603
3604 I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3605 POSTING_READ(GEN8_MASTER_IRQ);
3606
3607 return 0;
3608 }
3609
3610 static void gen8_irq_uninstall(struct drm_device *dev)
3611 {
3612 struct drm_i915_private *dev_priv = dev->dev_private;
3613
3614 if (!dev_priv)
3615 return;
3616
3617 gen8_irq_reset(dev);
3618 }
3619
3620 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3621 {
3622 /* Interrupt setup is already guaranteed to be single-threaded, this is
3623 * just to make the assert_spin_locked check happy. */
3624 spin_lock_irq(&dev_priv->irq_lock);
3625 if (dev_priv->display_irqs_enabled)
3626 valleyview_display_irqs_uninstall(dev_priv);
3627 spin_unlock_irq(&dev_priv->irq_lock);
3628
3629 vlv_display_irq_reset(dev_priv);
3630
3631 dev_priv->irq_mask = ~0;
3632 }
3633
3634 static void valleyview_irq_uninstall(struct drm_device *dev)
3635 {
3636 struct drm_i915_private *dev_priv = dev->dev_private;
3637
3638 if (!dev_priv)
3639 return;
3640
3641 I915_WRITE(VLV_MASTER_IER, 0);
3642
3643 gen5_gt_irq_reset(dev);
3644
3645 I915_WRITE(HWSTAM, 0xffffffff);
3646
3647 vlv_display_irq_uninstall(dev_priv);
3648 }
3649
3650 static void cherryview_irq_uninstall(struct drm_device *dev)
3651 {
3652 struct drm_i915_private *dev_priv = dev->dev_private;
3653
3654 if (!dev_priv)
3655 return;
3656
3657 I915_WRITE(GEN8_MASTER_IRQ, 0);
3658 POSTING_READ(GEN8_MASTER_IRQ);
3659
3660 gen8_gt_irq_reset(dev_priv);
3661
3662 GEN5_IRQ_RESET(GEN8_PCU_);
3663
3664 vlv_display_irq_uninstall(dev_priv);
3665 }
3666
3667 static void ironlake_irq_uninstall(struct drm_device *dev)
3668 {
3669 struct drm_i915_private *dev_priv = dev->dev_private;
3670
3671 if (!dev_priv)
3672 return;
3673
3674 ironlake_irq_reset(dev);
3675 }
3676
3677 static void i8xx_irq_preinstall(struct drm_device * dev)
3678 {
3679 struct drm_i915_private *dev_priv = dev->dev_private;
3680 int pipe;
3681
3682 for_each_pipe(dev_priv, pipe)
3683 I915_WRITE(PIPESTAT(pipe), 0);
3684 I915_WRITE16(IMR, 0xffff);
3685 I915_WRITE16(IER, 0x0);
3686 POSTING_READ16(IER);
3687 }
3688
3689 static int i8xx_irq_postinstall(struct drm_device *dev)
3690 {
3691 struct drm_i915_private *dev_priv = dev->dev_private;
3692
3693 I915_WRITE16(EMR,
3694 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3695
3696 /* Unmask the interrupts that we always want on. */
3697 dev_priv->irq_mask =
3698 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3699 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3700 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3701 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3702 I915_WRITE16(IMR, dev_priv->irq_mask);
3703
3704 I915_WRITE16(IER,
3705 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3706 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3707 I915_USER_INTERRUPT);
3708 POSTING_READ16(IER);
3709
3710 /* Interrupt setup is already guaranteed to be single-threaded, this is
3711 * just to make the assert_spin_locked check happy. */
3712 spin_lock_irq(&dev_priv->irq_lock);
3713 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3714 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3715 spin_unlock_irq(&dev_priv->irq_lock);
3716
3717 return 0;
3718 }
3719
3720 /*
3721 * Returns true when a page flip has completed.
3722 */
3723 static bool i8xx_handle_vblank(struct drm_device *dev,
3724 int plane, int pipe, u32 iir)
3725 {
3726 struct drm_i915_private *dev_priv = dev->dev_private;
3727 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3728
3729 if (!intel_pipe_handle_vblank(dev, pipe))
3730 return false;
3731
3732 if ((iir & flip_pending) == 0)
3733 goto check_page_flip;
3734
3735 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3736 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3737 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3738 * the flip is completed (no longer pending). Since this doesn't raise
3739 * an interrupt per se, we watch for the change at vblank.
3740 */
3741 if (I915_READ16(ISR) & flip_pending)
3742 goto check_page_flip;
3743
3744 intel_prepare_page_flip(dev, plane);
3745 intel_finish_page_flip(dev, pipe);
3746 return true;
3747
3748 check_page_flip:
3749 intel_check_page_flip(dev, pipe);
3750 return false;
3751 }
3752
3753 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3754 {
3755 struct drm_device *dev = arg;
3756 struct drm_i915_private *dev_priv = dev->dev_private;
3757 u16 iir, new_iir;
3758 u32 pipe_stats[2];
3759 int pipe;
3760 u16 flip_mask =
3761 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3762 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3763
3764 if (!intel_irqs_enabled(dev_priv))
3765 return IRQ_NONE;
3766
3767 iir = I915_READ16(IIR);
3768 if (iir == 0)
3769 return IRQ_NONE;
3770
3771 while (iir & ~flip_mask) {
3772 /* Can't rely on pipestat interrupt bit in iir as it might
3773 * have been cleared after the pipestat interrupt was received.
3774 * It doesn't set the bit in iir again, but it still produces
3775 * interrupts (for non-MSI).
3776 */
3777 spin_lock(&dev_priv->irq_lock);
3778 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3779 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3780
3781 for_each_pipe(dev_priv, pipe) {
3782 int reg = PIPESTAT(pipe);
3783 pipe_stats[pipe] = I915_READ(reg);
3784
3785 /*
3786 * Clear the PIPE*STAT regs before the IIR
3787 */
3788 if (pipe_stats[pipe] & 0x8000ffff)
3789 I915_WRITE(reg, pipe_stats[pipe]);
3790 }
3791 spin_unlock(&dev_priv->irq_lock);
3792
3793 I915_WRITE16(IIR, iir & ~flip_mask);
3794 new_iir = I915_READ16(IIR); /* Flush posted writes */
3795
3796 if (iir & I915_USER_INTERRUPT)
3797 notify_ring(&dev_priv->ring[RCS]);
3798
3799 for_each_pipe(dev_priv, pipe) {
3800 int plane = pipe;
3801 if (HAS_FBC(dev))
3802 plane = !plane;
3803
3804 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3805 i8xx_handle_vblank(dev, plane, pipe, iir))
3806 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3807
3808 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3809 i9xx_pipe_crc_irq_handler(dev, pipe);
3810
3811 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3812 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3813 pipe);
3814 }
3815
3816 iir = new_iir;
3817 }
3818
3819 return IRQ_HANDLED;
3820 }
3821
3822 static void i8xx_irq_uninstall(struct drm_device * dev)
3823 {
3824 struct drm_i915_private *dev_priv = dev->dev_private;
3825 int pipe;
3826
3827 for_each_pipe(dev_priv, pipe) {
3828 /* Clear enable bits; then clear status bits */
3829 I915_WRITE(PIPESTAT(pipe), 0);
3830 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3831 }
3832 I915_WRITE16(IMR, 0xffff);
3833 I915_WRITE16(IER, 0x0);
3834 I915_WRITE16(IIR, I915_READ16(IIR));
3835 }
3836
3837 static void i915_irq_preinstall(struct drm_device * dev)
3838 {
3839 struct drm_i915_private *dev_priv = dev->dev_private;
3840 int pipe;
3841
3842 if (I915_HAS_HOTPLUG(dev)) {
3843 I915_WRITE(PORT_HOTPLUG_EN, 0);
3844 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3845 }
3846
3847 I915_WRITE16(HWSTAM, 0xeffe);
3848 for_each_pipe(dev_priv, pipe)
3849 I915_WRITE(PIPESTAT(pipe), 0);
3850 I915_WRITE(IMR, 0xffffffff);
3851 I915_WRITE(IER, 0x0);
3852 POSTING_READ(IER);
3853 }
3854
3855 static int i915_irq_postinstall(struct drm_device *dev)
3856 {
3857 struct drm_i915_private *dev_priv = dev->dev_private;
3858 u32 enable_mask;
3859
3860 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3861
3862 /* Unmask the interrupts that we always want on. */
3863 dev_priv->irq_mask =
3864 ~(I915_ASLE_INTERRUPT |
3865 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3866 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3867 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3868 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3869
3870 enable_mask =
3871 I915_ASLE_INTERRUPT |
3872 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3873 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3874 I915_USER_INTERRUPT;
3875
3876 if (I915_HAS_HOTPLUG(dev)) {
3877 I915_WRITE(PORT_HOTPLUG_EN, 0);
3878 POSTING_READ(PORT_HOTPLUG_EN);
3879
3880 /* Enable in IER... */
3881 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3882 /* and unmask in IMR */
3883 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3884 }
3885
3886 I915_WRITE(IMR, dev_priv->irq_mask);
3887 I915_WRITE(IER, enable_mask);
3888 POSTING_READ(IER);
3889
3890 i915_enable_asle_pipestat(dev);
3891
3892 /* Interrupt setup is already guaranteed to be single-threaded, this is
3893 * just to make the assert_spin_locked check happy. */
3894 spin_lock_irq(&dev_priv->irq_lock);
3895 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3896 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3897 spin_unlock_irq(&dev_priv->irq_lock);
3898
3899 return 0;
3900 }
3901
3902 /*
3903 * Returns true when a page flip has completed.
3904 */
3905 static bool i915_handle_vblank(struct drm_device *dev,
3906 int plane, int pipe, u32 iir)
3907 {
3908 struct drm_i915_private *dev_priv = dev->dev_private;
3909 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3910
3911 if (!intel_pipe_handle_vblank(dev, pipe))
3912 return false;
3913
3914 if ((iir & flip_pending) == 0)
3915 goto check_page_flip;
3916
3917 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3918 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3919 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3920 * the flip is completed (no longer pending). Since this doesn't raise
3921 * an interrupt per se, we watch for the change at vblank.
3922 */
3923 if (I915_READ(ISR) & flip_pending)
3924 goto check_page_flip;
3925
3926 intel_prepare_page_flip(dev, plane);
3927 intel_finish_page_flip(dev, pipe);
3928 return true;
3929
3930 check_page_flip:
3931 intel_check_page_flip(dev, pipe);
3932 return false;
3933 }
3934
3935 static irqreturn_t i915_irq_handler(int irq, void *arg)
3936 {
3937 struct drm_device *dev = arg;
3938 struct drm_i915_private *dev_priv = dev->dev_private;
3939 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3940 u32 flip_mask =
3941 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3942 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3943 int pipe, ret = IRQ_NONE;
3944
3945 if (!intel_irqs_enabled(dev_priv))
3946 return IRQ_NONE;
3947
3948 iir = I915_READ(IIR);
3949 do {
3950 bool irq_received = (iir & ~flip_mask) != 0;
3951 bool blc_event = false;
3952
3953 /* Can't rely on pipestat interrupt bit in iir as it might
3954 * have been cleared after the pipestat interrupt was received.
3955 * It doesn't set the bit in iir again, but it still produces
3956 * interrupts (for non-MSI).
3957 */
3958 spin_lock(&dev_priv->irq_lock);
3959 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3960 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3961
3962 for_each_pipe(dev_priv, pipe) {
3963 int reg = PIPESTAT(pipe);
3964 pipe_stats[pipe] = I915_READ(reg);
3965
3966 /* Clear the PIPE*STAT regs before the IIR */
3967 if (pipe_stats[pipe] & 0x8000ffff) {
3968 I915_WRITE(reg, pipe_stats[pipe]);
3969 irq_received = true;
3970 }
3971 }
3972 spin_unlock(&dev_priv->irq_lock);
3973
3974 if (!irq_received)
3975 break;
3976
3977 /* Consume port. Then clear IIR or we'll miss events */
3978 if (I915_HAS_HOTPLUG(dev) &&
3979 iir & I915_DISPLAY_PORT_INTERRUPT)
3980 i9xx_hpd_irq_handler(dev);
3981
3982 I915_WRITE(IIR, iir & ~flip_mask);
3983 new_iir = I915_READ(IIR); /* Flush posted writes */
3984
3985 if (iir & I915_USER_INTERRUPT)
3986 notify_ring(&dev_priv->ring[RCS]);
3987
3988 for_each_pipe(dev_priv, pipe) {
3989 int plane = pipe;
3990 if (HAS_FBC(dev))
3991 plane = !plane;
3992
3993 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3994 i915_handle_vblank(dev, plane, pipe, iir))
3995 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3996
3997 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3998 blc_event = true;
3999
4000 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4001 i9xx_pipe_crc_irq_handler(dev, pipe);
4002
4003 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4004 intel_cpu_fifo_underrun_irq_handler(dev_priv,
4005 pipe);
4006 }
4007
4008 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4009 intel_opregion_asle_intr(dev);
4010
4011 /* With MSI, interrupts are only generated when iir
4012 * transitions from zero to nonzero. If another bit got
4013 * set while we were handling the existing iir bits, then
4014 * we would never get another interrupt.
4015 *
4016 * This is fine on non-MSI as well, as if we hit this path
4017 * we avoid exiting the interrupt handler only to generate
4018 * another one.
4019 *
4020 * Note that for MSI this could cause a stray interrupt report
4021 * if an interrupt landed in the time between writing IIR and
4022 * the posting read. This should be rare enough to never
4023 * trigger the 99% of 100,000 interrupts test for disabling
4024 * stray interrupts.
4025 */
4026 ret = IRQ_HANDLED;
4027 iir = new_iir;
4028 } while (iir & ~flip_mask);
4029
4030 return ret;
4031 }
4032
4033 static void i915_irq_uninstall(struct drm_device * dev)
4034 {
4035 struct drm_i915_private *dev_priv = dev->dev_private;
4036 int pipe;
4037
4038 if (I915_HAS_HOTPLUG(dev)) {
4039 I915_WRITE(PORT_HOTPLUG_EN, 0);
4040 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4041 }
4042
4043 I915_WRITE16(HWSTAM, 0xffff);
4044 for_each_pipe(dev_priv, pipe) {
4045 /* Clear enable bits; then clear status bits */
4046 I915_WRITE(PIPESTAT(pipe), 0);
4047 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4048 }
4049 I915_WRITE(IMR, 0xffffffff);
4050 I915_WRITE(IER, 0x0);
4051
4052 I915_WRITE(IIR, I915_READ(IIR));
4053 }
4054
4055 static void i965_irq_preinstall(struct drm_device * dev)
4056 {
4057 struct drm_i915_private *dev_priv = dev->dev_private;
4058 int pipe;
4059
4060 I915_WRITE(PORT_HOTPLUG_EN, 0);
4061 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4062
4063 I915_WRITE(HWSTAM, 0xeffe);
4064 for_each_pipe(dev_priv, pipe)
4065 I915_WRITE(PIPESTAT(pipe), 0);
4066 I915_WRITE(IMR, 0xffffffff);
4067 I915_WRITE(IER, 0x0);
4068 POSTING_READ(IER);
4069 }
4070
4071 static int i965_irq_postinstall(struct drm_device *dev)
4072 {
4073 struct drm_i915_private *dev_priv = dev->dev_private;
4074 u32 enable_mask;
4075 u32 error_mask;
4076
4077 /* Unmask the interrupts that we always want on. */
4078 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4079 I915_DISPLAY_PORT_INTERRUPT |
4080 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4081 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4082 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4083 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4084 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4085
4086 enable_mask = ~dev_priv->irq_mask;
4087 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4088 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4089 enable_mask |= I915_USER_INTERRUPT;
4090
4091 if (IS_G4X(dev))
4092 enable_mask |= I915_BSD_USER_INTERRUPT;
4093
4094 /* Interrupt setup is already guaranteed to be single-threaded, this is
4095 * just to make the assert_spin_locked check happy. */
4096 spin_lock_irq(&dev_priv->irq_lock);
4097 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4098 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4099 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4100 spin_unlock_irq(&dev_priv->irq_lock);
4101
4102 /*
4103 * Enable some error detection, note the instruction error mask
4104 * bit is reserved, so we leave it masked.
4105 */
4106 if (IS_G4X(dev)) {
4107 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4108 GM45_ERROR_MEM_PRIV |
4109 GM45_ERROR_CP_PRIV |
4110 I915_ERROR_MEMORY_REFRESH);
4111 } else {
4112 error_mask = ~(I915_ERROR_PAGE_TABLE |
4113 I915_ERROR_MEMORY_REFRESH);
4114 }
4115 I915_WRITE(EMR, error_mask);
4116
4117 I915_WRITE(IMR, dev_priv->irq_mask);
4118 I915_WRITE(IER, enable_mask);
4119 POSTING_READ(IER);
4120
4121 I915_WRITE(PORT_HOTPLUG_EN, 0);
4122 POSTING_READ(PORT_HOTPLUG_EN);
4123
4124 i915_enable_asle_pipestat(dev);
4125
4126 return 0;
4127 }
4128
4129 static void i915_hpd_irq_setup(struct drm_device *dev)
4130 {
4131 struct drm_i915_private *dev_priv = dev->dev_private;
4132 u32 hotplug_en;
4133
4134 assert_spin_locked(&dev_priv->irq_lock);
4135
4136 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
4137 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
4138 /* Note HDMI and DP share hotplug bits */
4139 /* enable bits are the same for all generations */
4140 hotplug_en |= intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4141 /* Programming the CRT detection parameters tends
4142 to generate a spurious hotplug event about three
4143 seconds later. So just do it once.
4144 */
4145 if (IS_G4X(dev))
4146 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4147 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
4148 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4149
4150 /* Ignore TV since it's buggy */
4151 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
4152 }
4153
4154 static irqreturn_t i965_irq_handler(int irq, void *arg)
4155 {
4156 struct drm_device *dev = arg;
4157 struct drm_i915_private *dev_priv = dev->dev_private;
4158 u32 iir, new_iir;
4159 u32 pipe_stats[I915_MAX_PIPES];
4160 int ret = IRQ_NONE, pipe;
4161 u32 flip_mask =
4162 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4163 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4164
4165 if (!intel_irqs_enabled(dev_priv))
4166 return IRQ_NONE;
4167
4168 iir = I915_READ(IIR);
4169
4170 for (;;) {
4171 bool irq_received = (iir & ~flip_mask) != 0;
4172 bool blc_event = false;
4173
4174 /* Can't rely on pipestat interrupt bit in iir as it might
4175 * have been cleared after the pipestat interrupt was received.
4176 * It doesn't set the bit in iir again, but it still produces
4177 * interrupts (for non-MSI).
4178 */
4179 spin_lock(&dev_priv->irq_lock);
4180 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4181 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4182
4183 for_each_pipe(dev_priv, pipe) {
4184 int reg = PIPESTAT(pipe);
4185 pipe_stats[pipe] = I915_READ(reg);
4186
4187 /*
4188 * Clear the PIPE*STAT regs before the IIR
4189 */
4190 if (pipe_stats[pipe] & 0x8000ffff) {
4191 I915_WRITE(reg, pipe_stats[pipe]);
4192 irq_received = true;
4193 }
4194 }
4195 spin_unlock(&dev_priv->irq_lock);
4196
4197 if (!irq_received)
4198 break;
4199
4200 ret = IRQ_HANDLED;
4201
4202 /* Consume port. Then clear IIR or we'll miss events */
4203 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4204 i9xx_hpd_irq_handler(dev);
4205
4206 I915_WRITE(IIR, iir & ~flip_mask);
4207 new_iir = I915_READ(IIR); /* Flush posted writes */
4208
4209 if (iir & I915_USER_INTERRUPT)
4210 notify_ring(&dev_priv->ring[RCS]);
4211 if (iir & I915_BSD_USER_INTERRUPT)
4212 notify_ring(&dev_priv->ring[VCS]);
4213
4214 for_each_pipe(dev_priv, pipe) {
4215 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4216 i915_handle_vblank(dev, pipe, pipe, iir))
4217 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4218
4219 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4220 blc_event = true;
4221
4222 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4223 i9xx_pipe_crc_irq_handler(dev, pipe);
4224
4225 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4226 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4227 }
4228
4229 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4230 intel_opregion_asle_intr(dev);
4231
4232 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4233 gmbus_irq_handler(dev);
4234
4235 /* With MSI, interrupts are only generated when iir
4236 * transitions from zero to nonzero. If another bit got
4237 * set while we were handling the existing iir bits, then
4238 * we would never get another interrupt.
4239 *
4240 * This is fine on non-MSI as well, as if we hit this path
4241 * we avoid exiting the interrupt handler only to generate
4242 * another one.
4243 *
4244 * Note that for MSI this could cause a stray interrupt report
4245 * if an interrupt landed in the time between writing IIR and
4246 * the posting read. This should be rare enough to never
4247 * trigger the 99% of 100,000 interrupts test for disabling
4248 * stray interrupts.
4249 */
4250 iir = new_iir;
4251 }
4252
4253 return ret;
4254 }
4255
4256 static void i965_irq_uninstall(struct drm_device * dev)
4257 {
4258 struct drm_i915_private *dev_priv = dev->dev_private;
4259 int pipe;
4260
4261 if (!dev_priv)
4262 return;
4263
4264 I915_WRITE(PORT_HOTPLUG_EN, 0);
4265 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4266
4267 I915_WRITE(HWSTAM, 0xffffffff);
4268 for_each_pipe(dev_priv, pipe)
4269 I915_WRITE(PIPESTAT(pipe), 0);
4270 I915_WRITE(IMR, 0xffffffff);
4271 I915_WRITE(IER, 0x0);
4272
4273 for_each_pipe(dev_priv, pipe)
4274 I915_WRITE(PIPESTAT(pipe),
4275 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4276 I915_WRITE(IIR, I915_READ(IIR));
4277 }
4278
4279 /**
4280 * intel_irq_init - initializes irq support
4281 * @dev_priv: i915 device instance
4282 *
4283 * This function initializes all the irq support including work items, timers
4284 * and all the vtables. It does not setup the interrupt itself though.
4285 */
4286 void intel_irq_init(struct drm_i915_private *dev_priv)
4287 {
4288 struct drm_device *dev = dev_priv->dev;
4289
4290 intel_hpd_init_work(dev_priv);
4291
4292 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4293 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4294
4295 /* Let's track the enabled rps events */
4296 if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4297 /* WaGsvRC0ResidencyMethod:vlv */
4298 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4299 else
4300 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4301
4302 INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4303 i915_hangcheck_elapsed);
4304
4305 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4306
4307 if (IS_GEN2(dev_priv)) {
4308 dev->max_vblank_count = 0;
4309 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4310 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4311 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4312 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4313 } else {
4314 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4315 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4316 }
4317
4318 /*
4319 * Opt out of the vblank disable timer on everything except gen2.
4320 * Gen2 doesn't have a hardware frame counter and so depends on
4321 * vblank interrupts to produce sane vblank seuquence numbers.
4322 */
4323 if (!IS_GEN2(dev_priv))
4324 dev->vblank_disable_immediate = true;
4325
4326 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4327 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4328
4329 if (IS_CHERRYVIEW(dev_priv)) {
4330 dev->driver->irq_handler = cherryview_irq_handler;
4331 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4332 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4333 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4334 dev->driver->enable_vblank = valleyview_enable_vblank;
4335 dev->driver->disable_vblank = valleyview_disable_vblank;
4336 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4337 } else if (IS_VALLEYVIEW(dev_priv)) {
4338 dev->driver->irq_handler = valleyview_irq_handler;
4339 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4340 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4341 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4342 dev->driver->enable_vblank = valleyview_enable_vblank;
4343 dev->driver->disable_vblank = valleyview_disable_vblank;
4344 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4345 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4346 dev->driver->irq_handler = gen8_irq_handler;
4347 dev->driver->irq_preinstall = gen8_irq_reset;
4348 dev->driver->irq_postinstall = gen8_irq_postinstall;
4349 dev->driver->irq_uninstall = gen8_irq_uninstall;
4350 dev->driver->enable_vblank = gen8_enable_vblank;
4351 dev->driver->disable_vblank = gen8_disable_vblank;
4352 if (IS_BROXTON(dev))
4353 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4354 else if (HAS_PCH_SPT(dev))
4355 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4356 else
4357 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4358 } else if (HAS_PCH_SPLIT(dev)) {
4359 dev->driver->irq_handler = ironlake_irq_handler;
4360 dev->driver->irq_preinstall = ironlake_irq_reset;
4361 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4362 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4363 dev->driver->enable_vblank = ironlake_enable_vblank;
4364 dev->driver->disable_vblank = ironlake_disable_vblank;
4365 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4366 } else {
4367 if (INTEL_INFO(dev_priv)->gen == 2) {
4368 dev->driver->irq_preinstall = i8xx_irq_preinstall;
4369 dev->driver->irq_postinstall = i8xx_irq_postinstall;
4370 dev->driver->irq_handler = i8xx_irq_handler;
4371 dev->driver->irq_uninstall = i8xx_irq_uninstall;
4372 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4373 dev->driver->irq_preinstall = i915_irq_preinstall;
4374 dev->driver->irq_postinstall = i915_irq_postinstall;
4375 dev->driver->irq_uninstall = i915_irq_uninstall;
4376 dev->driver->irq_handler = i915_irq_handler;
4377 } else {
4378 dev->driver->irq_preinstall = i965_irq_preinstall;
4379 dev->driver->irq_postinstall = i965_irq_postinstall;
4380 dev->driver->irq_uninstall = i965_irq_uninstall;
4381 dev->driver->irq_handler = i965_irq_handler;
4382 }
4383 if (I915_HAS_HOTPLUG(dev_priv))
4384 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4385 dev->driver->enable_vblank = i915_enable_vblank;
4386 dev->driver->disable_vblank = i915_disable_vblank;
4387 }
4388 }
4389
4390 /**
4391 * intel_irq_install - enables the hardware interrupt
4392 * @dev_priv: i915 device instance
4393 *
4394 * This function enables the hardware interrupt handling, but leaves the hotplug
4395 * handling still disabled. It is called after intel_irq_init().
4396 *
4397 * In the driver load and resume code we need working interrupts in a few places
4398 * but don't want to deal with the hassle of concurrent probe and hotplug
4399 * workers. Hence the split into this two-stage approach.
4400 */
4401 int intel_irq_install(struct drm_i915_private *dev_priv)
4402 {
4403 /*
4404 * We enable some interrupt sources in our postinstall hooks, so mark
4405 * interrupts as enabled _before_ actually enabling them to avoid
4406 * special cases in our ordering checks.
4407 */
4408 dev_priv->pm.irqs_enabled = true;
4409
4410 return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4411 }
4412
4413 /**
4414 * intel_irq_uninstall - finilizes all irq handling
4415 * @dev_priv: i915 device instance
4416 *
4417 * This stops interrupt and hotplug handling and unregisters and frees all
4418 * resources acquired in the init functions.
4419 */
4420 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4421 {
4422 drm_irq_uninstall(dev_priv->dev);
4423 intel_hpd_cancel_work(dev_priv);
4424 dev_priv->pm.irqs_enabled = false;
4425 }
4426
4427 /**
4428 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4429 * @dev_priv: i915 device instance
4430 *
4431 * This function is used to disable interrupts at runtime, both in the runtime
4432 * pm and the system suspend/resume code.
4433 */
4434 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4435 {
4436 dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4437 dev_priv->pm.irqs_enabled = false;
4438 synchronize_irq(dev_priv->dev->irq);
4439 }
4440
4441 /**
4442 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4443 * @dev_priv: i915 device instance
4444 *
4445 * This function is used to enable interrupts at runtime, both in the runtime
4446 * pm and the system suspend/resume code.
4447 */
4448 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4449 {
4450 dev_priv->pm.irqs_enabled = true;
4451 dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4452 dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4453 }
Linux kernel - Deliverables
This page took 0.122662 seconds and 5 git commands to generate.