projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: extract ibx_display_interrupt_update
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_irq.c
CommitLineData
0d6aa60b 1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
1da177e4 2 */
0d6aa60b 3/*
1da177e4
LT		 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
bc54fd1a
DA		 6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
0d6aa60b 27 */
1da177e4 28
a70491cc
JP		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
63eeaf38 31#include <linux/sysrq.h>
5a0e3ad6 32#include <linux/slab.h>
760285e7
DH		 33#include <drm/drmP.h>
34#include <drm/i915_drm.h>
1da177e4 35#include "i915_drv.h"
1c5d22f7 36#include "i915_trace.h"
79e53945 37#include "intel_drv.h"
1da177e4 38
e5868a31
EE		 39static const u32 hpd_ibx[] = {
40 [HPD_CRT] = SDE_CRT_HOTPLUG,
41 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
42 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
43 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
44 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
45};
46
47static const u32 hpd_cpt[] = {
48 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
73c352a2 49 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
e5868a31
EE		 50 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
51 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
52 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
53};
54
55static const u32 hpd_mask_i915[] = {
56 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
57 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
58 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
59 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
60 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
61 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
62};
63
64static const u32 hpd_status_gen4[] = {
65 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
66 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
67 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
68 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
69 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
70 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
71};
72
e5868a31
EE		 73static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
74 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
75 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
76 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
77 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
78 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
79 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
80};
81
036a4a7d 82/* For display hotplug interrupt */
995b6762 83static void
f2b115e6 84ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 85{
4bc9d430
DV		 86 assert_spin_locked(&dev_priv->irq_lock);
87
1ec14ad3
CW		 88 if ((dev_priv->irq_mask & mask) != 0) {
89 dev_priv->irq_mask &= ~mask;
90 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 91 POSTING_READ(DEIMR);
036a4a7d
ZW		 92 }
93}
94
0ff9800a 95static void
f2b115e6 96ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 97{
4bc9d430
DV		 98 assert_spin_locked(&dev_priv->irq_lock);
99
1ec14ad3
CW		 100 if ((dev_priv->irq_mask & mask) != mask) {
101 dev_priv->irq_mask |= mask;
102 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 103 POSTING_READ(DEIMR);
036a4a7d
ZW		 104 }
105}
106
8664281b
PZ		 107static bool ivb_can_enable_err_int(struct drm_device *dev)
108{
109 struct drm_i915_private *dev_priv = dev->dev_private;
110 struct intel_crtc *crtc;
111 enum pipe pipe;
112
4bc9d430
DV		 113 assert_spin_locked(&dev_priv->irq_lock);
114
8664281b
PZ		 115 for_each_pipe(pipe) {
116 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
117
118 if (crtc->cpu_fifo_underrun_disabled)
119 return false;
120 }
121
122 return true;
123}
124
125static bool cpt_can_enable_serr_int(struct drm_device *dev)
126{
127 struct drm_i915_private *dev_priv = dev->dev_private;
128 enum pipe pipe;
129 struct intel_crtc *crtc;
130
fee884ed
DV		 131 assert_spin_locked(&dev_priv->irq_lock);
132
8664281b
PZ		 133 for_each_pipe(pipe) {
134 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
135
136 if (crtc->pch_fifo_underrun_disabled)
137 return false;
138 }
139
140 return true;
141}
142
143static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
144 enum pipe pipe, bool enable)
145{
146 struct drm_i915_private *dev_priv = dev->dev_private;
147 uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
148 DE_PIPEB_FIFO_UNDERRUN;
149
150 if (enable)
151 ironlake_enable_display_irq(dev_priv, bit);
152 else
153 ironlake_disable_display_irq(dev_priv, bit);
154}
155
156static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
157 bool enable)
158{
159 struct drm_i915_private *dev_priv = dev->dev_private;
160
161 if (enable) {
162 if (!ivb_can_enable_err_int(dev))
163 return;
164
165 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN_A |
166 ERR_INT_FIFO_UNDERRUN_B |
167 ERR_INT_FIFO_UNDERRUN_C);
168
169 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
170 } else {
171 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
172 }
173}
174
fee884ed
DV		 175/**
176 * ibx_display_interrupt_update - update SDEIMR
177 * @dev_priv: driver private
178 * @interrupt_mask: mask of interrupt bits to update
179 * @enabled_irq_mask: mask of interrupt bits to enable
180 */
181static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
182 uint32_t interrupt_mask,
183 uint32_t enabled_irq_mask)
184{
185 uint32_t sdeimr = I915_READ(SDEIMR);
186 sdeimr &= ~interrupt_mask;
187 sdeimr |= (~enabled_irq_mask & interrupt_mask);
188
189 assert_spin_locked(&dev_priv->irq_lock);
190
191 I915_WRITE(SDEIMR, sdeimr);
192 POSTING_READ(SDEIMR);
193}
194#define ibx_enable_display_interrupt(dev_priv, bits) \
195 ibx_display_interrupt_update((dev_priv), (bits), (bits))
196#define ibx_disable_display_interrupt(dev_priv, bits) \
197 ibx_display_interrupt_update((dev_priv), (bits), 0)
198
8664281b
PZ		 199static void ibx_set_fifo_underrun_reporting(struct intel_crtc *crtc,
200 bool enable)
201{
202 struct drm_device *dev = crtc->base.dev;
203 struct drm_i915_private *dev_priv = dev->dev_private;
204 uint32_t bit = (crtc->pipe == PIPE_A) ? SDE_TRANSA_FIFO_UNDER :
205 SDE_TRANSB_FIFO_UNDER;
206
207 if (enable)
fee884ed 208 ibx_enable_display_interrupt(dev_priv, bit);
8664281b 209 else
fee884ed 210 ibx_disable_display_interrupt(dev_priv, bit);
8664281b
PZ		 211}
212
213static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
214 enum transcoder pch_transcoder,
215 bool enable)
216{
217 struct drm_i915_private *dev_priv = dev->dev_private;
218
219 if (enable) {
220 if (!cpt_can_enable_serr_int(dev))
221 return;
222
223 I915_WRITE(SERR_INT, SERR_INT_TRANS_A_FIFO_UNDERRUN |
224 SERR_INT_TRANS_B_FIFO_UNDERRUN |
225 SERR_INT_TRANS_C_FIFO_UNDERRUN);
226
fee884ed 227 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
8664281b 228 } else {
fee884ed 229 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
8664281b 230 }
8664281b
PZ		 231}
232
233/**
234 * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
235 * @dev: drm device
236 * @pipe: pipe
237 * @enable: true if we want to report FIFO underrun errors, false otherwise
238 *
239 * This function makes us disable or enable CPU fifo underruns for a specific
240 * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
241 * reporting for one pipe may also disable all the other CPU error interruts for
242 * the other pipes, due to the fact that there's just one interrupt mask/enable
243 * bit for all the pipes.
244 *
245 * Returns the previous state of underrun reporting.
246 */
247bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
248 enum pipe pipe, bool enable)
249{
250 struct drm_i915_private *dev_priv = dev->dev_private;
251 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
252 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
253 unsigned long flags;
254 bool ret;
255
256 spin_lock_irqsave(&dev_priv->irq_lock, flags);
257
258 ret = !intel_crtc->cpu_fifo_underrun_disabled;
259
260 if (enable == ret)
261 goto done;
262
263 intel_crtc->cpu_fifo_underrun_disabled = !enable;
264
265 if (IS_GEN5(dev) || IS_GEN6(dev))
266 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
267 else if (IS_GEN7(dev))
268 ivybridge_set_fifo_underrun_reporting(dev, enable);
269
270done:
271 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
272 return ret;
273}
274
275/**
276 * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
277 * @dev: drm device
278 * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
279 * @enable: true if we want to report FIFO underrun errors, false otherwise
280 *
281 * This function makes us disable or enable PCH fifo underruns for a specific
282 * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
283 * underrun reporting for one transcoder may also disable all the other PCH
284 * error interruts for the other transcoders, due to the fact that there's just
285 * one interrupt mask/enable bit for all the transcoders.
286 *
287 * Returns the previous state of underrun reporting.
288 */
289bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
290 enum transcoder pch_transcoder,
291 bool enable)
292{
293 struct drm_i915_private *dev_priv = dev->dev_private;
294 enum pipe p;
295 struct drm_crtc *crtc;
296 struct intel_crtc *intel_crtc;
297 unsigned long flags;
298 bool ret;
299
300 if (HAS_PCH_LPT(dev)) {
301 crtc = NULL;
302 for_each_pipe(p) {
303 struct drm_crtc *c = dev_priv->pipe_to_crtc_mapping[p];
304 if (intel_pipe_has_type(c, INTEL_OUTPUT_ANALOG)) {
305 crtc = c;
306 break;
307 }
308 }
309 if (!crtc) {
310 DRM_ERROR("PCH FIFO underrun, but no CRTC using the PCH found\n");
311 return false;
312 }
313 } else {
314 crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
315 }
316 intel_crtc = to_intel_crtc(crtc);
317
318 spin_lock_irqsave(&dev_priv->irq_lock, flags);
319
320 ret = !intel_crtc->pch_fifo_underrun_disabled;
321
322 if (enable == ret)
323 goto done;
324
325 intel_crtc->pch_fifo_underrun_disabled = !enable;
326
327 if (HAS_PCH_IBX(dev))
328 ibx_set_fifo_underrun_reporting(intel_crtc, enable);
329 else
330 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
331
332done:
333 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
334 return ret;
335}
336
337
7c463586
KP		 338void
339i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
340{
46c06a30
VS		 341 u32 reg = PIPESTAT(pipe);
342 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 343
b79480ba
DV		 344 assert_spin_locked(&dev_priv->irq_lock);
345
46c06a30
VS		 346 if ((pipestat & mask) == mask)
347 return;
348
349 /* Enable the interrupt, clear any pending status */
350 pipestat |= mask | (mask >> 16);
351 I915_WRITE(reg, pipestat);
352 POSTING_READ(reg);
7c463586
KP		 353}
354
355void
356i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
357{
46c06a30
VS		 358 u32 reg = PIPESTAT(pipe);
359 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 360
b79480ba
DV		 361 assert_spin_locked(&dev_priv->irq_lock);
362
46c06a30
VS		 363 if ((pipestat & mask) == 0)
364 return;
365
366 pipestat &= ~mask;
367 I915_WRITE(reg, pipestat);
368 POSTING_READ(reg);
7c463586
KP		 369}
370
01c66889 371/**
f49e38dd 372 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
01c66889 373 */
f49e38dd 374static void i915_enable_asle_pipestat(struct drm_device *dev)
01c66889 375{
1ec14ad3
CW		 376 drm_i915_private_t *dev_priv = dev->dev_private;
377 unsigned long irqflags;
378
f49e38dd
JN		 379 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
380 return;
381
1ec14ad3 382 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
01c66889 383
f898780b
JN		 384 i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
385 if (INTEL_INFO(dev)->gen >= 4)
386 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
1ec14ad3
CW		 387
388 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
01c66889
ZY		 389}
390
0a3e67a4
JB		 391/**
392 * i915_pipe_enabled - check if a pipe is enabled
393 * @dev: DRM device
394 * @pipe: pipe to check
395 *
396 * Reading certain registers when the pipe is disabled can hang the chip.
397 * Use this routine to make sure the PLL is running and the pipe is active
398 * before reading such registers if unsure.
399 */
400static int
401i915_pipe_enabled(struct drm_device *dev, int pipe)
402{
403 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
702e7a56 404
a01025af
DV		 405 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
406 /* Locking is horribly broken here, but whatever. */
407 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
408 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
71f8ba6b 409
a01025af
DV		 410 return intel_crtc->active;
411 } else {
412 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
413 }
0a3e67a4
JB		 414}
415
42f52ef8
KP		 416/* Called from drm generic code, passed a 'crtc', which
417 * we use as a pipe index
418 */
f71d4af4 419static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
0a3e67a4
JB		 420{
421 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
422 unsigned long high_frame;
423 unsigned long low_frame;
5eddb70b 424 u32 high1, high2, low;
0a3e67a4
JB		 425
426 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 427 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 428 "pipe %c\n", pipe_name(pipe));
0a3e67a4
JB		 429 return 0;
430 }
431
9db4a9c7
JB		 432 high_frame = PIPEFRAME(pipe);
433 low_frame = PIPEFRAMEPIXEL(pipe);
5eddb70b 434
0a3e67a4
JB		 435 /*
436 * High & low register fields aren't synchronized, so make sure
437 * we get a low value that's stable across two reads of the high
438 * register.
439 */
440 do {
5eddb70b
CW		 441 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
442 low = I915_READ(low_frame) & PIPE_FRAME_LOW_MASK;
443 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
0a3e67a4
JB		 444 } while (high1 != high2);
445
5eddb70b
CW		 446 high1 >>= PIPE_FRAME_HIGH_SHIFT;
447 low >>= PIPE_FRAME_LOW_SHIFT;
448 return (high1 << 8) | low;
0a3e67a4
JB		 449}
450
f71d4af4 451static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
9880b7a5
JB		 452{
453 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 454 int reg = PIPE_FRMCOUNT_GM45(pipe);
9880b7a5
JB		 455
456 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 457 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 458 "pipe %c\n", pipe_name(pipe));
9880b7a5
JB		 459 return 0;
460 }
461
462 return I915_READ(reg);
463}
464
f71d4af4 465static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
0af7e4df
MK		 466 int *vpos, int *hpos)
467{
468 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
469 u32 vbl = 0, position = 0;
470 int vbl_start, vbl_end, htotal, vtotal;
471 bool in_vbl = true;
472 int ret = 0;
fe2b8f9d
PZ		 473 enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
474 pipe);
0af7e4df
MK		 475
476 if (!i915_pipe_enabled(dev, pipe)) {
477 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9db4a9c7 478 "pipe %c\n", pipe_name(pipe));
0af7e4df
MK		 479 return 0;
480 }
481
482 /* Get vtotal. */
fe2b8f9d 483 vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 484
485 if (INTEL_INFO(dev)->gen >= 4) {
486 /* No obvious pixelcount register. Only query vertical
487 * scanout position from Display scan line register.
488 */
489 position = I915_READ(PIPEDSL(pipe));
490
491 /* Decode into vertical scanout position. Don't have
492 * horizontal scanout position.
493 */
494 *vpos = position & 0x1fff;
495 *hpos = 0;
496 } else {
497 /* Have access to pixelcount since start of frame.
498 * We can split this into vertical and horizontal
499 * scanout position.
500 */
501 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
502
fe2b8f9d 503 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 504 *vpos = position / htotal;
505 *hpos = position - (*vpos * htotal);
506 }
507
508 /* Query vblank area. */
fe2b8f9d 509 vbl = I915_READ(VBLANK(cpu_transcoder));
0af7e4df
MK		 510
511 /* Test position against vblank region. */
512 vbl_start = vbl & 0x1fff;
513 vbl_end = (vbl >> 16) & 0x1fff;
514
515 if ((*vpos < vbl_start) || (*vpos > vbl_end))
516 in_vbl = false;
517
518 /* Inside "upper part" of vblank area? Apply corrective offset: */
519 if (in_vbl && (*vpos >= vbl_start))
520 *vpos = *vpos - vtotal;
521
522 /* Readouts valid? */
523 if (vbl > 0)
524 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
525
526 /* In vblank? */
527 if (in_vbl)
528 ret |= DRM_SCANOUTPOS_INVBL;
529
530 return ret;
531}
532
f71d4af4 533static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
0af7e4df
MK		 534 int *max_error,
535 struct timeval *vblank_time,
536 unsigned flags)
537{
4041b853 538 struct drm_crtc *crtc;
0af7e4df 539
7eb552ae 540 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
4041b853 541 DRM_ERROR("Invalid crtc %d\n", pipe);
0af7e4df
MK		 542 return -EINVAL;
543 }
544
545 /* Get drm_crtc to timestamp: */
4041b853
CW		 546 crtc = intel_get_crtc_for_pipe(dev, pipe);
547 if (crtc == NULL) {
548 DRM_ERROR("Invalid crtc %d\n", pipe);
549 return -EINVAL;
550 }
551
552 if (!crtc->enabled) {
553 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
554 return -EBUSY;
555 }
0af7e4df
MK		 556
557 /* Helper routine in DRM core does all the work: */
4041b853
CW		 558 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
559 vblank_time, flags,
560 crtc);
0af7e4df
MK		 561}
562
321a1b30
EE		 563static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
564{
565 enum drm_connector_status old_status;
566
567 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
568 old_status = connector->status;
569
570 connector->status = connector->funcs->detect(connector, false);
571 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
572 connector->base.id,
573 drm_get_connector_name(connector),
574 old_status, connector->status);
575 return (old_status != connector->status);
576}
577
5ca58282
JB		 578/*
579 * Handle hotplug events outside the interrupt handler proper.
580 */
ac4c16c5
EE		 581#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
582
5ca58282
JB		 583static void i915_hotplug_work_func(struct work_struct *work)
584{
585 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
586 hotplug_work);
587 struct drm_device *dev = dev_priv->dev;
c31c4ba3 588 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed
EE		 589 struct intel_connector *intel_connector;
590 struct intel_encoder *intel_encoder;
591 struct drm_connector *connector;
592 unsigned long irqflags;
593 bool hpd_disabled = false;
321a1b30 594 bool changed = false;
142e2398 595 u32 hpd_event_bits;
4ef69c7a 596
52d7eced
DV		 597 /* HPD irq before everything is fully set up. */
598 if (!dev_priv->enable_hotplug_processing)
599 return;
600
a65e34c7 601 mutex_lock(&mode_config->mutex);
e67189ab
JB		 602 DRM_DEBUG_KMS("running encoder hotplug functions\n");
603
cd569aed 604 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
142e2398
EE		 605
606 hpd_event_bits = dev_priv->hpd_event_bits;
607 dev_priv->hpd_event_bits = 0;
cd569aed
EE		 608 list_for_each_entry(connector, &mode_config->connector_list, head) {
609 intel_connector = to_intel_connector(connector);
610 intel_encoder = intel_connector->encoder;
611 if (intel_encoder->hpd_pin > HPD_NONE &&
612 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
613 connector->polled == DRM_CONNECTOR_POLL_HPD) {
614 DRM_INFO("HPD interrupt storm detected on connector %s: "
615 "switching from hotplug detection to polling\n",
616 drm_get_connector_name(connector));
617 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
618 connector->polled = DRM_CONNECTOR_POLL_CONNECT
619 | DRM_CONNECTOR_POLL_DISCONNECT;
620 hpd_disabled = true;
621 }
142e2398
EE		 622 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
623 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
624 drm_get_connector_name(connector), intel_encoder->hpd_pin);
625 }
cd569aed
EE		 626 }
627 /* if there were no outputs to poll, poll was disabled,
628 * therefore make sure it's enabled when disabling HPD on
629 * some connectors */
ac4c16c5 630 if (hpd_disabled) {
cd569aed 631 drm_kms_helper_poll_enable(dev);
ac4c16c5
EE		 632 mod_timer(&dev_priv->hotplug_reenable_timer,
633 jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
634 }
cd569aed
EE		 635
636 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
637
321a1b30
EE		 638 list_for_each_entry(connector, &mode_config->connector_list, head) {
639 intel_connector = to_intel_connector(connector);
640 intel_encoder = intel_connector->encoder;
641 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
642 if (intel_encoder->hot_plug)
643 intel_encoder->hot_plug(intel_encoder);
644 if (intel_hpd_irq_event(dev, connector))
645 changed = true;
646 }
647 }
40ee3381
KP		 648 mutex_unlock(&mode_config->mutex);
649
321a1b30
EE		 650 if (changed)
651 drm_kms_helper_hotplug_event(dev);
5ca58282
JB		 652}
653
73edd18f 654static void ironlake_handle_rps_change(struct drm_device *dev)
f97108d1
JB		 655{
656 drm_i915_private_t *dev_priv = dev->dev_private;
b5b72e89 657 u32 busy_up, busy_down, max_avg, min_avg;
9270388e
DV		 658 u8 new_delay;
659 unsigned long flags;
660
661 spin_lock_irqsave(&mchdev_lock, flags);
f97108d1 662
73edd18f
DV		 663 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
664
20e4d407 665 new_delay = dev_priv->ips.cur_delay;
9270388e 666
7648fa99 667 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
b5b72e89
MG		 668 busy_up = I915_READ(RCPREVBSYTUPAVG);
669 busy_down = I915_READ(RCPREVBSYTDNAVG);
f97108d1
JB		 670 max_avg = I915_READ(RCBMAXAVG);
671 min_avg = I915_READ(RCBMINAVG);
672
673 /* Handle RCS change request from hw */
b5b72e89 674 if (busy_up > max_avg) {
20e4d407
DV		 675 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
676 new_delay = dev_priv->ips.cur_delay - 1;
677 if (new_delay < dev_priv->ips.max_delay)
678 new_delay = dev_priv->ips.max_delay;
b5b72e89 679 } else if (busy_down < min_avg) {
20e4d407
DV		 680 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
681 new_delay = dev_priv->ips.cur_delay + 1;
682 if (new_delay > dev_priv->ips.min_delay)
683 new_delay = dev_priv->ips.min_delay;
f97108d1
JB		 684 }
685
7648fa99 686 if (ironlake_set_drps(dev, new_delay))
20e4d407 687 dev_priv->ips.cur_delay = new_delay;
f97108d1 688
9270388e
DV		 689 spin_unlock_irqrestore(&mchdev_lock, flags);
690
f97108d1
JB		 691 return;
692}
693
549f7365
CW		 694static void notify_ring(struct drm_device *dev,
695 struct intel_ring_buffer *ring)
696{
697 struct drm_i915_private *dev_priv = dev->dev_private;
9862e600 698
475553de
CW		 699 if (ring->obj == NULL)
700 return;
701
b2eadbc8 702 trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
9862e600 703
549f7365 704 wake_up_all(&ring->irq_queue);
3e0dc6b0 705 if (i915_enable_hangcheck) {
99584db3 706 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
cecc21fe 707 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
3e0dc6b0 708 }
549f7365
CW		 709}
710
4912d041 711static void gen6_pm_rps_work(struct work_struct *work)
3b8d8d91 712{
4912d041 713 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
c6a828d3 714 rps.work);
4912d041 715 u32 pm_iir, pm_imr;
7b9e0ae6 716 u8 new_delay;
4912d041 717
c6a828d3
DV		 718 spin_lock_irq(&dev_priv->rps.lock);
719 pm_iir = dev_priv->rps.pm_iir;
720 dev_priv->rps.pm_iir = 0;
4912d041 721 pm_imr = I915_READ(GEN6_PMIMR);
4848405c
BW		 722 /* Make sure not to corrupt PMIMR state used by ringbuffer code */
723 I915_WRITE(GEN6_PMIMR, pm_imr & ~GEN6_PM_RPS_EVENTS);
c6a828d3 724 spin_unlock_irq(&dev_priv->rps.lock);
3b8d8d91 725
4848405c 726 if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
3b8d8d91
JB		 727 return;
728
4fc688ce 729 mutex_lock(&dev_priv->rps.hw_lock);
7b9e0ae6 730
7425034a 731 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
c6a828d3 732 new_delay = dev_priv->rps.cur_delay + 1;
7425034a
VS		 733
734 /*
735 * For better performance, jump directly
736 * to RPe if we're below it.
737 */
738 if (IS_VALLEYVIEW(dev_priv->dev) &&
739 dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
740 new_delay = dev_priv->rps.rpe_delay;
741 } else
c6a828d3 742 new_delay = dev_priv->rps.cur_delay - 1;
3b8d8d91 743
79249636
BW		 744 /* sysfs frequency interfaces may have snuck in while servicing the
745 * interrupt
746 */
d8289c9e
VS		 747 if (new_delay >= dev_priv->rps.min_delay &&
748 new_delay <= dev_priv->rps.max_delay) {
0a073b84
JB		 749 if (IS_VALLEYVIEW(dev_priv->dev))
750 valleyview_set_rps(dev_priv->dev, new_delay);
751 else
752 gen6_set_rps(dev_priv->dev, new_delay);
79249636 753 }
3b8d8d91 754
52ceb908
JB		 755 if (IS_VALLEYVIEW(dev_priv->dev)) {
756 /*
757 * On VLV, when we enter RC6 we may not be at the minimum
758 * voltage level, so arm a timer to check. It should only
759 * fire when there's activity or once after we've entered
760 * RC6, and then won't be re-armed until the next RPS interrupt.
761 */
762 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
763 msecs_to_jiffies(100));
764 }
765
4fc688ce 766 mutex_unlock(&dev_priv->rps.hw_lock);
3b8d8d91
JB		 767}
768
e3689190
BW		 769
770/**
771 * ivybridge_parity_work - Workqueue called when a parity error interrupt
772 * occurred.
773 * @work: workqueue struct
774 *
775 * Doesn't actually do anything except notify userspace. As a consequence of
776 * this event, userspace should try to remap the bad rows since statistically
777 * it is likely the same row is more likely to go bad again.
778 */
779static void ivybridge_parity_work(struct work_struct *work)
780{
781 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
a4da4fa4 782 l3_parity.error_work);
e3689190
BW		 783 u32 error_status, row, bank, subbank;
784 char *parity_event[5];
785 uint32_t misccpctl;
786 unsigned long flags;
787
788 /* We must turn off DOP level clock gating to access the L3 registers.
789 * In order to prevent a get/put style interface, acquire struct mutex
790 * any time we access those registers.
791 */
792 mutex_lock(&dev_priv->dev->struct_mutex);
793
794 misccpctl = I915_READ(GEN7_MISCCPCTL);
795 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
796 POSTING_READ(GEN7_MISCCPCTL);
797
798 error_status = I915_READ(GEN7_L3CDERRST1);
799 row = GEN7_PARITY_ERROR_ROW(error_status);
800 bank = GEN7_PARITY_ERROR_BANK(error_status);
801 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
802
803 I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
804 GEN7_L3CDERRST1_ENABLE);
805 POSTING_READ(GEN7_L3CDERRST1);
806
807 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
808
809 spin_lock_irqsave(&dev_priv->irq_lock, flags);
cc609d5d 810 dev_priv->gt_irq_mask &= ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
e3689190
BW		 811 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
812 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
813
814 mutex_unlock(&dev_priv->dev->struct_mutex);
815
816 parity_event[0] = "L3_PARITY_ERROR=1";
817 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
818 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
819 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
820 parity_event[4] = NULL;
821
822 kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
823 KOBJ_CHANGE, parity_event);
824
825 DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
826 row, bank, subbank);
827
828 kfree(parity_event[3]);
829 kfree(parity_event[2]);
830 kfree(parity_event[1]);
831}
832
d2ba8470 833static void ivybridge_handle_parity_error(struct drm_device *dev)
e3689190
BW		 834{
835 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
836 unsigned long flags;
837
e1ef7cc2 838 if (!HAS_L3_GPU_CACHE(dev))
e3689190
BW		 839 return;
840
841 spin_lock_irqsave(&dev_priv->irq_lock, flags);
cc609d5d 842 dev_priv->gt_irq_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
e3689190
BW		 843 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
844 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
845
a4da4fa4 846 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
e3689190
BW		 847}
848
e7b4c6b1
DV		 849static void snb_gt_irq_handler(struct drm_device *dev,
850 struct drm_i915_private *dev_priv,
851 u32 gt_iir)
852{
853
cc609d5d
BW		 854 if (gt_iir &
855 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
e7b4c6b1 856 notify_ring(dev, &dev_priv->ring[RCS]);
cc609d5d 857 if (gt_iir & GT_BSD_USER_INTERRUPT)
e7b4c6b1 858 notify_ring(dev, &dev_priv->ring[VCS]);
cc609d5d 859 if (gt_iir & GT_BLT_USER_INTERRUPT)
e7b4c6b1
DV		 860 notify_ring(dev, &dev_priv->ring[BCS]);
861
cc609d5d
BW		 862 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
863 GT_BSD_CS_ERROR_INTERRUPT |
864 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
e7b4c6b1
DV		 865 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
866 i915_handle_error(dev, false);
867 }
e3689190 868
cc609d5d 869 if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
e3689190 870 ivybridge_handle_parity_error(dev);
e7b4c6b1
DV		 871}
872
baf02a1f 873/* Legacy way of handling PM interrupts */
fc6826d1
CW		 874static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
875 u32 pm_iir)
876{
877 unsigned long flags;
878
879 /*
880 * IIR bits should never already be set because IMR should
881 * prevent an interrupt from being shown in IIR. The warning
882 * displays a case where we've unsafely cleared
c6a828d3 883 * dev_priv->rps.pm_iir. Although missing an interrupt of the same
fc6826d1
CW		 884 * type is not a problem, it displays a problem in the logic.
885 *
c6a828d3 886 * The mask bit in IMR is cleared by dev_priv->rps.work.
fc6826d1
CW		 887 */
888
c6a828d3 889 spin_lock_irqsave(&dev_priv->rps.lock, flags);
c6a828d3
DV		 890 dev_priv->rps.pm_iir |= pm_iir;
891 I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
fc6826d1 892 POSTING_READ(GEN6_PMIMR);
c6a828d3 893 spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
fc6826d1 894
c6a828d3 895 queue_work(dev_priv->wq, &dev_priv->rps.work);
fc6826d1
CW		 896}
897
b543fb04
EE		 898#define HPD_STORM_DETECT_PERIOD 1000
899#define HPD_STORM_THRESHOLD 5
900
10a504de 901static inline void intel_hpd_irq_handler(struct drm_device *dev,
22062dba
DV		 902 u32 hotplug_trigger,
903 const u32 *hpd)
b543fb04
EE		 904{
905 drm_i915_private_t *dev_priv = dev->dev_private;
b543fb04 906 int i;
10a504de 907 bool storm_detected = false;
b543fb04 908
91d131d2
DV		 909 if (!hotplug_trigger)
910 return;
911
b5ea2d56 912 spin_lock(&dev_priv->irq_lock);
b543fb04 913 for (i = 1; i < HPD_NUM_PINS; i++) {
821450c6 914
b543fb04
EE		 915 if (!(hpd[i] & hotplug_trigger) ||
916 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
917 continue;
918
bc5ead8c 919 dev_priv->hpd_event_bits |= (1 << i);
b543fb04
EE		 920 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
921 dev_priv->hpd_stats[i].hpd_last_jiffies
922 + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
923 dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
924 dev_priv->hpd_stats[i].hpd_cnt = 0;
925 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
926 dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
142e2398 927 dev_priv->hpd_event_bits &= ~(1 << i);
b543fb04 928 DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
10a504de 929 storm_detected = true;
b543fb04
EE		 930 } else {
931 dev_priv->hpd_stats[i].hpd_cnt++;
932 }
933 }
934
10a504de
DV		 935 if (storm_detected)
936 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 937 spin_unlock(&dev_priv->irq_lock);
5876fa0d
DV		 938
939 queue_work(dev_priv->wq,
940 &dev_priv->hotplug_work);
b543fb04
EE		 941}
942
515ac2bb
DV		 943static void gmbus_irq_handler(struct drm_device *dev)
944{
28c70f16
DV		 945 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
946
28c70f16 947 wake_up_all(&dev_priv->gmbus_wait_queue);
515ac2bb
DV		 948}
949
ce99c256
DV		 950static void dp_aux_irq_handler(struct drm_device *dev)
951{
9ee32fea
DV		 952 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
953
9ee32fea 954 wake_up_all(&dev_priv->gmbus_wait_queue);
ce99c256
DV		 955}
956
baf02a1f
BW		 957/* Unlike gen6_queue_rps_work() from which this function is originally derived,
958 * we must be able to deal with other PM interrupts. This is complicated because
959 * of the way in which we use the masks to defer the RPS work (which for
960 * posterity is necessary because of forcewake).
961 */
962static void hsw_pm_irq_handler(struct drm_i915_private *dev_priv,
963 u32 pm_iir)
964{
965 unsigned long flags;
966
967 spin_lock_irqsave(&dev_priv->rps.lock, flags);
4848405c 968 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
baf02a1f
BW		 969 if (dev_priv->rps.pm_iir) {
970 I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
971 /* never want to mask useful interrupts. (also posting read) */
4848405c 972 WARN_ON(I915_READ_NOTRACE(GEN6_PMIMR) & ~GEN6_PM_RPS_EVENTS);
baf02a1f
BW		 973 /* TODO: if queue_work is slow, move it out of the spinlock */
974 queue_work(dev_priv->wq, &dev_priv->rps.work);
975 }
976 spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
977
12638c57
BW		 978 if (pm_iir & ~GEN6_PM_RPS_EVENTS) {
979 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
980 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
981
982 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
983 DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
984 i915_handle_error(dev_priv->dev, false);
985 }
986 }
baf02a1f
BW		 987}
988
ff1f525e 989static irqreturn_t valleyview_irq_handler(int irq, void *arg)
7e231dbe
JB		 990{
991 struct drm_device *dev = (struct drm_device *) arg;
992 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
993 u32 iir, gt_iir, pm_iir;
994 irqreturn_t ret = IRQ_NONE;
995 unsigned long irqflags;
996 int pipe;
997 u32 pipe_stats[I915_MAX_PIPES];
7e231dbe
JB		 998
999 atomic_inc(&dev_priv->irq_received);
1000
7e231dbe
JB		 1001 while (true) {
1002 iir = I915_READ(VLV_IIR);
1003 gt_iir = I915_READ(GTIIR);
1004 pm_iir = I915_READ(GEN6_PMIIR);
1005
1006 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1007 goto out;
1008
1009 ret = IRQ_HANDLED;
1010
e7b4c6b1 1011 snb_gt_irq_handler(dev, dev_priv, gt_iir);
7e231dbe
JB		 1012
1013 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1014 for_each_pipe(pipe) {
1015 int reg = PIPESTAT(pipe);
1016 pipe_stats[pipe] = I915_READ(reg);
1017
1018 /*
1019 * Clear the PIPE*STAT regs before the IIR
1020 */
1021 if (pipe_stats[pipe] & 0x8000ffff) {
1022 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1023 DRM_DEBUG_DRIVER("pipe %c underrun\n",
1024 pipe_name(pipe));
1025 I915_WRITE(reg, pipe_stats[pipe]);
1026 }
1027 }
1028 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1029
31acc7f5
JB		 1030 for_each_pipe(pipe) {
1031 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1032 drm_handle_vblank(dev, pipe);
1033
1034 if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1035 intel_prepare_page_flip(dev, pipe);
1036 intel_finish_page_flip(dev, pipe);
1037 }
1038 }
1039
7e231dbe
JB		 1040 /* Consume port. Then clear IIR or we'll miss events */
1041 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1042 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 1043 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
7e231dbe
JB		 1044
1045 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1046 hotplug_status);
91d131d2
DV		 1047
1048 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1049
7e231dbe
JB		 1050 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1051 I915_READ(PORT_HOTPLUG_STAT);
1052 }
1053
515ac2bb
DV		 1054 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1055 gmbus_irq_handler(dev);
7e231dbe 1056
4848405c 1057 if (pm_iir & GEN6_PM_RPS_EVENTS)
fc6826d1 1058 gen6_queue_rps_work(dev_priv, pm_iir);
7e231dbe
JB		 1059
1060 I915_WRITE(GTIIR, gt_iir);
1061 I915_WRITE(GEN6_PMIIR, pm_iir);
1062 I915_WRITE(VLV_IIR, iir);
1063 }
1064
1065out:
1066 return ret;
1067}
1068
23e81d69 1069static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
776ad806
JB		 1070{
1071 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 1072 int pipe;
b543fb04 1073 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
776ad806 1074
91d131d2
DV		 1075 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1076
cfc33bf7
VS		 1077 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1078 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1079 SDE_AUDIO_POWER_SHIFT);
776ad806 1080 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
cfc33bf7
VS		 1081 port_name(port));
1082 }
776ad806 1083
ce99c256
DV		 1084 if (pch_iir & SDE_AUX_MASK)
1085 dp_aux_irq_handler(dev);
1086
776ad806 1087 if (pch_iir & SDE_GMBUS)
515ac2bb 1088 gmbus_irq_handler(dev);
776ad806
JB		 1089
1090 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1091 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1092
1093 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1094 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1095
1096 if (pch_iir & SDE_POISON)
1097 DRM_ERROR("PCH poison interrupt\n");
1098
9db4a9c7
JB		 1099 if (pch_iir & SDE_FDI_MASK)
1100 for_each_pipe(pipe)
1101 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1102 pipe_name(pipe),
1103 I915_READ(FDI_RX_IIR(pipe)));
776ad806
JB		 1104
1105 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1106 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1107
1108 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1109 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1110
776ad806 1111 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
8664281b
PZ		 1112 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1113 false))
1114 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1115
1116 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1117 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1118 false))
1119 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1120}
1121
1122static void ivb_err_int_handler(struct drm_device *dev)
1123{
1124 struct drm_i915_private *dev_priv = dev->dev_private;
1125 u32 err_int = I915_READ(GEN7_ERR_INT);
1126
de032bf4
PZ		 1127 if (err_int & ERR_INT_POISON)
1128 DRM_ERROR("Poison interrupt\n");
1129
8664281b
PZ		 1130 if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1131 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1132 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1133
1134 if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1135 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1136 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1137
1138 if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1139 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1140 DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1141
1142 I915_WRITE(GEN7_ERR_INT, err_int);
1143}
1144
1145static void cpt_serr_int_handler(struct drm_device *dev)
1146{
1147 struct drm_i915_private *dev_priv = dev->dev_private;
1148 u32 serr_int = I915_READ(SERR_INT);
1149
de032bf4
PZ		 1150 if (serr_int & SERR_INT_POISON)
1151 DRM_ERROR("PCH poison interrupt\n");
1152
8664281b
PZ		 1153 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1154 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1155 false))
1156 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1157
1158 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1159 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1160 false))
1161 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1162
1163 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1164 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1165 false))
1166 DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1167
1168 I915_WRITE(SERR_INT, serr_int);
776ad806
JB		 1169}
1170
23e81d69
AJ		 1171static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1172{
1173 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1174 int pipe;
b543fb04 1175 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
23e81d69 1176
91d131d2
DV		 1177 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1178
cfc33bf7
VS		 1179 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1180 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1181 SDE_AUDIO_POWER_SHIFT_CPT);
1182 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1183 port_name(port));
1184 }
23e81d69
AJ		 1185
1186 if (pch_iir & SDE_AUX_MASK_CPT)
ce99c256 1187 dp_aux_irq_handler(dev);
23e81d69
AJ		 1188
1189 if (pch_iir & SDE_GMBUS_CPT)
515ac2bb 1190 gmbus_irq_handler(dev);
23e81d69
AJ		 1191
1192 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1193 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1194
1195 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1196 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1197
1198 if (pch_iir & SDE_FDI_MASK_CPT)
1199 for_each_pipe(pipe)
1200 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1201 pipe_name(pipe),
1202 I915_READ(FDI_RX_IIR(pipe)));
8664281b
PZ		 1203
1204 if (pch_iir & SDE_ERROR_CPT)
1205 cpt_serr_int_handler(dev);
23e81d69
AJ		 1206}
1207
ff1f525e 1208static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
b1f14ad0
JB		 1209{
1210 struct drm_device *dev = (struct drm_device *) arg;
1211 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
ab5c608b 1212 u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier = 0;
0e43406b
CW		 1213 irqreturn_t ret = IRQ_NONE;
1214 int i;
b1f14ad0
JB		 1215
1216 atomic_inc(&dev_priv->irq_received);
1217
8664281b
PZ		 1218 /* We get interrupts on unclaimed registers, so check for this before we
1219 * do any I915_{READ,WRITE}. */
1220 if (IS_HASWELL(dev) &&
1221 (I915_READ_NOTRACE(FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
1222 DRM_ERROR("Unclaimed register before interrupt\n");
1223 I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1224 }
1225
b1f14ad0
JB		 1226 /* disable master interrupt before clearing iir */
1227 de_ier = I915_READ(DEIER);
1228 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
b1f14ad0 1229
44498aea
PZ		 1230 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1231 * interrupts will will be stored on its back queue, and then we'll be
1232 * able to process them after we restore SDEIER (as soon as we restore
1233 * it, we'll get an interrupt if SDEIIR still has something to process
1234 * due to its back queue). */
ab5c608b
BW		 1235 if (!HAS_PCH_NOP(dev)) {
1236 sde_ier = I915_READ(SDEIER);
1237 I915_WRITE(SDEIER, 0);
1238 POSTING_READ(SDEIER);
1239 }
44498aea 1240
8664281b
PZ		 1241 /* On Haswell, also mask ERR_INT because we don't want to risk
1242 * generating "unclaimed register" interrupts from inside the interrupt
1243 * handler. */
4bc9d430
DV		 1244 if (IS_HASWELL(dev)) {
1245 spin_lock(&dev_priv->irq_lock);
8664281b 1246 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
4bc9d430
DV		 1247 spin_unlock(&dev_priv->irq_lock);
1248 }
8664281b 1249
b1f14ad0 1250 gt_iir = I915_READ(GTIIR);
0e43406b
CW		 1251 if (gt_iir) {
1252 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1253 I915_WRITE(GTIIR, gt_iir);
1254 ret = IRQ_HANDLED;
b1f14ad0
JB		 1255 }
1256
0e43406b
CW		 1257 de_iir = I915_READ(DEIIR);
1258 if (de_iir) {
8664281b
PZ		 1259 if (de_iir & DE_ERR_INT_IVB)
1260 ivb_err_int_handler(dev);
1261
ce99c256
DV		 1262 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1263 dp_aux_irq_handler(dev);
1264
0e43406b 1265 if (de_iir & DE_GSE_IVB)
81a07809 1266 intel_opregion_asle_intr(dev);
0e43406b
CW		 1267
1268 for (i = 0; i < 3; i++) {
74d44445
DV		 1269 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1270 drm_handle_vblank(dev, i);
0e43406b
CW		 1271 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1272 intel_prepare_page_flip(dev, i);
1273 intel_finish_page_flip_plane(dev, i);
1274 }
0e43406b 1275 }
b615b57a 1276
0e43406b 1277 /* check event from PCH */
ab5c608b 1278 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
0e43406b 1279 u32 pch_iir = I915_READ(SDEIIR);
b1f14ad0 1280
23e81d69 1281 cpt_irq_handler(dev, pch_iir);
b1f14ad0 1282
0e43406b
CW		 1283 /* clear PCH hotplug event before clear CPU irq */
1284 I915_WRITE(SDEIIR, pch_iir);
1285 }
b615b57a 1286
0e43406b
CW		 1287 I915_WRITE(DEIIR, de_iir);
1288 ret = IRQ_HANDLED;
b1f14ad0
JB		 1289 }
1290
0e43406b
CW		 1291 pm_iir = I915_READ(GEN6_PMIIR);
1292 if (pm_iir) {
baf02a1f
BW		 1293 if (IS_HASWELL(dev))
1294 hsw_pm_irq_handler(dev_priv, pm_iir);
4848405c 1295 else if (pm_iir & GEN6_PM_RPS_EVENTS)
0e43406b
CW		 1296 gen6_queue_rps_work(dev_priv, pm_iir);
1297 I915_WRITE(GEN6_PMIIR, pm_iir);
1298 ret = IRQ_HANDLED;
1299 }
b1f14ad0 1300
4bc9d430
DV		 1301 if (IS_HASWELL(dev)) {
1302 spin_lock(&dev_priv->irq_lock);
1303 if (ivb_can_enable_err_int(dev))
1304 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1305 spin_unlock(&dev_priv->irq_lock);
1306 }
8664281b 1307
b1f14ad0
JB		 1308 I915_WRITE(DEIER, de_ier);
1309 POSTING_READ(DEIER);
ab5c608b
BW		 1310 if (!HAS_PCH_NOP(dev)) {
1311 I915_WRITE(SDEIER, sde_ier);
1312 POSTING_READ(SDEIER);
1313 }
b1f14ad0
JB		 1314
1315 return ret;
1316}
1317
e7b4c6b1
DV		 1318static void ilk_gt_irq_handler(struct drm_device *dev,
1319 struct drm_i915_private *dev_priv,
1320 u32 gt_iir)
1321{
cc609d5d
BW		 1322 if (gt_iir &
1323 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
e7b4c6b1 1324 notify_ring(dev, &dev_priv->ring[RCS]);
cc609d5d 1325 if (gt_iir & ILK_BSD_USER_INTERRUPT)
e7b4c6b1
DV		 1326 notify_ring(dev, &dev_priv->ring[VCS]);
1327}
1328
ff1f525e 1329static irqreturn_t ironlake_irq_handler(int irq, void *arg)
036a4a7d 1330{
4697995b 1331 struct drm_device *dev = (struct drm_device *) arg;
036a4a7d
ZW		 1332 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1333 int ret = IRQ_NONE;
44498aea 1334 u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier;
881f47b6 1335
4697995b
JB		 1336 atomic_inc(&dev_priv->irq_received);
1337
2d109a84
ZN		 1338 /* disable master interrupt before clearing iir */
1339 de_ier = I915_READ(DEIER);
1340 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
3143a2bf 1341 POSTING_READ(DEIER);
2d109a84 1342
44498aea
PZ		 1343 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1344 * interrupts will will be stored on its back queue, and then we'll be
1345 * able to process them after we restore SDEIER (as soon as we restore
1346 * it, we'll get an interrupt if SDEIIR still has something to process
1347 * due to its back queue). */
1348 sde_ier = I915_READ(SDEIER);
1349 I915_WRITE(SDEIER, 0);
1350 POSTING_READ(SDEIER);
1351
036a4a7d
ZW		 1352 de_iir = I915_READ(DEIIR);
1353 gt_iir = I915_READ(GTIIR);
3b8d8d91 1354 pm_iir = I915_READ(GEN6_PMIIR);
036a4a7d 1355
acd15b6c 1356 if (de_iir == 0 && gt_iir == 0 && (!IS_GEN6(dev) || pm_iir == 0))
c7c85101 1357 goto done;
036a4a7d 1358
c7c85101 1359 ret = IRQ_HANDLED;
036a4a7d 1360
e7b4c6b1
DV		 1361 if (IS_GEN5(dev))
1362 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1363 else
1364 snb_gt_irq_handler(dev, dev_priv, gt_iir);
01c66889 1365
ce99c256
DV		 1366 if (de_iir & DE_AUX_CHANNEL_A)
1367 dp_aux_irq_handler(dev);
1368
c7c85101 1369 if (de_iir & DE_GSE)
81a07809 1370 intel_opregion_asle_intr(dev);
c650156a 1371
74d44445
DV		 1372 if (de_iir & DE_PIPEA_VBLANK)
1373 drm_handle_vblank(dev, 0);
1374
1375 if (de_iir & DE_PIPEB_VBLANK)
1376 drm_handle_vblank(dev, 1);
1377
de032bf4
PZ		 1378 if (de_iir & DE_POISON)
1379 DRM_ERROR("Poison interrupt\n");
1380
8664281b
PZ		 1381 if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1382 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1383 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1384
1385 if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1386 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1387 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1388
f072d2e7 1389 if (de_iir & DE_PLANEA_FLIP_DONE) {
013d5aa2 1390 intel_prepare_page_flip(dev, 0);
2bbda389 1391 intel_finish_page_flip_plane(dev, 0);
f072d2e7 1392 }
013d5aa2 1393
f072d2e7 1394 if (de_iir & DE_PLANEB_FLIP_DONE) {
013d5aa2 1395 intel_prepare_page_flip(dev, 1);
2bbda389 1396 intel_finish_page_flip_plane(dev, 1);
f072d2e7 1397 }
013d5aa2 1398
c7c85101 1399 /* check event from PCH */
776ad806 1400 if (de_iir & DE_PCH_EVENT) {
acd15b6c
DV		 1401 u32 pch_iir = I915_READ(SDEIIR);
1402
23e81d69
AJ		 1403 if (HAS_PCH_CPT(dev))
1404 cpt_irq_handler(dev, pch_iir);
1405 else
1406 ibx_irq_handler(dev, pch_iir);
acd15b6c
DV		 1407
1408 /* should clear PCH hotplug event before clear CPU irq */
1409 I915_WRITE(SDEIIR, pch_iir);
776ad806 1410 }
036a4a7d 1411
73edd18f
DV		 1412 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1413 ironlake_handle_rps_change(dev);
f97108d1 1414
4848405c 1415 if (IS_GEN6(dev) && pm_iir & GEN6_PM_RPS_EVENTS)
fc6826d1 1416 gen6_queue_rps_work(dev_priv, pm_iir);
3b8d8d91 1417
c7c85101
ZN		 1418 I915_WRITE(GTIIR, gt_iir);
1419 I915_WRITE(DEIIR, de_iir);
4912d041 1420 I915_WRITE(GEN6_PMIIR, pm_iir);
c7c85101
ZN		 1421
1422done:
2d109a84 1423 I915_WRITE(DEIER, de_ier);
3143a2bf 1424 POSTING_READ(DEIER);
44498aea
PZ		 1425 I915_WRITE(SDEIER, sde_ier);
1426 POSTING_READ(SDEIER);
2d109a84 1427
036a4a7d
ZW		 1428 return ret;
1429}
1430
8a905236
JB		 1431/**
1432 * i915_error_work_func - do process context error handling work
1433 * @work: work struct
1434 *
1435 * Fire an error uevent so userspace can see that a hang or error
1436 * was detected.
1437 */
1438static void i915_error_work_func(struct work_struct *work)
1439{
1f83fee0
DV		 1440 struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1441 work);
1442 drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1443 gpu_error);
8a905236 1444 struct drm_device *dev = dev_priv->dev;
f69061be 1445 struct intel_ring_buffer *ring;
f316a42c
BG		 1446 char *error_event[] = { "ERROR=1", NULL };
1447 char *reset_event[] = { "RESET=1", NULL };
1448 char *reset_done_event[] = { "ERROR=0", NULL };
f69061be 1449 int i, ret;
8a905236 1450
f316a42c
BG		 1451 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1452
7db0ba24
DV		 1453 /*
1454 * Note that there's only one work item which does gpu resets, so we
1455 * need not worry about concurrent gpu resets potentially incrementing
1456 * error->reset_counter twice. We only need to take care of another
1457 * racing irq/hangcheck declaring the gpu dead for a second time. A
1458 * quick check for that is good enough: schedule_work ensures the
1459 * correct ordering between hang detection and this work item, and since
1460 * the reset in-progress bit is only ever set by code outside of this
1461 * work we don't need to worry about any other races.
1462 */
1463 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
f803aa55 1464 DRM_DEBUG_DRIVER("resetting chip\n");
7db0ba24
DV		 1465 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1466 reset_event);
1f83fee0 1467
f69061be
DV		 1468 ret = i915_reset(dev);
1469
1470 if (ret == 0) {
1471 /*
1472 * After all the gem state is reset, increment the reset
1473 * counter and wake up everyone waiting for the reset to
1474 * complete.
1475 *
1476 * Since unlock operations are a one-sided barrier only,
1477 * we need to insert a barrier here to order any seqno
1478 * updates before
1479 * the counter increment.
1480 */
1481 smp_mb__before_atomic_inc();
1482 atomic_inc(&dev_priv->gpu_error.reset_counter);
1483
1484 kobject_uevent_env(&dev->primary->kdev.kobj,
1485 KOBJ_CHANGE, reset_done_event);
1f83fee0
DV		 1486 } else {
1487 atomic_set(&error->reset_counter, I915_WEDGED);
f316a42c 1488 }
1f83fee0 1489
f69061be
DV		 1490 for_each_ring(ring, dev_priv, i)
1491 wake_up_all(&ring->irq_queue);
1492
96a02917
VS		 1493 intel_display_handle_reset(dev);
1494
1f83fee0 1495 wake_up_all(&dev_priv->gpu_error.reset_queue);
f316a42c 1496 }
8a905236
JB		 1497}
1498
85f9e50d
DV		 1499/* NB: please notice the memset */
1500static void i915_get_extra_instdone(struct drm_device *dev,
1501 uint32_t *instdone)
1502{
1503 struct drm_i915_private *dev_priv = dev->dev_private;
1504 memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
1505
1506 switch(INTEL_INFO(dev)->gen) {
1507 case 2:
1508 case 3:
1509 instdone[0] = I915_READ(INSTDONE);
1510 break;
1511 case 4:
1512 case 5:
1513 case 6:
1514 instdone[0] = I915_READ(INSTDONE_I965);
1515 instdone[1] = I915_READ(INSTDONE1);
1516 break;
1517 default:
1518 WARN_ONCE(1, "Unsupported platform\n");
1519 case 7:
1520 instdone[0] = I915_READ(GEN7_INSTDONE_1);
1521 instdone[1] = I915_READ(GEN7_SC_INSTDONE);
1522 instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
1523 instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
1524 break;
1525 }
1526}
1527
3bd3c932 1528#ifdef CONFIG_DEBUG_FS
9df30794 1529static struct drm_i915_error_object *
d0d045e8
BW		 1530i915_error_object_create_sized(struct drm_i915_private *dev_priv,
1531 struct drm_i915_gem_object *src,
1532 const int num_pages)
9df30794
CW		 1533{
1534 struct drm_i915_error_object *dst;
d0d045e8 1535 int i;
e56660dd 1536 u32 reloc_offset;
9df30794 1537
05394f39 1538 if (src == NULL || src->pages == NULL)
9df30794
CW		 1539 return NULL;
1540
d0d045e8 1541 dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
9df30794
CW		 1542 if (dst == NULL)
1543 return NULL;
1544
f343c5f6 1545 reloc_offset = dst->gtt_offset = i915_gem_obj_ggtt_offset(src);
d0d045e8 1546 for (i = 0; i < num_pages; i++) {
788885ae 1547 unsigned long flags;
e56660dd 1548 void *d;
788885ae 1549
e56660dd 1550 d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
9df30794
CW		 1551 if (d == NULL)
1552 goto unwind;
e56660dd 1553
788885ae 1554 local_irq_save(flags);
5d4545ae 1555 if (reloc_offset < dev_priv->gtt.mappable_end &&
74898d7e 1556 src->has_global_gtt_mapping) {
172975aa
CW		 1557 void __iomem *s;
1558
1559 /* Simply ignore tiling or any overlapping fence.
1560 * It's part of the error state, and this hopefully
1561 * captures what the GPU read.
1562 */
1563
5d4545ae 1564 s = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
172975aa
CW		 1565 reloc_offset);
1566 memcpy_fromio(d, s, PAGE_SIZE);
1567 io_mapping_unmap_atomic(s);
960e3564
CW		 1568 } else if (src->stolen) {
1569 unsigned long offset;
1570
1571 offset = dev_priv->mm.stolen_base;
1572 offset += src->stolen->start;
1573 offset += i << PAGE_SHIFT;
1574
1a240d4d 1575 memcpy_fromio(d, (void __iomem *) offset, PAGE_SIZE);
172975aa 1576 } else {
9da3da66 1577 struct page *page;
172975aa
CW		 1578 void *s;
1579
9da3da66 1580 page = i915_gem_object_get_page(src, i);
172975aa 1581
9da3da66
CW		 1582 drm_clflush_pages(&page, 1);
1583
1584 s = kmap_atomic(page);
172975aa
CW		 1585 memcpy(d, s, PAGE_SIZE);
1586 kunmap_atomic(s);
1587
9da3da66 1588 drm_clflush_pages(&page, 1);
172975aa 1589 }
788885ae 1590 local_irq_restore(flags);
e56660dd 1591
9da3da66 1592 dst->pages[i] = d;
e56660dd
CW		 1593
1594 reloc_offset += PAGE_SIZE;
9df30794 1595 }
d0d045e8 1596 dst->page_count = num_pages;
9df30794
CW		 1597
1598 return dst;
1599
1600unwind:
9da3da66
CW		 1601 while (i--)
1602 kfree(dst->pages[i]);
9df30794
CW		 1603 kfree(dst);
1604 return NULL;
1605}
d0d045e8
BW		 1606#define i915_error_object_create(dev_priv, src) \
1607 i915_error_object_create_sized((dev_priv), (src), \
1608 (src)->base.size>>PAGE_SHIFT)
9df30794
CW		 1609
1610static void
1611i915_error_object_free(struct drm_i915_error_object *obj)
1612{
1613 int page;
1614
1615 if (obj == NULL)
1616 return;
1617
1618 for (page = 0; page < obj->page_count; page++)
1619 kfree(obj->pages[page]);
1620
1621 kfree(obj);
1622}
1623
742cbee8
DV		 1624void
1625i915_error_state_free(struct kref *error_ref)
9df30794 1626{
742cbee8
DV		 1627 struct drm_i915_error_state *error = container_of(error_ref,
1628 typeof(*error), ref);
e2f973d5
CW		 1629 int i;
1630
52d39a21
CW		 1631 for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
1632 i915_error_object_free(error->ring[i].batchbuffer);
1633 i915_error_object_free(error->ring[i].ringbuffer);
7ed73da0 1634 i915_error_object_free(error->ring[i].ctx);
52d39a21
CW		 1635 kfree(error->ring[i].requests);
1636 }
e2f973d5 1637
9df30794 1638 kfree(error->active_bo);
6ef3d427 1639 kfree(error->overlay);
7ed73da0 1640 kfree(error->display);
9df30794
CW		 1641 kfree(error);
1642}
1b50247a
CW		 1643static void capture_bo(struct drm_i915_error_buffer *err,
1644 struct drm_i915_gem_object *obj)
1645{
1646 err->size = obj->base.size;
1647 err->name = obj->base.name;
0201f1ec
CW		 1648 err->rseqno = obj->last_read_seqno;
1649 err->wseqno = obj->last_write_seqno;
f343c5f6 1650 err->gtt_offset = i915_gem_obj_ggtt_offset(obj);
1b50247a
CW		 1651 err->read_domains = obj->base.read_domains;
1652 err->write_domain = obj->base.write_domain;
1653 err->fence_reg = obj->fence_reg;
1654 err->pinned = 0;
1655 if (obj->pin_count > 0)
1656 err->pinned = 1;
1657 if (obj->user_pin_count > 0)
1658 err->pinned = -1;
1659 err->tiling = obj->tiling_mode;
1660 err->dirty = obj->dirty;
1661 err->purgeable = obj->madv != I915_MADV_WILLNEED;
1662 err->ring = obj->ring ? obj->ring->id : -1;
1663 err->cache_level = obj->cache_level;
1664}
9df30794 1665
1b50247a
CW		 1666static u32 capture_active_bo(struct drm_i915_error_buffer *err,
1667 int count, struct list_head *head)
c724e8a9
CW		 1668{
1669 struct drm_i915_gem_object *obj;
1670 int i = 0;
1671
1672 list_for_each_entry(obj, head, mm_list) {
1b50247a 1673 capture_bo(err++, obj);
c724e8a9
CW		 1674 if (++i == count)
1675 break;
1b50247a
CW		 1676 }
1677
1678 return i;
1679}
1680
1681static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
1682 int count, struct list_head *head)
1683{
1684 struct drm_i915_gem_object *obj;
1685 int i = 0;
1686
35c20a60 1687 list_for_each_entry(obj, head, global_list) {
1b50247a
CW		 1688 if (obj->pin_count == 0)
1689 continue;
c724e8a9 1690
1b50247a
CW		 1691 capture_bo(err++, obj);
1692 if (++i == count)
1693 break;
c724e8a9
CW		 1694 }
1695
1696 return i;
1697}
1698
748ebc60
CW		 1699static void i915_gem_record_fences(struct drm_device *dev,
1700 struct drm_i915_error_state *error)
1701{
1702 struct drm_i915_private *dev_priv = dev->dev_private;
1703 int i;
1704
1705 /* Fences */
1706 switch (INTEL_INFO(dev)->gen) {
775d17b6 1707 case 7:
748ebc60 1708 case 6:
42b5aeab 1709 for (i = 0; i < dev_priv->num_fence_regs; i++)
748ebc60
CW		 1710 error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
1711 break;
1712 case 5:
1713 case 4:
1714 for (i = 0; i < 16; i++)
1715 error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
1716 break;
1717 case 3:
1718 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
1719 for (i = 0; i < 8; i++)
1720 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
1721 case 2:
1722 for (i = 0; i < 8; i++)
1723 error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
1724 break;
1725
7dbf9d6e
BW		 1726 default:
1727 BUG();
748ebc60
CW		 1728 }
1729}
1730
bcfb2e28
CW		 1731static struct drm_i915_error_object *
1732i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
1733 struct intel_ring_buffer *ring)
1734{
1735 struct drm_i915_gem_object *obj;
1736 u32 seqno;
1737
1738 if (!ring->get_seqno)
1739 return NULL;
1740
b45305fc
DV		 1741 if (HAS_BROKEN_CS_TLB(dev_priv->dev)) {
1742 u32 acthd = I915_READ(ACTHD);
1743
1744 if (WARN_ON(ring->id != RCS))
1745 return NULL;
1746
1747 obj = ring->private;
f343c5f6
BW		 1748 if (acthd >= i915_gem_obj_ggtt_offset(obj) &&
1749 acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size)
b45305fc
DV		 1750 return i915_error_object_create(dev_priv, obj);
1751 }
1752
b2eadbc8 1753 seqno = ring->get_seqno(ring, false);
bcfb2e28
CW		 1754 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
1755 if (obj->ring != ring)
1756 continue;
1757
0201f1ec 1758 if (i915_seqno_passed(seqno, obj->last_read_seqno))
bcfb2e28
CW		 1759 continue;
1760
1761 if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
1762 continue;
1763
1764 /* We need to copy these to an anonymous buffer as the simplest
1765 * method to avoid being overwritten by userspace.
1766 */
1767 return i915_error_object_create(dev_priv, obj);
1768 }
1769
1770 return NULL;
1771}
1772
d27b1e0e
DV		 1773static void i915_record_ring_state(struct drm_device *dev,
1774 struct drm_i915_error_state *error,
1775 struct intel_ring_buffer *ring)
1776{
1777 struct drm_i915_private *dev_priv = dev->dev_private;
1778
33f3f518 1779 if (INTEL_INFO(dev)->gen >= 6) {
12f55818 1780 error->rc_psmi[ring->id] = I915_READ(ring->mmio_base + 0x50);
33f3f518 1781 error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring));
7e3b8737
DV		 1782 error->semaphore_mboxes[ring->id][0]
1783 = I915_READ(RING_SYNC_0(ring->mmio_base));
1784 error->semaphore_mboxes[ring->id][1]
1785 = I915_READ(RING_SYNC_1(ring->mmio_base));
df2b23d9
CW		 1786 error->semaphore_seqno[ring->id][0] = ring->sync_seqno[0];
1787 error->semaphore_seqno[ring->id][1] = ring->sync_seqno[1];
33f3f518 1788 }
c1cd90ed 1789
d27b1e0e 1790 if (INTEL_INFO(dev)->gen >= 4) {
9d2f41fa 1791 error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
d27b1e0e
DV		 1792 error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base));
1793 error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
1794 error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
c1cd90ed 1795 error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base));
050ee91f 1796 if (ring->id == RCS)
d27b1e0e 1797 error->bbaddr = I915_READ64(BB_ADDR);
d27b1e0e 1798 } else {
9d2f41fa 1799 error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
d27b1e0e
DV		 1800 error->ipeir[ring->id] = I915_READ(IPEIR);
1801 error->ipehr[ring->id] = I915_READ(IPEHR);
1802 error->instdone[ring->id] = I915_READ(INSTDONE);
d27b1e0e
DV		 1803 }
1804
9574b3fe 1805 error->waiting[ring->id] = waitqueue_active(&ring->irq_queue);
c1cd90ed 1806 error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
b2eadbc8 1807 error->seqno[ring->id] = ring->get_seqno(ring, false);
d27b1e0e 1808 error->acthd[ring->id] = intel_ring_get_active_head(ring);
c1cd90ed
DV		 1809 error->head[ring->id] = I915_READ_HEAD(ring);
1810 error->tail[ring->id] = I915_READ_TAIL(ring);
0f3b6849 1811 error->ctl[ring->id] = I915_READ_CTL(ring);
7e3b8737
DV		 1812
1813 error->cpu_ring_head[ring->id] = ring->head;
1814 error->cpu_ring_tail[ring->id] = ring->tail;
d27b1e0e
DV		 1815}
1816
8c123e54
BW		 1817
1818static void i915_gem_record_active_context(struct intel_ring_buffer *ring,
1819 struct drm_i915_error_state *error,
1820 struct drm_i915_error_ring *ering)
1821{
1822 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1823 struct drm_i915_gem_object *obj;
1824
1825 /* Currently render ring is the only HW context user */
1826 if (ring->id != RCS || !error->ccid)
1827 return;
1828
35c20a60 1829 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
f343c5f6 1830 if ((error->ccid & PAGE_MASK) == i915_gem_obj_ggtt_offset(obj)) {
8c123e54
BW		 1831 ering->ctx = i915_error_object_create_sized(dev_priv,
1832 obj, 1);
3ef8fb5a 1833 break;
8c123e54
BW		 1834 }
1835 }
1836}
1837
52d39a21
CW		 1838static void i915_gem_record_rings(struct drm_device *dev,
1839 struct drm_i915_error_state *error)
1840{
1841 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513 1842 struct intel_ring_buffer *ring;
52d39a21
CW		 1843 struct drm_i915_gem_request *request;
1844 int i, count;
1845
b4519513 1846 for_each_ring(ring, dev_priv, i) {
52d39a21
CW		 1847 i915_record_ring_state(dev, error, ring);
1848
1849 error->ring[i].batchbuffer =
1850 i915_error_first_batchbuffer(dev_priv, ring);
1851
1852 error->ring[i].ringbuffer =
1853 i915_error_object_create(dev_priv, ring->obj);
1854
8c123e54
BW		 1855
1856 i915_gem_record_active_context(ring, error, &error->ring[i]);
1857
52d39a21
CW		 1858 count = 0;
1859 list_for_each_entry(request, &ring->request_list, list)
1860 count++;
1861
1862 error->ring[i].num_requests = count;
1863 error->ring[i].requests =
1864 kmalloc(count*sizeof(struct drm_i915_error_request),
1865 GFP_ATOMIC);
1866 if (error->ring[i].requests == NULL) {
1867 error->ring[i].num_requests = 0;
1868 continue;
1869 }
1870
1871 count = 0;
1872 list_for_each_entry(request, &ring->request_list, list) {
1873 struct drm_i915_error_request *erq;
1874
1875 erq = &error->ring[i].requests[count++];
1876 erq->seqno = request->seqno;
1877 erq->jiffies = request->emitted_jiffies;
ee4f42b1 1878 erq->tail = request->tail;
52d39a21
CW		 1879 }
1880 }
1881}
1882
26b7c224
BW		 1883static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
1884 struct drm_i915_error_state *error)
1885{
1886 struct drm_i915_gem_object *obj;
1887 int i;
1888
1889 i = 0;
1890 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
1891 i++;
1892 error->active_bo_count = i;
1893 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1894 if (obj->pin_count)
1895 i++;
1896 error->pinned_bo_count = i - error->active_bo_count;
1897
1898 if (i) {
1899 error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
1900 GFP_ATOMIC);
1901 if (error->active_bo)
1902 error->pinned_bo =
1903 error->active_bo + error->active_bo_count;
1904 }
1905
1906 if (error->active_bo)
1907 error->active_bo_count =
1908 capture_active_bo(error->active_bo,
1909 error->active_bo_count,
1910 &dev_priv->mm.active_list);
1911
1912 if (error->pinned_bo)
1913 error->pinned_bo_count =
1914 capture_pinned_bo(error->pinned_bo,
1915 error->pinned_bo_count,
1916 &dev_priv->mm.bound_list);
1917}
1918
8a905236
JB		 1919/**
1920 * i915_capture_error_state - capture an error record for later analysis
1921 * @dev: drm device
1922 *
1923 * Should be called when an error is detected (either a hang or an error
1924 * interrupt) to capture error state from the time of the error. Fills
1925 * out a structure which becomes available in debugfs for user level tools
1926 * to pick up.
1927 */
63eeaf38
JB		 1928static void i915_capture_error_state(struct drm_device *dev)
1929{
1930 struct drm_i915_private *dev_priv = dev->dev_private;
1931 struct drm_i915_error_state *error;
1932 unsigned long flags;
26b7c224 1933 int pipe;
63eeaf38 1934
99584db3
DV		 1935 spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
1936 error = dev_priv->gpu_error.first_error;
1937 spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
9df30794
CW		 1938 if (error)
1939 return;
63eeaf38 1940
9db4a9c7 1941 /* Account for pipe specific data like PIPE*STAT */
33f3f518 1942 error = kzalloc(sizeof(*error), GFP_ATOMIC);
63eeaf38 1943 if (!error) {
9df30794
CW		 1944 DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
1945 return;
63eeaf38
JB		 1946 }
1947
5d83d294 1948 DRM_INFO("capturing error event; look for more information in "
ef86ddce 1949 "/sys/class/drm/card%d/error\n", dev->primary->index);
2fa772f3 1950
742cbee8 1951 kref_init(&error->ref);
63eeaf38
JB		 1952 error->eir = I915_READ(EIR);
1953 error->pgtbl_er = I915_READ(PGTBL_ER);
211816ec
BW		 1954 if (HAS_HW_CONTEXTS(dev))
1955 error->ccid = I915_READ(CCID);
be998e2e
BW		 1956
1957 if (HAS_PCH_SPLIT(dev))
1958 error->ier = I915_READ(DEIER) | I915_READ(GTIER);
1959 else if (IS_VALLEYVIEW(dev))
1960 error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
1961 else if (IS_GEN2(dev))
1962 error->ier = I915_READ16(IER);
1963 else
1964 error->ier = I915_READ(IER);
1965
0f3b6849
CW		 1966 if (INTEL_INFO(dev)->gen >= 6)
1967 error->derrmr = I915_READ(DERRMR);
1968
1969 if (IS_VALLEYVIEW(dev))
1970 error->forcewake = I915_READ(FORCEWAKE_VLV);
1971 else if (INTEL_INFO(dev)->gen >= 7)
1972 error->forcewake = I915_READ(FORCEWAKE_MT);
1973 else if (INTEL_INFO(dev)->gen == 6)
1974 error->forcewake = I915_READ(FORCEWAKE);
1975
4f3308b9
PZ		 1976 if (!HAS_PCH_SPLIT(dev))
1977 for_each_pipe(pipe)
1978 error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
d27b1e0e 1979
33f3f518 1980 if (INTEL_INFO(dev)->gen >= 6) {
f406839f 1981 error->error = I915_READ(ERROR_GEN6);
33f3f518
DV		 1982 error->done_reg = I915_READ(DONE_REG);
1983 }
d27b1e0e 1984
71e172e8
BW		 1985 if (INTEL_INFO(dev)->gen == 7)
1986 error->err_int = I915_READ(GEN7_ERR_INT);
1987
050ee91f
BW		 1988 i915_get_extra_instdone(dev, error->extra_instdone);
1989
26b7c224 1990 i915_gem_capture_buffers(dev_priv, error);
748ebc60 1991 i915_gem_record_fences(dev, error);
52d39a21 1992 i915_gem_record_rings(dev, error);
9df30794 1993
9df30794
CW		 1994 do_gettimeofday(&error->time);
1995
6ef3d427 1996 error->overlay = intel_overlay_capture_error_state(dev);
c4a1d9e4 1997 error->display = intel_display_capture_error_state(dev);
6ef3d427 1998
99584db3
DV		 1999 spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
2000 if (dev_priv->gpu_error.first_error == NULL) {
2001 dev_priv->gpu_error.first_error = error;
9df30794
CW		 2002 error = NULL;
2003 }
99584db3 2004 spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
9df30794
CW		 2005
2006 if (error)
742cbee8 2007 i915_error_state_free(&error->ref);
9df30794
CW		 2008}
2009
2010void i915_destroy_error_state(struct drm_device *dev)
2011{
2012 struct drm_i915_private *dev_priv = dev->dev_private;
2013 struct drm_i915_error_state *error;
6dc0e816 2014 unsigned long flags;
9df30794 2015
99584db3
DV		 2016 spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
2017 error = dev_priv->gpu_error.first_error;
2018 dev_priv->gpu_error.first_error = NULL;
2019 spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
9df30794
CW		 2020
2021 if (error)
742cbee8 2022 kref_put(&error->ref, i915_error_state_free);
63eeaf38 2023}
3bd3c932
CW		 2024#else
2025#define i915_capture_error_state(x)
2026#endif
63eeaf38 2027
35aed2e6 2028static void i915_report_and_clear_eir(struct drm_device *dev)
8a905236
JB		 2029{
2030 struct drm_i915_private *dev_priv = dev->dev_private;
bd9854f9 2031 uint32_t instdone[I915_NUM_INSTDONE_REG];
8a905236 2032 u32 eir = I915_READ(EIR);
050ee91f 2033 int pipe, i;
8a905236 2034
35aed2e6
CW		 2035 if (!eir)
2036 return;
8a905236 2037
a70491cc 2038 pr_err("render error detected, EIR: 0x%08x\n", eir);
8a905236 2039
bd9854f9
BW		 2040 i915_get_extra_instdone(dev, instdone);
2041
8a905236
JB		 2042 if (IS_G4X(dev)) {
2043 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2044 u32 ipeir = I915_READ(IPEIR_I965);
2045
a70491cc
JP		 2046 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2047 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
050ee91f
BW		 2048 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2049 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a70491cc 2050 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 2051 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 2052 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 2053 POSTING_READ(IPEIR_I965);
8a905236
JB		 2054 }
2055 if (eir & GM45_ERROR_PAGE_TABLE) {
2056 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 2057 pr_err("page table error\n");
2058 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 2059 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 2060 POSTING_READ(PGTBL_ER);
8a905236
JB		 2061 }
2062 }
2063
a6c45cf0 2064 if (!IS_GEN2(dev)) {
8a905236
JB		 2065 if (eir & I915_ERROR_PAGE_TABLE) {
2066 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 2067 pr_err("page table error\n");
2068 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 2069 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 2070 POSTING_READ(PGTBL_ER);
8a905236
JB		 2071 }
2072 }
2073
2074 if (eir & I915_ERROR_MEMORY_REFRESH) {
a70491cc 2075 pr_err("memory refresh error:\n");
9db4a9c7 2076 for_each_pipe(pipe)
a70491cc 2077 pr_err("pipe %c stat: 0x%08x\n",
9db4a9c7 2078 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
8a905236
JB		 2079 /* pipestat has already been acked */
2080 }
2081 if (eir & I915_ERROR_INSTRUCTION) {
a70491cc
JP		 2082 pr_err("instruction error\n");
2083 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
050ee91f
BW		 2084 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2085 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a6c45cf0 2086 if (INTEL_INFO(dev)->gen < 4) {
8a905236
JB		 2087 u32 ipeir = I915_READ(IPEIR);
2088
a70491cc
JP		 2089 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
2090 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
a70491cc 2091 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
8a905236 2092 I915_WRITE(IPEIR, ipeir);
3143a2bf 2093 POSTING_READ(IPEIR);
8a905236
JB		 2094 } else {
2095 u32 ipeir = I915_READ(IPEIR_I965);
2096
a70491cc
JP		 2097 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2098 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
a70491cc 2099 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 2100 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 2101 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 2102 POSTING_READ(IPEIR_I965);
8a905236
JB		 2103 }
2104 }
2105
2106 I915_WRITE(EIR, eir);
3143a2bf 2107 POSTING_READ(EIR);
8a905236
JB		 2108 eir = I915_READ(EIR);
2109 if (eir) {
2110 /*
2111 * some errors might have become stuck,
2112 * mask them.
2113 */
2114 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2115 I915_WRITE(EMR, I915_READ(EMR) | eir);
2116 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2117 }
35aed2e6
CW		 2118}
2119
2120/**
2121 * i915_handle_error - handle an error interrupt
2122 * @dev: drm device
2123 *
2124 * Do some basic checking of regsiter state at error interrupt time and
2125 * dump it to the syslog. Also call i915_capture_error_state() to make
2126 * sure we get a record and make it available in debugfs. Fire a uevent
2127 * so userspace knows something bad happened (should trigger collection
2128 * of a ring dump etc.).
2129 */
527f9e90 2130void i915_handle_error(struct drm_device *dev, bool wedged)
35aed2e6
CW		 2131{
2132 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513
CW		 2133 struct intel_ring_buffer *ring;
2134 int i;
35aed2e6
CW		 2135
2136 i915_capture_error_state(dev);
2137 i915_report_and_clear_eir(dev);
8a905236 2138
ba1234d1 2139 if (wedged) {
f69061be
DV		 2140 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2141 &dev_priv->gpu_error.reset_counter);
ba1234d1 2142
11ed50ec 2143 /*
1f83fee0
DV		 2144 * Wakeup waiting processes so that the reset work item
2145 * doesn't deadlock trying to grab various locks.
11ed50ec 2146 */
b4519513
CW		 2147 for_each_ring(ring, dev_priv, i)
2148 wake_up_all(&ring->irq_queue);
11ed50ec
BG		 2149 }
2150
99584db3 2151 queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
8a905236
JB		 2152}
2153
21ad8330 2154static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
4e5359cd
SF		 2155{
2156 drm_i915_private_t *dev_priv = dev->dev_private;
2157 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
2158 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
05394f39 2159 struct drm_i915_gem_object *obj;
4e5359cd
SF		 2160 struct intel_unpin_work *work;
2161 unsigned long flags;
2162 bool stall_detected;
2163
2164 /* Ignore early vblank irqs */
2165 if (intel_crtc == NULL)
2166 return;
2167
2168 spin_lock_irqsave(&dev->event_lock, flags);
2169 work = intel_crtc->unpin_work;
2170
e7d841ca
CW		 2171 if (work == NULL ||
2172 atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
2173 !work->enable_stall_check) {
4e5359cd
SF		 2174 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
2175 spin_unlock_irqrestore(&dev->event_lock, flags);
2176 return;
2177 }
2178
2179 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
05394f39 2180 obj = work->pending_flip_obj;
a6c45cf0 2181 if (INTEL_INFO(dev)->gen >= 4) {
9db4a9c7 2182 int dspsurf = DSPSURF(intel_crtc->plane);
446f2545 2183 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
f343c5f6 2184 i915_gem_obj_ggtt_offset(obj);
4e5359cd 2185 } else {
9db4a9c7 2186 int dspaddr = DSPADDR(intel_crtc->plane);
f343c5f6 2187 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
01f2c773 2188 crtc->y * crtc->fb->pitches[0] +
4e5359cd
SF		 2189 crtc->x * crtc->fb->bits_per_pixel/8);
2190 }
2191
2192 spin_unlock_irqrestore(&dev->event_lock, flags);
2193
2194 if (stall_detected) {
2195 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
2196 intel_prepare_page_flip(dev, intel_crtc->plane);
2197 }
2198}
2199
42f52ef8
KP		 2200/* Called from drm generic code, passed 'crtc' which
2201 * we use as a pipe index
2202 */
f71d4af4 2203static int i915_enable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 2204{
2205 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 2206 unsigned long irqflags;
71e0ffa5 2207
5eddb70b 2208 if (!i915_pipe_enabled(dev, pipe))
71e0ffa5 2209 return -EINVAL;
0a3e67a4 2210
1ec14ad3 2211 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f796cf8f 2212 if (INTEL_INFO(dev)->gen >= 4)
7c463586
KP		 2213 i915_enable_pipestat(dev_priv, pipe,
2214 PIPE_START_VBLANK_INTERRUPT_ENABLE);
e9d21d7f 2215 else
7c463586
KP		 2216 i915_enable_pipestat(dev_priv, pipe,
2217 PIPE_VBLANK_INTERRUPT_ENABLE);
8692d00e
CW		 2218
2219 /* maintain vblank delivery even in deep C-states */
2220 if (dev_priv->info->gen == 3)
6b26c86d 2221 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1ec14ad3 2222 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
8692d00e 2223
0a3e67a4
JB		 2224 return 0;
2225}
2226
f71d4af4 2227static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 2228{
2229 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2230 unsigned long irqflags;
2231
2232 if (!i915_pipe_enabled(dev, pipe))
2233 return -EINVAL;
2234
2235 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2236 ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
0206e353 2237 DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
f796cf8f
JB		 2238 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2239
2240 return 0;
2241}
2242
f71d4af4 2243static int ivybridge_enable_vblank(struct drm_device *dev, int pipe)
b1f14ad0
JB		 2244{
2245 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2246 unsigned long irqflags;
2247
2248 if (!i915_pipe_enabled(dev, pipe))
2249 return -EINVAL;
2250
2251 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b615b57a
CW		 2252 ironlake_enable_display_irq(dev_priv,
2253 DE_PIPEA_VBLANK_IVB << (5 * pipe));
b1f14ad0
JB		 2254 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2255
2256 return 0;
2257}
2258
7e231dbe
JB		 2259static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2260{
2261 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2262 unsigned long irqflags;
31acc7f5 2263 u32 imr;
7e231dbe
JB		 2264
2265 if (!i915_pipe_enabled(dev, pipe))
2266 return -EINVAL;
2267
2268 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
7e231dbe 2269 imr = I915_READ(VLV_IMR);
31acc7f5 2270 if (pipe == 0)
7e231dbe 2271 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 2272 else
7e231dbe 2273 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 2274 I915_WRITE(VLV_IMR, imr);
31acc7f5
JB		 2275 i915_enable_pipestat(dev_priv, pipe,
2276 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe
JB		 2277 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2278
2279 return 0;
2280}
2281
42f52ef8
KP		 2282/* Called from drm generic code, passed 'crtc' which
2283 * we use as a pipe index
2284 */
f71d4af4 2285static void i915_disable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 2286{
2287 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 2288 unsigned long irqflags;
0a3e67a4 2289
1ec14ad3 2290 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
8692d00e 2291 if (dev_priv->info->gen == 3)
6b26c86d 2292 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
8692d00e 2293
f796cf8f
JB		 2294 i915_disable_pipestat(dev_priv, pipe,
2295 PIPE_VBLANK_INTERRUPT_ENABLE |
2296 PIPE_START_VBLANK_INTERRUPT_ENABLE);
2297 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2298}
2299
f71d4af4 2300static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 2301{
2302 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2303 unsigned long irqflags;
2304
2305 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2306 ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
0206e353 2307 DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
1ec14ad3 2308 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
0a3e67a4
JB		 2309}
2310
f71d4af4 2311static void ivybridge_disable_vblank(struct drm_device *dev, int pipe)
b1f14ad0
JB		 2312{
2313 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2314 unsigned long irqflags;
2315
2316 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b615b57a
CW		 2317 ironlake_disable_display_irq(dev_priv,
2318 DE_PIPEA_VBLANK_IVB << (pipe * 5));
b1f14ad0
JB		 2319 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2320}
2321
7e231dbe
JB		 2322static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2323{
2324 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2325 unsigned long irqflags;
31acc7f5 2326 u32 imr;
7e231dbe
JB		 2327
2328 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5
JB		 2329 i915_disable_pipestat(dev_priv, pipe,
2330 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe 2331 imr = I915_READ(VLV_IMR);
31acc7f5 2332 if (pipe == 0)
7e231dbe 2333 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 2334 else
7e231dbe 2335 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 2336 I915_WRITE(VLV_IMR, imr);
7e231dbe
JB		 2337 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2338}
2339
893eead0
CW		 2340static u32
2341ring_last_seqno(struct intel_ring_buffer *ring)
852835f3 2342{
893eead0
CW		 2343 return list_entry(ring->request_list.prev,
2344 struct drm_i915_gem_request, list)->seqno;
2345}
2346
9107e9d2
CW		 2347static bool
2348ring_idle(struct intel_ring_buffer *ring, u32 seqno)
2349{
2350 return (list_empty(&ring->request_list) ||
2351 i915_seqno_passed(seqno, ring_last_seqno(ring)));
f65d9421
BG		 2352}
2353
6274f212
CW		 2354static struct intel_ring_buffer *
2355semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
a24a11e6
CW		 2356{
2357 struct drm_i915_private *dev_priv = ring->dev->dev_private;
6274f212 2358 u32 cmd, ipehr, acthd, acthd_min;
a24a11e6
CW		 2359
2360 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2361 if ((ipehr & ~(0x3 << 16)) !=
2362 (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
6274f212 2363 return NULL;
a24a11e6
CW		 2364
2365 /* ACTHD is likely pointing to the dword after the actual command,
2366 * so scan backwards until we find the MBOX.
2367 */
6274f212 2368 acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
a24a11e6
CW		 2369 acthd_min = max((int)acthd - 3 * 4, 0);
2370 do {
2371 cmd = ioread32(ring->virtual_start + acthd);
2372 if (cmd == ipehr)
2373 break;
2374
2375 acthd -= 4;
2376 if (acthd < acthd_min)
6274f212 2377 return NULL;
a24a11e6
CW		 2378 } while (1);
2379
6274f212
CW		 2380 *seqno = ioread32(ring->virtual_start+acthd+4)+1;
2381 return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
a24a11e6
CW		 2382}
2383
6274f212
CW		 2384static int semaphore_passed(struct intel_ring_buffer *ring)
2385{
2386 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2387 struct intel_ring_buffer *signaller;
2388 u32 seqno, ctl;
2389
2390 ring->hangcheck.deadlock = true;
2391
2392 signaller = semaphore_waits_for(ring, &seqno);
2393 if (signaller == NULL || signaller->hangcheck.deadlock)
2394 return -1;
2395
2396 /* cursory check for an unkickable deadlock */
2397 ctl = I915_READ_CTL(signaller);
2398 if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
2399 return -1;
2400
2401 return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
2402}
2403
2404static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2405{
2406 struct intel_ring_buffer *ring;
2407 int i;
2408
2409 for_each_ring(ring, dev_priv, i)
2410 ring->hangcheck.deadlock = false;
2411}
2412
ad8beaea
MK		 2413static enum intel_ring_hangcheck_action
2414ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1ec14ad3
CW		 2415{
2416 struct drm_device *dev = ring->dev;
2417 struct drm_i915_private *dev_priv = dev->dev_private;
9107e9d2
CW		 2418 u32 tmp;
2419
6274f212
CW		 2420 if (ring->hangcheck.acthd != acthd)
2421 return active;
2422
9107e9d2 2423 if (IS_GEN2(dev))
6274f212 2424 return hung;
9107e9d2
CW		 2425
2426 /* Is the chip hanging on a WAIT_FOR_EVENT?
2427 * If so we can simply poke the RB_WAIT bit
2428 * and break the hang. This should work on
2429 * all but the second generation chipsets.
2430 */
2431 tmp = I915_READ_CTL(ring);
1ec14ad3
CW		 2432 if (tmp & RING_WAIT) {
2433 DRM_ERROR("Kicking stuck wait on %s\n",
2434 ring->name);
2435 I915_WRITE_CTL(ring, tmp);
6274f212
CW		 2436 return kick;
2437 }
2438
2439 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2440 switch (semaphore_passed(ring)) {
2441 default:
2442 return hung;
2443 case 1:
2444 DRM_ERROR("Kicking stuck semaphore on %s\n",
2445 ring->name);
2446 I915_WRITE_CTL(ring, tmp);
2447 return kick;
2448 case 0:
2449 return wait;
2450 }
9107e9d2 2451 }
ed5cbb03 2452
6274f212 2453 return hung;
ed5cbb03
MK		 2454}
2455
f65d9421
BG		 2456/**
2457 * This is called when the chip hasn't reported back with completed
05407ff8
MK		 2458 * batchbuffers in a long time. We keep track per ring seqno progress and
2459 * if there are no progress, hangcheck score for that ring is increased.
2460 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2461 * we kick the ring. If we see no progress on three subsequent calls
2462 * we assume chip is wedged and try to fix it by resetting the chip.
f65d9421
BG		 2463 */
2464void i915_hangcheck_elapsed(unsigned long data)
2465{
2466 struct drm_device *dev = (struct drm_device *)data;
2467 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 2468 struct intel_ring_buffer *ring;
b4519513 2469 int i;
05407ff8 2470 int busy_count = 0, rings_hung = 0;
9107e9d2
CW		 2471 bool stuck[I915_NUM_RINGS] = { 0 };
2472#define BUSY 1
2473#define KICK 5
2474#define HUNG 20
2475#define FIRE 30
893eead0 2476
3e0dc6b0
BW		 2477 if (!i915_enable_hangcheck)
2478 return;
2479
b4519513 2480 for_each_ring(ring, dev_priv, i) {
05407ff8 2481 u32 seqno, acthd;
9107e9d2 2482 bool busy = true;
05407ff8 2483
6274f212
CW		 2484 semaphore_clear_deadlocks(dev_priv);
2485
05407ff8
MK		 2486 seqno = ring->get_seqno(ring, false);
2487 acthd = intel_ring_get_active_head(ring);
b4519513 2488
9107e9d2
CW		 2489 if (ring->hangcheck.seqno == seqno) {
2490 if (ring_idle(ring, seqno)) {
2491 if (waitqueue_active(&ring->irq_queue)) {
2492 /* Issue a wake-up to catch stuck h/w. */
2493 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2494 ring->name);
2495 wake_up_all(&ring->irq_queue);
2496 ring->hangcheck.score += HUNG;
2497 } else
2498 busy = false;
05407ff8 2499 } else {
9107e9d2
CW		 2500 int score;
2501
6274f212
CW		 2502 /* We always increment the hangcheck score
2503 * if the ring is busy and still processing
2504 * the same request, so that no single request
2505 * can run indefinitely (such as a chain of
2506 * batches). The only time we do not increment
2507 * the hangcheck score on this ring, if this
2508 * ring is in a legitimate wait for another
2509 * ring. In that case the waiting ring is a
2510 * victim and we want to be sure we catch the
2511 * right culprit. Then every time we do kick
2512 * the ring, add a small increment to the
2513 * score so that we can catch a batch that is
2514 * being repeatedly kicked and so responsible
2515 * for stalling the machine.
2516 */
ad8beaea
MK		 2517 ring->hangcheck.action = ring_stuck(ring,
2518 acthd);
2519
2520 switch (ring->hangcheck.action) {
6274f212
CW		 2521 case wait:
2522 score = 0;
2523 break;
2524 case active:
9107e9d2 2525 score = BUSY;
6274f212
CW		 2526 break;
2527 case kick:
2528 score = KICK;
2529 break;
2530 case hung:
2531 score = HUNG;
2532 stuck[i] = true;
2533 break;
2534 }
9107e9d2 2535 ring->hangcheck.score += score;
05407ff8 2536 }
9107e9d2
CW		 2537 } else {
2538 /* Gradually reduce the count so that we catch DoS
2539 * attempts across multiple batches.
2540 */
2541 if (ring->hangcheck.score > 0)
2542 ring->hangcheck.score--;
d1e61e7f
CW		 2543 }
2544
05407ff8
MK		 2545 ring->hangcheck.seqno = seqno;
2546 ring->hangcheck.acthd = acthd;
9107e9d2 2547 busy_count += busy;
893eead0 2548 }
b9201c14 2549
92cab734 2550 for_each_ring(ring, dev_priv, i) {
9107e9d2 2551 if (ring->hangcheck.score > FIRE) {
acd78c11 2552 DRM_ERROR("%s on %s\n",
05407ff8 2553 stuck[i] ? "stuck" : "no progress",
a43adf07
CW		 2554 ring->name);
2555 rings_hung++;
92cab734
MK		 2556 }
2557 }
2558
05407ff8
MK		 2559 if (rings_hung)
2560 return i915_handle_error(dev, true);
f65d9421 2561
05407ff8
MK		 2562 if (busy_count)
2563 /* Reset timer case chip hangs without another request
2564 * being added */
2565 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2566 round_jiffies_up(jiffies +
2567 DRM_I915_HANGCHECK_JIFFIES));
f65d9421
BG		 2568}
2569
91738a95
PZ		 2570static void ibx_irq_preinstall(struct drm_device *dev)
2571{
2572 struct drm_i915_private *dev_priv = dev->dev_private;
2573
2574 if (HAS_PCH_NOP(dev))
2575 return;
2576
2577 /* south display irq */
2578 I915_WRITE(SDEIMR, 0xffffffff);
2579 /*
2580 * SDEIER is also touched by the interrupt handler to work around missed
2581 * PCH interrupts. Hence we can't update it after the interrupt handler
2582 * is enabled - instead we unconditionally enable all PCH interrupt
2583 * sources here, but then only unmask them as needed with SDEIMR.
2584 */
2585 I915_WRITE(SDEIER, 0xffffffff);
2586 POSTING_READ(SDEIER);
2587}
2588
1da177e4
LT		 2589/* drm_dma.h hooks
2590*/
f71d4af4 2591static void ironlake_irq_preinstall(struct drm_device *dev)
036a4a7d
ZW		 2592{
2593 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2594
4697995b
JB		 2595 atomic_set(&dev_priv->irq_received, 0);
2596
036a4a7d 2597 I915_WRITE(HWSTAM, 0xeffe);
bdfcdb63 2598
036a4a7d
ZW		 2599 /* XXX hotplug from PCH */
2600
2601 I915_WRITE(DEIMR, 0xffffffff);
2602 I915_WRITE(DEIER, 0x0);
3143a2bf 2603 POSTING_READ(DEIER);
036a4a7d
ZW		 2604
2605 /* and GT */
2606 I915_WRITE(GTIMR, 0xffffffff);
2607 I915_WRITE(GTIER, 0x0);
3143a2bf 2608 POSTING_READ(GTIER);
c650156a 2609
91738a95 2610 ibx_irq_preinstall(dev);
7d99163d
BW		 2611}
2612
2613static void ivybridge_irq_preinstall(struct drm_device *dev)
2614{
2615 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2616
2617 atomic_set(&dev_priv->irq_received, 0);
2618
2619 I915_WRITE(HWSTAM, 0xeffe);
2620
2621 /* XXX hotplug from PCH */
2622
2623 I915_WRITE(DEIMR, 0xffffffff);
2624 I915_WRITE(DEIER, 0x0);
2625 POSTING_READ(DEIER);
2626
2627 /* and GT */
2628 I915_WRITE(GTIMR, 0xffffffff);
2629 I915_WRITE(GTIER, 0x0);
2630 POSTING_READ(GTIER);
2631
eda63ffb
BW		 2632 /* Power management */
2633 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2634 I915_WRITE(GEN6_PMIER, 0x0);
2635 POSTING_READ(GEN6_PMIER);
2636
91738a95 2637 ibx_irq_preinstall(dev);
036a4a7d
ZW		 2638}
2639
7e231dbe
JB		 2640static void valleyview_irq_preinstall(struct drm_device *dev)
2641{
2642 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2643 int pipe;
2644
2645 atomic_set(&dev_priv->irq_received, 0);
2646
7e231dbe
JB		 2647 /* VLV magic */
2648 I915_WRITE(VLV_IMR, 0);
2649 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2650 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2651 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2652
7e231dbe
JB		 2653 /* and GT */
2654 I915_WRITE(GTIIR, I915_READ(GTIIR));
2655 I915_WRITE(GTIIR, I915_READ(GTIIR));
2656 I915_WRITE(GTIMR, 0xffffffff);
2657 I915_WRITE(GTIER, 0x0);
2658 POSTING_READ(GTIER);
2659
2660 I915_WRITE(DPINVGTT, 0xff);
2661
2662 I915_WRITE(PORT_HOTPLUG_EN, 0);
2663 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2664 for_each_pipe(pipe)
2665 I915_WRITE(PIPESTAT(pipe), 0xffff);
2666 I915_WRITE(VLV_IIR, 0xffffffff);
2667 I915_WRITE(VLV_IMR, 0xffffffff);
2668 I915_WRITE(VLV_IER, 0x0);
2669 POSTING_READ(VLV_IER);
2670}
2671
82a28bcf 2672static void ibx_hpd_irq_setup(struct drm_device *dev)
7fe0b973
KP		 2673{
2674 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf
DV		 2675 struct drm_mode_config *mode_config = &dev->mode_config;
2676 struct intel_encoder *intel_encoder;
fee884ed 2677 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
82a28bcf
DV		 2678
2679 if (HAS_PCH_IBX(dev)) {
fee884ed 2680 hotplug_irqs = SDE_HOTPLUG_MASK;
82a28bcf 2681 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2682 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2683 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
82a28bcf 2684 } else {
fee884ed 2685 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
82a28bcf 2686 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2687 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2688 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
82a28bcf 2689 }
7fe0b973 2690
fee884ed 2691 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
82a28bcf
DV		 2692
2693 /*
2694 * Enable digital hotplug on the PCH, and configure the DP short pulse
2695 * duration to 2ms (which is the minimum in the Display Port spec)
2696 *
2697 * This register is the same on all known PCH chips.
2698 */
7fe0b973
KP		 2699 hotplug = I915_READ(PCH_PORT_HOTPLUG);
2700 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2701 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2702 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2703 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2704 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2705}
2706
d46da437
PZ		 2707static void ibx_irq_postinstall(struct drm_device *dev)
2708{
2709 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf 2710 u32 mask;
e5868a31 2711
692a04cf
DV		 2712 if (HAS_PCH_NOP(dev))
2713 return;
2714
8664281b
PZ		 2715 if (HAS_PCH_IBX(dev)) {
2716 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
de032bf4 2717 SDE_TRANSA_FIFO_UNDER | SDE_POISON;
8664281b
PZ		 2718 } else {
2719 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2720
2721 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2722 }
ab5c608b 2723
d46da437
PZ		 2724 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2725 I915_WRITE(SDEIMR, ~mask);
d46da437
PZ		 2726}
2727
f71d4af4 2728static int ironlake_irq_postinstall(struct drm_device *dev)
036a4a7d 2729{
4bc9d430
DV		 2730 unsigned long irqflags;
2731
036a4a7d
ZW		 2732 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2733 /* enable kind of interrupts always enabled */
013d5aa2 2734 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
ce99c256 2735 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
8664281b 2736 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
de032bf4 2737 DE_PIPEA_FIFO_UNDERRUN | DE_POISON;
cc609d5d 2738 u32 gt_irqs;
036a4a7d 2739
1ec14ad3 2740 dev_priv->irq_mask = ~display_mask;
036a4a7d
ZW		 2741
2742 /* should always can generate irq */
2743 I915_WRITE(DEIIR, I915_READ(DEIIR));
1ec14ad3 2744 I915_WRITE(DEIMR, dev_priv->irq_mask);
6005ce42
DV		 2745 I915_WRITE(DEIER, display_mask |
2746 DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT);
3143a2bf 2747 POSTING_READ(DEIER);
036a4a7d 2748
1ec14ad3 2749 dev_priv->gt_irq_mask = ~0;
036a4a7d
ZW		 2750
2751 I915_WRITE(GTIIR, I915_READ(GTIIR));
1ec14ad3 2752 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
881f47b6 2753
cc609d5d
BW		 2754 gt_irqs = GT_RENDER_USER_INTERRUPT;
2755
1ec14ad3 2756 if (IS_GEN6(dev))
cc609d5d 2757 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
1ec14ad3 2758 else
cc609d5d
BW		 2759 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2760 ILK_BSD_USER_INTERRUPT;
2761
2762 I915_WRITE(GTIER, gt_irqs);
3143a2bf 2763 POSTING_READ(GTIER);
036a4a7d 2764
d46da437 2765 ibx_irq_postinstall(dev);
7fe0b973 2766
f97108d1 2767 if (IS_IRONLAKE_M(dev)) {
6005ce42
DV		 2768 /* Enable PCU event interrupts
2769 *
2770 * spinlocking not required here for correctness since interrupt
4bc9d430
DV		 2771 * setup is guaranteed to run in single-threaded context. But we
2772 * need it to make the assert_spin_locked happy. */
2773 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f97108d1 2774 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
4bc9d430 2775 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
f97108d1
JB		 2776 }
2777
036a4a7d
ZW		 2778 return 0;
2779}
2780
f71d4af4 2781static int ivybridge_irq_postinstall(struct drm_device *dev)
b1f14ad0
JB		 2782{
2783 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2784 /* enable kind of interrupts always enabled */
b615b57a
CW		 2785 u32 display_mask =
2786 DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB |
2787 DE_PLANEC_FLIP_DONE_IVB |
2788 DE_PLANEB_FLIP_DONE_IVB |
ce99c256 2789 DE_PLANEA_FLIP_DONE_IVB |
8664281b
PZ		 2790 DE_AUX_CHANNEL_A_IVB |
2791 DE_ERR_INT_IVB;
12638c57 2792 u32 pm_irqs = GEN6_PM_RPS_EVENTS;
cc609d5d 2793 u32 gt_irqs;
b1f14ad0 2794
b1f14ad0
JB		 2795 dev_priv->irq_mask = ~display_mask;
2796
2797 /* should always can generate irq */
8664281b 2798 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
b1f14ad0
JB		 2799 I915_WRITE(DEIIR, I915_READ(DEIIR));
2800 I915_WRITE(DEIMR, dev_priv->irq_mask);
b615b57a
CW		 2801 I915_WRITE(DEIER,
2802 display_mask |
2803 DE_PIPEC_VBLANK_IVB |
2804 DE_PIPEB_VBLANK_IVB |
2805 DE_PIPEA_VBLANK_IVB);
b1f14ad0
JB		 2806 POSTING_READ(DEIER);
2807
cc609d5d 2808 dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
b1f14ad0
JB		 2809
2810 I915_WRITE(GTIIR, I915_READ(GTIIR));
2811 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2812
cc609d5d
BW		 2813 gt_irqs = GT_RENDER_USER_INTERRUPT | GT_BSD_USER_INTERRUPT |
2814 GT_BLT_USER_INTERRUPT | GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2815 I915_WRITE(GTIER, gt_irqs);
b1f14ad0
JB		 2816 POSTING_READ(GTIER);
2817
12638c57
BW		 2818 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
2819 if (HAS_VEBOX(dev))
2820 pm_irqs |= PM_VEBOX_USER_INTERRUPT |
2821 PM_VEBOX_CS_ERROR_INTERRUPT;
2822
2823 /* Our enable/disable rps functions may touch these registers so
2824 * make sure to set a known state for only the non-RPS bits.
2825 * The RMW is extra paranoia since this should be called after being set
2826 * to a known state in preinstall.
2827 * */
2828 I915_WRITE(GEN6_PMIMR,
2829 (I915_READ(GEN6_PMIMR) | ~GEN6_PM_RPS_EVENTS) & ~pm_irqs);
2830 I915_WRITE(GEN6_PMIER,
2831 (I915_READ(GEN6_PMIER) & GEN6_PM_RPS_EVENTS) | pm_irqs);
2832 POSTING_READ(GEN6_PMIER);
eda63ffb 2833
d46da437 2834 ibx_irq_postinstall(dev);
7fe0b973 2835
b1f14ad0
JB		 2836 return 0;
2837}
2838
7e231dbe
JB		 2839static int valleyview_irq_postinstall(struct drm_device *dev)
2840{
2841 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
cc609d5d 2842 u32 gt_irqs;
7e231dbe 2843 u32 enable_mask;
31acc7f5 2844 u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
b79480ba 2845 unsigned long irqflags;
7e231dbe
JB		 2846
2847 enable_mask = I915_DISPLAY_PORT_INTERRUPT;
31acc7f5
JB		 2848 enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2849 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2850 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
7e231dbe
JB		 2851 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2852
31acc7f5
JB		 2853 /*
2854 *Leave vblank interrupts masked initially. enable/disable will
2855 * toggle them based on usage.
2856 */
2857 dev_priv->irq_mask = (~enable_mask) |
2858 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2859 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 2860
20afbda2
DV		 2861 I915_WRITE(PORT_HOTPLUG_EN, 0);
2862 POSTING_READ(PORT_HOTPLUG_EN);
2863
7e231dbe
JB		 2864 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2865 I915_WRITE(VLV_IER, enable_mask);
2866 I915_WRITE(VLV_IIR, 0xffffffff);
2867 I915_WRITE(PIPESTAT(0), 0xffff);
2868 I915_WRITE(PIPESTAT(1), 0xffff);
2869 POSTING_READ(VLV_IER);
2870
b79480ba
DV		 2871 /* Interrupt setup is already guaranteed to be single-threaded, this is
2872 * just to make the assert_spin_locked check happy. */
2873 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5 2874 i915_enable_pipestat(dev_priv, 0, pipestat_enable);
515ac2bb 2875 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
31acc7f5 2876 i915_enable_pipestat(dev_priv, 1, pipestat_enable);
b79480ba 2877 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
31acc7f5 2878
7e231dbe
JB		 2879 I915_WRITE(VLV_IIR, 0xffffffff);
2880 I915_WRITE(VLV_IIR, 0xffffffff);
2881
7e231dbe 2882 I915_WRITE(GTIIR, I915_READ(GTIIR));
31acc7f5 2883 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
3bcedbe5 2884
cc609d5d
BW		 2885 gt_irqs = GT_RENDER_USER_INTERRUPT | GT_BSD_USER_INTERRUPT |
2886 GT_BLT_USER_INTERRUPT;
2887 I915_WRITE(GTIER, gt_irqs);
7e231dbe
JB		 2888 POSTING_READ(GTIER);
2889
2890 /* ack & enable invalid PTE error interrupts */
2891#if 0 /* FIXME: add support to irq handler for checking these bits */
2892 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2893 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2894#endif
2895
2896 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
20afbda2
DV		 2897
2898 return 0;
2899}
2900
7e231dbe
JB		 2901static void valleyview_irq_uninstall(struct drm_device *dev)
2902{
2903 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2904 int pipe;
2905
2906 if (!dev_priv)
2907 return;
2908
ac4c16c5
EE		 2909 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2910
7e231dbe
JB		 2911 for_each_pipe(pipe)
2912 I915_WRITE(PIPESTAT(pipe), 0xffff);
2913
2914 I915_WRITE(HWSTAM, 0xffffffff);
2915 I915_WRITE(PORT_HOTPLUG_EN, 0);
2916 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2917 for_each_pipe(pipe)
2918 I915_WRITE(PIPESTAT(pipe), 0xffff);
2919 I915_WRITE(VLV_IIR, 0xffffffff);
2920 I915_WRITE(VLV_IMR, 0xffffffff);
2921 I915_WRITE(VLV_IER, 0x0);
2922 POSTING_READ(VLV_IER);
2923}
2924
f71d4af4 2925static void ironlake_irq_uninstall(struct drm_device *dev)
036a4a7d
ZW		 2926{
2927 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
4697995b
JB		 2928
2929 if (!dev_priv)
2930 return;
2931
ac4c16c5
EE		 2932 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2933
036a4a7d
ZW		 2934 I915_WRITE(HWSTAM, 0xffffffff);
2935
2936 I915_WRITE(DEIMR, 0xffffffff);
2937 I915_WRITE(DEIER, 0x0);
2938 I915_WRITE(DEIIR, I915_READ(DEIIR));
8664281b
PZ		 2939 if (IS_GEN7(dev))
2940 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
036a4a7d
ZW		 2941
2942 I915_WRITE(GTIMR, 0xffffffff);
2943 I915_WRITE(GTIER, 0x0);
2944 I915_WRITE(GTIIR, I915_READ(GTIIR));
192aac1f 2945
ab5c608b
BW		 2946 if (HAS_PCH_NOP(dev))
2947 return;
2948
192aac1f
KP		 2949 I915_WRITE(SDEIMR, 0xffffffff);
2950 I915_WRITE(SDEIER, 0x0);
2951 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
8664281b
PZ		 2952 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2953 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
036a4a7d
ZW		 2954}
2955
a266c7d5 2956static void i8xx_irq_preinstall(struct drm_device * dev)
1da177e4
LT		 2957{
2958 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 2959 int pipe;
91e3738e 2960
a266c7d5 2961 atomic_set(&dev_priv->irq_received, 0);
5ca58282 2962
9db4a9c7
JB		 2963 for_each_pipe(pipe)
2964 I915_WRITE(PIPESTAT(pipe), 0);
a266c7d5
CW		 2965 I915_WRITE16(IMR, 0xffff);
2966 I915_WRITE16(IER, 0x0);
2967 POSTING_READ16(IER);
c2798b19
CW		 2968}
2969
2970static int i8xx_irq_postinstall(struct drm_device *dev)
2971{
2972 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2973
c2798b19
CW		 2974 I915_WRITE16(EMR,
2975 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2976
2977 /* Unmask the interrupts that we always want on. */
2978 dev_priv->irq_mask =
2979 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2980 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2981 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2982 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2983 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2984 I915_WRITE16(IMR, dev_priv->irq_mask);
2985
2986 I915_WRITE16(IER,
2987 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2988 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2989 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2990 I915_USER_INTERRUPT);
2991 POSTING_READ16(IER);
2992
2993 return 0;
2994}
2995
90a72f87
VS		 2996/*
2997 * Returns true when a page flip has completed.
2998 */
2999static bool i8xx_handle_vblank(struct drm_device *dev,
3000 int pipe, u16 iir)
3001{
3002 drm_i915_private_t *dev_priv = dev->dev_private;
3003 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
3004
3005 if (!drm_handle_vblank(dev, pipe))
3006 return false;
3007
3008 if ((iir & flip_pending) == 0)
3009 return false;
3010
3011 intel_prepare_page_flip(dev, pipe);
3012
3013 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3014 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3015 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3016 * the flip is completed (no longer pending). Since this doesn't raise
3017 * an interrupt per se, we watch for the change at vblank.
3018 */
3019 if (I915_READ16(ISR) & flip_pending)
3020 return false;
3021
3022 intel_finish_page_flip(dev, pipe);
3023
3024 return true;
3025}
3026
ff1f525e 3027static irqreturn_t i8xx_irq_handler(int irq, void *arg)
c2798b19
CW		 3028{
3029 struct drm_device *dev = (struct drm_device *) arg;
3030 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
c2798b19
CW		 3031 u16 iir, new_iir;
3032 u32 pipe_stats[2];
3033 unsigned long irqflags;
3034 int irq_received;
3035 int pipe;
3036 u16 flip_mask =
3037 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3038 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3039
3040 atomic_inc(&dev_priv->irq_received);
3041
3042 iir = I915_READ16(IIR);
3043 if (iir == 0)
3044 return IRQ_NONE;
3045
3046 while (iir & ~flip_mask) {
3047 /* Can't rely on pipestat interrupt bit in iir as it might
3048 * have been cleared after the pipestat interrupt was received.
3049 * It doesn't set the bit in iir again, but it still produces
3050 * interrupts (for non-MSI).
3051 */
3052 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3053 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3054 i915_handle_error(dev, false);
3055
3056 for_each_pipe(pipe) {
3057 int reg = PIPESTAT(pipe);
3058 pipe_stats[pipe] = I915_READ(reg);
3059
3060 /*
3061 * Clear the PIPE*STAT regs before the IIR
3062 */
3063 if (pipe_stats[pipe] & 0x8000ffff) {
3064 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3065 DRM_DEBUG_DRIVER("pipe %c underrun\n",
3066 pipe_name(pipe));
3067 I915_WRITE(reg, pipe_stats[pipe]);
3068 irq_received = 1;
3069 }
3070 }
3071 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3072
3073 I915_WRITE16(IIR, iir & ~flip_mask);
3074 new_iir = I915_READ16(IIR); /* Flush posted writes */
3075
d05c617e 3076 i915_update_dri1_breadcrumb(dev);
c2798b19
CW		 3077
3078 if (iir & I915_USER_INTERRUPT)
3079 notify_ring(dev, &dev_priv->ring[RCS]);
3080
3081 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3082 i8xx_handle_vblank(dev, 0, iir))
3083 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
c2798b19
CW		 3084
3085 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3086 i8xx_handle_vblank(dev, 1, iir))
3087 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
c2798b19
CW		 3088
3089 iir = new_iir;
3090 }
3091
3092 return IRQ_HANDLED;
3093}
3094
3095static void i8xx_irq_uninstall(struct drm_device * dev)
3096{
3097 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3098 int pipe;
3099
c2798b19
CW		 3100 for_each_pipe(pipe) {
3101 /* Clear enable bits; then clear status bits */
3102 I915_WRITE(PIPESTAT(pipe), 0);
3103 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3104 }
3105 I915_WRITE16(IMR, 0xffff);
3106 I915_WRITE16(IER, 0x0);
3107 I915_WRITE16(IIR, I915_READ16(IIR));
3108}
3109
a266c7d5
CW		 3110static void i915_irq_preinstall(struct drm_device * dev)
3111{
3112 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3113 int pipe;
3114
3115 atomic_set(&dev_priv->irq_received, 0);
3116
3117 if (I915_HAS_HOTPLUG(dev)) {
3118 I915_WRITE(PORT_HOTPLUG_EN, 0);
3119 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3120 }
3121
00d98ebd 3122 I915_WRITE16(HWSTAM, 0xeffe);
a266c7d5
CW		 3123 for_each_pipe(pipe)
3124 I915_WRITE(PIPESTAT(pipe), 0);
3125 I915_WRITE(IMR, 0xffffffff);
3126 I915_WRITE(IER, 0x0);
3127 POSTING_READ(IER);
3128}
3129
3130static int i915_irq_postinstall(struct drm_device *dev)
3131{
3132 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
38bde180 3133 u32 enable_mask;
a266c7d5 3134
38bde180
CW		 3135 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3136
3137 /* Unmask the interrupts that we always want on. */
3138 dev_priv->irq_mask =
3139 ~(I915_ASLE_INTERRUPT |
3140 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3141 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3142 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3143 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3144 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3145
3146 enable_mask =
3147 I915_ASLE_INTERRUPT |
3148 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3149 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3150 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
3151 I915_USER_INTERRUPT;
3152
a266c7d5 3153 if (I915_HAS_HOTPLUG(dev)) {
20afbda2
DV		 3154 I915_WRITE(PORT_HOTPLUG_EN, 0);
3155 POSTING_READ(PORT_HOTPLUG_EN);
3156
a266c7d5
CW		 3157 /* Enable in IER... */
3158 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3159 /* and unmask in IMR */
3160 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3161 }
3162
a266c7d5
CW		 3163 I915_WRITE(IMR, dev_priv->irq_mask);
3164 I915_WRITE(IER, enable_mask);
3165 POSTING_READ(IER);
3166
f49e38dd 3167 i915_enable_asle_pipestat(dev);
20afbda2
DV		 3168
3169 return 0;
3170}
3171
90a72f87
VS		 3172/*
3173 * Returns true when a page flip has completed.
3174 */
3175static bool i915_handle_vblank(struct drm_device *dev,
3176 int plane, int pipe, u32 iir)
3177{
3178 drm_i915_private_t *dev_priv = dev->dev_private;
3179 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3180
3181 if (!drm_handle_vblank(dev, pipe))
3182 return false;
3183
3184 if ((iir & flip_pending) == 0)
3185 return false;
3186
3187 intel_prepare_page_flip(dev, plane);
3188
3189 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3190 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3191 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3192 * the flip is completed (no longer pending). Since this doesn't raise
3193 * an interrupt per se, we watch for the change at vblank.
3194 */
3195 if (I915_READ(ISR) & flip_pending)
3196 return false;
3197
3198 intel_finish_page_flip(dev, pipe);
3199
3200 return true;
3201}
3202
ff1f525e 3203static irqreturn_t i915_irq_handler(int irq, void *arg)
a266c7d5
CW		 3204{
3205 struct drm_device *dev = (struct drm_device *) arg;
3206 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
8291ee90 3207 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
a266c7d5 3208 unsigned long irqflags;
38bde180
CW		 3209 u32 flip_mask =
3210 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3211 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
38bde180 3212 int pipe, ret = IRQ_NONE;
a266c7d5
CW		 3213
3214 atomic_inc(&dev_priv->irq_received);
3215
3216 iir = I915_READ(IIR);
38bde180
CW		 3217 do {
3218 bool irq_received = (iir & ~flip_mask) != 0;
8291ee90 3219 bool blc_event = false;
a266c7d5
CW		 3220
3221 /* Can't rely on pipestat interrupt bit in iir as it might
3222 * have been cleared after the pipestat interrupt was received.
3223 * It doesn't set the bit in iir again, but it still produces
3224 * interrupts (for non-MSI).
3225 */
3226 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3227 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3228 i915_handle_error(dev, false);
3229
3230 for_each_pipe(pipe) {
3231 int reg = PIPESTAT(pipe);
3232 pipe_stats[pipe] = I915_READ(reg);
3233
38bde180 3234 /* Clear the PIPE*STAT regs before the IIR */
a266c7d5
CW		 3235 if (pipe_stats[pipe] & 0x8000ffff) {
3236 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3237 DRM_DEBUG_DRIVER("pipe %c underrun\n",
3238 pipe_name(pipe));
3239 I915_WRITE(reg, pipe_stats[pipe]);
38bde180 3240 irq_received = true;
a266c7d5
CW		 3241 }
3242 }
3243 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3244
3245 if (!irq_received)
3246 break;
3247
a266c7d5
CW		 3248 /* Consume port. Then clear IIR or we'll miss events */
3249 if ((I915_HAS_HOTPLUG(dev)) &&
3250 (iir & I915_DISPLAY_PORT_INTERRUPT)) {
3251 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 3252 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
a266c7d5
CW		 3253
3254 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
3255 hotplug_status);
91d131d2
DV		 3256
3257 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
3258
a266c7d5 3259 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
38bde180 3260 POSTING_READ(PORT_HOTPLUG_STAT);
a266c7d5
CW		 3261 }
3262
38bde180 3263 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 3264 new_iir = I915_READ(IIR); /* Flush posted writes */
3265
a266c7d5
CW		 3266 if (iir & I915_USER_INTERRUPT)
3267 notify_ring(dev, &dev_priv->ring[RCS]);
a266c7d5 3268
a266c7d5 3269 for_each_pipe(pipe) {
38bde180
CW		 3270 int plane = pipe;
3271 if (IS_MOBILE(dev))
3272 plane = !plane;
90a72f87 3273
8291ee90 3274 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3275 i915_handle_vblank(dev, plane, pipe, iir))
3276 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
a266c7d5
CW		 3277
3278 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3279 blc_event = true;
3280 }
3281
a266c7d5
CW		 3282 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3283 intel_opregion_asle_intr(dev);
3284
3285 /* With MSI, interrupts are only generated when iir
3286 * transitions from zero to nonzero. If another bit got
3287 * set while we were handling the existing iir bits, then
3288 * we would never get another interrupt.
3289 *
3290 * This is fine on non-MSI as well, as if we hit this path
3291 * we avoid exiting the interrupt handler only to generate
3292 * another one.
3293 *
3294 * Note that for MSI this could cause a stray interrupt report
3295 * if an interrupt landed in the time between writing IIR and
3296 * the posting read. This should be rare enough to never
3297 * trigger the 99% of 100,000 interrupts test for disabling
3298 * stray interrupts.
3299 */
38bde180 3300 ret = IRQ_HANDLED;
a266c7d5 3301 iir = new_iir;
38bde180 3302 } while (iir & ~flip_mask);
a266c7d5 3303
d05c617e 3304 i915_update_dri1_breadcrumb(dev);
8291ee90 3305
a266c7d5
CW		 3306 return ret;
3307}
3308
3309static void i915_irq_uninstall(struct drm_device * dev)
3310{
3311 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3312 int pipe;
3313
ac4c16c5
EE		 3314 del_timer_sync(&dev_priv->hotplug_reenable_timer);
3315
a266c7d5
CW		 3316 if (I915_HAS_HOTPLUG(dev)) {
3317 I915_WRITE(PORT_HOTPLUG_EN, 0);
3318 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3319 }
3320
00d98ebd 3321 I915_WRITE16(HWSTAM, 0xffff);
55b39755
CW		 3322 for_each_pipe(pipe) {
3323 /* Clear enable bits; then clear status bits */
a266c7d5 3324 I915_WRITE(PIPESTAT(pipe), 0);
55b39755
CW		 3325 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3326 }
a266c7d5
CW		 3327 I915_WRITE(IMR, 0xffffffff);
3328 I915_WRITE(IER, 0x0);
3329
a266c7d5
CW		 3330 I915_WRITE(IIR, I915_READ(IIR));
3331}
3332
3333static void i965_irq_preinstall(struct drm_device * dev)
3334{
3335 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3336 int pipe;
3337
3338 atomic_set(&dev_priv->irq_received, 0);
3339
adca4730
CW		 3340 I915_WRITE(PORT_HOTPLUG_EN, 0);
3341 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 3342
3343 I915_WRITE(HWSTAM, 0xeffe);
3344 for_each_pipe(pipe)
3345 I915_WRITE(PIPESTAT(pipe), 0);
3346 I915_WRITE(IMR, 0xffffffff);
3347 I915_WRITE(IER, 0x0);
3348 POSTING_READ(IER);
3349}
3350
3351static int i965_irq_postinstall(struct drm_device *dev)
3352{
3353 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
bbba0a97 3354 u32 enable_mask;
a266c7d5 3355 u32 error_mask;
b79480ba 3356 unsigned long irqflags;
a266c7d5 3357
a266c7d5 3358 /* Unmask the interrupts that we always want on. */
bbba0a97 3359 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
adca4730 3360 I915_DISPLAY_PORT_INTERRUPT |
bbba0a97
CW		 3361 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3362 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3363 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3364 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3365 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3366
3367 enable_mask = ~dev_priv->irq_mask;
21ad8330
VS		 3368 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3369 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
bbba0a97
CW		 3370 enable_mask |= I915_USER_INTERRUPT;
3371
3372 if (IS_G4X(dev))
3373 enable_mask |= I915_BSD_USER_INTERRUPT;
a266c7d5 3374
b79480ba
DV		 3375 /* Interrupt setup is already guaranteed to be single-threaded, this is
3376 * just to make the assert_spin_locked check happy. */
3377 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
515ac2bb 3378 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
b79480ba 3379 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
a266c7d5 3380
a266c7d5
CW		 3381 /*
3382 * Enable some error detection, note the instruction error mask
3383 * bit is reserved, so we leave it masked.
3384 */
3385 if (IS_G4X(dev)) {
3386 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3387 GM45_ERROR_MEM_PRIV |
3388 GM45_ERROR_CP_PRIV |
3389 I915_ERROR_MEMORY_REFRESH);
3390 } else {
3391 error_mask = ~(I915_ERROR_PAGE_TABLE |
3392 I915_ERROR_MEMORY_REFRESH);
3393 }
3394 I915_WRITE(EMR, error_mask);
3395
3396 I915_WRITE(IMR, dev_priv->irq_mask);
3397 I915_WRITE(IER, enable_mask);
3398 POSTING_READ(IER);
3399
20afbda2
DV		 3400 I915_WRITE(PORT_HOTPLUG_EN, 0);
3401 POSTING_READ(PORT_HOTPLUG_EN);
3402
f49e38dd 3403 i915_enable_asle_pipestat(dev);
20afbda2
DV		 3404
3405 return 0;
3406}
3407
bac56d5b 3408static void i915_hpd_irq_setup(struct drm_device *dev)
20afbda2
DV		 3409{
3410 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e5868a31 3411 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed 3412 struct intel_encoder *intel_encoder;
20afbda2
DV		 3413 u32 hotplug_en;
3414
b5ea2d56
DV		 3415 assert_spin_locked(&dev_priv->irq_lock);
3416
bac56d5b
EE		 3417 if (I915_HAS_HOTPLUG(dev)) {
3418 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3419 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3420 /* Note HDMI and DP share hotplug bits */
e5868a31 3421 /* enable bits are the same for all generations */
cd569aed
EE		 3422 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
3423 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3424 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
bac56d5b
EE		 3425 /* Programming the CRT detection parameters tends
3426 to generate a spurious hotplug event about three
3427 seconds later. So just do it once.
3428 */
3429 if (IS_G4X(dev))
3430 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
85fc95ba 3431 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
bac56d5b 3432 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
a266c7d5 3433
bac56d5b
EE		 3434 /* Ignore TV since it's buggy */
3435 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3436 }
a266c7d5
CW		 3437}
3438
ff1f525e 3439static irqreturn_t i965_irq_handler(int irq, void *arg)
a266c7d5
CW		 3440{
3441 struct drm_device *dev = (struct drm_device *) arg;
3442 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
a266c7d5
CW		 3443 u32 iir, new_iir;
3444 u32 pipe_stats[I915_MAX_PIPES];
a266c7d5
CW		 3445 unsigned long irqflags;
3446 int irq_received;
3447 int ret = IRQ_NONE, pipe;
21ad8330
VS		 3448 u32 flip_mask =
3449 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3450 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
a266c7d5
CW		 3451
3452 atomic_inc(&dev_priv->irq_received);
3453
3454 iir = I915_READ(IIR);
3455
a266c7d5 3456 for (;;) {
2c8ba29f
CW		 3457 bool blc_event = false;
3458
21ad8330 3459 irq_received = (iir & ~flip_mask) != 0;
a266c7d5
CW		 3460
3461 /* Can't rely on pipestat interrupt bit in iir as it might
3462 * have been cleared after the pipestat interrupt was received.
3463 * It doesn't set the bit in iir again, but it still produces
3464 * interrupts (for non-MSI).
3465 */
3466 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3467 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3468 i915_handle_error(dev, false);
3469
3470 for_each_pipe(pipe) {
3471 int reg = PIPESTAT(pipe);
3472 pipe_stats[pipe] = I915_READ(reg);
3473
3474 /*
3475 * Clear the PIPE*STAT regs before the IIR
3476 */
3477 if (pipe_stats[pipe] & 0x8000ffff) {
3478 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3479 DRM_DEBUG_DRIVER("pipe %c underrun\n",
3480 pipe_name(pipe));
3481 I915_WRITE(reg, pipe_stats[pipe]);
3482 irq_received = 1;
3483 }
3484 }
3485 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3486
3487 if (!irq_received)
3488 break;
3489
3490 ret = IRQ_HANDLED;
3491
3492 /* Consume port. Then clear IIR or we'll miss events */
adca4730 3493 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
a266c7d5 3494 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04
EE		 3495 u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
3496 HOTPLUG_INT_STATUS_G4X :
4f7fd709 3497 HOTPLUG_INT_STATUS_I915);
a266c7d5
CW		 3498
3499 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
3500 hotplug_status);
91d131d2
DV		 3501
3502 intel_hpd_irq_handler(dev, hotplug_trigger,
3503 IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
3504
a266c7d5
CW		 3505 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
3506 I915_READ(PORT_HOTPLUG_STAT);
3507 }
3508
21ad8330 3509 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 3510 new_iir = I915_READ(IIR); /* Flush posted writes */
3511
a266c7d5
CW		 3512 if (iir & I915_USER_INTERRUPT)
3513 notify_ring(dev, &dev_priv->ring[RCS]);
3514 if (iir & I915_BSD_USER_INTERRUPT)
3515 notify_ring(dev, &dev_priv->ring[VCS]);
3516
a266c7d5 3517 for_each_pipe(pipe) {
2c8ba29f 3518 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3519 i915_handle_vblank(dev, pipe, pipe, iir))
3520 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
a266c7d5
CW		 3521
3522 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3523 blc_event = true;
3524 }
3525
3526
3527 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3528 intel_opregion_asle_intr(dev);
3529
515ac2bb
DV		 3530 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
3531 gmbus_irq_handler(dev);
3532
a266c7d5
CW		 3533 /* With MSI, interrupts are only generated when iir
3534 * transitions from zero to nonzero. If another bit got
3535 * set while we were handling the existing iir bits, then
3536 * we would never get another interrupt.
3537 *
3538 * This is fine on non-MSI as well, as if we hit this path
3539 * we avoid exiting the interrupt handler only to generate
3540 * another one.
3541 *
3542 * Note that for MSI this could cause a stray interrupt report
3543 * if an interrupt landed in the time between writing IIR and
3544 * the posting read. This should be rare enough to never
3545 * trigger the 99% of 100,000 interrupts test for disabling
3546 * stray interrupts.
3547 */
3548 iir = new_iir;
3549 }
3550
d05c617e 3551 i915_update_dri1_breadcrumb(dev);
2c8ba29f 3552
a266c7d5
CW		 3553 return ret;
3554}
3555
3556static void i965_irq_uninstall(struct drm_device * dev)
3557{
3558 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3559 int pipe;
3560
3561 if (!dev_priv)
3562 return;
3563
ac4c16c5
EE		 3564 del_timer_sync(&dev_priv->hotplug_reenable_timer);
3565
adca4730
CW		 3566 I915_WRITE(PORT_HOTPLUG_EN, 0);
3567 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 3568
3569 I915_WRITE(HWSTAM, 0xffffffff);
3570 for_each_pipe(pipe)
3571 I915_WRITE(PIPESTAT(pipe), 0);
3572 I915_WRITE(IMR, 0xffffffff);
3573 I915_WRITE(IER, 0x0);
3574
3575 for_each_pipe(pipe)
3576 I915_WRITE(PIPESTAT(pipe),
3577 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3578 I915_WRITE(IIR, I915_READ(IIR));
3579}
3580
ac4c16c5
EE		 3581static void i915_reenable_hotplug_timer_func(unsigned long data)
3582{
3583 drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3584 struct drm_device *dev = dev_priv->dev;
3585 struct drm_mode_config *mode_config = &dev->mode_config;
3586 unsigned long irqflags;
3587 int i;
3588
3589 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3590 for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3591 struct drm_connector *connector;
3592
3593 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3594 continue;
3595
3596 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3597
3598 list_for_each_entry(connector, &mode_config->connector_list, head) {
3599 struct intel_connector *intel_connector = to_intel_connector(connector);
3600
3601 if (intel_connector->encoder->hpd_pin == i) {
3602 if (connector->polled != intel_connector->polled)
3603 DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3604 drm_get_connector_name(connector));
3605 connector->polled = intel_connector->polled;
3606 if (!connector->polled)
3607 connector->polled = DRM_CONNECTOR_POLL_HPD;
3608 }
3609 }
3610 }
3611 if (dev_priv->display.hpd_irq_setup)
3612 dev_priv->display.hpd_irq_setup(dev);
3613 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3614}
3615
f71d4af4
JB		 3616void intel_irq_init(struct drm_device *dev)
3617{
8b2e326d
CW		 3618 struct drm_i915_private *dev_priv = dev->dev_private;
3619
3620 INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
99584db3 3621 INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
c6a828d3 3622 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
a4da4fa4 3623 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
8b2e326d 3624
99584db3
DV		 3625 setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3626 i915_hangcheck_elapsed,
61bac78e 3627 (unsigned long) dev);
ac4c16c5
EE		 3628 setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3629 (unsigned long) dev_priv);
61bac78e 3630
97a19a24 3631 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
9ee32fea 3632
f71d4af4
JB		 3633 dev->driver->get_vblank_counter = i915_get_vblank_counter;
3634 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
7d4e146f 3635 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
f71d4af4
JB		 3636 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3637 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3638 }
3639
c3613de9
KP		 3640 if (drm_core_check_feature(dev, DRIVER_MODESET))
3641 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3642 else
3643 dev->driver->get_vblank_timestamp = NULL;
f71d4af4
JB		 3644 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3645
7e231dbe
JB		 3646 if (IS_VALLEYVIEW(dev)) {
3647 dev->driver->irq_handler = valleyview_irq_handler;
3648 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3649 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3650 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3651 dev->driver->enable_vblank = valleyview_enable_vblank;
3652 dev->driver->disable_vblank = valleyview_disable_vblank;
fa00abe0 3653 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4a06e201 3654 } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
7d99163d 3655 /* Share uninstall handlers with ILK/SNB */
f71d4af4 3656 dev->driver->irq_handler = ivybridge_irq_handler;
7d99163d 3657 dev->driver->irq_preinstall = ivybridge_irq_preinstall;
f71d4af4
JB		 3658 dev->driver->irq_postinstall = ivybridge_irq_postinstall;
3659 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3660 dev->driver->enable_vblank = ivybridge_enable_vblank;
3661 dev->driver->disable_vblank = ivybridge_disable_vblank;
82a28bcf 3662 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4
JB		 3663 } else if (HAS_PCH_SPLIT(dev)) {
3664 dev->driver->irq_handler = ironlake_irq_handler;
3665 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3666 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3667 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3668 dev->driver->enable_vblank = ironlake_enable_vblank;
3669 dev->driver->disable_vblank = ironlake_disable_vblank;
82a28bcf 3670 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4 3671 } else {
c2798b19
CW		 3672 if (INTEL_INFO(dev)->gen == 2) {
3673 dev->driver->irq_preinstall = i8xx_irq_preinstall;
3674 dev->driver->irq_postinstall = i8xx_irq_postinstall;
3675 dev->driver->irq_handler = i8xx_irq_handler;
3676 dev->driver->irq_uninstall = i8xx_irq_uninstall;
a266c7d5
CW		 3677 } else if (INTEL_INFO(dev)->gen == 3) {
3678 dev->driver->irq_preinstall = i915_irq_preinstall;
3679 dev->driver->irq_postinstall = i915_irq_postinstall;
3680 dev->driver->irq_uninstall = i915_irq_uninstall;
3681 dev->driver->irq_handler = i915_irq_handler;
20afbda2 3682 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3683 } else {
a266c7d5
CW		 3684 dev->driver->irq_preinstall = i965_irq_preinstall;
3685 dev->driver->irq_postinstall = i965_irq_postinstall;
3686 dev->driver->irq_uninstall = i965_irq_uninstall;
3687 dev->driver->irq_handler = i965_irq_handler;
bac56d5b 3688 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3689 }
f71d4af4
JB		 3690 dev->driver->enable_vblank = i915_enable_vblank;
3691 dev->driver->disable_vblank = i915_disable_vblank;
3692 }
3693}
20afbda2
DV		 3694
3695void intel_hpd_init(struct drm_device *dev)
3696{
3697 struct drm_i915_private *dev_priv = dev->dev_private;
821450c6
EE		 3698 struct drm_mode_config *mode_config = &dev->mode_config;
3699 struct drm_connector *connector;
b5ea2d56 3700 unsigned long irqflags;
821450c6 3701 int i;
20afbda2 3702
821450c6
EE		 3703 for (i = 1; i < HPD_NUM_PINS; i++) {
3704 dev_priv->hpd_stats[i].hpd_cnt = 0;
3705 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3706 }
3707 list_for_each_entry(connector, &mode_config->connector_list, head) {
3708 struct intel_connector *intel_connector = to_intel_connector(connector);
3709 connector->polled = intel_connector->polled;
3710 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3711 connector->polled = DRM_CONNECTOR_POLL_HPD;
3712 }
b5ea2d56
DV		 3713
3714 /* Interrupt setup is already guaranteed to be single-threaded, this is
3715 * just to make the assert_spin_locked checks happy. */
3716 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
20afbda2
DV		 3717 if (dev_priv->display.hpd_irq_setup)
3718 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 3719 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
20afbda2 3720}
Linux kernel - Deliverables
This page took 1.005735 seconds and 5 git commands to generate.