drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43 static void
  44 assert_device_not_suspended(struct drm_i915_private *dev_priv)
  45 {
  46         WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
  47              "Device suspended\n");
  48 }
  49
  50 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  51 {
  52         u32 gt_thread_status_mask;
  53
  54         if (IS_HASWELL(dev_priv->dev))
  55                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  56         else
  57                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  58
  59         /* w/a for a sporadic read returning 0 by waiting for the GT
  60          * thread to wake up.
  61          */
  62         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  63                 DRM_ERROR("GT thread status wait timed out\n");
  64 }
  65
  66 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  67 {
  68         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  69         /* something from same cacheline, but !FORCEWAKE */
  70         __raw_posting_read(dev_priv, ECOBUS);
  71 }
  72
  73 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  74                                                         int fw_engine)
  75 {
  76         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  77                             FORCEWAKE_ACK_TIMEOUT_MS))
  78                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  79
  80         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  81         /* something from same cacheline, but !FORCEWAKE */
  82         __raw_posting_read(dev_priv, ECOBUS);
  83
  84         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  85                             FORCEWAKE_ACK_TIMEOUT_MS))
  86                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  87
  88         /* WaRsForcewakeWaitTC0:snb */
  89         __gen6_gt_wait_for_thread_c0(dev_priv);
  90 }
  91
  92 static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  93 {
  94         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  95         /* something from same cacheline, but !FORCEWAKE_MT */
  96         __raw_posting_read(dev_priv, ECOBUS);
  97 }
  98
  99 static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
 100                                                         int fw_engine)
 101 {
 102         u32 forcewake_ack;
 103
 104         if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))
 105                 forcewake_ack = FORCEWAKE_ACK_HSW;
 106         else
 107                 forcewake_ack = FORCEWAKE_MT_ACK;
 108
 109         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 110                             FORCEWAKE_ACK_TIMEOUT_MS))
 111                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 112
 113         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 114                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 115         /* something from same cacheline, but !FORCEWAKE_MT */
 116         __raw_posting_read(dev_priv, ECOBUS);
 117
 118         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 119                             FORCEWAKE_ACK_TIMEOUT_MS))
 120                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 121
 122         /* WaRsForcewakeWaitTC0:ivb,hsw */
 123         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 124                 __gen6_gt_wait_for_thread_c0(dev_priv);
 125 }
 126
 127 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 128 {
 129         u32 gtfifodbg;
 130
 131         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 132         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 133                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 134 }
 135
 136 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 137                                                         int fw_engine)
 138 {
 139         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 140         /* something from same cacheline, but !FORCEWAKE */
 141         __raw_posting_read(dev_priv, ECOBUS);
 142         gen6_gt_check_fifodbg(dev_priv);
 143 }
 144
 145 static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 146                                                         int fw_engine)
 147 {
 148         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 149                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 150         /* something from same cacheline, but !FORCEWAKE_MT */
 151         __raw_posting_read(dev_priv, ECOBUS);
 152
 153         if (IS_GEN7(dev_priv->dev))
 154                 gen6_gt_check_fifodbg(dev_priv);
 155 }
 156
 157 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 158 {
 159         int ret = 0;
 160
 161         /* On VLV, FIFO will be shared by both SW and HW.
 162          * So, we need to read the FREE_ENTRIES everytime */
 163         if (IS_VALLEYVIEW(dev_priv->dev))
 164                 dev_priv->uncore.fifo_count =
 165                         __raw_i915_read32(dev_priv, GTFIFOCTL) &
 166                                                 GT_FIFO_FREE_ENTRIES_MASK;
 167
 168         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 169                 int loop = 500;
 170                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 171                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 172                         udelay(10);
 173                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 174                 }
 175                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 176                         ++ret;
 177                 dev_priv->uncore.fifo_count = fifo;
 178         }
 179         dev_priv->uncore.fifo_count--;
 180
 181         return ret;
 182 }
 183
 184 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 185 {
 186         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 187                            _MASKED_BIT_DISABLE(0xffff));
 188         __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 189                            _MASKED_BIT_DISABLE(0xffff));
 190         /* something from same cacheline, but !FORCEWAKE_VLV */
 191         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 192 }
 193
 194 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 195                                                 int fw_engine)
 196 {
 197         /*
 198          * WaRsDontPollForAckOnClearingFWBits:vlv
 199          * Hardware clears ack bits lazily (only when all ack
 200          * bits become 0) so don't poll for individiual ack
 201          * bits to be clear here like on other platforms.
 202          */
 203
 204         /* Check for Render Engine */
 205         if (FORCEWAKE_RENDER & fw_engine) {
 206
 207                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 208                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 209
 210                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 211                                                 FORCEWAKE_ACK_VLV) &
 212                                                 FORCEWAKE_KERNEL),
 213                                         FORCEWAKE_ACK_TIMEOUT_MS))
 214                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 215         }
 216
 217         /* Check for Media Engine */
 218         if (FORCEWAKE_MEDIA & fw_engine) {
 219
 220                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 221                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 222
 223                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 224                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 225                                                 FORCEWAKE_KERNEL),
 226                                         FORCEWAKE_ACK_TIMEOUT_MS))
 227                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 228         }
 229
 230         /* WaRsForcewakeWaitTC0:vlv */
 231         if (!IS_CHERRYVIEW(dev_priv->dev))
 232                 __gen6_gt_wait_for_thread_c0(dev_priv);
 233 }
 234
 235 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 236                                         int fw_engine)
 237 {
 238
 239         /* Check for Render Engine */
 240         if (FORCEWAKE_RENDER & fw_engine)
 241                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 242                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 243
 244
 245         /* Check for Media Engine */
 246         if (FORCEWAKE_MEDIA & fw_engine)
 247                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 248                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 249
 250         /* something from same cacheline, but !FORCEWAKE_VLV */
 251         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 252         if (!IS_CHERRYVIEW(dev_priv->dev))
 253                 gen6_gt_check_fifodbg(dev_priv);
 254 }
 255
 256 static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 257 {
 258         unsigned long irqflags;
 259
 260         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 261
 262         if (fw_engine & FORCEWAKE_RENDER &&
 263             dev_priv->uncore.fw_rendercount++ != 0)
 264                 fw_engine &= ~FORCEWAKE_RENDER;
 265         if (fw_engine & FORCEWAKE_MEDIA &&
 266             dev_priv->uncore.fw_mediacount++ != 0)
 267                 fw_engine &= ~FORCEWAKE_MEDIA;
 268
 269         if (fw_engine)
 270                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
 271
 272         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 273 }
 274
 275 static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 276 {
 277         unsigned long irqflags;
 278
 279         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 280
 281         if (fw_engine & FORCEWAKE_RENDER) {
 282                 WARN_ON(!dev_priv->uncore.fw_rendercount);
 283                 if (--dev_priv->uncore.fw_rendercount != 0)
 284                         fw_engine &= ~FORCEWAKE_RENDER;
 285         }
 286
 287         if (fw_engine & FORCEWAKE_MEDIA) {
 288                 WARN_ON(!dev_priv->uncore.fw_mediacount);
 289                 if (--dev_priv->uncore.fw_mediacount != 0)
 290                         fw_engine &= ~FORCEWAKE_MEDIA;
 291         }
 292
 293         if (fw_engine)
 294                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
 295
 296         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 297 }
 298
 299 static void gen6_force_wake_timer(unsigned long arg)
 300 {
 301         struct drm_i915_private *dev_priv = (void *)arg;
 302         unsigned long irqflags;
 303
 304         assert_device_not_suspended(dev_priv);
 305
 306         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 307         WARN_ON(!dev_priv->uncore.forcewake_count);
 308
 309         if (--dev_priv->uncore.forcewake_count == 0)
 310                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 311         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 312
 313         intel_runtime_pm_put(dev_priv);
 314 }
 315
 316 void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 317 {
 318         struct drm_i915_private *dev_priv = dev->dev_private;
 319         unsigned long irqflags;
 320
 321         if (del_timer_sync(&dev_priv->uncore.force_wake_timer))
 322                 gen6_force_wake_timer((unsigned long)dev_priv);
 323
 324         /* Hold uncore.lock across reset to prevent any register access
 325          * with forcewake not set correctly
 326          */
 327         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 328
 329         if (IS_VALLEYVIEW(dev))
 330                 vlv_force_wake_reset(dev_priv);
 331         else if (IS_GEN6(dev) || IS_GEN7(dev))
 332                 __gen6_gt_force_wake_reset(dev_priv);
 333
 334         if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
 335                 __gen7_gt_force_wake_mt_reset(dev_priv);
 336
 337         if (restore) { /* If reset with a user forcewake, try to restore */
 338                 unsigned fw = 0;
 339
 340                 if (IS_VALLEYVIEW(dev)) {
 341                         if (dev_priv->uncore.fw_rendercount)
 342                                 fw |= FORCEWAKE_RENDER;
 343
 344                         if (dev_priv->uncore.fw_mediacount)
 345                                 fw |= FORCEWAKE_MEDIA;
 346                 } else {
 347                         if (dev_priv->uncore.forcewake_count)
 348                                 fw = FORCEWAKE_ALL;
 349                 }
 350
 351                 if (fw)
 352                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 353
 354                 if (IS_GEN6(dev) || IS_GEN7(dev))
 355                         dev_priv->uncore.fifo_count =
 356                                 __raw_i915_read32(dev_priv, GTFIFOCTL) &
 357                                 GT_FIFO_FREE_ENTRIES_MASK;
 358         }
 359
 360         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 361 }
 362
 363 static void __intel_uncore_early_sanitize(struct drm_device *dev,
 364                                           bool restore_forcewake)
 365 {
 366         struct drm_i915_private *dev_priv = dev->dev_private;
 367
 368         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 369                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 370
 371         if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) &&
 372             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 373                 /* The docs do not explain exactly how the calculation can be
 374                  * made. It is somewhat guessable, but for now, it's always
 375                  * 128MB.
 376                  * NB: We can't write IDICR yet because we do not have gt funcs
 377                  * set up */
 378                 dev_priv->ellc_size = 128;
 379                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 380         }
 381
 382         /* clear out old GT FIFO errors */
 383         if (IS_GEN6(dev) || IS_GEN7(dev))
 384                 __raw_i915_write32(dev_priv, GTFIFODBG,
 385                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 386
 387         intel_uncore_forcewake_reset(dev, restore_forcewake);
 388 }
 389
 390 void intel_uncore_early_sanitize(struct drm_device *dev, bool restore_forcewake)
 391 {
 392         __intel_uncore_early_sanitize(dev, restore_forcewake);
 393         i915_check_and_clear_faults(dev);
 394 }
 395
 396 void intel_uncore_sanitize(struct drm_device *dev)
 397 {
 398         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 399         intel_disable_gt_powersave(dev);
 400 }
 401
 402 /*
 403  * Generally this is called implicitly by the register read function. However,
 404  * if some sequence requires the GT to not power down then this function should
 405  * be called at the beginning of the sequence followed by a call to
 406  * gen6_gt_force_wake_put() at the end of the sequence.
 407  */
 408 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 409 {
 410         unsigned long irqflags;
 411
 412         if (!dev_priv->uncore.funcs.force_wake_get)
 413                 return;
 414
 415         intel_runtime_pm_get(dev_priv);
 416
 417         /* Redirect to VLV specific routine */
 418         if (IS_VALLEYVIEW(dev_priv->dev))
 419                 return vlv_force_wake_get(dev_priv, fw_engine);
 420
 421         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 422         if (dev_priv->uncore.forcewake_count++ == 0)
 423                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 424         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 425 }
 426
 427 /*
 428  * see gen6_gt_force_wake_get()
 429  */
 430 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 431 {
 432         unsigned long irqflags;
 433         bool delayed = false;
 434
 435         if (!dev_priv->uncore.funcs.force_wake_put)
 436                 return;
 437
 438         /* Redirect to VLV specific routine */
 439         if (IS_VALLEYVIEW(dev_priv->dev)) {
 440                 vlv_force_wake_put(dev_priv, fw_engine);
 441                 goto out;
 442         }
 443
 444
 445         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 446         WARN_ON(!dev_priv->uncore.forcewake_count);
 447
 448         if (--dev_priv->uncore.forcewake_count == 0) {
 449                 dev_priv->uncore.forcewake_count++;
 450                 delayed = true;
 451                 mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
 452                                  jiffies + 1);
 453         }
 454         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 455
 456 out:
 457         if (!delayed)
 458                 intel_runtime_pm_put(dev_priv);
 459 }
 460
 461 void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
 462 {
 463         if (!dev_priv->uncore.funcs.force_wake_get)
 464                 return;
 465
 466         WARN_ON(dev_priv->uncore.forcewake_count > 0);
 467 }
 468
 469 /* We give fast paths for the really cool registers */
 470 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 471          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 472
 473 #define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
 474
 475 #define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \
 476         (REG_RANGE((reg), 0x2000, 0x4000) || \
 477          REG_RANGE((reg), 0x5000, 0x8000) || \
 478          REG_RANGE((reg), 0xB000, 0x12000) || \
 479          REG_RANGE((reg), 0x2E000, 0x30000))
 480
 481 #define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg) \
 482         (REG_RANGE((reg), 0x12000, 0x14000) || \
 483          REG_RANGE((reg), 0x22000, 0x24000) || \
 484          REG_RANGE((reg), 0x30000, 0x40000))
 485
 486 #define FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg) \
 487         (REG_RANGE((reg), 0x2000, 0x4000) || \
 488          REG_RANGE((reg), 0x5000, 0x8000) || \
 489          REG_RANGE((reg), 0x8300, 0x8500) || \
 490          REG_RANGE((reg), 0xB000, 0xC000) || \
 491          REG_RANGE((reg), 0xE000, 0xE800))
 492
 493 #define FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg) \
 494         (REG_RANGE((reg), 0x8800, 0x8900) || \
 495          REG_RANGE((reg), 0xD000, 0xD800) || \
 496          REG_RANGE((reg), 0x12000, 0x14000) || \
 497          REG_RANGE((reg), 0x1A000, 0x1C000) || \
 498          REG_RANGE((reg), 0x1E800, 0x1EA00) || \
 499          REG_RANGE((reg), 0x30000, 0x40000))
 500
 501 #define FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg) \
 502         (REG_RANGE((reg), 0x4000, 0x5000) || \
 503          REG_RANGE((reg), 0x8000, 0x8300) || \
 504          REG_RANGE((reg), 0x8500, 0x8600) || \
 505          REG_RANGE((reg), 0x9000, 0xB000) || \
 506          REG_RANGE((reg), 0xC000, 0xC800) || \
 507          REG_RANGE((reg), 0xF000, 0x10000) || \
 508          REG_RANGE((reg), 0x14000, 0x14400) || \
 509          REG_RANGE((reg), 0x22000, 0x24000))
 510
 511 static void
 512 ilk_dummy_write(struct drm_i915_private *dev_priv)
 513 {
 514         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 515          * the chip from rc6 before touching it for real. MI_MODE is masked,
 516          * hence harmless to write 0 into. */
 517         __raw_i915_write32(dev_priv, MI_MODE, 0);
 518 }
 519
 520 static void
 521 hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read,
 522                         bool before)
 523 {
 524         const char *op = read ? "reading" : "writing to";
 525         const char *when = before ? "before" : "after";
 526
 527         if (!i915.mmio_debug)
 528                 return;
 529
 530         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 531                 WARN(1, "Unclaimed register detected %s %s register 0x%x\n",
 532                      when, op, reg);
 533                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 534         }
 535 }
 536
 537 static void
 538 hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv)
 539 {
 540         if (i915.mmio_debug)
 541                 return;
 542
 543         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 544                 DRM_ERROR("Unclaimed register detected. Please use the i915.mmio_debug=1 to debug this problem.");
 545                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 546         }
 547 }
 548
 549 #define REG_READ_HEADER(x) \
 550         unsigned long irqflags; \
 551         u##x val = 0; \
 552         assert_device_not_suspended(dev_priv); \
 553         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 554
 555 #define REG_READ_FOOTER \
 556         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 557         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 558         return val
 559
 560 #define __gen4_read(x) \
 561 static u##x \
 562 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 563         REG_READ_HEADER(x); \
 564         val = __raw_i915_read##x(dev_priv, reg); \
 565         REG_READ_FOOTER; \
 566 }
 567
 568 #define __gen5_read(x) \
 569 static u##x \
 570 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 571         REG_READ_HEADER(x); \
 572         ilk_dummy_write(dev_priv); \
 573         val = __raw_i915_read##x(dev_priv, reg); \
 574         REG_READ_FOOTER; \
 575 }
 576
 577 #define __gen6_read(x) \
 578 static u##x \
 579 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 580         REG_READ_HEADER(x); \
 581         hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
 582         if (dev_priv->uncore.forcewake_count == 0 && \
 583             NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 584                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 585                                                       FORCEWAKE_ALL); \
 586                 val = __raw_i915_read##x(dev_priv, reg); \
 587                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 588                                                       FORCEWAKE_ALL); \
 589         } else { \
 590                 val = __raw_i915_read##x(dev_priv, reg); \
 591         } \
 592         hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
 593         REG_READ_FOOTER; \
 594 }
 595
 596 #define __vlv_read(x) \
 597 static u##x \
 598 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 599         unsigned fwengine = 0; \
 600         REG_READ_HEADER(x); \
 601         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
 602                 if (dev_priv->uncore.fw_rendercount == 0) \
 603                         fwengine = FORCEWAKE_RENDER; \
 604         } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
 605                 if (dev_priv->uncore.fw_mediacount == 0) \
 606                         fwengine = FORCEWAKE_MEDIA; \
 607         }  \
 608         if (fwengine) \
 609                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 610         val = __raw_i915_read##x(dev_priv, reg); \
 611         if (fwengine) \
 612                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 613         REG_READ_FOOTER; \
 614 }
 615
 616 #define __chv_read(x) \
 617 static u##x \
 618 chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 619         unsigned fwengine = 0; \
 620         REG_READ_HEADER(x); \
 621         if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 622                 if (dev_priv->uncore.fw_rendercount == 0) \
 623                         fwengine = FORCEWAKE_RENDER; \
 624         } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 625                 if (dev_priv->uncore.fw_mediacount == 0) \
 626                         fwengine = FORCEWAKE_MEDIA; \
 627         } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 628                 if (dev_priv->uncore.fw_rendercount == 0) \
 629                         fwengine |= FORCEWAKE_RENDER; \
 630                 if (dev_priv->uncore.fw_mediacount == 0) \
 631                         fwengine |= FORCEWAKE_MEDIA; \
 632         } \
 633         if (fwengine) \
 634                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 635         val = __raw_i915_read##x(dev_priv, reg); \
 636         if (fwengine) \
 637                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 638         REG_READ_FOOTER; \
 639 }
 640
 641 __chv_read(8)
 642 __chv_read(16)
 643 __chv_read(32)
 644 __chv_read(64)
 645 __vlv_read(8)
 646 __vlv_read(16)
 647 __vlv_read(32)
 648 __vlv_read(64)
 649 __gen6_read(8)
 650 __gen6_read(16)
 651 __gen6_read(32)
 652 __gen6_read(64)
 653 __gen5_read(8)
 654 __gen5_read(16)
 655 __gen5_read(32)
 656 __gen5_read(64)
 657 __gen4_read(8)
 658 __gen4_read(16)
 659 __gen4_read(32)
 660 __gen4_read(64)
 661
 662 #undef __chv_read
 663 #undef __vlv_read
 664 #undef __gen6_read
 665 #undef __gen5_read
 666 #undef __gen4_read
 667 #undef REG_READ_FOOTER
 668 #undef REG_READ_HEADER
 669
 670 #define REG_WRITE_HEADER \
 671         unsigned long irqflags; \
 672         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 673         assert_device_not_suspended(dev_priv); \
 674         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 675
 676 #define REG_WRITE_FOOTER \
 677         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
 678
 679 #define __gen4_write(x) \
 680 static void \
 681 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 682         REG_WRITE_HEADER; \
 683         __raw_i915_write##x(dev_priv, reg, val); \
 684         REG_WRITE_FOOTER; \
 685 }
 686
 687 #define __gen5_write(x) \
 688 static void \
 689 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 690         REG_WRITE_HEADER; \
 691         ilk_dummy_write(dev_priv); \
 692         __raw_i915_write##x(dev_priv, reg, val); \
 693         REG_WRITE_FOOTER; \
 694 }
 695
 696 #define __gen6_write(x) \
 697 static void \
 698 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 699         u32 __fifo_ret = 0; \
 700         REG_WRITE_HEADER; \
 701         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 702                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 703         } \
 704         __raw_i915_write##x(dev_priv, reg, val); \
 705         if (unlikely(__fifo_ret)) { \
 706                 gen6_gt_check_fifodbg(dev_priv); \
 707         } \
 708         REG_WRITE_FOOTER; \
 709 }
 710
 711 #define __hsw_write(x) \
 712 static void \
 713 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 714         u32 __fifo_ret = 0; \
 715         REG_WRITE_HEADER; \
 716         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 717                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 718         } \
 719         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 720         __raw_i915_write##x(dev_priv, reg, val); \
 721         if (unlikely(__fifo_ret)) { \
 722                 gen6_gt_check_fifodbg(dev_priv); \
 723         } \
 724         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 725         hsw_unclaimed_reg_detect(dev_priv); \
 726         REG_WRITE_FOOTER; \
 727 }
 728
 729 static const u32 gen8_shadowed_regs[] = {
 730         FORCEWAKE_MT,
 731         GEN6_RPNSWREQ,
 732         GEN6_RC_VIDEO_FREQ,
 733         RING_TAIL(RENDER_RING_BASE),
 734         RING_TAIL(GEN6_BSD_RING_BASE),
 735         RING_TAIL(VEBOX_RING_BASE),
 736         RING_TAIL(BLT_RING_BASE),
 737         /* TODO: Other registers are not yet used */
 738 };
 739
 740 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 741 {
 742         int i;
 743         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 744                 if (reg == gen8_shadowed_regs[i])
 745                         return true;
 746
 747         return false;
 748 }
 749
 750 #define __gen8_write(x) \
 751 static void \
 752 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 753         REG_WRITE_HEADER; \
 754         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 755         if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
 756                 if (dev_priv->uncore.forcewake_count == 0) \
 757                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 758                                                               FORCEWAKE_ALL); \
 759                 __raw_i915_write##x(dev_priv, reg, val); \
 760                 if (dev_priv->uncore.forcewake_count == 0) \
 761                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 762                                                               FORCEWAKE_ALL); \
 763         } else { \
 764                 __raw_i915_write##x(dev_priv, reg, val); \
 765         } \
 766         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 767         hsw_unclaimed_reg_detect(dev_priv); \
 768         REG_WRITE_FOOTER; \
 769 }
 770
 771 #define __chv_write(x) \
 772 static void \
 773 chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 774         unsigned fwengine = 0; \
 775         bool shadowed = is_gen8_shadowed(dev_priv, reg); \
 776         REG_WRITE_HEADER; \
 777         if (!shadowed) { \
 778                 if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 779                         if (dev_priv->uncore.fw_rendercount == 0) \
 780                                 fwengine = FORCEWAKE_RENDER; \
 781                 } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 782                         if (dev_priv->uncore.fw_mediacount == 0) \
 783                                 fwengine = FORCEWAKE_MEDIA; \
 784                 } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 785                         if (dev_priv->uncore.fw_rendercount == 0) \
 786                                 fwengine |= FORCEWAKE_RENDER; \
 787                         if (dev_priv->uncore.fw_mediacount == 0) \
 788                                 fwengine |= FORCEWAKE_MEDIA; \
 789                 } \
 790         } \
 791         if (fwengine) \
 792                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 793         __raw_i915_write##x(dev_priv, reg, val); \
 794         if (fwengine) \
 795                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 796         REG_WRITE_FOOTER; \
 797 }
 798
 799 __chv_write(8)
 800 __chv_write(16)
 801 __chv_write(32)
 802 __chv_write(64)
 803 __gen8_write(8)
 804 __gen8_write(16)
 805 __gen8_write(32)
 806 __gen8_write(64)
 807 __hsw_write(8)
 808 __hsw_write(16)
 809 __hsw_write(32)
 810 __hsw_write(64)
 811 __gen6_write(8)
 812 __gen6_write(16)
 813 __gen6_write(32)
 814 __gen6_write(64)
 815 __gen5_write(8)
 816 __gen5_write(16)
 817 __gen5_write(32)
 818 __gen5_write(64)
 819 __gen4_write(8)
 820 __gen4_write(16)
 821 __gen4_write(32)
 822 __gen4_write(64)
 823
 824 #undef __chv_write
 825 #undef __gen8_write
 826 #undef __hsw_write
 827 #undef __gen6_write
 828 #undef __gen5_write
 829 #undef __gen4_write
 830 #undef REG_WRITE_FOOTER
 831 #undef REG_WRITE_HEADER
 832
 833 #define ASSIGN_WRITE_MMIO_VFUNCS(x) \
 834 do { \
 835         dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
 836         dev_priv->uncore.funcs.mmio_writew = x##_write16; \
 837         dev_priv->uncore.funcs.mmio_writel = x##_write32; \
 838         dev_priv->uncore.funcs.mmio_writeq = x##_write64; \
 839 } while (0)
 840
 841 #define ASSIGN_READ_MMIO_VFUNCS(x) \
 842 do { \
 843         dev_priv->uncore.funcs.mmio_readb = x##_read8; \
 844         dev_priv->uncore.funcs.mmio_readw = x##_read16; \
 845         dev_priv->uncore.funcs.mmio_readl = x##_read32; \
 846         dev_priv->uncore.funcs.mmio_readq = x##_read64; \
 847 } while (0)
 848
 849 void intel_uncore_init(struct drm_device *dev)
 850 {
 851         struct drm_i915_private *dev_priv = dev->dev_private;
 852
 853         setup_timer(&dev_priv->uncore.force_wake_timer,
 854                     gen6_force_wake_timer, (unsigned long)dev_priv);
 855
 856         __intel_uncore_early_sanitize(dev, false);
 857
 858         if (IS_VALLEYVIEW(dev)) {
 859                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
 860                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
 861         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 862                 dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
 863                 dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
 864         } else if (IS_IVYBRIDGE(dev)) {
 865                 u32 ecobus;
 866
 867                 /* IVB configs may use multi-threaded forcewake */
 868
 869                 /* A small trick here - if the bios hasn't configured
 870                  * MT forcewake, and if the device is in RC6, then
 871                  * force_wake_mt_get will not wake the device and the
 872                  * ECOBUS read will return zero. Which will be
 873                  * (correctly) interpreted by the test below as MT
 874                  * forcewake being disabled.
 875                  */
 876                 mutex_lock(&dev->struct_mutex);
 877                 __gen7_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
 878                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
 879                 __gen7_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
 880                 mutex_unlock(&dev->struct_mutex);
 881
 882                 if (ecobus & FORCEWAKE_MT_ENABLE) {
 883                         dev_priv->uncore.funcs.force_wake_get =
 884                                 __gen7_gt_force_wake_mt_get;
 885                         dev_priv->uncore.funcs.force_wake_put =
 886                                 __gen7_gt_force_wake_mt_put;
 887                 } else {
 888                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 889                         DRM_INFO("when using vblank-synced partial screen updates.\n");
 890                         dev_priv->uncore.funcs.force_wake_get =
 891                                 __gen6_gt_force_wake_get;
 892                         dev_priv->uncore.funcs.force_wake_put =
 893                                 __gen6_gt_force_wake_put;
 894                 }
 895         } else if (IS_GEN6(dev)) {
 896                 dev_priv->uncore.funcs.force_wake_get =
 897                         __gen6_gt_force_wake_get;
 898                 dev_priv->uncore.funcs.force_wake_put =
 899                         __gen6_gt_force_wake_put;
 900         }
 901
 902         switch (INTEL_INFO(dev)->gen) {
 903         default:
 904                 if (IS_CHERRYVIEW(dev)) {
 905                         ASSIGN_WRITE_MMIO_VFUNCS(chv);
 906                         ASSIGN_READ_MMIO_VFUNCS(chv);
 907
 908                 } else {
 909                         ASSIGN_WRITE_MMIO_VFUNCS(gen8);
 910                         ASSIGN_READ_MMIO_VFUNCS(gen6);
 911                 }
 912                 break;
 913         case 7:
 914         case 6:
 915                 if (IS_HASWELL(dev)) {
 916                         ASSIGN_WRITE_MMIO_VFUNCS(hsw);
 917                 } else {
 918                         ASSIGN_WRITE_MMIO_VFUNCS(gen6);
 919                 }
 920
 921                 if (IS_VALLEYVIEW(dev)) {
 922                         ASSIGN_READ_MMIO_VFUNCS(vlv);
 923                 } else {
 924                         ASSIGN_READ_MMIO_VFUNCS(gen6);
 925                 }
 926                 break;
 927         case 5:
 928                 ASSIGN_WRITE_MMIO_VFUNCS(gen5);
 929                 ASSIGN_READ_MMIO_VFUNCS(gen5);
 930                 break;
 931         case 4:
 932         case 3:
 933         case 2:
 934                 ASSIGN_WRITE_MMIO_VFUNCS(gen4);
 935                 ASSIGN_READ_MMIO_VFUNCS(gen4);
 936                 break;
 937         }
 938
 939         i915_check_and_clear_faults(dev);
 940 }
 941 #undef ASSIGN_WRITE_MMIO_VFUNCS
 942 #undef ASSIGN_READ_MMIO_VFUNCS
 943
 944 void intel_uncore_fini(struct drm_device *dev)
 945 {
 946         /* Paranoia: make sure we have disabled everything before we exit. */
 947         intel_uncore_sanitize(dev);
 948         intel_uncore_forcewake_reset(dev, false);
 949 }
 950
 951 #define GEN_RANGE(l, h) GENMASK(h, l)
 952
 953 static const struct register_whitelist {
 954         uint64_t offset;
 955         uint32_t size;
 956         /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 957         uint32_t gen_bitmask;
 958 } whitelist[] = {
 959         { RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 9) },
 960 };
 961
 962 int i915_reg_read_ioctl(struct drm_device *dev,
 963                         void *data, struct drm_file *file)
 964 {
 965         struct drm_i915_private *dev_priv = dev->dev_private;
 966         struct drm_i915_reg_read *reg = data;
 967         struct register_whitelist const *entry = whitelist;
 968         int i, ret = 0;
 969
 970         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 971                 if (entry->offset == reg->offset &&
 972                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 973                         break;
 974         }
 975
 976         if (i == ARRAY_SIZE(whitelist))
 977                 return -EINVAL;
 978
 979         intel_runtime_pm_get(dev_priv);
 980
 981         switch (entry->size) {
 982         case 8:
 983                 reg->val = I915_READ64(reg->offset);
 984                 break;
 985         case 4:
 986                 reg->val = I915_READ(reg->offset);
 987                 break;
 988         case 2:
 989                 reg->val = I915_READ16(reg->offset);
 990                 break;
 991         case 1:
 992                 reg->val = I915_READ8(reg->offset);
 993                 break;
 994         default:
 995                 WARN_ON(1);
 996                 ret = -EINVAL;
 997                 goto out;
 998         }
 999
1000 out:
1001         intel_runtime_pm_put(dev_priv);
1002         return ret;
1003 }
1004
1005 int i915_get_reset_stats_ioctl(struct drm_device *dev,
1006                                void *data, struct drm_file *file)
1007 {
1008         struct drm_i915_private *dev_priv = dev->dev_private;
1009         struct drm_i915_reset_stats *args = data;
1010         struct i915_ctx_hang_stats *hs;
1011         struct intel_context *ctx;
1012         int ret;
1013
1014         if (args->flags || args->pad)
1015                 return -EINVAL;
1016
1017         if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
1018                 return -EPERM;
1019
1020         ret = mutex_lock_interruptible(&dev->struct_mutex);
1021         if (ret)
1022                 return ret;
1023
1024         ctx = i915_gem_context_get(file->driver_priv, args->ctx_id);
1025         if (IS_ERR(ctx)) {
1026                 mutex_unlock(&dev->struct_mutex);
1027                 return PTR_ERR(ctx);
1028         }
1029         hs = &ctx->hang_stats;
1030
1031         if (capable(CAP_SYS_ADMIN))
1032                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
1033         else
1034                 args->reset_count = 0;
1035
1036         args->batch_active = hs->batch_active;
1037         args->batch_pending = hs->batch_pending;
1038
1039         mutex_unlock(&dev->struct_mutex);
1040
1041         return 0;
1042 }
1043
1044 static int i965_reset_complete(struct drm_device *dev)
1045 {
1046         u8 gdrst;
1047         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
1048         return (gdrst & GRDOM_RESET_ENABLE) == 0;
1049 }
1050
1051 static int i965_do_reset(struct drm_device *dev)
1052 {
1053         int ret;
1054
1055         /* FIXME: i965g/gm need a display save/restore for gpu reset. */
1056         return -ENODEV;
1057
1058         /*
1059          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
1060          * well as the reset bit (GR/bit 0).  Setting the GR bit
1061          * triggers the reset; when done, the hardware will clear it.
1062          */
1063         pci_write_config_byte(dev->pdev, I965_GDRST,
1064                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1065         ret =  wait_for(i965_reset_complete(dev), 500);
1066         if (ret)
1067                 return ret;
1068
1069         pci_write_config_byte(dev->pdev, I965_GDRST,
1070                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1071
1072         ret =  wait_for(i965_reset_complete(dev), 500);
1073         if (ret)
1074                 return ret;
1075
1076         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1077
1078         return 0;
1079 }
1080
1081 static int g4x_do_reset(struct drm_device *dev)
1082 {
1083         struct drm_i915_private *dev_priv = dev->dev_private;
1084         int ret;
1085
1086         pci_write_config_byte(dev->pdev, I965_GDRST,
1087                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1088         ret =  wait_for(i965_reset_complete(dev), 500);
1089         if (ret)
1090                 return ret;
1091
1092         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1093         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1094         POSTING_READ(VDECCLK_GATE_D);
1095
1096         pci_write_config_byte(dev->pdev, I965_GDRST,
1097                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1098         ret =  wait_for(i965_reset_complete(dev), 500);
1099         if (ret)
1100                 return ret;
1101
1102         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1103         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1104         POSTING_READ(VDECCLK_GATE_D);
1105
1106         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1107
1108         return 0;
1109 }
1110
1111 static int ironlake_do_reset(struct drm_device *dev)
1112 {
1113         struct drm_i915_private *dev_priv = dev->dev_private;
1114         int ret;
1115
1116         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1117                    ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1118         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1119                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1120         if (ret)
1121                 return ret;
1122
1123         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1124                    ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1125         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1126                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1127         if (ret)
1128                 return ret;
1129
1130         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0);
1131
1132         return 0;
1133 }
1134
1135 static int gen6_do_reset(struct drm_device *dev)
1136 {
1137         struct drm_i915_private *dev_priv = dev->dev_private;
1138         int     ret;
1139
1140         /* Reset the chip */
1141
1142         /* GEN6_GDRST is not in the gt power well, no need to check
1143          * for fifo space for the write or forcewake the chip for
1144          * the read
1145          */
1146         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
1147
1148         /* Spin waiting for the device to ack the reset request */
1149         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
1150
1151         intel_uncore_forcewake_reset(dev, true);
1152
1153         return ret;
1154 }
1155
1156 int intel_gpu_reset(struct drm_device *dev)
1157 {
1158         if (INTEL_INFO(dev)->gen >= 6)
1159                 return gen6_do_reset(dev);
1160         else if (IS_GEN5(dev))
1161                 return ironlake_do_reset(dev);
1162         else if (IS_G4X(dev))
1163                 return g4x_do_reset(dev);
1164         else if (IS_GEN4(dev))
1165                 return i965_do_reset(dev);
1166         else
1167                 return -ENODEV;
1168 }
1169
1170 void intel_uncore_check_errors(struct drm_device *dev)
1171 {
1172         struct drm_i915_private *dev_priv = dev->dev_private;
1173
1174         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
1175             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
1176                 DRM_ERROR("Unclaimed register before interrupt\n");
1177                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1178         }
1179 }