drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43 static void
  44 assert_device_not_suspended(struct drm_i915_private *dev_priv)
  45 {
  46         WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
  47              "Device suspended\n");
  48 }
  49
  50 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  51 {
  52         u32 gt_thread_status_mask;
  53
  54         if (IS_HASWELL(dev_priv->dev))
  55                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  56         else
  57                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  58
  59         /* w/a for a sporadic read returning 0 by waiting for the GT
  60          * thread to wake up.
  61          */
  62         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  63                 DRM_ERROR("GT thread status wait timed out\n");
  64 }
  65
  66 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  67 {
  68         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  69         /* something from same cacheline, but !FORCEWAKE */
  70         __raw_posting_read(dev_priv, ECOBUS);
  71 }
  72
  73 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  74                                                         int fw_engine)
  75 {
  76         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  77                             FORCEWAKE_ACK_TIMEOUT_MS))
  78                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  79
  80         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  81         /* something from same cacheline, but !FORCEWAKE */
  82         __raw_posting_read(dev_priv, ECOBUS);
  83
  84         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  85                             FORCEWAKE_ACK_TIMEOUT_MS))
  86                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  87
  88         /* WaRsForcewakeWaitTC0:snb */
  89         __gen6_gt_wait_for_thread_c0(dev_priv);
  90 }
  91
  92 static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  93 {
  94         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  95         /* something from same cacheline, but !FORCEWAKE_MT */
  96         __raw_posting_read(dev_priv, ECOBUS);
  97 }
  98
  99 static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
 100                                                         int fw_engine)
 101 {
 102         u32 forcewake_ack;
 103
 104         if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))
 105                 forcewake_ack = FORCEWAKE_ACK_HSW;
 106         else
 107                 forcewake_ack = FORCEWAKE_MT_ACK;
 108
 109         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 110                             FORCEWAKE_ACK_TIMEOUT_MS))
 111                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 112
 113         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 114                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 115         /* something from same cacheline, but !FORCEWAKE_MT */
 116         __raw_posting_read(dev_priv, ECOBUS);
 117
 118         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 119                             FORCEWAKE_ACK_TIMEOUT_MS))
 120                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 121
 122         /* WaRsForcewakeWaitTC0:ivb,hsw */
 123         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 124                 __gen6_gt_wait_for_thread_c0(dev_priv);
 125 }
 126
 127 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 128 {
 129         u32 gtfifodbg;
 130
 131         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 132         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 133                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 134 }
 135
 136 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 137                                                         int fw_engine)
 138 {
 139         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 140         /* something from same cacheline, but !FORCEWAKE */
 141         __raw_posting_read(dev_priv, ECOBUS);
 142         gen6_gt_check_fifodbg(dev_priv);
 143 }
 144
 145 static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 146                                                         int fw_engine)
 147 {
 148         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 149                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 150         /* something from same cacheline, but !FORCEWAKE_MT */
 151         __raw_posting_read(dev_priv, ECOBUS);
 152
 153         if (IS_GEN7(dev_priv->dev))
 154                 gen6_gt_check_fifodbg(dev_priv);
 155 }
 156
 157 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 158 {
 159         int ret = 0;
 160
 161         /* On VLV, FIFO will be shared by both SW and HW.
 162          * So, we need to read the FREE_ENTRIES everytime */
 163         if (IS_VALLEYVIEW(dev_priv->dev))
 164                 dev_priv->uncore.fifo_count =
 165                         __raw_i915_read32(dev_priv, GTFIFOCTL) &
 166                                                 GT_FIFO_FREE_ENTRIES_MASK;
 167
 168         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 169                 int loop = 500;
 170                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 171                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 172                         udelay(10);
 173                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 174                 }
 175                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 176                         ++ret;
 177                 dev_priv->uncore.fifo_count = fifo;
 178         }
 179         dev_priv->uncore.fifo_count--;
 180
 181         return ret;
 182 }
 183
 184 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 185 {
 186         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 187                            _MASKED_BIT_DISABLE(0xffff));
 188         __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 189                            _MASKED_BIT_DISABLE(0xffff));
 190         /* something from same cacheline, but !FORCEWAKE_VLV */
 191         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 192 }
 193
 194 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 195                                                 int fw_engine)
 196 {
 197         /*
 198          * WaRsDontPollForAckOnClearingFWBits:vlv
 199          * Hardware clears ack bits lazily (only when all ack
 200          * bits become 0) so don't poll for individiual ack
 201          * bits to be clear here like on other platforms.
 202          */
 203
 204         /* Check for Render Engine */
 205         if (FORCEWAKE_RENDER & fw_engine) {
 206
 207                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 208                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 209
 210                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 211                                                 FORCEWAKE_ACK_VLV) &
 212                                                 FORCEWAKE_KERNEL),
 213                                         FORCEWAKE_ACK_TIMEOUT_MS))
 214                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 215         }
 216
 217         /* Check for Media Engine */
 218         if (FORCEWAKE_MEDIA & fw_engine) {
 219
 220                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 221                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 222
 223                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 224                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 225                                                 FORCEWAKE_KERNEL),
 226                                         FORCEWAKE_ACK_TIMEOUT_MS))
 227                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 228         }
 229
 230         /* WaRsForcewakeWaitTC0:vlv */
 231         if (!IS_CHERRYVIEW(dev_priv->dev))
 232                 __gen6_gt_wait_for_thread_c0(dev_priv);
 233 }
 234
 235 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 236                                         int fw_engine)
 237 {
 238
 239         /* Check for Render Engine */
 240         if (FORCEWAKE_RENDER & fw_engine)
 241                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 242                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 243
 244
 245         /* Check for Media Engine */
 246         if (FORCEWAKE_MEDIA & fw_engine)
 247                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 248                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 249
 250         /* something from same cacheline, but !FORCEWAKE_VLV */
 251         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 252         if (!IS_CHERRYVIEW(dev_priv->dev))
 253                 gen6_gt_check_fifodbg(dev_priv);
 254 }
 255
 256 static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 257 {
 258         unsigned long irqflags;
 259
 260         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 261
 262         if (fw_engine & FORCEWAKE_RENDER &&
 263             dev_priv->uncore.fw_rendercount++ != 0)
 264                 fw_engine &= ~FORCEWAKE_RENDER;
 265         if (fw_engine & FORCEWAKE_MEDIA &&
 266             dev_priv->uncore.fw_mediacount++ != 0)
 267                 fw_engine &= ~FORCEWAKE_MEDIA;
 268
 269         if (fw_engine)
 270                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
 271
 272         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 273 }
 274
 275 static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 276 {
 277         unsigned long irqflags;
 278
 279         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 280
 281         if (fw_engine & FORCEWAKE_RENDER) {
 282                 WARN_ON(!dev_priv->uncore.fw_rendercount);
 283                 if (--dev_priv->uncore.fw_rendercount != 0)
 284                         fw_engine &= ~FORCEWAKE_RENDER;
 285         }
 286
 287         if (fw_engine & FORCEWAKE_MEDIA) {
 288                 WARN_ON(!dev_priv->uncore.fw_mediacount);
 289                 if (--dev_priv->uncore.fw_mediacount != 0)
 290                         fw_engine &= ~FORCEWAKE_MEDIA;
 291         }
 292
 293         if (fw_engine)
 294                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
 295
 296         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 297 }
 298
 299 static void gen6_force_wake_timer(unsigned long arg)
 300 {
 301         struct drm_i915_private *dev_priv = (void *)arg;
 302         unsigned long irqflags;
 303
 304         assert_device_not_suspended(dev_priv);
 305
 306         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 307         WARN_ON(!dev_priv->uncore.forcewake_count);
 308
 309         if (--dev_priv->uncore.forcewake_count == 0)
 310                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 311         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 312
 313         intel_runtime_pm_put(dev_priv);
 314 }
 315
 316 void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 317 {
 318         struct drm_i915_private *dev_priv = dev->dev_private;
 319         unsigned long irqflags;
 320
 321         if (del_timer_sync(&dev_priv->uncore.force_wake_timer))
 322                 gen6_force_wake_timer((unsigned long)dev_priv);
 323
 324         /* Hold uncore.lock across reset to prevent any register access
 325          * with forcewake not set correctly
 326          */
 327         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 328
 329         if (IS_VALLEYVIEW(dev))
 330                 vlv_force_wake_reset(dev_priv);
 331         else if (IS_GEN6(dev) || IS_GEN7(dev))
 332                 __gen6_gt_force_wake_reset(dev_priv);
 333
 334         if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
 335                 __gen7_gt_force_wake_mt_reset(dev_priv);
 336
 337         if (restore) { /* If reset with a user forcewake, try to restore */
 338                 unsigned fw = 0;
 339
 340                 if (IS_VALLEYVIEW(dev)) {
 341                         if (dev_priv->uncore.fw_rendercount)
 342                                 fw |= FORCEWAKE_RENDER;
 343
 344                         if (dev_priv->uncore.fw_mediacount)
 345                                 fw |= FORCEWAKE_MEDIA;
 346                 } else {
 347                         if (dev_priv->uncore.forcewake_count)
 348                                 fw = FORCEWAKE_ALL;
 349                 }
 350
 351                 if (fw)
 352                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 353
 354                 if (IS_GEN6(dev) || IS_GEN7(dev))
 355                         dev_priv->uncore.fifo_count =
 356                                 __raw_i915_read32(dev_priv, GTFIFOCTL) &
 357                                 GT_FIFO_FREE_ENTRIES_MASK;
 358         }
 359
 360         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 361 }
 362
 363 void intel_uncore_early_sanitize(struct drm_device *dev, bool restore_forcewake)
 364 {
 365         struct drm_i915_private *dev_priv = dev->dev_private;
 366
 367         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 368                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 369
 370         if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) &&
 371             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 372                 /* The docs do not explain exactly how the calculation can be
 373                  * made. It is somewhat guessable, but for now, it's always
 374                  * 128MB.
 375                  * NB: We can't write IDICR yet because we do not have gt funcs
 376                  * set up */
 377                 dev_priv->ellc_size = 128;
 378                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 379         }
 380
 381         /* clear out old GT FIFO errors */
 382         if (IS_GEN6(dev) || IS_GEN7(dev))
 383                 __raw_i915_write32(dev_priv, GTFIFODBG,
 384                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 385
 386         intel_uncore_forcewake_reset(dev, restore_forcewake);
 387 }
 388
 389 void intel_uncore_sanitize(struct drm_device *dev)
 390 {
 391         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 392         intel_disable_gt_powersave(dev);
 393 }
 394
 395 /*
 396  * Generally this is called implicitly by the register read function. However,
 397  * if some sequence requires the GT to not power down then this function should
 398  * be called at the beginning of the sequence followed by a call to
 399  * gen6_gt_force_wake_put() at the end of the sequence.
 400  */
 401 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 402 {
 403         unsigned long irqflags;
 404
 405         if (!dev_priv->uncore.funcs.force_wake_get)
 406                 return;
 407
 408         intel_runtime_pm_get(dev_priv);
 409
 410         /* Redirect to VLV specific routine */
 411         if (IS_VALLEYVIEW(dev_priv->dev))
 412                 return vlv_force_wake_get(dev_priv, fw_engine);
 413
 414         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 415         if (dev_priv->uncore.forcewake_count++ == 0)
 416                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 417         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 418 }
 419
 420 /*
 421  * see gen6_gt_force_wake_get()
 422  */
 423 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 424 {
 425         unsigned long irqflags;
 426         bool delayed = false;
 427
 428         if (!dev_priv->uncore.funcs.force_wake_put)
 429                 return;
 430
 431         /* Redirect to VLV specific routine */
 432         if (IS_VALLEYVIEW(dev_priv->dev)) {
 433                 vlv_force_wake_put(dev_priv, fw_engine);
 434                 goto out;
 435         }
 436
 437
 438         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 439         WARN_ON(!dev_priv->uncore.forcewake_count);
 440
 441         if (--dev_priv->uncore.forcewake_count == 0) {
 442                 dev_priv->uncore.forcewake_count++;
 443                 delayed = true;
 444                 mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
 445                                  jiffies + 1);
 446         }
 447         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 448
 449 out:
 450         if (!delayed)
 451                 intel_runtime_pm_put(dev_priv);
 452 }
 453
 454 void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
 455 {
 456         if (!dev_priv->uncore.funcs.force_wake_get)
 457                 return;
 458
 459         WARN_ON(dev_priv->uncore.forcewake_count > 0);
 460 }
 461
 462 /* We give fast paths for the really cool registers */
 463 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 464          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 465
 466 #define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
 467
 468 #define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \
 469         (REG_RANGE((reg), 0x2000, 0x4000) || \
 470          REG_RANGE((reg), 0x5000, 0x8000) || \
 471          REG_RANGE((reg), 0xB000, 0x12000) || \
 472          REG_RANGE((reg), 0x2E000, 0x30000))
 473
 474 #define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg) \
 475         (REG_RANGE((reg), 0x12000, 0x14000) || \
 476          REG_RANGE((reg), 0x22000, 0x24000) || \
 477          REG_RANGE((reg), 0x30000, 0x40000))
 478
 479 #define FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg) \
 480         (REG_RANGE((reg), 0x2000, 0x4000) || \
 481          REG_RANGE((reg), 0x5000, 0x8000) || \
 482          REG_RANGE((reg), 0x8300, 0x8500) || \
 483          REG_RANGE((reg), 0xB000, 0xC000) || \
 484          REG_RANGE((reg), 0xE000, 0xE800))
 485
 486 #define FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg) \
 487         (REG_RANGE((reg), 0x8800, 0x8900) || \
 488          REG_RANGE((reg), 0xD000, 0xD800) || \
 489          REG_RANGE((reg), 0x12000, 0x14000) || \
 490          REG_RANGE((reg), 0x1A000, 0x1C000) || \
 491          REG_RANGE((reg), 0x1E800, 0x1EA00) || \
 492          REG_RANGE((reg), 0x30000, 0x40000))
 493
 494 #define FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg) \
 495         (REG_RANGE((reg), 0x4000, 0x5000) || \
 496          REG_RANGE((reg), 0x8000, 0x8300) || \
 497          REG_RANGE((reg), 0x8500, 0x8600) || \
 498          REG_RANGE((reg), 0x9000, 0xB000) || \
 499          REG_RANGE((reg), 0xC000, 0xC800) || \
 500          REG_RANGE((reg), 0xF000, 0x10000) || \
 501          REG_RANGE((reg), 0x14000, 0x14400) || \
 502          REG_RANGE((reg), 0x22000, 0x24000))
 503
 504 static void
 505 ilk_dummy_write(struct drm_i915_private *dev_priv)
 506 {
 507         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 508          * the chip from rc6 before touching it for real. MI_MODE is masked,
 509          * hence harmless to write 0 into. */
 510         __raw_i915_write32(dev_priv, MI_MODE, 0);
 511 }
 512
 513 static void
 514 hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read,
 515                         bool before)
 516 {
 517         const char *op = read ? "reading" : "writing to";
 518         const char *when = before ? "before" : "after";
 519
 520         if (!i915.mmio_debug)
 521                 return;
 522
 523         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 524                 WARN(1, "Unclaimed register detected %s %s register 0x%x\n",
 525                      when, op, reg);
 526                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 527         }
 528 }
 529
 530 static void
 531 hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv)
 532 {
 533         if (i915.mmio_debug)
 534                 return;
 535
 536         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 537                 DRM_ERROR("Unclaimed register detected. Please use the i915.mmio_debug=1 to debug this problem.");
 538                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 539         }
 540 }
 541
 542 #define REG_READ_HEADER(x) \
 543         unsigned long irqflags; \
 544         u##x val = 0; \
 545         assert_device_not_suspended(dev_priv); \
 546         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 547
 548 #define REG_READ_FOOTER \
 549         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 550         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 551         return val
 552
 553 #define __gen4_read(x) \
 554 static u##x \
 555 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 556         REG_READ_HEADER(x); \
 557         val = __raw_i915_read##x(dev_priv, reg); \
 558         REG_READ_FOOTER; \
 559 }
 560
 561 #define __gen5_read(x) \
 562 static u##x \
 563 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 564         REG_READ_HEADER(x); \
 565         ilk_dummy_write(dev_priv); \
 566         val = __raw_i915_read##x(dev_priv, reg); \
 567         REG_READ_FOOTER; \
 568 }
 569
 570 #define __gen6_read(x) \
 571 static u##x \
 572 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 573         REG_READ_HEADER(x); \
 574         hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
 575         if (dev_priv->uncore.forcewake_count == 0 && \
 576             NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 577                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 578                                                       FORCEWAKE_ALL); \
 579                 val = __raw_i915_read##x(dev_priv, reg); \
 580                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 581                                                       FORCEWAKE_ALL); \
 582         } else { \
 583                 val = __raw_i915_read##x(dev_priv, reg); \
 584         } \
 585         hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
 586         REG_READ_FOOTER; \
 587 }
 588
 589 #define __vlv_read(x) \
 590 static u##x \
 591 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 592         unsigned fwengine = 0; \
 593         REG_READ_HEADER(x); \
 594         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
 595                 if (dev_priv->uncore.fw_rendercount == 0) \
 596                         fwengine = FORCEWAKE_RENDER; \
 597         } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
 598                 if (dev_priv->uncore.fw_mediacount == 0) \
 599                         fwengine = FORCEWAKE_MEDIA; \
 600         }  \
 601         if (fwengine) \
 602                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 603         val = __raw_i915_read##x(dev_priv, reg); \
 604         if (fwengine) \
 605                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 606         REG_READ_FOOTER; \
 607 }
 608
 609 #define __chv_read(x) \
 610 static u##x \
 611 chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 612         unsigned fwengine = 0; \
 613         REG_READ_HEADER(x); \
 614         if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 615                 if (dev_priv->uncore.fw_rendercount == 0) \
 616                         fwengine = FORCEWAKE_RENDER; \
 617         } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 618                 if (dev_priv->uncore.fw_mediacount == 0) \
 619                         fwengine = FORCEWAKE_MEDIA; \
 620         } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 621                 if (dev_priv->uncore.fw_rendercount == 0) \
 622                         fwengine |= FORCEWAKE_RENDER; \
 623                 if (dev_priv->uncore.fw_mediacount == 0) \
 624                         fwengine |= FORCEWAKE_MEDIA; \
 625         } \
 626         if (fwengine) \
 627                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 628         val = __raw_i915_read##x(dev_priv, reg); \
 629         if (fwengine) \
 630                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 631         REG_READ_FOOTER; \
 632 }
 633
 634 __chv_read(8)
 635 __chv_read(16)
 636 __chv_read(32)
 637 __chv_read(64)
 638 __vlv_read(8)
 639 __vlv_read(16)
 640 __vlv_read(32)
 641 __vlv_read(64)
 642 __gen6_read(8)
 643 __gen6_read(16)
 644 __gen6_read(32)
 645 __gen6_read(64)
 646 __gen5_read(8)
 647 __gen5_read(16)
 648 __gen5_read(32)
 649 __gen5_read(64)
 650 __gen4_read(8)
 651 __gen4_read(16)
 652 __gen4_read(32)
 653 __gen4_read(64)
 654
 655 #undef __chv_read
 656 #undef __vlv_read
 657 #undef __gen6_read
 658 #undef __gen5_read
 659 #undef __gen4_read
 660 #undef REG_READ_FOOTER
 661 #undef REG_READ_HEADER
 662
 663 #define REG_WRITE_HEADER \
 664         unsigned long irqflags; \
 665         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 666         assert_device_not_suspended(dev_priv); \
 667         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 668
 669 #define REG_WRITE_FOOTER \
 670         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
 671
 672 #define __gen4_write(x) \
 673 static void \
 674 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 675         REG_WRITE_HEADER; \
 676         __raw_i915_write##x(dev_priv, reg, val); \
 677         REG_WRITE_FOOTER; \
 678 }
 679
 680 #define __gen5_write(x) \
 681 static void \
 682 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 683         REG_WRITE_HEADER; \
 684         ilk_dummy_write(dev_priv); \
 685         __raw_i915_write##x(dev_priv, reg, val); \
 686         REG_WRITE_FOOTER; \
 687 }
 688
 689 #define __gen6_write(x) \
 690 static void \
 691 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 692         u32 __fifo_ret = 0; \
 693         REG_WRITE_HEADER; \
 694         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 695                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 696         } \
 697         __raw_i915_write##x(dev_priv, reg, val); \
 698         if (unlikely(__fifo_ret)) { \
 699                 gen6_gt_check_fifodbg(dev_priv); \
 700         } \
 701         REG_WRITE_FOOTER; \
 702 }
 703
 704 #define __hsw_write(x) \
 705 static void \
 706 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 707         u32 __fifo_ret = 0; \
 708         REG_WRITE_HEADER; \
 709         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 710                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 711         } \
 712         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 713         __raw_i915_write##x(dev_priv, reg, val); \
 714         if (unlikely(__fifo_ret)) { \
 715                 gen6_gt_check_fifodbg(dev_priv); \
 716         } \
 717         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 718         hsw_unclaimed_reg_detect(dev_priv); \
 719         REG_WRITE_FOOTER; \
 720 }
 721
 722 static const u32 gen8_shadowed_regs[] = {
 723         FORCEWAKE_MT,
 724         GEN6_RPNSWREQ,
 725         GEN6_RC_VIDEO_FREQ,
 726         RING_TAIL(RENDER_RING_BASE),
 727         RING_TAIL(GEN6_BSD_RING_BASE),
 728         RING_TAIL(VEBOX_RING_BASE),
 729         RING_TAIL(BLT_RING_BASE),
 730         /* TODO: Other registers are not yet used */
 731 };
 732
 733 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 734 {
 735         int i;
 736         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 737                 if (reg == gen8_shadowed_regs[i])
 738                         return true;
 739
 740         return false;
 741 }
 742
 743 #define __gen8_write(x) \
 744 static void \
 745 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 746         REG_WRITE_HEADER; \
 747         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 748         if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
 749                 if (dev_priv->uncore.forcewake_count == 0) \
 750                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 751                                                               FORCEWAKE_ALL); \
 752                 __raw_i915_write##x(dev_priv, reg, val); \
 753                 if (dev_priv->uncore.forcewake_count == 0) \
 754                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 755                                                               FORCEWAKE_ALL); \
 756         } else { \
 757                 __raw_i915_write##x(dev_priv, reg, val); \
 758         } \
 759         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 760         hsw_unclaimed_reg_detect(dev_priv); \
 761         REG_WRITE_FOOTER; \
 762 }
 763
 764 #define __chv_write(x) \
 765 static void \
 766 chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 767         unsigned fwengine = 0; \
 768         bool shadowed = is_gen8_shadowed(dev_priv, reg); \
 769         REG_WRITE_HEADER; \
 770         if (!shadowed) { \
 771                 if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 772                         if (dev_priv->uncore.fw_rendercount == 0) \
 773                                 fwengine = FORCEWAKE_RENDER; \
 774                 } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 775                         if (dev_priv->uncore.fw_mediacount == 0) \
 776                                 fwengine = FORCEWAKE_MEDIA; \
 777                 } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 778                         if (dev_priv->uncore.fw_rendercount == 0) \
 779                                 fwengine |= FORCEWAKE_RENDER; \
 780                         if (dev_priv->uncore.fw_mediacount == 0) \
 781                                 fwengine |= FORCEWAKE_MEDIA; \
 782                 } \
 783         } \
 784         if (fwengine) \
 785                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 786         __raw_i915_write##x(dev_priv, reg, val); \
 787         if (fwengine) \
 788                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 789         REG_WRITE_FOOTER; \
 790 }
 791
 792 __chv_write(8)
 793 __chv_write(16)
 794 __chv_write(32)
 795 __chv_write(64)
 796 __gen8_write(8)
 797 __gen8_write(16)
 798 __gen8_write(32)
 799 __gen8_write(64)
 800 __hsw_write(8)
 801 __hsw_write(16)
 802 __hsw_write(32)
 803 __hsw_write(64)
 804 __gen6_write(8)
 805 __gen6_write(16)
 806 __gen6_write(32)
 807 __gen6_write(64)
 808 __gen5_write(8)
 809 __gen5_write(16)
 810 __gen5_write(32)
 811 __gen5_write(64)
 812 __gen4_write(8)
 813 __gen4_write(16)
 814 __gen4_write(32)
 815 __gen4_write(64)
 816
 817 #undef __chv_write
 818 #undef __gen8_write
 819 #undef __hsw_write
 820 #undef __gen6_write
 821 #undef __gen5_write
 822 #undef __gen4_write
 823 #undef REG_WRITE_FOOTER
 824 #undef REG_WRITE_HEADER
 825
 826 void intel_uncore_init(struct drm_device *dev)
 827 {
 828         struct drm_i915_private *dev_priv = dev->dev_private;
 829
 830         setup_timer(&dev_priv->uncore.force_wake_timer,
 831                     gen6_force_wake_timer, (unsigned long)dev_priv);
 832
 833         intel_uncore_early_sanitize(dev, false);
 834
 835         if (IS_VALLEYVIEW(dev)) {
 836                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
 837                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
 838         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 839                 dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
 840                 dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
 841         } else if (IS_IVYBRIDGE(dev)) {
 842                 u32 ecobus;
 843
 844                 /* IVB configs may use multi-threaded forcewake */
 845
 846                 /* A small trick here - if the bios hasn't configured
 847                  * MT forcewake, and if the device is in RC6, then
 848                  * force_wake_mt_get will not wake the device and the
 849                  * ECOBUS read will return zero. Which will be
 850                  * (correctly) interpreted by the test below as MT
 851                  * forcewake being disabled.
 852                  */
 853                 mutex_lock(&dev->struct_mutex);
 854                 __gen7_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
 855                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
 856                 __gen7_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
 857                 mutex_unlock(&dev->struct_mutex);
 858
 859                 if (ecobus & FORCEWAKE_MT_ENABLE) {
 860                         dev_priv->uncore.funcs.force_wake_get =
 861                                 __gen7_gt_force_wake_mt_get;
 862                         dev_priv->uncore.funcs.force_wake_put =
 863                                 __gen7_gt_force_wake_mt_put;
 864                 } else {
 865                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 866                         DRM_INFO("when using vblank-synced partial screen updates.\n");
 867                         dev_priv->uncore.funcs.force_wake_get =
 868                                 __gen6_gt_force_wake_get;
 869                         dev_priv->uncore.funcs.force_wake_put =
 870                                 __gen6_gt_force_wake_put;
 871                 }
 872         } else if (IS_GEN6(dev)) {
 873                 dev_priv->uncore.funcs.force_wake_get =
 874                         __gen6_gt_force_wake_get;
 875                 dev_priv->uncore.funcs.force_wake_put =
 876                         __gen6_gt_force_wake_put;
 877         }
 878
 879         switch (INTEL_INFO(dev)->gen) {
 880         default:
 881                 if (IS_CHERRYVIEW(dev)) {
 882                         dev_priv->uncore.funcs.mmio_writeb  = chv_write8;
 883                         dev_priv->uncore.funcs.mmio_writew  = chv_write16;
 884                         dev_priv->uncore.funcs.mmio_writel  = chv_write32;
 885                         dev_priv->uncore.funcs.mmio_writeq  = chv_write64;
 886                         dev_priv->uncore.funcs.mmio_readb  = chv_read8;
 887                         dev_priv->uncore.funcs.mmio_readw  = chv_read16;
 888                         dev_priv->uncore.funcs.mmio_readl  = chv_read32;
 889                         dev_priv->uncore.funcs.mmio_readq  = chv_read64;
 890
 891                 } else {
 892                         dev_priv->uncore.funcs.mmio_writeb  = gen8_write8;
 893                         dev_priv->uncore.funcs.mmio_writew  = gen8_write16;
 894                         dev_priv->uncore.funcs.mmio_writel  = gen8_write32;
 895                         dev_priv->uncore.funcs.mmio_writeq  = gen8_write64;
 896                         dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 897                         dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 898                         dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 899                         dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 900                 }
 901                 break;
 902         case 7:
 903         case 6:
 904                 if (IS_HASWELL(dev)) {
 905                         dev_priv->uncore.funcs.mmio_writeb  = hsw_write8;
 906                         dev_priv->uncore.funcs.mmio_writew  = hsw_write16;
 907                         dev_priv->uncore.funcs.mmio_writel  = hsw_write32;
 908                         dev_priv->uncore.funcs.mmio_writeq  = hsw_write64;
 909                 } else {
 910                         dev_priv->uncore.funcs.mmio_writeb  = gen6_write8;
 911                         dev_priv->uncore.funcs.mmio_writew  = gen6_write16;
 912                         dev_priv->uncore.funcs.mmio_writel  = gen6_write32;
 913                         dev_priv->uncore.funcs.mmio_writeq  = gen6_write64;
 914                 }
 915
 916                 if (IS_VALLEYVIEW(dev)) {
 917                         dev_priv->uncore.funcs.mmio_readb  = vlv_read8;
 918                         dev_priv->uncore.funcs.mmio_readw  = vlv_read16;
 919                         dev_priv->uncore.funcs.mmio_readl  = vlv_read32;
 920                         dev_priv->uncore.funcs.mmio_readq  = vlv_read64;
 921                 } else {
 922                         dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 923                         dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 924                         dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 925                         dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 926                 }
 927                 break;
 928         case 5:
 929                 dev_priv->uncore.funcs.mmio_writeb  = gen5_write8;
 930                 dev_priv->uncore.funcs.mmio_writew  = gen5_write16;
 931                 dev_priv->uncore.funcs.mmio_writel  = gen5_write32;
 932                 dev_priv->uncore.funcs.mmio_writeq  = gen5_write64;
 933                 dev_priv->uncore.funcs.mmio_readb  = gen5_read8;
 934                 dev_priv->uncore.funcs.mmio_readw  = gen5_read16;
 935                 dev_priv->uncore.funcs.mmio_readl  = gen5_read32;
 936                 dev_priv->uncore.funcs.mmio_readq  = gen5_read64;
 937                 break;
 938         case 4:
 939         case 3:
 940         case 2:
 941                 dev_priv->uncore.funcs.mmio_writeb  = gen4_write8;
 942                 dev_priv->uncore.funcs.mmio_writew  = gen4_write16;
 943                 dev_priv->uncore.funcs.mmio_writel  = gen4_write32;
 944                 dev_priv->uncore.funcs.mmio_writeq  = gen4_write64;
 945                 dev_priv->uncore.funcs.mmio_readb  = gen4_read8;
 946                 dev_priv->uncore.funcs.mmio_readw  = gen4_read16;
 947                 dev_priv->uncore.funcs.mmio_readl  = gen4_read32;
 948                 dev_priv->uncore.funcs.mmio_readq  = gen4_read64;
 949                 break;
 950         }
 951 }
 952
 953 void intel_uncore_fini(struct drm_device *dev)
 954 {
 955         /* Paranoia: make sure we have disabled everything before we exit. */
 956         intel_uncore_sanitize(dev);
 957         intel_uncore_forcewake_reset(dev, false);
 958 }
 959
 960 #define GEN_RANGE(l, h) GENMASK(h, l)
 961
 962 static const struct register_whitelist {
 963         uint64_t offset;
 964         uint32_t size;
 965         /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 966         uint32_t gen_bitmask;
 967 } whitelist[] = {
 968         { RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 9) },
 969 };
 970
 971 int i915_reg_read_ioctl(struct drm_device *dev,
 972                         void *data, struct drm_file *file)
 973 {
 974         struct drm_i915_private *dev_priv = dev->dev_private;
 975         struct drm_i915_reg_read *reg = data;
 976         struct register_whitelist const *entry = whitelist;
 977         int i, ret = 0;
 978
 979         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 980                 if (entry->offset == reg->offset &&
 981                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 982                         break;
 983         }
 984
 985         if (i == ARRAY_SIZE(whitelist))
 986                 return -EINVAL;
 987
 988         intel_runtime_pm_get(dev_priv);
 989
 990         switch (entry->size) {
 991         case 8:
 992                 reg->val = I915_READ64(reg->offset);
 993                 break;
 994         case 4:
 995                 reg->val = I915_READ(reg->offset);
 996                 break;
 997         case 2:
 998                 reg->val = I915_READ16(reg->offset);
 999                 break;
1000         case 1:
1001                 reg->val = I915_READ8(reg->offset);
1002                 break;
1003         default:
1004                 WARN_ON(1);
1005                 ret = -EINVAL;
1006                 goto out;
1007         }
1008
1009 out:
1010         intel_runtime_pm_put(dev_priv);
1011         return ret;
1012 }
1013
1014 int i915_get_reset_stats_ioctl(struct drm_device *dev,
1015                                void *data, struct drm_file *file)
1016 {
1017         struct drm_i915_private *dev_priv = dev->dev_private;
1018         struct drm_i915_reset_stats *args = data;
1019         struct i915_ctx_hang_stats *hs;
1020         struct intel_context *ctx;
1021         int ret;
1022
1023         if (args->flags || args->pad)
1024                 return -EINVAL;
1025
1026         if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
1027                 return -EPERM;
1028
1029         ret = mutex_lock_interruptible(&dev->struct_mutex);
1030         if (ret)
1031                 return ret;
1032
1033         ctx = i915_gem_context_get(file->driver_priv, args->ctx_id);
1034         if (IS_ERR(ctx)) {
1035                 mutex_unlock(&dev->struct_mutex);
1036                 return PTR_ERR(ctx);
1037         }
1038         hs = &ctx->hang_stats;
1039
1040         if (capable(CAP_SYS_ADMIN))
1041                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
1042         else
1043                 args->reset_count = 0;
1044
1045         args->batch_active = hs->batch_active;
1046         args->batch_pending = hs->batch_pending;
1047
1048         mutex_unlock(&dev->struct_mutex);
1049
1050         return 0;
1051 }
1052
1053 static int i965_reset_complete(struct drm_device *dev)
1054 {
1055         u8 gdrst;
1056         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
1057         return (gdrst & GRDOM_RESET_ENABLE) == 0;
1058 }
1059
1060 static int i965_do_reset(struct drm_device *dev)
1061 {
1062         int ret;
1063
1064         /* FIXME: i965g/gm need a display save/restore for gpu reset. */
1065         return -ENODEV;
1066
1067         /*
1068          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
1069          * well as the reset bit (GR/bit 0).  Setting the GR bit
1070          * triggers the reset; when done, the hardware will clear it.
1071          */
1072         pci_write_config_byte(dev->pdev, I965_GDRST,
1073                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1074         ret =  wait_for(i965_reset_complete(dev), 500);
1075         if (ret)
1076                 return ret;
1077
1078         pci_write_config_byte(dev->pdev, I965_GDRST,
1079                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1080
1081         ret =  wait_for(i965_reset_complete(dev), 500);
1082         if (ret)
1083                 return ret;
1084
1085         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1086
1087         return 0;
1088 }
1089
1090 static int g4x_do_reset(struct drm_device *dev)
1091 {
1092         struct drm_i915_private *dev_priv = dev->dev_private;
1093         int ret;
1094
1095         pci_write_config_byte(dev->pdev, I965_GDRST,
1096                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1097         ret =  wait_for(i965_reset_complete(dev), 500);
1098         if (ret)
1099                 return ret;
1100
1101         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1102         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1103         POSTING_READ(VDECCLK_GATE_D);
1104
1105         pci_write_config_byte(dev->pdev, I965_GDRST,
1106                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1107         ret =  wait_for(i965_reset_complete(dev), 500);
1108         if (ret)
1109                 return ret;
1110
1111         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1112         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1113         POSTING_READ(VDECCLK_GATE_D);
1114
1115         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1116
1117         return 0;
1118 }
1119
1120 static int ironlake_do_reset(struct drm_device *dev)
1121 {
1122         struct drm_i915_private *dev_priv = dev->dev_private;
1123         int ret;
1124
1125         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1126                    ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1127         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1128                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1129         if (ret)
1130                 return ret;
1131
1132         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1133                    ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1134         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1135                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1136         if (ret)
1137                 return ret;
1138
1139         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0);
1140
1141         return 0;
1142 }
1143
1144 static int gen6_do_reset(struct drm_device *dev)
1145 {
1146         struct drm_i915_private *dev_priv = dev->dev_private;
1147         int     ret;
1148
1149         /* Reset the chip */
1150
1151         /* GEN6_GDRST is not in the gt power well, no need to check
1152          * for fifo space for the write or forcewake the chip for
1153          * the read
1154          */
1155         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
1156
1157         /* Spin waiting for the device to ack the reset request */
1158         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
1159
1160         intel_uncore_forcewake_reset(dev, true);
1161
1162         return ret;
1163 }
1164
1165 int intel_gpu_reset(struct drm_device *dev)
1166 {
1167         if (INTEL_INFO(dev)->gen >= 6)
1168                 return gen6_do_reset(dev);
1169         else if (IS_GEN5(dev))
1170                 return ironlake_do_reset(dev);
1171         else if (IS_G4X(dev))
1172                 return g4x_do_reset(dev);
1173         else if (IS_GEN4(dev))
1174                 return i965_do_reset(dev);
1175         else
1176                 return -ENODEV;
1177 }
1178
1179 void intel_uncore_check_errors(struct drm_device *dev)
1180 {
1181         struct drm_i915_private *dev_priv = dev->dev_private;
1182
1183         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
1184             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
1185                 DRM_ERROR("Unclaimed register before interrupt\n");
1186                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1187         }
1188 }