drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43 static void
  44 assert_device_not_suspended(struct drm_i915_private *dev_priv)
  45 {
  46         WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
  47              "Device suspended\n");
  48 }
  49
  50 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  51 {
  52         u32 gt_thread_status_mask;
  53
  54         if (IS_HASWELL(dev_priv->dev))
  55                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  56         else
  57                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  58
  59         /* w/a for a sporadic read returning 0 by waiting for the GT
  60          * thread to wake up.
  61          */
  62         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  63                 DRM_ERROR("GT thread status wait timed out\n");
  64 }
  65
  66 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  67 {
  68         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  69         /* something from same cacheline, but !FORCEWAKE */
  70         __raw_posting_read(dev_priv, ECOBUS);
  71 }
  72
  73 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  74                                                         int fw_engine)
  75 {
  76         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  77                             FORCEWAKE_ACK_TIMEOUT_MS))
  78                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  79
  80         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  81         /* something from same cacheline, but !FORCEWAKE */
  82         __raw_posting_read(dev_priv, ECOBUS);
  83
  84         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  85                             FORCEWAKE_ACK_TIMEOUT_MS))
  86                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  87
  88         /* WaRsForcewakeWaitTC0:snb */
  89         __gen6_gt_wait_for_thread_c0(dev_priv);
  90 }
  91
  92 static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  93 {
  94         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  95         /* something from same cacheline, but !FORCEWAKE_MT */
  96         __raw_posting_read(dev_priv, ECOBUS);
  97 }
  98
  99 static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
 100                                                         int fw_engine)
 101 {
 102         u32 forcewake_ack;
 103
 104         if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))
 105                 forcewake_ack = FORCEWAKE_ACK_HSW;
 106         else
 107                 forcewake_ack = FORCEWAKE_MT_ACK;
 108
 109         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 110                             FORCEWAKE_ACK_TIMEOUT_MS))
 111                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 112
 113         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 114                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 115         /* something from same cacheline, but !FORCEWAKE_MT */
 116         __raw_posting_read(dev_priv, ECOBUS);
 117
 118         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 119                             FORCEWAKE_ACK_TIMEOUT_MS))
 120                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 121
 122         /* WaRsForcewakeWaitTC0:ivb,hsw */
 123         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 124                 __gen6_gt_wait_for_thread_c0(dev_priv);
 125 }
 126
 127 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 128 {
 129         u32 gtfifodbg;
 130
 131         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 132         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 133                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 134 }
 135
 136 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 137                                                         int fw_engine)
 138 {
 139         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 140         /* something from same cacheline, but !FORCEWAKE */
 141         __raw_posting_read(dev_priv, ECOBUS);
 142         gen6_gt_check_fifodbg(dev_priv);
 143 }
 144
 145 static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 146                                                         int fw_engine)
 147 {
 148         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 149                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 150         /* something from same cacheline, but !FORCEWAKE_MT */
 151         __raw_posting_read(dev_priv, ECOBUS);
 152
 153         if (IS_GEN7(dev_priv->dev))
 154                 gen6_gt_check_fifodbg(dev_priv);
 155 }
 156
 157 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 158 {
 159         int ret = 0;
 160
 161         /* On VLV, FIFO will be shared by both SW and HW.
 162          * So, we need to read the FREE_ENTRIES everytime */
 163         if (IS_VALLEYVIEW(dev_priv->dev))
 164                 dev_priv->uncore.fifo_count =
 165                         __raw_i915_read32(dev_priv, GTFIFOCTL) &
 166                                                 GT_FIFO_FREE_ENTRIES_MASK;
 167
 168         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 169                 int loop = 500;
 170                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 171                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 172                         udelay(10);
 173                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 174                 }
 175                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 176                         ++ret;
 177                 dev_priv->uncore.fifo_count = fifo;
 178         }
 179         dev_priv->uncore.fifo_count--;
 180
 181         return ret;
 182 }
 183
 184 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 185 {
 186         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 187                            _MASKED_BIT_DISABLE(0xffff));
 188         __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 189                            _MASKED_BIT_DISABLE(0xffff));
 190         /* something from same cacheline, but !FORCEWAKE_VLV */
 191         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 192 }
 193
 194 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 195                                                 int fw_engine)
 196 {
 197         /*
 198          * WaRsDontPollForAckOnClearingFWBits:vlv
 199          * Hardware clears ack bits lazily (only when all ack
 200          * bits become 0) so don't poll for individiual ack
 201          * bits to be clear here like on other platforms.
 202          */
 203
 204         /* Check for Render Engine */
 205         if (FORCEWAKE_RENDER & fw_engine) {
 206
 207                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 208                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 209
 210                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 211                                                 FORCEWAKE_ACK_VLV) &
 212                                                 FORCEWAKE_KERNEL),
 213                                         FORCEWAKE_ACK_TIMEOUT_MS))
 214                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 215         }
 216
 217         /* Check for Media Engine */
 218         if (FORCEWAKE_MEDIA & fw_engine) {
 219
 220                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 221                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 222
 223                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 224                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 225                                                 FORCEWAKE_KERNEL),
 226                                         FORCEWAKE_ACK_TIMEOUT_MS))
 227                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 228         }
 229
 230         /* WaRsForcewakeWaitTC0:vlv */
 231         if (!IS_CHERRYVIEW(dev_priv->dev))
 232                 __gen6_gt_wait_for_thread_c0(dev_priv);
 233 }
 234
 235 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 236                                         int fw_engine)
 237 {
 238
 239         /* Check for Render Engine */
 240         if (FORCEWAKE_RENDER & fw_engine)
 241                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 242                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 243
 244
 245         /* Check for Media Engine */
 246         if (FORCEWAKE_MEDIA & fw_engine)
 247                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 248                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 249
 250         /* something from same cacheline, but !FORCEWAKE_VLV */
 251         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 252         if (!IS_CHERRYVIEW(dev_priv->dev))
 253                 gen6_gt_check_fifodbg(dev_priv);
 254 }
 255
 256 static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 257 {
 258         unsigned long irqflags;
 259
 260         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 261
 262         if (fw_engine & FORCEWAKE_RENDER &&
 263             dev_priv->uncore.fw_rendercount++ != 0)
 264                 fw_engine &= ~FORCEWAKE_RENDER;
 265         if (fw_engine & FORCEWAKE_MEDIA &&
 266             dev_priv->uncore.fw_mediacount++ != 0)
 267                 fw_engine &= ~FORCEWAKE_MEDIA;
 268
 269         if (fw_engine)
 270                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
 271
 272         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 273 }
 274
 275 static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 276 {
 277         unsigned long irqflags;
 278
 279         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 280
 281         if (fw_engine & FORCEWAKE_RENDER) {
 282                 WARN_ON(!dev_priv->uncore.fw_rendercount);
 283                 if (--dev_priv->uncore.fw_rendercount != 0)
 284                         fw_engine &= ~FORCEWAKE_RENDER;
 285         }
 286
 287         if (fw_engine & FORCEWAKE_MEDIA) {
 288                 WARN_ON(!dev_priv->uncore.fw_mediacount);
 289                 if (--dev_priv->uncore.fw_mediacount != 0)
 290                         fw_engine &= ~FORCEWAKE_MEDIA;
 291         }
 292
 293         if (fw_engine)
 294                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
 295
 296         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 297 }
 298
 299 static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 300 {
 301         __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
 302                         _MASKED_BIT_DISABLE(0xffff));
 303
 304         __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9,
 305                         _MASKED_BIT_DISABLE(0xffff));
 306
 307         __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9,
 308                         _MASKED_BIT_DISABLE(0xffff));
 309 }
 310
 311 static void
 312 __gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 313 {
 314         /* Check for Render Engine */
 315         if (FORCEWAKE_RENDER & fw_engine) {
 316                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 317                                                 FORCEWAKE_ACK_RENDER_GEN9) &
 318                                                 FORCEWAKE_KERNEL) == 0,
 319                                         FORCEWAKE_ACK_TIMEOUT_MS))
 320                         DRM_ERROR("Timed out: Render forcewake old ack to clear.\n");
 321
 322                 __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
 323                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 324
 325                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 326                                                 FORCEWAKE_ACK_RENDER_GEN9) &
 327                                                 FORCEWAKE_KERNEL),
 328                                         FORCEWAKE_ACK_TIMEOUT_MS))
 329                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 330         }
 331
 332         /* Check for Media Engine */
 333         if (FORCEWAKE_MEDIA & fw_engine) {
 334                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 335                                                 FORCEWAKE_ACK_MEDIA_GEN9) &
 336                                                 FORCEWAKE_KERNEL) == 0,
 337                                         FORCEWAKE_ACK_TIMEOUT_MS))
 338                         DRM_ERROR("Timed out: Media forcewake old ack to clear.\n");
 339
 340                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9,
 341                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 342
 343                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 344                                                 FORCEWAKE_ACK_MEDIA_GEN9) &
 345                                                 FORCEWAKE_KERNEL),
 346                                         FORCEWAKE_ACK_TIMEOUT_MS))
 347                         DRM_ERROR("Timed out: waiting for Media to ack.\n");
 348         }
 349
 350         /* Check for Blitter Engine */
 351         if (FORCEWAKE_BLITTER & fw_engine) {
 352                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 353                                                 FORCEWAKE_ACK_BLITTER_GEN9) &
 354                                                 FORCEWAKE_KERNEL) == 0,
 355                                         FORCEWAKE_ACK_TIMEOUT_MS))
 356                         DRM_ERROR("Timed out: Blitter forcewake old ack to clear.\n");
 357
 358                 __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9,
 359                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 360
 361                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 362                                                 FORCEWAKE_ACK_BLITTER_GEN9) &
 363                                                 FORCEWAKE_KERNEL),
 364                                         FORCEWAKE_ACK_TIMEOUT_MS))
 365                         DRM_ERROR("Timed out: waiting for Blitter to ack.\n");
 366         }
 367 }
 368
 369 static void
 370 __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 371 {
 372         /* Check for Render Engine */
 373         if (FORCEWAKE_RENDER & fw_engine)
 374                 __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
 375                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 376
 377         /* Check for Media Engine */
 378         if (FORCEWAKE_MEDIA & fw_engine)
 379                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9,
 380                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 381
 382         /* Check for Blitter Engine */
 383         if (FORCEWAKE_BLITTER & fw_engine)
 384                 __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9,
 385                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 386 }
 387
 388 static void
 389 gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 390 {
 391         unsigned long irqflags;
 392
 393         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 394
 395         if (FORCEWAKE_RENDER & fw_engine) {
 396                 if (dev_priv->uncore.fw_rendercount++ == 0)
 397                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 398                                                         FORCEWAKE_RENDER);
 399         }
 400
 401         if (FORCEWAKE_MEDIA & fw_engine) {
 402                 if (dev_priv->uncore.fw_mediacount++ == 0)
 403                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 404                                                         FORCEWAKE_MEDIA);
 405         }
 406
 407         if (FORCEWAKE_BLITTER & fw_engine) {
 408                 if (dev_priv->uncore.fw_blittercount++ == 0)
 409                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 410                                                         FORCEWAKE_BLITTER);
 411         }
 412
 413         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 414 }
 415
 416 static void
 417 gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 418 {
 419         unsigned long irqflags;
 420
 421         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 422
 423         if (FORCEWAKE_RENDER & fw_engine) {
 424                 WARN_ON(dev_priv->uncore.fw_rendercount == 0);
 425                 if (--dev_priv->uncore.fw_rendercount == 0)
 426                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 427                                                         FORCEWAKE_RENDER);
 428         }
 429
 430         if (FORCEWAKE_MEDIA & fw_engine) {
 431                 WARN_ON(dev_priv->uncore.fw_mediacount == 0);
 432                 if (--dev_priv->uncore.fw_mediacount == 0)
 433                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 434                                                         FORCEWAKE_MEDIA);
 435         }
 436
 437         if (FORCEWAKE_BLITTER & fw_engine) {
 438                 WARN_ON(dev_priv->uncore.fw_blittercount == 0);
 439                 if (--dev_priv->uncore.fw_blittercount == 0)
 440                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 441                                                         FORCEWAKE_BLITTER);
 442         }
 443
 444         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 445 }
 446
 447 static void gen6_force_wake_timer(unsigned long arg)
 448 {
 449         struct drm_i915_private *dev_priv = (void *)arg;
 450         unsigned long irqflags;
 451
 452         assert_device_not_suspended(dev_priv);
 453
 454         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 455         WARN_ON(!dev_priv->uncore.forcewake_count);
 456
 457         if (--dev_priv->uncore.forcewake_count == 0)
 458                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 459         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 460
 461         intel_runtime_pm_put(dev_priv);
 462 }
 463
 464 void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 465 {
 466         struct drm_i915_private *dev_priv = dev->dev_private;
 467         unsigned long irqflags;
 468
 469         if (del_timer_sync(&dev_priv->uncore.force_wake_timer))
 470                 gen6_force_wake_timer((unsigned long)dev_priv);
 471
 472         /* Hold uncore.lock across reset to prevent any register access
 473          * with forcewake not set correctly
 474          */
 475         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 476
 477         if (IS_VALLEYVIEW(dev))
 478                 vlv_force_wake_reset(dev_priv);
 479         else if (IS_GEN6(dev) || IS_GEN7(dev))
 480                 __gen6_gt_force_wake_reset(dev_priv);
 481
 482         if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
 483                 __gen7_gt_force_wake_mt_reset(dev_priv);
 484
 485         if (IS_GEN9(dev))
 486                 __gen9_gt_force_wake_mt_reset(dev_priv);
 487
 488         if (restore) { /* If reset with a user forcewake, try to restore */
 489                 unsigned fw = 0;
 490
 491                 if (IS_VALLEYVIEW(dev)) {
 492                         if (dev_priv->uncore.fw_rendercount)
 493                                 fw |= FORCEWAKE_RENDER;
 494
 495                         if (dev_priv->uncore.fw_mediacount)
 496                                 fw |= FORCEWAKE_MEDIA;
 497                 } else if (IS_GEN9(dev)) {
 498                         if (dev_priv->uncore.fw_rendercount)
 499                                 fw |= FORCEWAKE_RENDER;
 500
 501                         if (dev_priv->uncore.fw_mediacount)
 502                                 fw |= FORCEWAKE_MEDIA;
 503
 504                         if (dev_priv->uncore.fw_blittercount)
 505                                 fw |= FORCEWAKE_BLITTER;
 506                 } else {
 507                         if (dev_priv->uncore.forcewake_count)
 508                                 fw = FORCEWAKE_ALL;
 509                 }
 510
 511                 if (fw)
 512                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 513
 514                 if (IS_GEN6(dev) || IS_GEN7(dev))
 515                         dev_priv->uncore.fifo_count =
 516                                 __raw_i915_read32(dev_priv, GTFIFOCTL) &
 517                                 GT_FIFO_FREE_ENTRIES_MASK;
 518         }
 519
 520         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 521 }
 522
 523 static void __intel_uncore_early_sanitize(struct drm_device *dev,
 524                                           bool restore_forcewake)
 525 {
 526         struct drm_i915_private *dev_priv = dev->dev_private;
 527
 528         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 529                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 530
 531         if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) &&
 532             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 533                 /* The docs do not explain exactly how the calculation can be
 534                  * made. It is somewhat guessable, but for now, it's always
 535                  * 128MB.
 536                  * NB: We can't write IDICR yet because we do not have gt funcs
 537                  * set up */
 538                 dev_priv->ellc_size = 128;
 539                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 540         }
 541
 542         /* clear out old GT FIFO errors */
 543         if (IS_GEN6(dev) || IS_GEN7(dev))
 544                 __raw_i915_write32(dev_priv, GTFIFODBG,
 545                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 546
 547         intel_uncore_forcewake_reset(dev, restore_forcewake);
 548 }
 549
 550 void intel_uncore_early_sanitize(struct drm_device *dev, bool restore_forcewake)
 551 {
 552         __intel_uncore_early_sanitize(dev, restore_forcewake);
 553         i915_check_and_clear_faults(dev);
 554 }
 555
 556 void intel_uncore_sanitize(struct drm_device *dev)
 557 {
 558         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 559         intel_disable_gt_powersave(dev);
 560 }
 561
 562 /*
 563  * Generally this is called implicitly by the register read function. However,
 564  * if some sequence requires the GT to not power down then this function should
 565  * be called at the beginning of the sequence followed by a call to
 566  * gen6_gt_force_wake_put() at the end of the sequence.
 567  */
 568 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 569 {
 570         unsigned long irqflags;
 571
 572         if (!dev_priv->uncore.funcs.force_wake_get)
 573                 return;
 574
 575         intel_runtime_pm_get(dev_priv);
 576
 577         /* Redirect to Gen9 specific routine */
 578         if (IS_GEN9(dev_priv->dev))
 579                 return gen9_force_wake_get(dev_priv, fw_engine);
 580
 581         /* Redirect to VLV specific routine */
 582         if (IS_VALLEYVIEW(dev_priv->dev))
 583                 return vlv_force_wake_get(dev_priv, fw_engine);
 584
 585         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 586         if (dev_priv->uncore.forcewake_count++ == 0)
 587                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 588         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 589 }
 590
 591 /*
 592  * see gen6_gt_force_wake_get()
 593  */
 594 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 595 {
 596         unsigned long irqflags;
 597         bool delayed = false;
 598
 599         if (!dev_priv->uncore.funcs.force_wake_put)
 600                 return;
 601
 602         /* Redirect to Gen9 specific routine */
 603         if (IS_GEN9(dev_priv->dev)) {
 604                 gen9_force_wake_put(dev_priv, fw_engine);
 605                 goto out;
 606         }
 607
 608         /* Redirect to VLV specific routine */
 609         if (IS_VALLEYVIEW(dev_priv->dev)) {
 610                 vlv_force_wake_put(dev_priv, fw_engine);
 611                 goto out;
 612         }
 613
 614
 615         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 616         WARN_ON(!dev_priv->uncore.forcewake_count);
 617
 618         if (--dev_priv->uncore.forcewake_count == 0) {
 619                 dev_priv->uncore.forcewake_count++;
 620                 delayed = true;
 621                 mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
 622                                  jiffies + 1);
 623         }
 624         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 625
 626 out:
 627         if (!delayed)
 628                 intel_runtime_pm_put(dev_priv);
 629 }
 630
 631 void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
 632 {
 633         if (!dev_priv->uncore.funcs.force_wake_get)
 634                 return;
 635
 636         WARN_ON(dev_priv->uncore.forcewake_count > 0);
 637 }
 638
 639 /* We give fast paths for the really cool registers */
 640 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 641          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 642
 643 #define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
 644
 645 #define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \
 646         (REG_RANGE((reg), 0x2000, 0x4000) || \
 647          REG_RANGE((reg), 0x5000, 0x8000) || \
 648          REG_RANGE((reg), 0xB000, 0x12000) || \
 649          REG_RANGE((reg), 0x2E000, 0x30000))
 650
 651 #define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg) \
 652         (REG_RANGE((reg), 0x12000, 0x14000) || \
 653          REG_RANGE((reg), 0x22000, 0x24000) || \
 654          REG_RANGE((reg), 0x30000, 0x40000))
 655
 656 #define FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg) \
 657         (REG_RANGE((reg), 0x2000, 0x4000) || \
 658          REG_RANGE((reg), 0x5000, 0x8000) || \
 659          REG_RANGE((reg), 0x8300, 0x8500) || \
 660          REG_RANGE((reg), 0xB000, 0xC000) || \
 661          REG_RANGE((reg), 0xE000, 0xE800))
 662
 663 #define FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg) \
 664         (REG_RANGE((reg), 0x8800, 0x8900) || \
 665          REG_RANGE((reg), 0xD000, 0xD800) || \
 666          REG_RANGE((reg), 0x12000, 0x14000) || \
 667          REG_RANGE((reg), 0x1A000, 0x1C000) || \
 668          REG_RANGE((reg), 0x1E800, 0x1EA00) || \
 669          REG_RANGE((reg), 0x30000, 0x40000))
 670
 671 #define FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg) \
 672         (REG_RANGE((reg), 0x4000, 0x5000) || \
 673          REG_RANGE((reg), 0x8000, 0x8300) || \
 674          REG_RANGE((reg), 0x8500, 0x8600) || \
 675          REG_RANGE((reg), 0x9000, 0xB000) || \
 676          REG_RANGE((reg), 0xC000, 0xC800) || \
 677          REG_RANGE((reg), 0xF000, 0x10000) || \
 678          REG_RANGE((reg), 0x14000, 0x14400) || \
 679          REG_RANGE((reg), 0x22000, 0x24000))
 680
 681 static void
 682 ilk_dummy_write(struct drm_i915_private *dev_priv)
 683 {
 684         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 685          * the chip from rc6 before touching it for real. MI_MODE is masked,
 686          * hence harmless to write 0 into. */
 687         __raw_i915_write32(dev_priv, MI_MODE, 0);
 688 }
 689
 690 static void
 691 hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read,
 692                         bool before)
 693 {
 694         const char *op = read ? "reading" : "writing to";
 695         const char *when = before ? "before" : "after";
 696
 697         if (!i915.mmio_debug)
 698                 return;
 699
 700         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 701                 WARN(1, "Unclaimed register detected %s %s register 0x%x\n",
 702                      when, op, reg);
 703                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 704         }
 705 }
 706
 707 static void
 708 hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv)
 709 {
 710         if (i915.mmio_debug)
 711                 return;
 712
 713         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 714                 DRM_ERROR("Unclaimed register detected. Please use the i915.mmio_debug=1 to debug this problem.");
 715                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 716         }
 717 }
 718
 719 #define REG_READ_HEADER(x) \
 720         unsigned long irqflags; \
 721         u##x val = 0; \
 722         assert_device_not_suspended(dev_priv); \
 723         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 724
 725 #define REG_READ_FOOTER \
 726         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 727         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 728         return val
 729
 730 #define __gen4_read(x) \
 731 static u##x \
 732 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 733         REG_READ_HEADER(x); \
 734         val = __raw_i915_read##x(dev_priv, reg); \
 735         REG_READ_FOOTER; \
 736 }
 737
 738 #define __gen5_read(x) \
 739 static u##x \
 740 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 741         REG_READ_HEADER(x); \
 742         ilk_dummy_write(dev_priv); \
 743         val = __raw_i915_read##x(dev_priv, reg); \
 744         REG_READ_FOOTER; \
 745 }
 746
 747 #define __gen6_read(x) \
 748 static u##x \
 749 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 750         REG_READ_HEADER(x); \
 751         hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
 752         if (dev_priv->uncore.forcewake_count == 0 && \
 753             NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 754                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 755                                                       FORCEWAKE_ALL); \
 756                 val = __raw_i915_read##x(dev_priv, reg); \
 757                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 758                                                       FORCEWAKE_ALL); \
 759         } else { \
 760                 val = __raw_i915_read##x(dev_priv, reg); \
 761         } \
 762         hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
 763         REG_READ_FOOTER; \
 764 }
 765
 766 #define __vlv_read(x) \
 767 static u##x \
 768 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 769         unsigned fwengine = 0; \
 770         REG_READ_HEADER(x); \
 771         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
 772                 if (dev_priv->uncore.fw_rendercount == 0) \
 773                         fwengine = FORCEWAKE_RENDER; \
 774         } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
 775                 if (dev_priv->uncore.fw_mediacount == 0) \
 776                         fwengine = FORCEWAKE_MEDIA; \
 777         }  \
 778         if (fwengine) \
 779                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 780         val = __raw_i915_read##x(dev_priv, reg); \
 781         if (fwengine) \
 782                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 783         REG_READ_FOOTER; \
 784 }
 785
 786 #define __chv_read(x) \
 787 static u##x \
 788 chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 789         unsigned fwengine = 0; \
 790         REG_READ_HEADER(x); \
 791         if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 792                 if (dev_priv->uncore.fw_rendercount == 0) \
 793                         fwengine = FORCEWAKE_RENDER; \
 794         } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 795                 if (dev_priv->uncore.fw_mediacount == 0) \
 796                         fwengine = FORCEWAKE_MEDIA; \
 797         } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 798                 if (dev_priv->uncore.fw_rendercount == 0) \
 799                         fwengine |= FORCEWAKE_RENDER; \
 800                 if (dev_priv->uncore.fw_mediacount == 0) \
 801                         fwengine |= FORCEWAKE_MEDIA; \
 802         } \
 803         if (fwengine) \
 804                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 805         val = __raw_i915_read##x(dev_priv, reg); \
 806         if (fwengine) \
 807                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 808         REG_READ_FOOTER; \
 809 }
 810
 811 __chv_read(8)
 812 __chv_read(16)
 813 __chv_read(32)
 814 __chv_read(64)
 815 __vlv_read(8)
 816 __vlv_read(16)
 817 __vlv_read(32)
 818 __vlv_read(64)
 819 __gen6_read(8)
 820 __gen6_read(16)
 821 __gen6_read(32)
 822 __gen6_read(64)
 823 __gen5_read(8)
 824 __gen5_read(16)
 825 __gen5_read(32)
 826 __gen5_read(64)
 827 __gen4_read(8)
 828 __gen4_read(16)
 829 __gen4_read(32)
 830 __gen4_read(64)
 831
 832 #undef __chv_read
 833 #undef __vlv_read
 834 #undef __gen6_read
 835 #undef __gen5_read
 836 #undef __gen4_read
 837 #undef REG_READ_FOOTER
 838 #undef REG_READ_HEADER
 839
 840 #define REG_WRITE_HEADER \
 841         unsigned long irqflags; \
 842         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 843         assert_device_not_suspended(dev_priv); \
 844         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 845
 846 #define REG_WRITE_FOOTER \
 847         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
 848
 849 #define __gen4_write(x) \
 850 static void \
 851 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 852         REG_WRITE_HEADER; \
 853         __raw_i915_write##x(dev_priv, reg, val); \
 854         REG_WRITE_FOOTER; \
 855 }
 856
 857 #define __gen5_write(x) \
 858 static void \
 859 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 860         REG_WRITE_HEADER; \
 861         ilk_dummy_write(dev_priv); \
 862         __raw_i915_write##x(dev_priv, reg, val); \
 863         REG_WRITE_FOOTER; \
 864 }
 865
 866 #define __gen6_write(x) \
 867 static void \
 868 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 869         u32 __fifo_ret = 0; \
 870         REG_WRITE_HEADER; \
 871         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 872                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 873         } \
 874         __raw_i915_write##x(dev_priv, reg, val); \
 875         if (unlikely(__fifo_ret)) { \
 876                 gen6_gt_check_fifodbg(dev_priv); \
 877         } \
 878         REG_WRITE_FOOTER; \
 879 }
 880
 881 #define __hsw_write(x) \
 882 static void \
 883 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 884         u32 __fifo_ret = 0; \
 885         REG_WRITE_HEADER; \
 886         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 887                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 888         } \
 889         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 890         __raw_i915_write##x(dev_priv, reg, val); \
 891         if (unlikely(__fifo_ret)) { \
 892                 gen6_gt_check_fifodbg(dev_priv); \
 893         } \
 894         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 895         hsw_unclaimed_reg_detect(dev_priv); \
 896         REG_WRITE_FOOTER; \
 897 }
 898
 899 static const u32 gen8_shadowed_regs[] = {
 900         FORCEWAKE_MT,
 901         GEN6_RPNSWREQ,
 902         GEN6_RC_VIDEO_FREQ,
 903         RING_TAIL(RENDER_RING_BASE),
 904         RING_TAIL(GEN6_BSD_RING_BASE),
 905         RING_TAIL(VEBOX_RING_BASE),
 906         RING_TAIL(BLT_RING_BASE),
 907         /* TODO: Other registers are not yet used */
 908 };
 909
 910 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 911 {
 912         int i;
 913         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 914                 if (reg == gen8_shadowed_regs[i])
 915                         return true;
 916
 917         return false;
 918 }
 919
 920 #define __gen8_write(x) \
 921 static void \
 922 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 923         REG_WRITE_HEADER; \
 924         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
 925         if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
 926                 if (dev_priv->uncore.forcewake_count == 0) \
 927                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 928                                                               FORCEWAKE_ALL); \
 929                 __raw_i915_write##x(dev_priv, reg, val); \
 930                 if (dev_priv->uncore.forcewake_count == 0) \
 931                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 932                                                               FORCEWAKE_ALL); \
 933         } else { \
 934                 __raw_i915_write##x(dev_priv, reg, val); \
 935         } \
 936         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
 937         hsw_unclaimed_reg_detect(dev_priv); \
 938         REG_WRITE_FOOTER; \
 939 }
 940
 941 #define __chv_write(x) \
 942 static void \
 943 chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 944         unsigned fwengine = 0; \
 945         bool shadowed = is_gen8_shadowed(dev_priv, reg); \
 946         REG_WRITE_HEADER; \
 947         if (!shadowed) { \
 948                 if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
 949                         if (dev_priv->uncore.fw_rendercount == 0) \
 950                                 fwengine = FORCEWAKE_RENDER; \
 951                 } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
 952                         if (dev_priv->uncore.fw_mediacount == 0) \
 953                                 fwengine = FORCEWAKE_MEDIA; \
 954                 } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
 955                         if (dev_priv->uncore.fw_rendercount == 0) \
 956                                 fwengine |= FORCEWAKE_RENDER; \
 957                         if (dev_priv->uncore.fw_mediacount == 0) \
 958                                 fwengine |= FORCEWAKE_MEDIA; \
 959                 } \
 960         } \
 961         if (fwengine) \
 962                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
 963         __raw_i915_write##x(dev_priv, reg, val); \
 964         if (fwengine) \
 965                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
 966         REG_WRITE_FOOTER; \
 967 }
 968
 969 __chv_write(8)
 970 __chv_write(16)
 971 __chv_write(32)
 972 __chv_write(64)
 973 __gen8_write(8)
 974 __gen8_write(16)
 975 __gen8_write(32)
 976 __gen8_write(64)
 977 __hsw_write(8)
 978 __hsw_write(16)
 979 __hsw_write(32)
 980 __hsw_write(64)
 981 __gen6_write(8)
 982 __gen6_write(16)
 983 __gen6_write(32)
 984 __gen6_write(64)
 985 __gen5_write(8)
 986 __gen5_write(16)
 987 __gen5_write(32)
 988 __gen5_write(64)
 989 __gen4_write(8)
 990 __gen4_write(16)
 991 __gen4_write(32)
 992 __gen4_write(64)
 993
 994 #undef __chv_write
 995 #undef __gen8_write
 996 #undef __hsw_write
 997 #undef __gen6_write
 998 #undef __gen5_write
 999 #undef __gen4_write
1000 #undef REG_WRITE_FOOTER
1001 #undef REG_WRITE_HEADER
1002
1003 #define ASSIGN_WRITE_MMIO_VFUNCS(x) \
1004 do { \
1005         dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
1006         dev_priv->uncore.funcs.mmio_writew = x##_write16; \
1007         dev_priv->uncore.funcs.mmio_writel = x##_write32; \
1008         dev_priv->uncore.funcs.mmio_writeq = x##_write64; \
1009 } while (0)
1010
1011 #define ASSIGN_READ_MMIO_VFUNCS(x) \
1012 do { \
1013         dev_priv->uncore.funcs.mmio_readb = x##_read8; \
1014         dev_priv->uncore.funcs.mmio_readw = x##_read16; \
1015         dev_priv->uncore.funcs.mmio_readl = x##_read32; \
1016         dev_priv->uncore.funcs.mmio_readq = x##_read64; \
1017 } while (0)
1018
1019 void intel_uncore_init(struct drm_device *dev)
1020 {
1021         struct drm_i915_private *dev_priv = dev->dev_private;
1022
1023         setup_timer(&dev_priv->uncore.force_wake_timer,
1024                     gen6_force_wake_timer, (unsigned long)dev_priv);
1025
1026         __intel_uncore_early_sanitize(dev, false);
1027
1028         if (IS_GEN9(dev)) {
1029                 dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get;
1030                 dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put;
1031         } else if (IS_VALLEYVIEW(dev)) {
1032                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
1033                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
1034         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
1035                 dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
1036                 dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
1037         } else if (IS_IVYBRIDGE(dev)) {
1038                 u32 ecobus;
1039
1040                 /* IVB configs may use multi-threaded forcewake */
1041
1042                 /* A small trick here - if the bios hasn't configured
1043                  * MT forcewake, and if the device is in RC6, then
1044                  * force_wake_mt_get will not wake the device and the
1045                  * ECOBUS read will return zero. Which will be
1046                  * (correctly) interpreted by the test below as MT
1047                  * forcewake being disabled.
1048                  */
1049                 mutex_lock(&dev->struct_mutex);
1050                 __gen7_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
1051                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1052                 __gen7_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
1053                 mutex_unlock(&dev->struct_mutex);
1054
1055                 if (ecobus & FORCEWAKE_MT_ENABLE) {
1056                         dev_priv->uncore.funcs.force_wake_get =
1057                                 __gen7_gt_force_wake_mt_get;
1058                         dev_priv->uncore.funcs.force_wake_put =
1059                                 __gen7_gt_force_wake_mt_put;
1060                 } else {
1061                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
1062                         DRM_INFO("when using vblank-synced partial screen updates.\n");
1063                         dev_priv->uncore.funcs.force_wake_get =
1064                                 __gen6_gt_force_wake_get;
1065                         dev_priv->uncore.funcs.force_wake_put =
1066                                 __gen6_gt_force_wake_put;
1067                 }
1068         } else if (IS_GEN6(dev)) {
1069                 dev_priv->uncore.funcs.force_wake_get =
1070                         __gen6_gt_force_wake_get;
1071                 dev_priv->uncore.funcs.force_wake_put =
1072                         __gen6_gt_force_wake_put;
1073         }
1074
1075         switch (INTEL_INFO(dev)->gen) {
1076         default:
1077                 if (IS_CHERRYVIEW(dev)) {
1078                         ASSIGN_WRITE_MMIO_VFUNCS(chv);
1079                         ASSIGN_READ_MMIO_VFUNCS(chv);
1080
1081                 } else {
1082                         ASSIGN_WRITE_MMIO_VFUNCS(gen8);
1083                         ASSIGN_READ_MMIO_VFUNCS(gen6);
1084                 }
1085                 break;
1086         case 7:
1087         case 6:
1088                 if (IS_HASWELL(dev)) {
1089                         ASSIGN_WRITE_MMIO_VFUNCS(hsw);
1090                 } else {
1091                         ASSIGN_WRITE_MMIO_VFUNCS(gen6);
1092                 }
1093
1094                 if (IS_VALLEYVIEW(dev)) {
1095                         ASSIGN_READ_MMIO_VFUNCS(vlv);
1096                 } else {
1097                         ASSIGN_READ_MMIO_VFUNCS(gen6);
1098                 }
1099                 break;
1100         case 5:
1101                 ASSIGN_WRITE_MMIO_VFUNCS(gen5);
1102                 ASSIGN_READ_MMIO_VFUNCS(gen5);
1103                 break;
1104         case 4:
1105         case 3:
1106         case 2:
1107                 ASSIGN_WRITE_MMIO_VFUNCS(gen4);
1108                 ASSIGN_READ_MMIO_VFUNCS(gen4);
1109                 break;
1110         }
1111
1112         i915_check_and_clear_faults(dev);
1113 }
1114 #undef ASSIGN_WRITE_MMIO_VFUNCS
1115 #undef ASSIGN_READ_MMIO_VFUNCS
1116
1117 void intel_uncore_fini(struct drm_device *dev)
1118 {
1119         /* Paranoia: make sure we have disabled everything before we exit. */
1120         intel_uncore_sanitize(dev);
1121         intel_uncore_forcewake_reset(dev, false);
1122 }
1123
1124 #define GEN_RANGE(l, h) GENMASK(h, l)
1125
1126 static const struct register_whitelist {
1127         uint64_t offset;
1128         uint32_t size;
1129         /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
1130         uint32_t gen_bitmask;
1131 } whitelist[] = {
1132         { RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 9) },
1133 };
1134
1135 int i915_reg_read_ioctl(struct drm_device *dev,
1136                         void *data, struct drm_file *file)
1137 {
1138         struct drm_i915_private *dev_priv = dev->dev_private;
1139         struct drm_i915_reg_read *reg = data;
1140         struct register_whitelist const *entry = whitelist;
1141         int i, ret = 0;
1142
1143         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
1144                 if (entry->offset == reg->offset &&
1145                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
1146                         break;
1147         }
1148
1149         if (i == ARRAY_SIZE(whitelist))
1150                 return -EINVAL;
1151
1152         intel_runtime_pm_get(dev_priv);
1153
1154         switch (entry->size) {
1155         case 8:
1156                 reg->val = I915_READ64(reg->offset);
1157                 break;
1158         case 4:
1159                 reg->val = I915_READ(reg->offset);
1160                 break;
1161         case 2:
1162                 reg->val = I915_READ16(reg->offset);
1163                 break;
1164         case 1:
1165                 reg->val = I915_READ8(reg->offset);
1166                 break;
1167         default:
1168                 WARN_ON(1);
1169                 ret = -EINVAL;
1170                 goto out;
1171         }
1172
1173 out:
1174         intel_runtime_pm_put(dev_priv);
1175         return ret;
1176 }
1177
1178 int i915_get_reset_stats_ioctl(struct drm_device *dev,
1179                                void *data, struct drm_file *file)
1180 {
1181         struct drm_i915_private *dev_priv = dev->dev_private;
1182         struct drm_i915_reset_stats *args = data;
1183         struct i915_ctx_hang_stats *hs;
1184         struct intel_context *ctx;
1185         int ret;
1186
1187         if (args->flags || args->pad)
1188                 return -EINVAL;
1189
1190         if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
1191                 return -EPERM;
1192
1193         ret = mutex_lock_interruptible(&dev->struct_mutex);
1194         if (ret)
1195                 return ret;
1196
1197         ctx = i915_gem_context_get(file->driver_priv, args->ctx_id);
1198         if (IS_ERR(ctx)) {
1199                 mutex_unlock(&dev->struct_mutex);
1200                 return PTR_ERR(ctx);
1201         }
1202         hs = &ctx->hang_stats;
1203
1204         if (capable(CAP_SYS_ADMIN))
1205                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
1206         else
1207                 args->reset_count = 0;
1208
1209         args->batch_active = hs->batch_active;
1210         args->batch_pending = hs->batch_pending;
1211
1212         mutex_unlock(&dev->struct_mutex);
1213
1214         return 0;
1215 }
1216
1217 static int i965_reset_complete(struct drm_device *dev)
1218 {
1219         u8 gdrst;
1220         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
1221         return (gdrst & GRDOM_RESET_ENABLE) == 0;
1222 }
1223
1224 static int i965_do_reset(struct drm_device *dev)
1225 {
1226         int ret;
1227
1228         /* FIXME: i965g/gm need a display save/restore for gpu reset. */
1229         return -ENODEV;
1230
1231         /*
1232          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
1233          * well as the reset bit (GR/bit 0).  Setting the GR bit
1234          * triggers the reset; when done, the hardware will clear it.
1235          */
1236         pci_write_config_byte(dev->pdev, I965_GDRST,
1237                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1238         ret =  wait_for(i965_reset_complete(dev), 500);
1239         if (ret)
1240                 return ret;
1241
1242         pci_write_config_byte(dev->pdev, I965_GDRST,
1243                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1244
1245         ret =  wait_for(i965_reset_complete(dev), 500);
1246         if (ret)
1247                 return ret;
1248
1249         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1250
1251         return 0;
1252 }
1253
1254 static int g4x_do_reset(struct drm_device *dev)
1255 {
1256         struct drm_i915_private *dev_priv = dev->dev_private;
1257         int ret;
1258
1259         pci_write_config_byte(dev->pdev, I965_GDRST,
1260                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1261         ret =  wait_for(i965_reset_complete(dev), 500);
1262         if (ret)
1263                 return ret;
1264
1265         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1266         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1267         POSTING_READ(VDECCLK_GATE_D);
1268
1269         pci_write_config_byte(dev->pdev, I965_GDRST,
1270                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1271         ret =  wait_for(i965_reset_complete(dev), 500);
1272         if (ret)
1273                 return ret;
1274
1275         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1276         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1277         POSTING_READ(VDECCLK_GATE_D);
1278
1279         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
1280
1281         return 0;
1282 }
1283
1284 static int ironlake_do_reset(struct drm_device *dev)
1285 {
1286         struct drm_i915_private *dev_priv = dev->dev_private;
1287         int ret;
1288
1289         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1290                    ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1291         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1292                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1293         if (ret)
1294                 return ret;
1295
1296         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
1297                    ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1298         ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) &
1299                         ILK_GRDOM_RESET_ENABLE) == 0, 500);
1300         if (ret)
1301                 return ret;
1302
1303         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0);
1304
1305         return 0;
1306 }
1307
1308 static int gen6_do_reset(struct drm_device *dev)
1309 {
1310         struct drm_i915_private *dev_priv = dev->dev_private;
1311         int     ret;
1312
1313         /* Reset the chip */
1314
1315         /* GEN6_GDRST is not in the gt power well, no need to check
1316          * for fifo space for the write or forcewake the chip for
1317          * the read
1318          */
1319         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
1320
1321         /* Spin waiting for the device to ack the reset request */
1322         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
1323
1324         intel_uncore_forcewake_reset(dev, true);
1325
1326         return ret;
1327 }
1328
1329 int intel_gpu_reset(struct drm_device *dev)
1330 {
1331         if (INTEL_INFO(dev)->gen >= 6)
1332                 return gen6_do_reset(dev);
1333         else if (IS_GEN5(dev))
1334                 return ironlake_do_reset(dev);
1335         else if (IS_G4X(dev))
1336                 return g4x_do_reset(dev);
1337         else if (IS_GEN4(dev))
1338                 return i965_do_reset(dev);
1339         else
1340                 return -ENODEV;
1341 }
1342
1343 void intel_uncore_check_errors(struct drm_device *dev)
1344 {
1345         struct drm_i915_private *dev_priv = dev->dev_private;
1346
1347         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
1348             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
1349                 DRM_ERROR("Unclaimed register before interrupt\n");
1350                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1351         }
1352 }