+static void i915_gem_fini(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+
+ /*
+ * Neither the BIOS, ourselves or any other kernel
+ * expects the system to be in execlists mode on startup,
+ * so we need to reset the GPU back to legacy mode. And the only
+ * known way to disable logical contexts is through a GPU reset.
+ *
+ * So in order to leave the system in a known default configuration,
+ * always reset the GPU upon unload. Afterwards we then clean up the
+ * GEM state tracking, flushing off the requests and leaving the
+ * system in a known idle state.
+ *
+ * Note that is of the upmost importance that the GPU is idle and
+ * all stray writes are flushed *before* we dismantle the backing
+ * storage for the pinned objects.
+ *
+ * However, since we are uncertain that reseting the GPU on older
+ * machines is a good idea, we don't - just in case it leaves the
+ * machine in an unusable condition.
+ */
+ if (HAS_HW_CONTEXTS(dev)) {
+ int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
+ WARN_ON(reset && reset != -ENODEV);
+ }
+
+ mutex_lock(&dev->struct_mutex);
+ i915_gem_reset(dev);
+ i915_gem_cleanup_engines(dev);
+ i915_gem_context_fini(dev);
+ mutex_unlock(&dev->struct_mutex);
+
+ WARN_ON(!list_empty(&to_i915(dev)->context_list));
+}
+
+static int i915_load_modeset_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int ret;
+
+ if (i915_inject_load_failure())
+ return -ENODEV;
+
+ ret = intel_bios_init(dev_priv);
+ if (ret)
+ DRM_INFO("failed to find VBIOS tables\n");
+
+ /* If we have > 1 VGA cards, then we need to arbitrate access
+ * to the common VGA resources.
+ *
+ * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
+ * then we do not take part in VGA arbitration and the
+ * vga_client_register() fails with -ENODEV.
+ */
+ ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
+ if (ret && ret != -ENODEV)
+ goto out;
+
+ intel_register_dsm_handler();
+
+ ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
+ if (ret)
+ goto cleanup_vga_client;
+
+ /* must happen before intel_power_domains_init_hw() on VLV/CHV */
+ intel_update_rawclk(dev_priv);
+
+ intel_power_domains_init_hw(dev_priv, false);
+
+ intel_csr_ucode_init(dev_priv);
+
+ ret = intel_irq_install(dev_priv);
+ if (ret)
+ goto cleanup_csr;
+
+ intel_setup_gmbus(dev);
+
+ /* Important: The output setup functions called by modeset_init need
+ * working irqs for e.g. gmbus and dp aux transfers. */
+ intel_modeset_init(dev);
+
+ intel_guc_init(dev);
+
+ ret = i915_gem_init(dev);
+ if (ret)
+ goto cleanup_irq;
+
+ intel_modeset_gem_init(dev);
+
+ if (INTEL_INFO(dev)->num_pipes == 0)
+ return 0;
+
+ ret = intel_fbdev_init(dev);
+ if (ret)
+ goto cleanup_gem;
+
+ /* Only enable hotplug handling once the fbdev is fully set up. */
+ intel_hpd_init(dev_priv);
+
+ drm_kms_helper_poll_init(dev);
+
+ return 0;
+
+cleanup_gem:
+ i915_gem_fini(dev);
+cleanup_irq:
+ intel_guc_fini(dev);
+ drm_irq_uninstall(dev);
+ intel_teardown_gmbus(dev);
+cleanup_csr:
+ intel_csr_ucode_fini(dev_priv);
+ intel_power_domains_fini(dev_priv);
+ vga_switcheroo_unregister_client(dev->pdev);
+cleanup_vga_client:
+ vga_client_register(dev->pdev, NULL, NULL, NULL);
+out:
+ return ret;
+}
+
+#if IS_ENABLED(CONFIG_FB)
+static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
+{
+ struct apertures_struct *ap;
+ struct pci_dev *pdev = dev_priv->dev->pdev;
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
+ bool primary;
+ int ret;
+
+ ap = alloc_apertures(1);
+ if (!ap)
+ return -ENOMEM;
+
+ ap->ranges[0].base = ggtt->mappable_base;
+ ap->ranges[0].size = ggtt->mappable_end;
+
+ primary =
+ pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+
+ ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
+
+ kfree(ap);
+
+ return ret;
+}
+#else
+static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
+{
+ return 0;
+}
+#endif
+
+#if !defined(CONFIG_VGA_CONSOLE)
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+ return 0;
+}
+#elif !defined(CONFIG_DUMMY_CONSOLE)
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+ return -ENODEV;
+}
+#else
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+ int ret = 0;
+
+ DRM_INFO("Replacing VGA console driver\n");
+
+ console_lock();
+ if (con_is_bound(&vga_con))
+ ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
+ if (ret == 0) {
+ ret = do_unregister_con_driver(&vga_con);
+
+ /* Ignore "already unregistered". */
+ if (ret == -ENODEV)
+ ret = 0;
+ }
+ console_unlock();
+
+ return ret;
+}
+#endif
+
+static void i915_dump_device_info(struct drm_i915_private *dev_priv)
+{
+ const struct intel_device_info *info = &dev_priv->info;
+
+#define PRINT_S(name) "%s"
+#define SEP_EMPTY
+#define PRINT_FLAG(name) info->name ? #name "," : ""
+#define SEP_COMMA ,
+ DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
+ DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
+ info->gen,
+ dev_priv->dev->pdev->device,
+ dev_priv->dev->pdev->revision,
+ DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
+#undef PRINT_S
+#undef SEP_EMPTY
+#undef PRINT_FLAG
+#undef SEP_COMMA
+}
+
+static void cherryview_sseu_info_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_device_info *info;
+ u32 fuse, eu_dis;
+
+ info = (struct intel_device_info *)&dev_priv->info;
+ fuse = I915_READ(CHV_FUSE_GT);
+
+ info->slice_total = 1;
+
+ if (!(fuse & CHV_FGT_DISABLE_SS0)) {
+ info->subslice_per_slice++;
+ eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK |
+ CHV_FGT_EU_DIS_SS0_R1_MASK);
+ info->eu_total += 8 - hweight32(eu_dis);
+ }
+
+ if (!(fuse & CHV_FGT_DISABLE_SS1)) {
+ info->subslice_per_slice++;
+ eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK |
+ CHV_FGT_EU_DIS_SS1_R1_MASK);
+ info->eu_total += 8 - hweight32(eu_dis);
+ }
+
+ info->subslice_total = info->subslice_per_slice;
+ /*
+ * CHV expected to always have a uniform distribution of EU
+ * across subslices.
+ */
+ info->eu_per_subslice = info->subslice_total ?
+ info->eu_total / info->subslice_total :
+ 0;
+ /*
+ * CHV supports subslice power gating on devices with more than
+ * one subslice, and supports EU power gating on devices with
+ * more than one EU pair per subslice.
+ */
+ info->has_slice_pg = 0;
+ info->has_subslice_pg = (info->subslice_total > 1);
+ info->has_eu_pg = (info->eu_per_subslice > 2);
+}
+
+static void gen9_sseu_info_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_device_info *info;
+ int s_max = 3, ss_max = 4, eu_max = 8;
+ int s, ss;
+ u32 fuse2, s_enable, ss_disable, eu_disable;
+ u8 eu_mask = 0xff;
+
+ info = (struct intel_device_info *)&dev_priv->info;
+ fuse2 = I915_READ(GEN8_FUSE2);
+ s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >>
+ GEN8_F2_S_ENA_SHIFT;
+ ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >>
+ GEN9_F2_SS_DIS_SHIFT;
+
+ info->slice_total = hweight32(s_enable);
+ /*
+ * The subslice disable field is global, i.e. it applies
+ * to each of the enabled slices.
+ */
+ info->subslice_per_slice = ss_max - hweight32(ss_disable);
+ info->subslice_total = info->slice_total *
+ info->subslice_per_slice;
+
+ /*
+ * Iterate through enabled slices and subslices to
+ * count the total enabled EU.
+ */
+ for (s = 0; s < s_max; s++) {
+ if (!(s_enable & (0x1 << s)))
+ /* skip disabled slice */
+ continue;
+
+ eu_disable = I915_READ(GEN9_EU_DISABLE(s));
+ for (ss = 0; ss < ss_max; ss++) {
+ int eu_per_ss;
+
+ if (ss_disable & (0x1 << ss))
+ /* skip disabled subslice */
+ continue;
+
+ eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) &
+ eu_mask);
+
+ /*
+ * Record which subslice(s) has(have) 7 EUs. we
+ * can tune the hash used to spread work among
+ * subslices if they are unbalanced.
+ */
+ if (eu_per_ss == 7)
+ info->subslice_7eu[s] |= 1 << ss;
+
+ info->eu_total += eu_per_ss;
+ }
+ }
+
+ /*
+ * SKL is expected to always have a uniform distribution
+ * of EU across subslices with the exception that any one
+ * EU in any one subslice may be fused off for die
+ * recovery. BXT is expected to be perfectly uniform in EU
+ * distribution.
+ */
+ info->eu_per_subslice = info->subslice_total ?
+ DIV_ROUND_UP(info->eu_total,
+ info->subslice_total) : 0;
+ /*
+ * SKL supports slice power gating on devices with more than
+ * one slice, and supports EU power gating on devices with
+ * more than one EU pair per subslice. BXT supports subslice
+ * power gating on devices with more than one subslice, and
+ * supports EU power gating on devices with more than one EU
+ * pair per subslice.
+ */
+ info->has_slice_pg = ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
+ (info->slice_total > 1));
+ info->has_subslice_pg = (IS_BROXTON(dev) && (info->subslice_total > 1));
+ info->has_eu_pg = (info->eu_per_subslice > 2);
+
+ if (IS_BROXTON(dev)) {
+#define IS_SS_DISABLED(_ss_disable, ss) (_ss_disable & (0x1 << ss))
+ /*
+ * There is a HW issue in 2x6 fused down parts that requires
+ * Pooled EU to be enabled as a WA. The pool configuration
+ * changes depending upon which subslice is fused down. This
+ * doesn't affect if the device has all 3 subslices enabled.
+ */
+ /* WaEnablePooledEuFor2x6:bxt */
+ info->has_pooled_eu = ((info->subslice_per_slice == 3) ||
+ (info->subslice_per_slice == 2 &&
+ INTEL_REVID(dev) < BXT_REVID_C0));
+
+ info->min_eu_in_pool = 0;
+ if (info->has_pooled_eu) {
+ if (IS_SS_DISABLED(ss_disable, 0) ||
+ IS_SS_DISABLED(ss_disable, 2))
+ info->min_eu_in_pool = 3;
+ else if (IS_SS_DISABLED(ss_disable, 1))
+ info->min_eu_in_pool = 6;
+ else
+ info->min_eu_in_pool = 9;
+ }
+#undef IS_SS_DISABLED
+ }
+}
+
+static void broadwell_sseu_info_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_device_info *info;
+ const int s_max = 3, ss_max = 3, eu_max = 8;
+ int s, ss;
+ u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
+
+ fuse2 = I915_READ(GEN8_FUSE2);
+ s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
+ ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
+
+ eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
+ eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
+ ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) <<
+ (32 - GEN8_EU_DIS0_S1_SHIFT));
+ eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) |
+ ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
+ (32 - GEN8_EU_DIS1_S2_SHIFT));
+
+
+ info = (struct intel_device_info *)&dev_priv->info;
+ info->slice_total = hweight32(s_enable);
+
+ /*
+ * The subslice disable field is global, i.e. it applies
+ * to each of the enabled slices.
+ */
+ info->subslice_per_slice = ss_max - hweight32(ss_disable);
+ info->subslice_total = info->slice_total * info->subslice_per_slice;
+
+ /*
+ * Iterate through enabled slices and subslices to
+ * count the total enabled EU.
+ */
+ for (s = 0; s < s_max; s++) {
+ if (!(s_enable & (0x1 << s)))
+ /* skip disabled slice */
+ continue;
+
+ for (ss = 0; ss < ss_max; ss++) {
+ u32 n_disabled;
+
+ if (ss_disable & (0x1 << ss))
+ /* skip disabled subslice */
+ continue;
+
+ n_disabled = hweight8(eu_disable[s] >> (ss * eu_max));
+
+ /*
+ * Record which subslices have 7 EUs.
+ */
+ if (eu_max - n_disabled == 7)
+ info->subslice_7eu[s] |= 1 << ss;
+
+ info->eu_total += eu_max - n_disabled;
+ }
+ }
+
+ /*
+ * BDW is expected to always have a uniform distribution of EU across
+ * subslices with the exception that any one EU in any one subslice may
+ * be fused off for die recovery.
+ */
+ info->eu_per_subslice = info->subslice_total ?
+ DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
+
+ /*
+ * BDW supports slice power gating on devices with more than
+ * one slice.
+ */
+ info->has_slice_pg = (info->slice_total > 1);
+ info->has_subslice_pg = 0;
+ info->has_eu_pg = 0;
+}
+
+/*
+ * Determine various intel_device_info fields at runtime.
+ *
+ * Use it when either:
+ * - it's judged too laborious to fill n static structures with the limit
+ * when a simple if statement does the job,
+ * - run-time checks (eg read fuse/strap registers) are needed.
+ *
+ * This function needs to be called:
+ * - after the MMIO has been setup as we are reading registers,
+ * - after the PCH has been detected,
+ * - before the first usage of the fields it can tweak.
+ */
+static void intel_device_info_runtime_init(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_device_info *info;
+ enum pipe pipe;
+
+ info = (struct intel_device_info *)&dev_priv->info;
+
+ /*
+ * Skylake and Broxton currently don't expose the topmost plane as its
+ * use is exclusive with the legacy cursor and we only want to expose
+ * one of those, not both. Until we can safely expose the topmost plane
+ * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported,
+ * we don't expose the topmost plane at all to prevent ABI breakage
+ * down the line.
+ */
+ if (IS_BROXTON(dev)) {
+ info->num_sprites[PIPE_A] = 2;
+ info->num_sprites[PIPE_B] = 2;
+ info->num_sprites[PIPE_C] = 1;
+ } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+ for_each_pipe(dev_priv, pipe)
+ info->num_sprites[pipe] = 2;
+ else
+ for_each_pipe(dev_priv, pipe)
+ info->num_sprites[pipe] = 1;
+
+ if (i915.disable_display) {
+ DRM_INFO("Display disabled (module parameter)\n");
+ info->num_pipes = 0;
+ } else if (info->num_pipes > 0 &&
+ (IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) &&
+ HAS_PCH_SPLIT(dev)) {
+ u32 fuse_strap = I915_READ(FUSE_STRAP);
+ u32 sfuse_strap = I915_READ(SFUSE_STRAP);
+
+ /*
+ * SFUSE_STRAP is supposed to have a bit signalling the display
+ * is fused off. Unfortunately it seems that, at least in
+ * certain cases, fused off display means that PCH display
+ * reads don't land anywhere. In that case, we read 0s.
+ *
+ * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
+ * should be set when taking over after the firmware.
+ */
+ if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
+ sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
+ (dev_priv->pch_type == PCH_CPT &&
+ !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
+ DRM_INFO("Display fused off, disabling\n");
+ info->num_pipes = 0;
+ } else if (fuse_strap & IVB_PIPE_C_DISABLE) {
+ DRM_INFO("PipeC fused off\n");
+ info->num_pipes -= 1;
+ }
+ } else if (info->num_pipes > 0 && IS_GEN9(dev_priv)) {
+ u32 dfsm = I915_READ(SKL_DFSM);
+ u8 disabled_mask = 0;
+ bool invalid;
+ int num_bits;
+
+ if (dfsm & SKL_DFSM_PIPE_A_DISABLE)
+ disabled_mask |= BIT(PIPE_A);
+ if (dfsm & SKL_DFSM_PIPE_B_DISABLE)
+ disabled_mask |= BIT(PIPE_B);
+ if (dfsm & SKL_DFSM_PIPE_C_DISABLE)
+ disabled_mask |= BIT(PIPE_C);
+
+ num_bits = hweight8(disabled_mask);
+
+ switch (disabled_mask) {
+ case BIT(PIPE_A):
+ case BIT(PIPE_B):
+ case BIT(PIPE_A) | BIT(PIPE_B):
+ case BIT(PIPE_A) | BIT(PIPE_C):
+ invalid = true;
+ break;
+ default:
+ invalid = false;
+ }
+
+ if (num_bits > info->num_pipes || invalid)
+ DRM_ERROR("invalid pipe fuse configuration: 0x%x\n",
+ disabled_mask);
+ else
+ info->num_pipes -= num_bits;
+ }
+
+ /* Initialize slice/subslice/EU info */
+ if (IS_CHERRYVIEW(dev))
+ cherryview_sseu_info_init(dev);
+ else if (IS_BROADWELL(dev))
+ broadwell_sseu_info_init(dev);
+ else if (INTEL_INFO(dev)->gen >= 9)
+ gen9_sseu_info_init(dev);
+
+ info->has_snoop = !info->has_llc;
+
+ /* Snooping is broken on BXT A stepping. */
+ if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+ info->has_snoop = false;
+
+ DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total);
+ DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total);
+ DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice);
+ DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total);
+ DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice);
+ DRM_DEBUG_DRIVER("has slice power gating: %s\n",
+ info->has_slice_pg ? "y" : "n");
+ DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
+ info->has_subslice_pg ? "y" : "n");
+ DRM_DEBUG_DRIVER("has EU power gating: %s\n",
+ info->has_eu_pg ? "y" : "n");
+
+ i915.enable_execlists =
+ intel_sanitize_enable_execlists(dev_priv,
+ i915.enable_execlists);
+
+ /*
+ * i915.enable_ppgtt is read-only, so do an early pass to validate the
+ * user's requested state against the hardware/driver capabilities. We
+ * do this now so that we can print out any log messages once rather
+ * than every time we check intel_enable_ppgtt().
+ */
+ i915.enable_ppgtt =
+ intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
+ DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
+}
+
+static void intel_init_dpio(struct drm_i915_private *dev_priv)
+{
+ /*
+ * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
+ * CHV x1 PHY (DP/HDMI D)
+ * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
+ */
+ if (IS_CHERRYVIEW(dev_priv)) {
+ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
+ DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
+ } else if (IS_VALLEYVIEW(dev_priv)) {
+ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
+ }
+}
+
+static int i915_workqueues_init(struct drm_i915_private *dev_priv)
+{
+ /*
+ * The i915 workqueue is primarily used for batched retirement of
+ * requests (and thus managing bo) once the task has been completed
+ * by the GPU. i915_gem_retire_requests() is called directly when we
+ * need high-priority retirement, such as waiting for an explicit
+ * bo.
+ *
+ * It is also used for periodic low-priority events, such as
+ * idle-timers and recording error state.
+ *
+ * All tasks on the workqueue are expected to acquire the dev mutex
+ * so there is no point in running more than one instance of the
+ * workqueue at any time. Use an ordered one.
+ */
+ dev_priv->wq = alloc_ordered_workqueue("i915", 0);
+ if (dev_priv->wq == NULL)
+ goto out_err;
+
+ dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
+ if (dev_priv->hotplug.dp_wq == NULL)
+ goto out_free_wq;
+
+ dev_priv->gpu_error.hangcheck_wq =
+ alloc_ordered_workqueue("i915-hangcheck", 0);
+ if (dev_priv->gpu_error.hangcheck_wq == NULL)
+ goto out_free_dp_wq;
+
+ return 0;
+
+out_free_dp_wq:
+ destroy_workqueue(dev_priv->hotplug.dp_wq);
+out_free_wq:
+ destroy_workqueue(dev_priv->wq);
+out_err:
+ DRM_ERROR("Failed to allocate workqueues.\n");
+
+ return -ENOMEM;
+}
+
+static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
+{
+ destroy_workqueue(dev_priv->gpu_error.hangcheck_wq);
+ destroy_workqueue(dev_priv->hotplug.dp_wq);
+ destroy_workqueue(dev_priv->wq);
+}
+
+/**
+ * i915_driver_init_early - setup state not requiring device access
+ * @dev_priv: device private
+ *
+ * Initialize everything that is a "SW-only" state, that is state not
+ * requiring accessing the device or exposing the driver via kernel internal
+ * or userspace interfaces. Example steps belonging here: lock initialization,
+ * system memory allocation, setting up device specific attributes and
+ * function hooks not requiring accessing the device.
+ */
+static int i915_driver_init_early(struct drm_i915_private *dev_priv,
+ const struct pci_device_id *ent)
+{
+ const struct intel_device_info *match_info =
+ (struct intel_device_info *)ent->driver_data;
+ struct intel_device_info *device_info;
+ int ret = 0;
+
+ if (i915_inject_load_failure())
+ return -ENODEV;
+
+ /* Setup the write-once "constant" device info */
+ device_info = (struct intel_device_info *)&dev_priv->info;
+ memcpy(device_info, match_info, sizeof(*device_info));
+ device_info->device_id = dev_priv->drm.pdev->device;
+
+ BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
+ device_info->gen_mask = BIT(device_info->gen - 1);
+
+ spin_lock_init(&dev_priv->irq_lock);
+ spin_lock_init(&dev_priv->gpu_error.lock);
+ mutex_init(&dev_priv->backlight_lock);
+ spin_lock_init(&dev_priv->uncore.lock);
+ spin_lock_init(&dev_priv->mm.object_stat_lock);
+ spin_lock_init(&dev_priv->mmio_flip_lock);
+ mutex_init(&dev_priv->sb_lock);
+ mutex_init(&dev_priv->modeset_restore_lock);
+ mutex_init(&dev_priv->av_mutex);
+ mutex_init(&dev_priv->wm.wm_mutex);
+ mutex_init(&dev_priv->pps_mutex);
+
+ ret = i915_workqueues_init(dev_priv);
+ if (ret < 0)
+ return ret;
+
+ ret = intel_gvt_init(dev_priv);
+ if (ret < 0)
+ goto err_workqueues;
+
+ /* This must be called before any calls to HAS_PCH_* */
+ intel_detect_pch(&dev_priv->drm);
+
+ intel_pm_setup(&dev_priv->drm);
+ intel_init_dpio(dev_priv);
+ intel_power_domains_init(dev_priv);
+ intel_irq_init(dev_priv);
+ intel_init_display_hooks(dev_priv);
+ intel_init_clock_gating_hooks(dev_priv);
+ intel_init_audio_hooks(dev_priv);
+ i915_gem_load_init(&dev_priv->drm);
+
+ intel_display_crc_init(&dev_priv->drm);
+
+ i915_dump_device_info(dev_priv);
+
+ /* Not all pre-production machines fall into this category, only the
+ * very first ones. Almost everything should work, except for maybe
+ * suspend/resume. And we don't implement workarounds that affect only
+ * pre-production machines. */
+ if (IS_HSW_EARLY_SDV(dev_priv))
+ DRM_INFO("This is an early pre-production Haswell machine. "
+ "It may not be fully functional.\n");
+
+ return 0;
+
+err_workqueues:
+ i915_workqueues_cleanup(dev_priv);
+ return ret;
+}
+
+/**
+ * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
+{
+ i915_gem_load_cleanup(dev_priv->dev);
+ i915_workqueues_cleanup(dev_priv);
+}
+
+static int i915_mmio_setup(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ int mmio_bar;
+ int mmio_size;
+
+ mmio_bar = IS_GEN2(dev) ? 1 : 0;
+ /*
+ * Before gen4, the registers and the GTT are behind different BARs.
+ * However, from gen4 onwards, the registers and the GTT are shared
+ * in the same BAR, so we want to restrict this ioremap from
+ * clobbering the GTT which we want ioremap_wc instead. Fortunately,
+ * the register BAR remains the same size for all the earlier
+ * generations up to Ironlake.
+ */
+ if (INTEL_INFO(dev)->gen < 5)
+ mmio_size = 512 * 1024;
+ else
+ mmio_size = 2 * 1024 * 1024;
+ dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
+ if (dev_priv->regs == NULL) {
+ DRM_ERROR("failed to map registers\n");
+
+ return -EIO;
+ }
+
+ /* Try to make sure MCHBAR is enabled before poking at it */
+ intel_setup_mchbar(dev);
+
+ return 0;
+}
+
+static void i915_mmio_cleanup(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = to_i915(dev);
+
+ intel_teardown_mchbar(dev);
+ pci_iounmap(dev->pdev, dev_priv->regs);
+}
+
+/**
+ * i915_driver_init_mmio - setup device MMIO
+ * @dev_priv: device private
+ *
+ * Setup minimal device state necessary for MMIO accesses later in the
+ * initialization sequence. The setup here should avoid any other device-wide
+ * side effects or exposing the driver via kernel internal or user space
+ * interfaces.
+ */
+static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
+{
+ struct drm_device *dev = dev_priv->dev;
+ int ret;
+
+ if (i915_inject_load_failure())
+ return -ENODEV;
+
+ if (i915_get_bridge_dev(dev))
+ return -EIO;
+
+ ret = i915_mmio_setup(dev);
+ if (ret < 0)
+ goto put_bridge;
+
+ intel_uncore_init(dev_priv);
+
+ return 0;
+
+put_bridge:
+ pci_dev_put(dev_priv->bridge_dev);
+
+ return ret;
+}
+
+/**
+ * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
+{
+ struct drm_device *dev = dev_priv->dev;
+
+ intel_uncore_fini(dev_priv);
+ i915_mmio_cleanup(dev);
+ pci_dev_put(dev_priv->bridge_dev);
+}
+
+/**
+ * i915_driver_init_hw - setup state requiring device access
+ * @dev_priv: device private
+ *
+ * Setup state that requires accessing the device, but doesn't require
+ * exposing the driver via kernel internal or userspace interfaces.
+ */
+static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
+{
+ struct drm_device *dev = dev_priv->dev;
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
+ uint32_t aperture_size;
+ int ret;
+
+ if (i915_inject_load_failure())
+ return -ENODEV;
+
+ intel_device_info_runtime_init(dev);
+
+ ret = i915_ggtt_init_hw(dev);
+ if (ret)
+ return ret;
+
+ ret = i915_ggtt_enable_hw(dev);
+ if (ret) {
+ DRM_ERROR("failed to enable GGTT\n");
+ goto out_ggtt;
+ }
+
+ /* WARNING: Apparently we must kick fbdev drivers before vgacon,
+ * otherwise the vga fbdev driver falls over. */
+ ret = i915_kick_out_firmware_fb(dev_priv);
+ if (ret) {
+ DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
+ goto out_ggtt;
+ }
+
+ ret = i915_kick_out_vgacon(dev_priv);
+ if (ret) {
+ DRM_ERROR("failed to remove conflicting VGA console\n");
+ goto out_ggtt;
+ }
+
+ pci_set_master(dev->pdev);
+
+ /* overlay on gen2 is broken and can't address above 1G */
+ if (IS_GEN2(dev)) {
+ ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
+ if (ret) {
+ DRM_ERROR("failed to set DMA mask\n");
+
+ goto out_ggtt;
+ }
+ }
+
+
+ /* 965GM sometimes incorrectly writes to hardware status page (HWS)
+ * using 32bit addressing, overwriting memory if HWS is located
+ * above 4GB.
+ *
+ * The documentation also mentions an issue with undefined
+ * behaviour if any general state is accessed within a page above 4GB,
+ * which also needs to be handled carefully.
+ */
+ if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
+ ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
+
+ if (ret) {
+ DRM_ERROR("failed to set DMA mask\n");
+
+ goto out_ggtt;
+ }
+ }
+
+ aperture_size = ggtt->mappable_end;
+
+ ggtt->mappable =
+ io_mapping_create_wc(ggtt->mappable_base,
+ aperture_size);
+ if (!ggtt->mappable) {
+ ret = -EIO;
+ goto out_ggtt;
+ }
+
+ ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
+ aperture_size);
+
+ pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
+ PM_QOS_DEFAULT_VALUE);
+
+ intel_uncore_sanitize(dev_priv);
+
+ intel_opregion_setup(dev_priv);
+
+ i915_gem_load_init_fences(dev_priv);
+
+ /* On the 945G/GM, the chipset reports the MSI capability on the
+ * integrated graphics even though the support isn't actually there
+ * according to the published specs. It doesn't appear to function
+ * correctly in testing on 945G.
+ * This may be a side effect of MSI having been made available for PEG
+ * and the registers being closely associated.
+ *
+ * According to chipset errata, on the 965GM, MSI interrupts may
+ * be lost or delayed, but we use them anyways to avoid
+ * stuck interrupts on some machines.
+ */
+ if (!IS_I945G(dev) && !IS_I945GM(dev)) {
+ if (pci_enable_msi(dev->pdev) < 0)
+ DRM_DEBUG_DRIVER("can't enable MSI");
+ }
+
+ return 0;
+
+out_ggtt:
+ i915_ggtt_cleanup_hw(dev);
+
+ return ret;
+}
+
+/**
+ * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
+{
+ struct drm_device *dev = dev_priv->dev;
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
+
+ if (dev->pdev->msi_enabled)
+ pci_disable_msi(dev->pdev);