diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_pm.c')
| -rw-r--r-- | drivers/gpu/drm/i915/intel_pm.c | 1202 |
1 files changed, 826 insertions, 376 deletions
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index e1fc35a..5874716 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -92,12 +92,12 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int cfb_pitch; - int plane, i; + int i; u32 fbc_ctl; cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE; @@ -109,7 +109,6 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) cfb_pitch = (cfb_pitch / 32) - 1; else cfb_pitch = (cfb_pitch / 64) - 1; - plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; /* Clear old tags */ for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) @@ -120,7 +119,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) /* Set it up... */ fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; - fbc_ctl2 |= plane; + fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane); I915_WRITE(FBC_CONTROL2, fbc_ctl2); I915_WRITE(FBC_FENCE_OFF, crtc->y); } @@ -135,7 +134,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) fbc_ctl |= obj->fence_reg; I915_WRITE(FBC_CONTROL, fbc_ctl); - DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ", + DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n", cfb_pitch, crtc->y, plane_name(intel_crtc->plane)); } @@ -150,21 +149,23 @@ static void g4x_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; u32 dpfc_ctl; - dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg; - I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); I915_WRITE(DPFC_FENCE_YOFF, crtc->y); /* enable it... */ - I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN); + I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); } @@ -220,22 +221,20 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; u32 dpfc_ctl; - dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); - dpfc_ctl &= DPFC_RESERVED; - dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); - /* Set persistent mode for front-buffer rendering, ala X. */ - dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN; if (IS_GEN5(dev)) dpfc_ctl |= obj->fence_reg; - I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID); @@ -278,24 +277,31 @@ static void gen7_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + u32 dpfc_ctl; - I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj)); + dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; + dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; - I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X | - IVB_DPFC_CTL_FENCE_EN | - intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT); + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); if (IS_IVYBRIDGE(dev)) { /* WaFbcAsynchFlipDisableFbcQueue:ivb */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS); + I915_WRITE(ILK_DISPLAY_CHICKEN1, + I915_READ(ILK_DISPLAY_CHICKEN1) | + ILK_FBCQ_DIS); } else { - /* WaFbcAsynchFlipDisableFbcQueue:hsw */ - I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe), - HSW_BYPASS_FBC_QUEUE); + /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ + I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe), + I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) | + HSW_FBCQ_DIS); } I915_WRITE(SNB_DPFC_CTL_SA, @@ -330,11 +336,11 @@ static void intel_fbc_work_fn(struct work_struct *__work) /* Double check that we haven't switched fb without cancelling * the prior work. */ - if (work->crtc->fb == work->fb) { + if (work->crtc->primary->fb == work->fb) { dev_priv->display.enable_fbc(work->crtc); dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane; - dev_priv->fbc.fb_id = work->crtc->fb->base.id; + dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id; dev_priv->fbc.y = work->crtc->y; } @@ -387,7 +393,7 @@ static void intel_enable_fbc(struct drm_crtc *crtc) } work->crtc = crtc; - work->fb = crtc->fb; + work->fb = crtc->primary->fb; INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn); dev_priv->fbc.fbc_work = work; @@ -466,7 +472,7 @@ void intel_update_fbc(struct drm_device *dev) return; } - if (!i915_powersave) { + if (!i915.powersave) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); return; @@ -493,25 +499,25 @@ void intel_update_fbc(struct drm_device *dev) } } - if (!crtc || crtc->fb == NULL) { + if (!crtc || crtc->primary->fb == NULL) { if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT)) DRM_DEBUG_KMS("no output, disabling\n"); goto out_disable; } intel_crtc = to_intel_crtc(crtc); - fb = crtc->fb; + fb = crtc->primary->fb; intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; adjusted_mode = &intel_crtc->config.adjusted_mode; - if (i915_enable_fbc < 0 && + if (i915.enable_fbc < 0 && INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) { if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT)) DRM_DEBUG_KMS("disabled per chip default\n"); goto out_disable; } - if (!i915_enable_fbc) { + if (!i915.enable_fbc) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); goto out_disable; @@ -537,7 +543,7 @@ void intel_update_fbc(struct drm_device *dev) DRM_DEBUG_KMS("mode too large for compression, disabling\n"); goto out_disable; } - if ((INTEL_INFO(dev)->gen < 4 || IS_HASWELL(dev)) && + if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) && intel_crtc->plane != PLANE_A) { if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE)) DRM_DEBUG_KMS("plane not A, disabling compression\n"); @@ -617,7 +623,7 @@ out_disable: static void i915_pineview_get_mem_freq(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 tmp; tmp = I915_READ(CLKCFG); @@ -656,7 +662,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev) static void i915_ironlake_get_mem_freq(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); @@ -1035,7 +1041,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) crtc = single_enabled_crtc(dev); if (crtc) { const struct drm_display_mode *adjusted_mode; - int pixel_size = crtc->fb->bits_per_pixel / 8; + int pixel_size = crtc->primary->fb->bits_per_pixel / 8; int clock; adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; @@ -1115,7 +1121,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - pixel_size = crtc->fb->bits_per_pixel / 8; + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* Use the small buffer method to calculate plane watermark */ entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; @@ -1128,9 +1134,9 @@ static bool g4x_compute_wm0(struct drm_device *dev, *plane_wm = display->max_wm; /* Use the large buffer method to calculate cursor watermark */ - line_time_us = ((htotal * 1000) / clock); + line_time_us = max(htotal * 1000 / clock, 1); line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; - entries = line_count * 64 * pixel_size; + entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size; tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; if (tlb_miss > 0) entries += tlb_miss; @@ -1202,9 +1208,9 @@ static bool g4x_compute_srwm(struct drm_device *dev, clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - pixel_size = crtc->fb->bits_per_pixel / 8; + pixel_size = crtc->primary->fb->bits_per_pixel / 8; - line_time_us = (htotal * 1000) / clock; + line_time_us = max(htotal * 1000 / clock, 1); line_count = (latency_ns / line_time_us + 1000) / 1000; line_size = hdisplay * pixel_size; @@ -1216,7 +1222,7 @@ static bool g4x_compute_srwm(struct drm_device *dev, *display_wm = entries + display->guard_size; /* calculate the self-refresh watermark for display cursor */ - entries = line_count * pixel_size * 64; + entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width; entries = DIV_ROUND_UP(entries, cursor->cacheline_size); *cursor_wm = entries + cursor->guard_size; @@ -1241,7 +1247,7 @@ static bool vlv_compute_drain_latency(struct drm_device *dev, return false; clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */ + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ entries = (clock / 1000) * pixel_size; *plane_prec_mult = (entries > 256) ? @@ -1433,11 +1439,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - int pixel_size = crtc->fb->bits_per_pixel / 8; + int pixel_size = crtc->primary->fb->bits_per_pixel / 8; unsigned long line_time_us; int entries; - line_time_us = ((htotal * 1000) / clock); + line_time_us = max(htotal * 1000 / clock, 1); /* Use ns/us then divide to preserve precision */ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * @@ -1451,7 +1457,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) entries, srwm); entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * - pixel_size * 64; + pixel_size * to_intel_crtc(crtc)->cursor_width; entries = DIV_ROUND_UP(entries, i965_cursor_wm_info.cacheline_size); cursor_sr = i965_cursor_wm_info.fifo_size - @@ -1506,7 +1512,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) crtc = intel_get_crtc_for_plane(dev, 0); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode; - int cpp = crtc->fb->bits_per_pixel / 8; + int cpp = crtc->primary->fb->bits_per_pixel / 8; if (IS_GEN2(dev)) cpp = 4; @@ -1522,7 +1528,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) crtc = intel_get_crtc_for_plane(dev, 1); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode; - int cpp = crtc->fb->bits_per_pixel / 8; + int cpp = crtc->primary->fb->bits_per_pixel / 8; if (IS_GEN2(dev)) cpp = 4; @@ -1559,11 +1565,11 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w; - int pixel_size = enabled->fb->bits_per_pixel / 8; + int pixel_size = enabled->primary->fb->bits_per_pixel / 8; unsigned long line_time_us; int entries; - line_time_us = (htotal * 1000) / clock; + line_time_us = max(htotal * 1000 / clock, 1); /* Use ns/us then divide to preserve precision */ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * @@ -1886,7 +1892,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, } /* Calculate the maximum FBC watermark */ -static unsigned int ilk_fbc_wm_max(struct drm_device *dev) +static unsigned int ilk_fbc_wm_max(const struct drm_device *dev) { /* max that registers can hold */ if (INTEL_INFO(dev)->gen >= 8) @@ -1895,7 +1901,7 @@ static unsigned int ilk_fbc_wm_max(struct drm_device *dev) return 15; } -static void ilk_compute_wm_maximums(struct drm_device *dev, +static void ilk_compute_wm_maximums(const struct drm_device *dev, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, @@ -1948,7 +1954,7 @@ static bool ilk_validate_wm_level(int level, return ret; } -static void ilk_compute_wm_level(struct drm_i915_private *dev_priv, +static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, int level, const struct ilk_pipe_wm_parameters *p, struct intel_wm_level *result) @@ -2079,7 +2085,7 @@ static void intel_print_wm_latency(struct drm_device *dev, } } -static void intel_setup_wm_latency(struct drm_device *dev) +static void ilk_setup_wm_latency(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -2111,10 +2117,10 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc, if (p->active) { p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); - p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8; + p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8; p->cur.bytes_per_pixel = 4; p->pri.horiz_pixels = intel_crtc->config.pipe_src_w; - p->cur.horiz_pixels = 64; + p->cur.horiz_pixels = intel_crtc->cursor_width; /* TODO: for now, assume primary and cursor planes are always enabled. */ p->pri.enabled = true; p->cur.enabled = true; @@ -2123,7 +2129,7 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc, list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) config->num_pipes_active += intel_crtc_active(crtc); - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { + drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { struct intel_plane *intel_plane = to_intel_plane(plane); if (intel_plane->pipe == pipe) @@ -2140,7 +2146,7 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, struct intel_pipe_wm *pipe_wm) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_i915_private *dev_priv = dev->dev_private; int level, max_level = ilk_wm_max_level(dev); /* LP0 watermark maximums depend on this pipe alone */ struct intel_wm_config config = { @@ -2738,7 +2744,7 @@ intel_alloc_context_page(struct drm_device *dev) return NULL; } - ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false); + ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0); if (ret) { DRM_ERROR("failed to pin power context: %d\n", ret); goto err_unref; @@ -2753,7 +2759,7 @@ intel_alloc_context_page(struct drm_device *dev) return ctx; err_unpin: - i915_gem_object_unpin(ctx); + i915_gem_object_ggtt_unpin(ctx); err_unref: drm_gem_object_unreference(&ctx->base); return NULL; @@ -2901,9 +2907,9 @@ static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val) * the hw runs at the minimal clock before selecting the desired * frequency, if the down threshold expires in that window we will not * receive a down interrupt. */ - limits = dev_priv->rps.max_delay << 24; - if (val <= dev_priv->rps.min_delay) - limits |= dev_priv->rps.min_delay << 16; + limits = dev_priv->rps.max_freq_softlimit << 24; + if (val <= dev_priv->rps.min_freq_softlimit) + limits |= dev_priv->rps.min_freq_softlimit << 16; return limits; } @@ -2915,26 +2921,26 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) new_power = dev_priv->rps.power; switch (dev_priv->rps.power) { case LOW_POWER: - if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay) + if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) new_power = BETWEEN; break; case BETWEEN: - if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay) + if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) new_power = LOW_POWER; - else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay) + else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) new_power = HIGH_POWER; break; case HIGH_POWER: - if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay) + if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) new_power = BETWEEN; break; } /* Max/min bins are special */ - if (val == dev_priv->rps.min_delay) + if (val == dev_priv->rps.min_freq_softlimit) new_power = LOW_POWER; - if (val == dev_priv->rps.max_delay) + if (val == dev_priv->rps.max_freq_softlimit) new_power = HIGH_POWER; if (new_power == dev_priv->rps.power) return; @@ -3000,41 +3006,113 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) dev_priv->rps.last_adj = 0; } +static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) +{ + u32 mask = 0; + + if (val > dev_priv->rps.min_freq_softlimit) + mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; + if (val < dev_priv->rps.max_freq_softlimit) + mask |= GEN6_PM_RP_UP_THRESHOLD; + + /* IVB and SNB hard hangs on looping batchbuffer + * if GEN6_PM_UP_EI_EXPIRED is masked. + */ + if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev)) + mask |= GEN6_PM_RP_UP_EI_EXPIRED; + + return ~mask; +} + +/* gen6_set_rps is called to update the frequency request, but should also be + * called when the range (min_delay and max_delay) is modified so that we can + * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ void gen6_set_rps(struct drm_device *dev, u8 val) { struct drm_i915_private *dev_priv = dev->dev_private; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - WARN_ON(val > dev_priv->rps.max_delay); - WARN_ON(val < dev_priv->rps.min_delay); - - if (val == dev_priv->rps.cur_delay) - return; + WARN_ON(val > dev_priv->rps.max_freq_softlimit); + WARN_ON(val < dev_priv->rps.min_freq_softlimit); - gen6_set_rps_thresholds(dev_priv, val); + /* min/max delay may still have been modified so be sure to + * write the limits value. + */ + if (val != dev_priv->rps.cur_freq) { + gen6_set_rps_thresholds(dev_priv, val); - if (IS_HASWELL(dev)) - I915_WRITE(GEN6_RPNSWREQ, - HSW_FREQUENCY(val)); - else - I915_WRITE(GEN6_RPNSWREQ, - GEN6_FREQUENCY(val) | - GEN6_OFFSET(0) | - GEN6_AGGRESSIVE_TURBO); + if (IS_HASWELL(dev)) + I915_WRITE(GEN6_RPNSWREQ, + HSW_FREQUENCY(val)); + else + I915_WRITE(GEN6_RPNSWREQ, + GEN6_FREQUENCY(val) | + GEN6_OFFSET(0) | + GEN6_AGGRESSIVE_TURBO); + } /* Make sure we continue to get interrupts * until we hit the minimum or maximum frequencies. */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - gen6_rps_limits(dev_priv, val)); + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val)); + I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); POSTING_READ(GEN6_RPNSWREQ); - dev_priv->rps.cur_delay = val; - + dev_priv->rps.cur_freq = val; trace_intel_gpu_freq_change(val * 50); } +/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down + * + * * If Gfx is Idle, then + * 1. Mask Turbo interrupts + * 2. Bring up Gfx clock + * 3. Change the freq to Rpn and wait till P-Unit updates freq + * 4. Clear the Force GFX CLK ON bit so that Gfx can down + * 5. Unmask Turbo interrupts +*/ +static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) +{ + /* + * When we are idle. Drop to min voltage state. + */ + + if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit) + return; + + /* Mask turbo interrupt so that they will not come in between */ + I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); + + /* Bring up the Gfx clock */ + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, + I915_READ(VLV_GTLC_SURVIVABILITY_REG) | + VLV_GFX_CLK_FORCE_ON_BIT); + + if (wait_for(((VLV_GFX_CLK_STATUS_BIT & + I915_READ(VLV_GTLC_SURVIVABILITY_REG)) != 0), 5)) { + DRM_ERROR("GFX_CLK_ON request timed out\n"); + return; + } + + dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit; + + vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, + dev_priv->rps.min_freq_softlimit); + + if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) + & GENFREQSTATUS) == 0, 5)) + DRM_ERROR("timed out waiting for Punit\n"); + + /* Release the Gfx clock */ + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, + I915_READ(VLV_GTLC_SURVIVABILITY_REG) & + ~VLV_GFX_CLK_FORCE_ON_BIT); + + I915_WRITE(GEN6_PMINTRMSK, + gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); +} + void gen6_rps_idle(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -3042,9 +3120,9 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) - valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + vlv_set_rps_idle(dev_priv); else - gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3057,9 +3135,9 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) - valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); else - gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3070,21 +3148,20 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) struct drm_i915_private *dev_priv = dev->dev_private; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - WARN_ON(val > dev_priv->rps.max_delay); - WARN_ON(val < dev_priv->rps.min_delay); + WARN_ON(val > dev_priv->rps.max_freq_softlimit); + WARN_ON(val < dev_priv->rps.min_freq_softlimit); DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), - dev_priv->rps.cur_delay, + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), + dev_priv->rps.cur_freq, vlv_gpu_freq(dev_priv, val), val); - if (val == dev_priv->rps.cur_delay) - return; - - vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); + if (val != dev_priv->rps.cur_freq) + vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); - dev_priv->rps.cur_delay = val; + I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); + dev_priv->rps.cur_freq = val; trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val)); } @@ -3093,7 +3170,8 @@ static void gen6_disable_rps_interrupts(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); - I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS); + I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & + ~dev_priv->pm_rps_events); /* Complete PM interrupt masking here doesn't race with the rps work * item again unmasking PM interrupts because that is using a different * register (PMIMR) to mask PM interrupts. The only risk is in leaving @@ -3103,7 +3181,7 @@ static void gen6_disable_rps_interrupts(struct drm_device *dev) dev_priv->rps.pm_iir = 0; spin_unlock_irq(&dev_priv->irq_lock); - I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS); + I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); } static void gen6_disable_rps(struct drm_device *dev) @@ -3123,25 +3201,14 @@ static void valleyview_disable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); gen6_disable_rps_interrupts(dev); - - if (dev_priv->vlv_pctx) { - drm_gem_object_unreference(&dev_priv->vlv_pctx->base); - dev_priv->vlv_pctx = NULL; - } } static void intel_print_rc6_info(struct drm_device *dev, u32 mode) { - if (IS_GEN6(dev)) - DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n"); - - if (IS_HASWELL(dev)) - DRM_DEBUG_DRIVER("Haswell: only RC6 available\n"); - DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n", - (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); + (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); } int intel_enable_rc6(const struct drm_device *dev) @@ -3151,44 +3218,28 @@ int intel_enable_rc6(const struct drm_device *dev) return 0; /* Respect the kernel parameter if it is set */ - if (i915_enable_rc6 >= 0) - return i915_enable_rc6; + if (i915.enable_rc6 >= 0) + return i915.enable_rc6; /* Disable RC6 on Ironlake */ if (INTEL_INFO(dev)->gen == 5) return 0; - if (IS_HASWELL(dev)) - return INTEL_RC6_ENABLE; - - /* snb/ivb have more than one rc6 state. */ - if (INTEL_INFO(dev)->gen == 6) - return INTEL_RC6_ENABLE; + if (IS_IVYBRIDGE(dev)) + return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); - return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); + return INTEL_RC6_ENABLE; } static void gen6_enable_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 enabled_intrs; spin_lock_irq(&dev_priv->irq_lock); WARN_ON(dev_priv->rps.pm_iir); - snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS); - I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS); + snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); - - /* only unmask PM interrupts we need. Mask all others. */ - enabled_intrs = GEN6_PM_RPS_EVENTS; - - /* IVB and SNB hard hangs on looping batchbuffer - * if GEN6_PM_UP_EI_EXPIRED is masked. - */ - if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) - enabled_intrs |= GEN6_PM_RP_UP_EI_EXPIRED; - - I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs); } static void gen8_enable_rps(struct drm_device *dev) @@ -3222,10 +3273,10 @@ static void gen8_enable_rps(struct drm_device *dev) /* 3: Enable RC6 */ if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; - DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off"); + intel_print_rc6_info(dev, rc6_mask); I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | - GEN6_RC_CTL_EI_MODE(1) | - rc6_mask); + GEN6_RC_CTL_EI_MODE(1) | + rc6_mask); /* 4 Program defaults and thresholds for RPS*/ I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */ @@ -3235,8 +3286,8 @@ static void gen8_enable_rps(struct drm_device *dev) /* Docs recommend 900MHz, and 300 MHz respectively */ I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - dev_priv->rps.max_delay << 24 | - dev_priv->rps.min_delay << 16); + dev_priv->rps.max_freq_softlimit << 24 | + dev_priv->rps.min_freq_softlimit << 16); I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ @@ -3269,7 +3320,7 @@ static void gen6_enable_rps(struct drm_device *dev) struct intel_ring_buffer *ring; u32 rp_state_cap; u32 gt_perf_status; - u32 rc6vids, pcu_mbox, rc6_mask = 0; + u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; int rc6_mode; int i, ret; @@ -3295,13 +3346,23 @@ static void gen6_enable_rps(struct drm_device *dev) rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); - /* In units of 50MHz */ - dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff; - dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff; - dev_priv->rps.rp1_delay = (rp_state_cap >> 8) & 0xff; - dev_priv->rps.rp0_delay = (rp_state_cap >> 0) & 0xff; - dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay; - dev_priv->rps.cur_delay = 0; + /* All of these values are in units of 50MHz */ + dev_priv->rps.cur_freq = 0; + /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */ + dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; + dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; + dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; + /* XXX: only BYT has a special efficient freq */ + dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; + /* hw_max = RP0 until we check for overclocking */ + dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; + + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_freq_softlimit == 0) + dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; + + if (dev_priv->rps.min_freq_softlimit == 0) + dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; /* disable the counters and set deterministic thresholds */ I915_WRITE(GEN6_RC_CONTROL, 0); @@ -3350,21 +3411,19 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); - if (!ret) { - pcu_mbox = 0; - ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); - if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ - DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", - (dev_priv->rps.max_delay & 0xff) * 50, - (pcu_mbox & 0xff) * 50); - dev_priv->rps.hw_max = pcu_mbox & 0xff; - } - } else { + if (ret) DRM_DEBUG_DRIVER("Failed to set the min frequency\n"); + + ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); + if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ + DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", + (dev_priv->rps.max_freq_softlimit & 0xff) * 50, + (pcu_mbox & 0xff) * 50); + dev_priv->rps.max_freq = pcu_mbox & 0xff; } dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); gen6_enable_rps_interrupts(dev); @@ -3420,9 +3479,9 @@ void gen6_update_ring_freq(struct drm_device *dev) * to use for memory access. We do this by specifying the IA frequency * the PCU should use as a reference to determine the ring frequency. */ - for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay; + for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit; gpu_freq--) { - int diff = dev_priv->rps.max_delay - gpu_freq; + int diff = dev_priv->rps.max_freq_softlimit - gpu_freq; unsigned int ia_freq = 0, ring_freq = 0; if (INTEL_INFO(dev)->gen >= 8) { @@ -3485,6 +3544,15 @@ int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; } +/* Check that the pctx buffer wasn't move under us. */ +static void valleyview_check_pctx(struct drm_i915_private *dev_priv) +{ + unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; + + WARN_ON(pctx_addr != dev_priv->mm.stolen_base + + dev_priv->vlv_pctx->stolen->start); +} + static void valleyview_setup_pctx(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -3529,6 +3597,17 @@ out: dev_priv->vlv_pctx = pctx; } +static void valleyview_cleanup_pctx(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (WARN_ON(!dev_priv->vlv_pctx)) + return; + + drm_gem_object_unreference(&dev_priv->vlv_pctx->base); + dev_priv->vlv_pctx = NULL; +} + static void valleyview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -3538,6 +3617,8 @@ static void valleyview_enable_rps(struct drm_device *dev) WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); + valleyview_check_pctx(dev_priv); + if ((gtfifodbg = I915_READ(GTFIFODBG))) { DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", gtfifodbg); @@ -3588,32 +3669,39 @@ static void valleyview_enable_rps(struct drm_device *dev) DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no"); DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); - dev_priv->rps.cur_delay = (val >> 8) & 0xff; + dev_priv->rps.cur_freq = (val >> 8) & 0xff; DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), - dev_priv->rps.cur_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), + dev_priv->rps.cur_freq); - dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv); - dev_priv->rps.hw_max = dev_priv->rps.max_delay; + dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); + dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay), - dev_priv->rps.max_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq), + dev_priv->rps.max_freq); - dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv); + dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), - dev_priv->rps.rpe_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), + dev_priv->rps.efficient_freq); - dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv); + dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay), - dev_priv->rps.min_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), + dev_priv->rps.min_freq); + + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_freq_softlimit == 0) + dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; + + if (dev_priv->rps.min_freq_softlimit == 0) + dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), - dev_priv->rps.rpe_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), + dev_priv->rps.efficient_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); gen6_enable_rps_interrupts(dev); @@ -3625,13 +3713,13 @@ void ironlake_teardown_rc6(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; if (dev_priv->ips.renderctx) { - i915_gem_object_unpin(dev_priv->ips.renderctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx); drm_gem_object_unreference(&dev_priv->ips.renderctx->base); dev_priv->ips.renderctx = NULL; } if (dev_priv->ips.pwrctx) { - i915_gem_object_unpin(dev_priv->ips.pwrctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx); drm_gem_object_unreference(&dev_priv->ips.pwrctx->base); dev_priv->ips.pwrctx = NULL; } @@ -3823,9 +3911,10 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -3854,6 +3943,7 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv) static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { + struct drm_device *dev = dev_priv->dev; static const struct v_table { u16 vd; /* in .1 mil */ u16 vm; /* in .1 mil */ @@ -3987,7 +4077,7 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { 16000, 14875, }, { 16125, 15000, }, }; - if (dev_priv->info->is_mobile) + if (INTEL_INFO(dev)->is_mobile) return v_table[pxvid].vm; else return v_table[pxvid].vd; @@ -4030,7 +4120,9 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) void i915_update_gfx_val(struct drm_i915_private *dev_priv) { - if (dev_priv->info->gen != 5) + struct drm_device *dev = dev_priv->dev; + + if (INTEL_INFO(dev)->gen != 5) return; spin_lock_irq(&mchdev_lock); @@ -4047,7 +4139,7 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) assert_spin_locked(&mchdev_lock); - pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4)); + pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); pxvid = (pxvid >> 24) & 0x7f; ext_v = pvid_to_extvid(dev_priv, pxvid); @@ -4079,9 +4171,10 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -4270,6 +4363,7 @@ void intel_gpu_ips_teardown(void) i915_mch_dev = NULL; spin_unlock_irq(&mchdev_lock); } + static void intel_init_emon(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4341,6 +4435,18 @@ static void intel_init_emon(struct drm_device *dev) dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); } +void intel_init_gt_powersave(struct drm_device *dev) +{ + if (IS_VALLEYVIEW(dev)) + valleyview_setup_pctx(dev); +} + +void intel_cleanup_gt_powersave(struct drm_device *dev) +{ + if (IS_VALLEYVIEW(dev)) + valleyview_cleanup_pctx(dev); +} + void intel_disable_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4395,8 +4501,6 @@ void intel_enable_gt_powersave(struct drm_device *dev) ironlake_enable_rc6(dev); intel_init_emon(dev); } else if (IS_GEN6(dev) || IS_GEN7(dev)) { - if (IS_VALLEYVIEW(dev)) - valleyview_setup_pctx(dev); /* * PCU communication is slow and this doesn't need to be * done at any specific time, so do this out of our fast path @@ -4587,6 +4691,17 @@ static void gen6_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE)); + /* + * BSpec recoomends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN6_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + ilk_init_lp_watermarks(dev); I915_WRITE(CACHE_MODE_0, @@ -4607,17 +4722,24 @@ static void gen6_init_clock_gating(struct drm_device *dev) * According to the spec, bit 11 (RCCUNIT) must also be set, * but we didn't debug actual testcases to find it out. * - * Also apply WaDisableVDSUnitClockGating:snb and - * WaDisableRCPBUnitClockGating:snb. + * WaDisableRCCUnitClockGating:snb + * WaDisableRCPBUnitClockGating:snb */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - /* Bspec says we need to always set all mask bits. */ - I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) | - _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL); + /* WaStripsFansDisableFastClipPerformanceFix:snb */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); + + /* + * Bspec says: + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and + * 3DSTATE_SF number of SF output attributes is more than 16." + */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); /* * According to the spec the following bits should be @@ -4643,11 +4765,6 @@ static void gen6_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* The default value should be 0x200 according to docs, but the two - * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */ - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff)); - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI)); - cpt_init_clock_gating(dev); gen6_check_mch_setup(dev); @@ -4657,14 +4774,17 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) { uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); + /* + * WaVSThreadDispatchOverride:ivb,vlv + * + * This actually overrides the dispatch + * mode for all thread types. + */ reg &= ~GEN7_FF_SCHED_MASK; reg |= GEN7_FF_TS_SCHED_HW; reg |= GEN7_FF_VS_SCHED_HW; reg |= GEN7_FF_DS_SCHED_HW; - if (IS_HASWELL(dev_priv->dev)) - reg &= ~GEN7_FF_VS_REF_CNT_FFME; - I915_WRITE(GEN7_FF_THREAD_MODE, reg); } @@ -4702,7 +4822,7 @@ static void lpt_suspend_hw(struct drm_device *dev) static void gen8_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - enum pipe i; + enum pipe pipe; I915_WRITE(WM3_LP_ILK, 0); I915_WRITE(WM2_LP_ILK, 0); @@ -4711,8 +4831,19 @@ static void gen8_init_clock_gating(struct drm_device *dev) /* FIXME(BDW): Check all the w/a, some might only apply to * pre-production hw. */ - WARN(!i915_preliminary_hw_support, - "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n"); + /* WaDisablePartialInstShootdown:bdw */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); + + /* WaDisableThreadStallDopClockGating:bdw */ + /* FIXME: Unclear whether we really need this on production bdw. */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); + + /* + * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for + * pre-production hardware + */ I915_WRITE(HALF_SLICE_CHICKEN3, _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS)); I915_WRITE(HALF_SLICE_CHICKEN3, @@ -4736,10 +4867,10 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ - for_each_pipe(i) { - I915_WRITE(CHICKEN_PIPESL_1(i), - I915_READ(CHICKEN_PIPESL_1(i) | - DPRS_MASK_VBLANK_SRD)); + for_each_pipe(pipe) { + I915_WRITE(CHICKEN_PIPESL_1(pipe), + I915_READ(CHICKEN_PIPESL_1(pipe)) | + BDW_DPRS_MASK_VBLANK_SRD); } /* Use Force Non-Coherent whenever executing a 3D context. This is a @@ -4755,6 +4886,28 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN7_FF_THREAD_MODE, I915_READ(GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + + I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); + + /* WaDisableSDEUnitClockGating:bdw */ + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* Wa4x4STCOptimizationDisable:bdw */ + I915_WRITE(CACHE_MODE_1, + _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE)); } static void haswell_init_clock_gating(struct drm_device *dev) @@ -4763,21 +4916,6 @@ static void haswell_init_clock_gating(struct drm_device *dev) ilk_init_lp_watermarks(dev); - /* According to the spec, bit 13 (RCZUNIT) must be set on IVB. - * This implements the WaDisableRCZUnitClockGating:hsw workaround. - */ - I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE); - - /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:hsw */ - I915_WRITE(GEN7_L3CNTLREG1, - GEN7_WA_FOR_GEN7_L3_CONTROL); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, - GEN7_WA_L3_CHICKEN_MODE); - /* L3 caching of data atomics doesn't work -- disable it. */ I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); I915_WRITE(HSW_ROW_CHICKEN3, @@ -4789,12 +4927,28 @@ static void haswell_init_clock_gating(struct drm_device *dev) GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* WaVSRefCountFullforceMissDisable:hsw */ - gen7_setup_fixed_func_scheduler(dev_priv); + I915_WRITE(GEN7_FF_THREAD_MODE, + I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); + + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); /* WaDisable4x2SubspanOptimization:hsw */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + /* WaSwitchSolVfFArbitrationPriority:hsw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -4827,9 +4981,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - else - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2, - _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, @@ -4843,31 +4994,24 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - else + else { + /* must write both registers */ + I915_WRITE(GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); I915_WRITE(GEN7_ROW_CHICKEN2_GT2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - + } /* WaForceL3Serialization:ivb */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & ~L3SQ_URB_READ_CAM_MATCH_DISABLE); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:ivb workaround. */ I915_WRITE(GEN6_UCGCTL2, - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); /* This is required by WaCatErrorRejectionIssue:ivb */ I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, @@ -4876,13 +5020,29 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* WaVSRefCountFullforceMissDisable:ivb */ gen7_setup_fixed_func_scheduler(dev_priv); + if (0) { /* causes HiZ corruption on ivb:gt1 */ + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); + } + /* WaDisable4x2SubspanOptimization:ivb */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); snpcr &= ~GEN6_MBC_SNPCR_MASK; snpcr |= GEN6_MBC_SNPCR_MED; @@ -4904,13 +5064,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev) mutex_unlock(&dev_priv->rps.hw_lock); switch ((val >> 6) & 3) { case 0: - dev_priv->mem_freq = 800; - break; case 1: - dev_priv->mem_freq = 1066; + dev_priv->mem_freq = 800; break; case 2: - dev_priv->mem_freq = 1333; + dev_priv->mem_freq = 1066; break; case 3: dev_priv->mem_freq = 1333; @@ -4929,19 +5087,12 @@ static void valleyview_init_clock_gating(struct drm_device *dev) CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); + /* WaPsdDispatchEnable:vlv */ /* WaDisablePSDDualDispatchEnable:vlv */ I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:vlv */ - I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); - /* WaForceL3Serialization:vlv */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & ~L3SQ_URB_READ_CAM_MATCH_DISABLE); @@ -4955,51 +5106,39 @@ static void valleyview_init_clock_gating(struct drm_device *dev) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + gen7_setup_fixed_func_scheduler(dev_priv); + + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:vlv workaround. - * - * Also apply WaDisableVDSUnitClockGating:vlv and - * WaDisableRCPBUnitClockGating:vlv. */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | - GEN7_TDLUNIT_CLOCK_GATE_DISABLE | - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); + /* WaDisableL3Bank2xClockGate:vlv */ I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE); I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); + /* + * BSpec says this must be set, even though + * WaDisable4x2SubspanOptimization isn't listed for VLV. + */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); /* + * WaIncreaseL3CreditsForVLVB0:vlv + * This is the hardware default actually. + */ + I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); + + /* * WaDisableVLVClockGating_VBIIssue:vlv * Disable clock gating on th GCFG unit to prevent a delay * in the reporting of vblank events. */ - I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff); - - /* Conservative clock gating settings for now */ - I915_WRITE(0x9400, 0xffffffff); - I915_WRITE(0x9404, 0xffffffff); - I915_WRITE(0x9408, 0xffffffff); - I915_WRITE(0x940c, 0xffffffff); - I915_WRITE(0x9410, 0xffffffff); - I915_WRITE(0x9414, 0xffffffff); - I915_WRITE(0x9418, 0xffffffff); + I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); } static void g4x_init_clock_gating(struct drm_device *dev) @@ -5114,19 +5253,16 @@ void intel_suspend_hw(struct drm_device *dev) * enable it, so check if it's enabled and also check if we've requested it to * be enabled. */ -static bool hsw_power_well_enabled(struct drm_device *dev, +static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(HSW_PWR_WELL_DRIVER) == (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); } -bool intel_display_power_enabled_sw(struct drm_device *dev, +bool intel_display_power_enabled_sw(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; power_domains = &dev_priv->power_domains; @@ -5134,15 +5270,17 @@ bool intel_display_power_enabled_sw(struct drm_device *dev, return power_domains->domain_use_count[domain]; } -bool intel_display_power_enabled(struct drm_device *dev, +bool intel_display_power_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; bool is_enabled; int i; + if (dev_priv->pm.suspended) + return false; + power_domains = &dev_priv->power_domains; is_enabled = true; @@ -5152,7 +5290,7 @@ bool intel_display_power_enabled(struct drm_device *dev, if (power_well->always_on) continue; - if (!power_well->is_enabled(dev, power_well)) { + if (!power_well->ops->is_enabled(dev_priv, power_well)) { is_enabled = false; break; } @@ -5162,6 +5300,12 @@ bool intel_display_power_enabled(struct drm_device *dev, return is_enabled; } +/* + * Starting with Haswell, we have a "Power Down Well" that can be turned off + * when not needed anymore. We have 4 registers that can request the power well + * to be enabled, and it will only be disabled if none of the registers is + * requesting it to be enabled. + */ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -5198,10 +5342,17 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) } } +static void reset_vblank_counter(struct drm_device *dev, enum pipe pipe) +{ + assert_spin_locked(&dev->vbl_lock); + + dev->vblank[pipe].last = 0; +} + static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; - enum pipe p; + enum pipe pipe; unsigned long irqflags; /* @@ -5212,21 +5363,18 @@ static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv) * FIXME: Should we do this in general in drm_vblank_post_modeset? */ spin_lock_irqsave(&dev->vbl_lock, irqflags); - for_each_pipe(p) - if (p != PIPE_A) - dev->vblank[p].last = 0; + for_each_pipe(pipe) + if (pipe != PIPE_A) + reset_vblank_counter(dev, pipe); spin_unlock_irqrestore(&dev->vbl_lock, irqflags); } -static void hsw_set_power_well(struct drm_device *dev, +static void hsw_set_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; bool is_enabled, enable_requested; uint32_t tmp; - WARN_ON(dev_priv->pc8.enabled); - tmp = I915_READ(HSW_PWR_WELL_DRIVER); is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED; enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST; @@ -5255,55 +5403,229 @@ static void hsw_set_power_well(struct drm_device *dev, } } -static void __intel_power_well_get(struct drm_device *dev, +static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; + hsw_set_power_well(dev_priv, power_well, power_well->count > 0); - if (!power_well->count++ && power_well->set) { - hsw_disable_package_c8(dev_priv); - power_well->set(dev, power_well, true); - } + /* + * We're taking over the BIOS, so clear any requests made by it since + * the driver is in charge now. + */ + if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) + I915_WRITE(HSW_PWR_WELL_BIOS, 0); +} + +static void hsw_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + hsw_set_power_well(dev_priv, power_well, true); } -static void __intel_power_well_put(struct drm_device *dev, +static void hsw_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; + hsw_set_power_well(dev_priv, power_well, false); +} + +static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ +} + +static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + return true; +} + +static void vlv_set_power_well(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well, bool enable) +{ + enum punit_power_well power_well_id = power_well->data; + u32 mask; + u32 state; + u32 ctrl; + + mask = PUNIT_PWRGT_MASK(power_well_id); + state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) : + PUNIT_PWRGT_PWR_GATE(power_well_id); + + mutex_lock(&dev_priv->rps.hw_lock); + +#define COND \ + ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state) + + if (COND) + goto out; + + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL); + ctrl &= ~mask; + ctrl |= state; + vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl); + + if (wait_for(COND, 100)) + DRM_ERROR("timout setting power well state %08x (%08x)\n", + state, + vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL)); + +#undef COND + +out: + mutex_unlock(&dev_priv->rps.hw_lock); +} + +static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + vlv_set_power_well(dev_priv, power_well, power_well->count > 0); +} + +static void vlv_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + vlv_set_power_well(dev_priv, power_well, true); +} + +static void vlv_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + vlv_set_power_well(dev_priv, power_well, false); +} + +static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + int power_well_id = power_well->data; + bool enabled = false; + u32 mask; + u32 state; + u32 ctrl; + + mask = PUNIT_PWRGT_MASK(power_well_id); + ctrl = PUNIT_PWRGT_PWR_ON(power_well_id); + + mutex_lock(&dev_priv->rps.hw_lock); + + state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask; + /* + * We only ever set the power-on and power-gate states, anything + * else is unexpected. + */ + WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) && + state != PUNIT_PWRGT_PWR_GATE(power_well_id)); + if (state == ctrl) + enabled = true; + + /* + * A transient state at this point would mean some unexpected party + * is poking at the power controls too. + */ + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask; + WARN_ON(ctrl != state); + + mutex_unlock(&dev_priv->rps.hw_lock); + + return enabled; +} - WARN_ON(!power_well->count); +static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + + vlv_set_power_well(dev_priv, power_well, true); + + spin_lock_irq(&dev_priv->irq_lock); + valleyview_enable_display_irqs(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); + + /* + * During driver initialization we need to defer enabling hotplug + * processing until fbdev is set up. + */ + if (dev_priv->enable_hotplug_processing) + intel_hpd_init(dev_priv->dev); + + i915_redisable_vga_power_on(dev_priv->dev); +} + +static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + struct drm_device *dev = dev_priv->dev; + enum pipe pipe; + + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + + spin_lock_irq(&dev_priv->irq_lock); + for_each_pipe(pipe) + __intel_set_cpu_fifo_underrun_reporting(dev, pipe, false); + + valleyview_disable_display_irqs(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); + + spin_lock_irq(&dev->vbl_lock); + for_each_pipe(pipe) + reset_vblank_counter(dev, pipe); + spin_unlock_irq(&dev->vbl_lock); + + vlv_set_power_well(dev_priv, power_well, false); +} - if (!--power_well->count && power_well->set && - i915_disable_power_well) { - power_well->set(dev, power_well, false); - hsw_enable_package_c8(dev_priv); +static void check_power_well_state(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + bool enabled = power_well->ops->is_enabled(dev_priv, power_well); + + if (power_well->always_on || !i915.disable_power_well) { + if (!enabled) + goto mismatch; + + return; } + + if (enabled != (power_well->count > 0)) + goto mismatch; + + return; + +mismatch: + WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n", + power_well->name, power_well->always_on, enabled, + power_well->count, i915.disable_power_well); } -void intel_display_power_get(struct drm_device *dev, +void intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; int i; + intel_runtime_pm_get(dev_priv); + power_domains = &dev_priv->power_domains; mutex_lock(&power_domains->lock); - for_each_power_well(i, power_well, BIT(domain), power_domains) - __intel_power_well_get(dev, power_well); + for_each_power_well(i, power_well, BIT(domain), power_domains) { + if (!power_well->count++) { + DRM_DEBUG_KMS("enabling %s\n", power_well->name); + power_well->ops->enable(dev_priv, power_well); + } + + check_power_well_state(dev_priv, power_well); + } power_domains->domain_use_count[domain]++; mutex_unlock(&power_domains->lock); } -void intel_display_power_put(struct drm_device *dev, +void intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; int i; @@ -5315,10 +5637,20 @@ void intel_display_power_put(struct drm_device *dev, WARN_ON(!power_domains->domain_use_count[domain]); power_domains->domain_use_count[domain]--; - for_each_power_well_rev(i, power_well, BIT(domain), power_domains) - __intel_power_well_put(dev, power_well); + for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { + WARN_ON(!power_well->count); + + if (!--power_well->count && i915.disable_power_well) { + DRM_DEBUG_KMS("disabling %s\n", power_well->name); + power_well->ops->disable(dev_priv, power_well); + } + + check_power_well_state(dev_priv, power_well); + } mutex_unlock(&power_domains->lock); + + intel_runtime_pm_put(dev_priv); } static struct i915_power_domains *hsw_pwr; @@ -5333,7 +5665,7 @@ void i915_request_power_well(void) dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - intel_display_power_get(dev_priv->dev, POWER_DOMAIN_AUDIO); + intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); } EXPORT_SYMBOL_GPL(i915_request_power_well); @@ -5347,29 +5679,99 @@ void i915_release_power_well(void) dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - intel_display_power_put(dev_priv->dev, POWER_DOMAIN_AUDIO); + intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); } EXPORT_SYMBOL_GPL(i915_release_power_well); +#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) + +#define HSW_ALWAYS_ON_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PIPE_A) | \ + BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_CRT) | \ + BIT(POWER_DOMAIN_INIT)) +#define HSW_DISPLAY_POWER_DOMAINS ( \ + (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \ + BIT(POWER_DOMAIN_INIT)) + +#define BDW_ALWAYS_ON_POWER_DOMAINS ( \ + HSW_ALWAYS_ON_POWER_DOMAINS | \ + BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER)) +#define BDW_DISPLAY_POWER_DOMAINS ( \ + (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT) +#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK + +#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_CRT) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { + .sync_hw = i9xx_always_on_power_well_noop, + .enable = i9xx_always_on_power_well_noop, + .disable = i9xx_always_on_power_well_noop, + .is_enabled = i9xx_always_on_power_well_enabled, +}; + static struct i915_power_well i9xx_always_on_power_well[] = { { .name = "always-on", .always_on = 1, .domains = POWER_DOMAIN_MASK, + .ops = &i9xx_always_on_power_well_ops, }, }; +static const struct i915_power_well_ops hsw_power_well_ops = { + .sync_hw = hsw_power_well_sync_hw, + .enable = hsw_power_well_enable, + .disable = hsw_power_well_disable, + .is_enabled = hsw_power_well_enabled, +}; + static struct i915_power_well hsw_power_wells[] = { { .name = "always-on", .always_on = 1, .domains = HSW_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, }, { .name = "display", - .domains = POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS, - .is_enabled = hsw_power_well_enabled, - .set = hsw_set_power_well, + .domains = HSW_DISPLAY_POWER_DOMAINS, + .ops = &hsw_power_well_ops, }, }; @@ -5378,12 +5780,83 @@ static struct i915_power_well bdw_power_wells[] = { .name = "always-on", .always_on = 1, .domains = BDW_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, }, { .name = "display", - .domains = POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS, - .is_enabled = hsw_power_well_enabled, - .set = hsw_set_power_well, + .domains = BDW_DISPLAY_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + }, +}; + +static const struct i915_power_well_ops vlv_display_power_well_ops = { + .sync_hw = vlv_power_well_sync_hw, + .enable = vlv_display_power_well_enable, + .disable = vlv_display_power_well_disable, + .is_enabled = vlv_power_well_enabled, +}; + +static const struct i915_power_well_ops vlv_dpio_power_well_ops = { + .sync_hw = vlv_power_well_sync_hw, + .enable = vlv_power_well_enable, + .disable = vlv_power_well_disable, + .is_enabled = vlv_power_well_enabled, +}; + +static struct i915_power_well vlv_power_wells[] = { + { + .name = "always-on", + .always_on = 1, + .domains = VLV_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, + }, + { + .name = "display", + .domains = VLV_DISPLAY_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DISP2D, + .ops = &vlv_display_power_well_ops, + }, + { + .name = "dpio-common", + .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .ops = &vlv_dpio_power_well_ops, + }, + { + .name = "dpio-tx-b-01", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, + }, + { + .name = "dpio-tx-b-23", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, + }, + { + .name = "dpio-tx-c-01", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, + }, + { + .name = "dpio-tx-c-23", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, }, }; @@ -5392,9 +5865,8 @@ static struct i915_power_well bdw_power_wells[] = { (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ }) -int intel_power_domains_init(struct drm_device *dev) +int intel_power_domains_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; mutex_init(&power_domains->lock); @@ -5403,12 +5875,14 @@ int intel_power_domains_init(struct drm_device *dev) * The enabling order will be from lower to higher indexed wells, * the disabling order is reversed. */ - if (IS_HASWELL(dev)) { + if (IS_HASWELL(dev_priv->dev)) { set_power_wells(power_domains, hsw_power_wells); hsw_pwr = power_domains; - } else if (IS_BROADWELL(dev)) { + } else if (IS_BROADWELL(dev_priv->dev)) { set_power_wells(power_domains, bdw_power_wells); hsw_pwr = power_domains; + } else if (IS_VALLEYVIEW(dev_priv->dev)) { + set_power_wells(power_domains, vlv_power_wells); } else { set_power_wells(power_domains, i9xx_always_on_power_well); } @@ -5416,58 +5890,38 @@ int intel_power_domains_init(struct drm_device *dev) return 0; } -void intel_power_domains_remove(struct drm_device *dev) +void intel_power_domains_remove(struct drm_i915_private *dev_priv) { hsw_pwr = NULL; } -static void intel_power_domains_resume(struct drm_device *dev) +static void intel_power_domains_resume(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *power_well; int i; mutex_lock(&power_domains->lock); - for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { - if (power_well->set) - power_well->set(dev, power_well, power_well->count > 0); - } + for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) + power_well->ops->sync_hw(dev_priv, power_well); mutex_unlock(&power_domains->lock); } -/* - * Starting with Haswell, we have a "Power Down Well" that can be turned off - * when not needed anymore. We have 4 registers that can request the power well - * to be enabled, and it will only be disabled if none of the registers is - * requesting it to be enabled. - */ -void intel_power_domains_init_hw(struct drm_device *dev) +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* For now, we need the power well to be always enabled. */ - intel_display_set_init_power(dev, true); - intel_power_domains_resume(dev); - - if (!(IS_HASWELL(dev) || IS_BROADWELL(dev))) - return; - - /* We're taking over the BIOS, so clear any requests made by it since - * the driver is in charge now. */ - if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) - I915_WRITE(HSW_PWR_WELL_BIOS, 0); + intel_display_set_init_power(dev_priv, true); + intel_power_domains_resume(dev_priv); } -/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */ void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv) { - hsw_disable_package_c8(dev_priv); + intel_runtime_pm_get(dev_priv); } void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) { - hsw_enable_package_c8(dev_priv); + intel_runtime_pm_put(dev_priv); } void intel_runtime_pm_get(struct drm_i915_private *dev_priv) @@ -5499,8 +5953,6 @@ void intel_init_runtime_pm(struct drm_i915_private *dev_priv) struct drm_device *dev = dev_priv->dev; struct device *device = &dev->pdev->dev; - dev_priv->pm.suspended = false; - if (!HAS_RUNTIME_PM(dev)) return; @@ -5509,6 +5961,8 @@ void intel_init_runtime_pm(struct drm_i915_private *dev_priv) pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ pm_runtime_mark_last_busy(device); pm_runtime_use_autosuspend(device); + + pm_runtime_put_autosuspend(device); } void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) @@ -5560,7 +6014,7 @@ void intel_init_pm(struct drm_device *dev) /* For FIFO watermark updates */ if (HAS_PCH_SPLIT(dev)) { - intel_setup_wm_latency(dev); + ilk_setup_wm_latency(dev); if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || @@ -5731,13 +6185,9 @@ void intel_pm_setup(struct drm_device *dev) mutex_init(&dev_priv->rps.hw_lock); - mutex_init(&dev_priv->pc8.lock); - dev_priv->pc8.requirements_met = false; - dev_priv->pc8.gpu_idle = false; - dev_priv->pc8.irqs_disabled = false; - dev_priv->pc8.enabled = false; - dev_priv->pc8.disable_count = 2; /* requirements_met + gpu_idle */ - INIT_DELAYED_WORK(&dev_priv->pc8.enable_work, hsw_enable_pc8_work); INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, intel_gen6_powersave_work); + + dev_priv->pm.suspended = false; + dev_priv->pm.irqs_disabled = false; } |
