AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2025-01-20 09:58:04 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

drm/i915: Extract GT render sleep (rc6) management

Browse Source 
Continuing the theme of breaking intel_pm.c up in a reasonable chunk of
powermanagement utilities, pull out the rc6 setup into its GT handler.

Based on a patch by Chris Wilson.

Signed-off-by: Andi Shyti <andi.shyti@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190919143840.20384-1-andi.shyti@intel.com
Link: https://patchwork.freedesktop.org/patch/msgid/20190927110849.28734-1-chris@chris-wilson.co.uk
This commit is contained in:
Andi Shyti 2019-09-27 12:08:49 +01:00 committed by Chris Wilson
parent a3f56e7da5
commit c113236718
18 changed files with 917 additions and 844 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/gpu/drm/i915/Makefile
View File

@ -85,6 +85,7 @@ gt-y += \
gt/intel_gt_pm_irq.o \
gt/intel_hangcheck.o \
gt/intel_lrc.o \
gt/intel_rc6.o \
gt/intel_renderstate.o \
gt/intel_reset.o \
gt/intel_ringbuffer.o \

2
drivers/gpu/drm/i915/gem/i915_gem_pm.c
View File

@ -137,7 +137,6 @@ static bool switch_to_kernel_context_sync(struct intel_gt *gt)
bool i915_gem_load_power_context(struct drm_i915_private *i915)
{
intel_gt_pm_enable(&i915->gt);
return switch_to_kernel_context_sync(&i915->gt);
}
@ -188,6 +187,7 @@ void i915_gem_suspend(struct drm_i915_private *i915)
i915_gem_drain_freed_objects(i915);
intel_uc_suspend(&i915->gt.uc);
intel_gt_suspend(&i915->gt);
}
static struct drm_i915_gem_object *first_mm_object(struct list_head *list)

1
drivers/gpu/drm/i915/gt/intel_engine_pm.c
View File

@ -11,6 +11,7 @@
#include "intel_engine_pool.h"
#include "intel_gt.h"
#include "intel_gt_pm.h"
#include "intel_rc6.h"
static int __engine_unpark(struct intel_wakeref *wf)
{

5
drivers/gpu/drm/i915/gt/intel_gt.c
View File

@ -7,6 +7,7 @@
#include "intel_gt.h"
#include "intel_gt_pm.h"
#include "intel_mocs.h"
#include "intel_rc6.h"
#include "intel_uncore.h"
#include "intel_pm.h"
@ -369,6 +370,8 @@ int intel_gt_init(struct intel_gt *gt)
if (err)
return err;
intel_gt_pm_init(gt);
return 0;
}
@ -387,8 +390,8 @@ void intel_gt_driver_release(struct intel_gt *gt)
{
/* Paranoia: make sure we have disabled everything before we exit. */
intel_gt_pm_disable(gt);
intel_gt_pm_fini(gt);
intel_cleanup_gt_powersave(gt->i915);
intel_gt_fini_scratch(gt);
}

94
drivers/gpu/drm/i915/gt/intel_gt_pm.c
View File

@ -11,6 +11,7 @@
#include "intel_gt.h"
#include "intel_gt_pm.h"
#include "intel_pm.h"
#include "intel_rc6.h"
#include "intel_wakeref.h"
static void pm_notify(struct intel_gt *gt, int state)
@ -90,6 +91,16 @@ void intel_gt_pm_init_early(struct intel_gt *gt)
BLOCKING_INIT_NOTIFIER_HEAD(&gt->pm_notifications);
}
void intel_gt_pm_init(struct intel_gt *gt)
{
/*
* Enabling power-management should be "self-healing". If we cannot
* enable a feature, simply leave it disabled with a notice to the
* user.
*/
intel_rc6_init(&gt->rc6);
}
static bool reset_engines(struct intel_gt *gt)
{
if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
@ -124,42 +135,16 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force)
__intel_engine_reset(engine, false);
}
static bool is_mock_device(const struct intel_gt *gt)
{
return I915_SELFTEST_ONLY(gt->awake == -1);
}
void intel_gt_pm_enable(struct intel_gt *gt)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* Powersaving is controlled by the host when inside a VM */
if (intel_vgpu_active(gt->i915))
return;
if (is_mock_device(gt))
return;
intel_gt_pm_get(gt);
for_each_engine(engine, gt->i915, id) {
intel_engine_pm_get(engine);
engine->serial++; /* force kernel context reload */
intel_engine_pm_put(engine);
}
intel_gt_pm_put(gt);
}
void intel_gt_pm_disable(struct intel_gt *gt)
{
if (is_mock_device(gt))
return;
intel_sanitize_gt_powersave(gt->i915);
}
void intel_gt_pm_fini(struct intel_gt *gt)
{
intel_rc6_fini(&gt->rc6);
}
int intel_gt_resume(struct intel_gt *gt)
{
struct intel_engine_cs *engine;
@ -173,6 +158,9 @@ int intel_gt_resume(struct intel_gt *gt)
* allowing us to fixup the user contexts on their first pin.
*/
intel_gt_pm_get(gt);
intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
intel_rc6_sanitize(&gt->rc6);
for_each_engine(engine, gt->i915, id) {
struct intel_context *ce;
@ -197,11 +185,51 @@ int intel_gt_resume(struct intel_gt *gt)
break;
}
}
intel_rc6_enable(&gt->rc6);
intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
intel_gt_pm_put(gt);
return err;
}
static void wait_for_idle(struct intel_gt *gt)
{
mutex_lock(&gt->i915->drm.struct_mutex); /* XXX */
do {
if (i915_gem_wait_for_idle(gt->i915,
I915_WAIT_LOCKED,
I915_GEM_IDLE_TIMEOUT) == -ETIME) {
/* XXX hide warning from gem_eio */
if (i915_modparams.reset) {
dev_err(gt->i915->drm.dev,
"Failed to idle engines, declaring wedged!\n");
GEM_TRACE_DUMP();
}
/*
* Forcibly cancel outstanding work and leave
* the gpu quiet.
*/
intel_gt_set_wedged(gt);
}
} while (i915_retire_requests(gt->i915));
mutex_unlock(&gt->i915->drm.struct_mutex);
intel_gt_pm_wait_for_idle(gt);
}
void intel_gt_suspend(struct intel_gt *gt)
{
intel_wakeref_t wakeref;
/* We expect to be idle already; but also want to be independent */
wait_for_idle(gt);
with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
intel_rc6_disable(&gt->rc6);
}
void intel_gt_runtime_suspend(struct intel_gt *gt)
{
intel_uc_runtime_suspend(&gt->uc);
@ -213,3 +241,7 @@ int intel_gt_runtime_resume(struct intel_gt *gt)
return intel_uc_runtime_resume(&gt->uc);
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftest_gt_pm.c"
#endif

11
drivers/gpu/drm/i915/gt/intel_gt_pm.h
View File

@ -43,12 +43,21 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt)
}
void intel_gt_pm_init_early(struct intel_gt *gt);
void intel_gt_pm_enable(struct intel_gt *gt);
void intel_gt_pm_init(struct intel_gt *gt);
void intel_gt_pm_disable(struct intel_gt *gt);
void intel_gt_pm_fini(struct intel_gt *gt);
void intel_gt_sanitize(struct intel_gt *gt, bool force);
int intel_gt_resume(struct intel_gt *gt);
void intel_gt_suspend(struct intel_gt *gt);
void intel_gt_runtime_suspend(struct intel_gt *gt);
int intel_gt_runtime_resume(struct intel_gt *gt);
static inline bool is_mock_gt(const struct intel_gt *gt)
{
return I915_SELFTEST_ONLY(gt->awake == -1);
}
#endif /* INTEL_GT_PM_H */

3
drivers/gpu/drm/i915/gt/intel_gt_types.h
View File

@ -18,6 +18,7 @@
#include "i915_vma.h"
#include "intel_engine_types.h"
#include "intel_reset_types.h"
#include "intel_rc6_types.h"
#include "intel_wakeref.h"
struct drm_i915_private;
@ -67,6 +68,8 @@ struct intel_gt {
*/
intel_wakeref_t awake;
struct intel_rc6 rc6;
struct blocking_notifier_head pm_notifications;
ktime_t last_init_time;

712
drivers/gpu/drm/i915/gt/intel_rc6.c Normal file
View File

@ -0,0 +1,712 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#include <linux/pm_runtime.h>
#include "i915_drv.h"
#include "intel_gt.h"
#include "intel_gt_pm.h"
#include "intel_rc6.h"
#include "intel_sideband.h"
/**
* DOC: RC6
*
* RC6 is a special power stage which allows the GPU to enter an very
* low-voltage mode when idle, using down to 0V while at this stage. This
* stage is entered automatically when the GPU is idle when RC6 support is
* enabled, and as soon as new workload arises GPU wakes up automatically as
* well.
*
* There are different RC6 modes available in Intel GPU, which differentiate
* among each other with the latency required to enter and leave RC6 and
* voltage consumed by the GPU in different states.
*
* The combination of the following flags define which states GPU is allowed
* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
* RC6pp is deepest RC6. Their support by hardware varies according to the
* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
* which brings the most power savings; deeper states save more power, but
* require higher latency to switch to and wake up.
*/
static struct intel_gt *rc6_to_gt(struct intel_rc6 *rc6)
{
return container_of(rc6, struct intel_gt, rc6);
}
static struct intel_uncore *rc6_to_uncore(struct intel_rc6 *rc)
{
return rc6_to_gt(rc)->uncore;
}
static struct drm_i915_private *rc6_to_i915(struct intel_rc6 *rc)
{
return rc6_to_gt(rc)->i915;
}
static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
{
intel_uncore_write_fw(uncore, reg, val);
}
static void gen11_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* 2b: Program RC6 thresholds.*/
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, rc6_to_gt(rc6)->i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GUC_MAX_IDLE_COUNT, 0xA);
set(uncore, GEN6_RC_SLEEP, 0);
set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
/*
* 2c: Program Coarse Power Gating Policies.
*
* Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
* use instead is a more conservative estimate for the maximum time
* it takes us to service a CS interrupt and submit a new ELSP - that
* is the time which the GPU is idle waiting for the CPU to select the
* next request to execute. If the idle hysteresis is less than that
* interrupt service latency, the hardware will automatically gate
* the power well and we will then incur the wake up cost on top of
* the service latency. A similar guide from plane_state is that we
* do not want the enable hysteresis to less than the wakeup latency.
*
* igt/gem_exec_nop/sequential provides a rough estimate for the
* service latency, and puts it around 10us for Broadwell (and other
* big core) and around 40us for Broxton (and other low power cores).
* [Note that for legacy ringbuffer submission, this is less than 1us!]
* However, the wakeup latency on Broxton is closer to 100us. To be
* conservative, we have to factor in a context switch on top (due
* to ksoftirqd).
*/
set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
/* 3a: Enable RC6 */
set(uncore, GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN6_RC_CTL_RC6_ENABLE |
GEN6_RC_CTL_EI_MODE(1));
set(uncore, GEN9_PG_ENABLE,
GEN9_RENDER_PG_ENABLE |
GEN9_MEDIA_PG_ENABLE |
GEN11_MEDIA_SAMPLER_PG_ENABLE);
}
static void gen9_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 rc6_mode;
/* 2b: Program RC6 thresholds.*/
if (INTEL_GEN(rc6_to_i915(rc6)) >= 10) {
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
} else if (IS_SKYLAKE(rc6_to_i915(rc6))) {
/*
* WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
* when CPG is enabled
*/
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
} else {
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
}
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, rc6_to_gt(rc6)->i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GUC_MAX_IDLE_COUNT, 0xA);
set(uncore, GEN6_RC_SLEEP, 0);
/*
* 2c: Program Coarse Power Gating Policies.
*
* Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
* use instead is a more conservative estimate for the maximum time
* it takes us to service a CS interrupt and submit a new ELSP - that
* is the time which the GPU is idle waiting for the CPU to select the
* next request to execute. If the idle hysteresis is less than that
* interrupt service latency, the hardware will automatically gate
* the power well and we will then incur the wake up cost on top of
* the service latency. A similar guide from plane_state is that we
* do not want the enable hysteresis to less than the wakeup latency.
*
* igt/gem_exec_nop/sequential provides a rough estimate for the
* service latency, and puts it around 10us for Broadwell (and other
* big core) and around 40us for Broxton (and other low power cores).
* [Note that for legacy ringbuffer submission, this is less than 1us!]
* However, the wakeup latency on Broxton is closer to 100us. To be
* conservative, we have to factor in a context switch on top (due
* to ksoftirqd).
*/
set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
/* 3a: Enable RC6 */
set(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
/* WaRsUseTimeoutMode:cnl (pre-prod) */
if (IS_CNL_REVID(rc6_to_i915(rc6), CNL_REVID_A0, CNL_REVID_C0))
rc6_mode = GEN7_RC_CTL_TO_MODE;
else
rc6_mode = GEN6_RC_CTL_EI_MODE(1);
set(uncore, GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN6_RC_CTL_RC6_ENABLE |
rc6_mode);
set(uncore, GEN9_PG_ENABLE,
GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
}
static void gen8_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* 2b: Program RC6 thresholds.*/
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, rc6_to_gt(rc6)->i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GEN6_RC_SLEEP, 0);
set(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
/* 3: Enable RC6 */
set(uncore, GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN7_RC_CTL_TO_MODE |
GEN6_RC_CTL_RC6_ENABLE);
}
static void gen6_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct drm_i915_private *i915 = rc6_to_i915(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 rc6vids, rc6_mask;
int ret;
set(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
set(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000);
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25);
for_each_engine(engine, i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GEN6_RC_SLEEP, 0);
set(uncore, GEN6_RC1e_THRESHOLD, 1000);
if (IS_IVYBRIDGE(i915))
set(uncore, GEN6_RC6_THRESHOLD, 125000);
else
set(uncore, GEN6_RC6_THRESHOLD, 50000);
set(uncore, GEN6_RC6p_THRESHOLD, 150000);
set(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */
/* We don't use those on Haswell */
rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
if (HAS_RC6p(i915))
rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
if (HAS_RC6pp(i915))
rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
set(uncore, GEN6_RC_CONTROL,
rc6_mask |
GEN6_RC_CTL_EI_MODE(1) |
GEN6_RC_CTL_HW_ENABLE);
rc6vids = 0;
ret = sandybridge_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS,
&rc6vids, NULL);
if (IS_GEN(i915, 6) && ret) {
DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
} else if (IS_GEN(i915, 6) &&
(GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
rc6vids &= 0xffff00;
rc6vids |= GEN6_ENCODE_RC6_VID(450);
ret = sandybridge_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
if (ret)
DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
}
}
/* Check that the pcbr address is not empty. */
static int chv_rc6_init(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
resource_size_t pctx_paddr, paddr;
resource_size_t pctx_size = 32 * SZ_1K;
u32 pcbr;
pcbr = intel_uncore_read(uncore, VLV_PCBR);
if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
paddr = rc6_to_i915(rc6)->dsm.end + 1 - pctx_size;
GEM_BUG_ON(paddr > U32_MAX);
pctx_paddr = (paddr & ~4095);
intel_uncore_write(uncore, VLV_PCBR, pctx_paddr);
}
return 0;
}
static int vlv_rc6_init(struct intel_rc6 *rc6)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct drm_i915_gem_object *pctx;
resource_size_t pctx_paddr;
resource_size_t pctx_size = 24 * SZ_1K;
u32 pcbr;
pcbr = intel_uncore_read(uncore, VLV_PCBR);
if (pcbr) {
/* BIOS set it up already, grab the pre-alloc'd space */
resource_size_t pcbr_offset;
pcbr_offset = (pcbr & ~4095) - i915->dsm.start;
pctx = i915_gem_object_create_stolen_for_preallocated(i915,
pcbr_offset,
I915_GTT_OFFSET_NONE,
pctx_size);
if (!pctx)
return -ENOMEM;
goto out;
}
DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
/*
* From the Gunit register HAS:
* The Gfx driver is expected to program this register and ensure
* proper allocation within Gfx stolen memory. For example, this
* register should be programmed such than the PCBR range does not
* overlap with other ranges, such as the frame buffer, protected
* memory, or any other relevant ranges.
*/
pctx = i915_gem_object_create_stolen(i915, pctx_size);
if (!pctx) {
DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
return -ENOMEM;
}
GEM_BUG_ON(range_overflows_t(u64,
i915->dsm.start,
pctx->stolen->start,
U32_MAX));
pctx_paddr = i915->dsm.start + pctx->stolen->start;
intel_uncore_write(uncore, VLV_PCBR, pctx_paddr);
out:
rc6->pctx = pctx;
return 0;
}
static void chv_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* 2a: Program RC6 thresholds.*/
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, rc6_to_gt(rc6)->i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GEN6_RC_SLEEP, 0);
/* TO threshold set to 500 us (0x186 * 1.28 us) */
set(uncore, GEN6_RC6_THRESHOLD, 0x186);
/* Allows RC6 residency counter to work */
set(uncore, VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
VLV_MEDIA_RC6_COUNT_EN |
VLV_RENDER_RC6_COUNT_EN));
/* 3: Enable RC6 */
set(uncore, GEN6_RC_CONTROL, GEN7_RC_CTL_TO_MODE);
}
static void vlv_rc6_enable(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct intel_engine_cs *engine;
enum intel_engine_id id;
set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000);
set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25);
for_each_engine(engine, rc6_to_gt(rc6)->i915, id)
set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
set(uncore, GEN6_RC6_THRESHOLD, 0x557);
/* Allows RC6 residency counter to work */
set(uncore, VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
VLV_MEDIA_RC0_COUNT_EN |
VLV_RENDER_RC0_COUNT_EN |
VLV_MEDIA_RC6_COUNT_EN |
VLV_RENDER_RC6_COUNT_EN));
set(uncore, GEN6_RC_CONTROL,
GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
}
static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6)
{
struct intel_uncore *uncore = rc6_to_uncore(rc6);
struct drm_i915_private *i915 = rc6_to_i915(rc6);
u32 rc6_ctx_base, rc_ctl, rc_sw_target;
bool enable_rc6 = true;
rc_ctl = intel_uncore_read(uncore, GEN6_RC_CONTROL);
rc_sw_target = intel_uncore_read(uncore, GEN6_RC_STATE);
rc_sw_target &= RC_SW_TARGET_STATE_MASK;
rc_sw_target >>= RC_SW_TARGET_STATE_SHIFT;
DRM_DEBUG_DRIVER("BIOS enabled RC states: "
"HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
rc_sw_target);
if (!(intel_uncore_read(uncore, RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
enable_rc6 = false;
}
/*
* The exact context size is not known for BXT, so assume a page size
* for this check.
*/
rc6_ctx_base =
intel_uncore_read(uncore, RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
if (!(rc6_ctx_base >= i915->dsm_reserved.start &&
rc6_ctx_base + PAGE_SIZE < i915->dsm_reserved.end)) {
DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
enable_rc6 = false;
}
if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1 &&
(intel_uncore_read(uncore, PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1 &&
(intel_uncore_read(uncore, PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1 &&
(intel_uncore_read(uncore, PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1)) {
DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
enable_rc6 = false;
}
if (!intel_uncore_read(uncore, GEN8_PUSHBUS_CONTROL) ||
!intel_uncore_read(uncore, GEN8_PUSHBUS_ENABLE) ||
!intel_uncore_read(uncore, GEN8_PUSHBUS_SHIFT)) {
DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
enable_rc6 = false;
}
if (!intel_uncore_read(uncore, GEN6_GFXPAUSE)) {
DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
enable_rc6 = false;
}
if (!intel_uncore_read(uncore, GEN8_MISC_CTRL0)) {
DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
enable_rc6 = false;
}
return enable_rc6;
}
static bool rc6_supported(struct intel_rc6 *rc6)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
if (!HAS_RC6(i915))
return false;
if (intel_vgpu_active(i915))
return false;
if (is_mock_gt(rc6_to_gt(rc6)))
return false;
if (IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(rc6)) {
dev_notice(i915->drm.dev,
"RC6 and powersaving disabled by BIOS\n");
return false;
}
return true;
}
static void rpm_get(struct intel_rc6 *rc6)
{
GEM_BUG_ON(rc6->wakeref);
pm_runtime_get_sync(&rc6_to_i915(rc6)->drm.pdev->dev);
rc6->wakeref = true;
}
static void rpm_put(struct intel_rc6 *rc6)
{
GEM_BUG_ON(!rc6->wakeref);
pm_runtime_put(&rc6_to_i915(rc6)->drm.pdev->dev);
rc6->wakeref = false;
}
static void __intel_rc6_disable(struct intel_rc6 *rc6)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
struct intel_uncore *uncore = rc6_to_uncore(rc6);
intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
if (INTEL_GEN(i915) >= 9)
set(uncore, GEN9_PG_ENABLE, 0);
set(uncore, GEN6_RC_CONTROL, 0);
set(uncore, GEN6_RC_STATE, 0);
intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
}
void intel_rc6_init(struct intel_rc6 *rc6)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
int err;
/* Disable runtime-pm until we can save the GPU state with rc6 pctx */
rpm_get(rc6);
if (!rc6_supported(rc6))
return;
if (IS_CHERRYVIEW(i915))
err = chv_rc6_init(rc6);
else if (IS_VALLEYVIEW(i915))
err = vlv_rc6_init(rc6);
else
err = 0;
/* Sanitize rc6, ensure it is disabled before we are ready. */
__intel_rc6_disable(rc6);
rc6->supported = err == 0;
}
void intel_rc6_sanitize(struct intel_rc6 *rc6)
{
if (rc6->supported)
__intel_rc6_disable(rc6);
}
void intel_rc6_enable(struct intel_rc6 *rc6)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
struct intel_uncore *uncore = rc6_to_uncore(rc6);
if (!rc6->supported)
return;
GEM_BUG_ON(rc6->enabled);
intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
if (IS_CHERRYVIEW(i915))
chv_rc6_enable(rc6);
else if (IS_VALLEYVIEW(i915))
vlv_rc6_enable(rc6);
else if (INTEL_GEN(i915) >= 11)
gen11_rc6_enable(rc6);
else if (INTEL_GEN(i915) >= 9)
gen9_rc6_enable(rc6);
else if (IS_BROADWELL(i915))
gen8_rc6_enable(rc6);
else if (INTEL_GEN(i915) >= 6)
gen6_rc6_enable(rc6);
intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
/* rc6 is ready, runtime-pm is go! */
rpm_put(rc6);
rc6->enabled = true;
}
void intel_rc6_disable(struct intel_rc6 *rc6)
{
if (!rc6->enabled)
return;
rpm_get(rc6);
rc6->enabled = false;
__intel_rc6_disable(rc6);
}
void intel_rc6_fini(struct intel_rc6 *rc6)
{
struct drm_i915_gem_object *pctx;
intel_rc6_disable(rc6);
pctx = fetch_and_zero(&rc6->pctx);
if (pctx)
i915_gem_object_put(pctx);
if (rc6->wakeref)
rpm_put(rc6);
}
static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg)
{
u32 lower, upper, tmp;
int loop = 2;
/*
* The register accessed do not need forcewake. We borrow
* uncore lock to prevent concurrent access to range reg.
*/
lockdep_assert_held(&uncore->lock);
/*
* vlv and chv residency counters are 40 bits in width.
* With a control bit, we can choose between upper or lower
* 32bit window into this counter.
*
* Although we always use the counter in high-range mode elsewhere,
* userspace may attempt to read the value before rc6 is initialised,
* before we have set the default VLV_COUNTER_CONTROL value. So always
* set the high bit to be safe.
*/
set(uncore, VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
upper = intel_uncore_read_fw(uncore, reg);
do {
tmp = upper;
set(uncore, VLV_COUNTER_CONTROL,
_MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
lower = intel_uncore_read_fw(uncore, reg);
set(uncore, VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
upper = intel_uncore_read_fw(uncore, reg);
} while (upper != tmp && --loop);
/*
* Everywhere else we always use VLV_COUNTER_CONTROL with the
* VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
* now.
*/
return lower | (u64)upper << 8;
}
u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, const i915_reg_t reg)
{
struct drm_i915_private *i915 = rc6_to_i915(rc6);
struct intel_uncore *uncore = rc6_to_uncore(rc6);
u64 time_hw, prev_hw, overflow_hw;
unsigned int fw_domains;
unsigned long flags;
unsigned int i;
u32 mul, div;
if (!rc6->supported)
return 0;
/*
* Store previous hw counter values for counter wrap-around handling.
*
* There are only four interesting registers and they live next to each
* other so we can use the relative address, compared to the smallest
* one as the index into driver storage.
*/
i = (i915_mmio_reg_offset(reg) -
i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
if (WARN_ON_ONCE(i >= ARRAY_SIZE(rc6->cur_residency)))
return 0;
fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
spin_lock_irqsave(&uncore->lock, flags);
intel_uncore_forcewake_get__locked(uncore, fw_domains);
/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
mul = 1000000;
div = i915->czclk_freq;
overflow_hw = BIT_ULL(40);
time_hw = vlv_residency_raw(uncore, reg);
} else {
/* 833.33ns units on Gen9LP, 1.28us elsewhere. */
if (IS_GEN9_LP(i915)) {
mul = 10000;
div = 12;
} else {
mul = 1280;
div = 1;
}
overflow_hw = BIT_ULL(32);
time_hw = intel_uncore_read_fw(uncore, reg);
}
/*
* Counter wrap handling.
*
* But relying on a sufficient frequency of queries otherwise counters
* can still wrap.
*/
prev_hw = rc6->prev_hw_residency[i];
rc6->prev_hw_residency[i] = time_hw;
/* RC6 delta from last sample. */
if (time_hw >= prev_hw)
time_hw -= prev_hw;
else
time_hw += overflow_hw - prev_hw;
/* Add delta to RC6 extended raw driver copy. */
time_hw += rc6->cur_residency[i];
rc6->cur_residency[i] = time_hw;
intel_uncore_forcewake_put__locked(uncore, fw_domains);
spin_unlock_irqrestore(&uncore->lock, flags);
return mul_u64_u32_div(time_hw, mul, div);
}
u64 intel_rc6_residency_us(struct intel_rc6 *rc6, i915_reg_t reg)
{
return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(rc6, reg), 1000);
}

25
drivers/gpu/drm/i915/gt/intel_rc6.h Normal file
View File

@ -0,0 +1,25 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#ifndef INTEL_RC6_H
#define INTEL_RC6_H
#include "i915_reg.h"
struct intel_engine_cs;
struct intel_rc6;
void intel_rc6_init(struct intel_rc6 *rc6);
void intel_rc6_fini(struct intel_rc6 *rc6);
void intel_rc6_sanitize(struct intel_rc6 *rc6);
void intel_rc6_enable(struct intel_rc6 *rc6);
void intel_rc6_disable(struct intel_rc6 *rc6);
u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, i915_reg_t reg);
u64 intel_rc6_residency_us(struct intel_rc6 *rc6, i915_reg_t reg);
#endif /* INTEL_RC6_H */

28
drivers/gpu/drm/i915/gt/intel_rc6_types.h Normal file
View File

@ -0,0 +1,28 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#ifndef INTEL_RC6_TYPES_H
#define INTEL_RC6_TYPES_H
#include <linux/spinlock.h>
#include <linux/types.h>
#include "intel_engine_types.h"
struct drm_i915_gem_object;
struct intel_rc6 {
u64 prev_hw_residency[4];
u64 cur_residency[4];
struct drm_i915_gem_object *pctx;
bool supported : 1;
bool enabled : 1;
bool wakeref : 1;
};
#endif /* INTEL_RC6_TYPES_H */

50
drivers/gpu/drm/i915/gt/selftest_gt_pm.c Normal file
View File

@ -0,0 +1,50 @@
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
static int live_gt_resume(void *arg)
{
struct intel_gt *gt = arg;
IGT_TIMEOUT(end_time);
int err;
/* Do several suspend/resume cycles to check we don't explode! */
do {
intel_gt_suspend(gt);
if (gt->rc6.enabled) {
pr_err("rc6 still enabled after suspend!\n");
intel_gt_set_wedged_on_init(gt);
err = -EINVAL;
break;
}
err = intel_gt_resume(gt);
if (err)
break;
if (gt->rc6.supported && !gt->rc6.enabled) {
pr_err("rc6 not enabled upon resume!\n");
intel_gt_set_wedged_on_init(gt);
err = -EINVAL;
break;
}
} while (!__igt_timeout(end_time, NULL));
return err;
}
int intel_gt_pm_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
SUBTEST(live_gt_resume),
};
if (intel_gt_is_wedged(&i915->gt))
return 0;
return intel_gt_live_subtests(tests, &i915->gt);
}

11
drivers/gpu/drm/i915/i915_debugfs.c
View File

@ -42,6 +42,7 @@
#include "gem/i915_gem_context.h"
#include "gt/intel_gt_pm.h"
#include "gt/intel_reset.h"
#include "gt/intel_rc6.h"
#include "gt/uc/intel_guc_submission.h"
#include "i915_debugfs.h"
@ -1168,11 +1169,13 @@ static void print_rc6_res(struct seq_file *m,
const char *title,
const i915_reg_t reg)
{
struct drm_i915_private *dev_priv = node_to_i915(m->private);
struct drm_i915_private *i915 = node_to_i915(m->private);
intel_wakeref_t wakeref;
seq_printf(m, "%s %u (%llu us)\n",
title, I915_READ(reg),
intel_rc6_residency_us(dev_priv, reg));
with_intel_runtime_pm(&i915->runtime_pm, wakeref)
seq_printf(m, "%s %u (%llu us)\n", title,
intel_uncore_read(&i915->uncore, reg),
intel_rc6_residency_us(&i915->gt.rc6, reg));
}
static int vlv_drpc_info(struct seq_file *m)

7
drivers/gpu/drm/i915/i915_drv.h
View File

@ -599,19 +599,12 @@ struct intel_rps {
struct intel_rps_ei ei;
};
struct intel_rc6 {
bool enabled;
u64 prev_hw_residency[4];
u64 cur_residency[4];
};
struct intel_llc_pstate {
bool enabled;
};
struct intel_gen6_power_mgmt {
struct intel_rps rps;
struct intel_rc6 rc6;
struct intel_llc_pstate llc_pstate;
};

9
drivers/gpu/drm/i915/i915_pmu.c
View File

@ -11,6 +11,7 @@
#include "gt/intel_engine_pm.h"
#include "gt/intel_engine_user.h"
#include "gt/intel_gt_pm.h"
#include "gt/intel_rc6.h"
#include "i915_drv.h"
#include "i915_pmu.h"
@ -116,21 +117,21 @@ static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active)
return enable;
}
static u64 __get_rc6(const struct intel_gt *gt)
static u64 __get_rc6(struct intel_gt *gt)
{
struct drm_i915_private *i915 = gt->i915;
u64 val;
val = intel_rc6_residency_ns(i915,
val = intel_rc6_residency_ns(&gt->rc6,
IS_VALLEYVIEW(i915) ?
VLV_GT_RENDER_RC6 :
GEN6_GT_GFX_RC6);
if (HAS_RC6p(i915))
val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
val += intel_rc6_residency_ns(&gt->rc6, GEN6_GT_GFX_RC6p);
if (HAS_RC6pp(i915))
val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
val += intel_rc6_residency_ns(&gt->rc6, GEN6_GT_GFX_RC6pp);
return val;
}

4
drivers/gpu/drm/i915/i915_sysfs.c
View File

@ -30,6 +30,8 @@
#include <linux/stat.h>
#include <linux/sysfs.h>
#include "gt/intel_rc6.h"
#include "i915_drv.h"
#include "i915_sysfs.h"
#include "intel_pm.h"
@ -49,7 +51,7 @@ static u32 calc_residency(struct drm_i915_private *dev_priv,
u64 res = 0;
with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref)
res = intel_rc6_residency_us(dev_priv, reg);
res = intel_rc6_residency_us(&dev_priv->gt.rc6, reg);
return DIV_ROUND_CLOSEST_ULL(res, 1000);
}

794
drivers/gpu/drm/i915/intel_pm.c
View File

@ -45,26 +45,6 @@
#include "intel_sideband.h"
#include "../../../platform/x86/intel_ips.h"
/**
* DOC: RC6
*
* RC6 is a special power stage which allows the GPU to enter an very
* low-voltage mode when idle, using down to 0V while at this stage. This
* stage is entered automatically when the GPU is idle when RC6 support is
* enabled, and as soon as new workload arises GPU wakes up automatically as well.
*
* There are different RC6 modes available in Intel GPU, which differentiate
* among each other with the latency required to enter and leave RC6 and
* voltage consumed by the GPU in different states.
*
* The combination of the following flags define which states GPU is allowed
* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
* RC6pp is deepest RC6. Their support by hardware varies according to the
* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
* which brings the most power savings; deeper states save more power, but
* require higher latency to switch to and wake up.
*/
static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
{
if (HAS_LLC(dev_priv)) {
@ -6918,142 +6898,27 @@ int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
return err;
}
static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RC_CONTROL, 0);
I915_WRITE(GEN9_PG_ENABLE, 0);
}
static void gen9_disable_rps(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RP_CONTROL, 0);
}
static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RC_CONTROL, 0);
}
static void gen6_disable_rps(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
I915_WRITE(GEN6_RP_CONTROL, 0);
}
static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RC_CONTROL, 0);
}
static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RP_CONTROL, 0);
}
static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
{
/* We're doing forcewake before Disabling RC6,
* This what the BIOS expects when going into suspend */
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
I915_WRITE(GEN6_RC_CONTROL, 0);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
{
I915_WRITE(GEN6_RP_CONTROL, 0);
}
static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
{
bool enable_rc6 = true;
unsigned long rc6_ctx_base;
u32 rc_ctl;
int rc_sw_target;
rc_ctl = I915_READ(GEN6_RC_CONTROL);
rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
RC_SW_TARGET_STATE_SHIFT;
DRM_DEBUG_DRIVER("BIOS enabled RC states: "
"HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
rc_sw_target);
if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
enable_rc6 = false;
}
/*
* The exact context size is not known for BXT, so assume a page size
* for this check.
*/
rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
(rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
enable_rc6 = false;
}
if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
enable_rc6 = false;
}
if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
!I915_READ(GEN8_PUSHBUS_ENABLE) ||
!I915_READ(GEN8_PUSHBUS_SHIFT)) {
DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
enable_rc6 = false;
}
if (!I915_READ(GEN6_GFXPAUSE)) {
DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
enable_rc6 = false;
}
if (!I915_READ(GEN8_MISC_CTRL0)) {
DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
enable_rc6 = false;
}
return enable_rc6;
}
static bool sanitize_rc6(struct drm_i915_private *i915)
{
struct intel_device_info *info = mkwrite_device_info(i915);
/* Powersaving is controlled by the host when inside a VM */
if (intel_vgpu_active(i915)) {
info->has_rc6 = 0;
info->has_rps = false;
}
if (info->has_rc6 &&
IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
DRM_INFO("RC6 disabled by BIOS\n");
info->has_rc6 = 0;
}
/*
* We assume that we do not have any deep rc6 levels if we don't have
* have the previous rc6 level supported, i.e. we use HAS_RC6()
* as the initial coarse check for rc6 in general, moving on to
* progressively finer/deeper levels.
*/
if (!info->has_rc6 && info->has_rc6p)
info->has_rc6p = 0;
return info->has_rc6;
}
static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
{
struct intel_rps *rps = &dev_priv->gt_pm.rps;
@ -7140,203 +7005,6 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen11_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* 1a: Software RC state - RC0 */
I915_WRITE(GEN6_RC_STATE, 0);
/*
* 1b: Get forcewake during program sequence. Although the driver
* hasn't enabled a state yet where we need forcewake, BIOS may have.
*/
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* 2a: Disable RC states. */
I915_WRITE(GEN6_RC_CONTROL, 0);
/* 2b: Program RC6 thresholds.*/
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
if (HAS_GT_UC(dev_priv))
I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
I915_WRITE(GEN6_RC_SLEEP, 0);
I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
/*
* 2c: Program Coarse Power Gating Policies.
*
* Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
* use instead is a more conservative estimate for the maximum time
* it takes us to service a CS interrupt and submit a new ELSP - that
* is the time which the GPU is idle waiting for the CPU to select the
* next request to execute. If the idle hysteresis is less than that
* interrupt service latency, the hardware will automatically gate
* the power well and we will then incur the wake up cost on top of
* the service latency. A similar guide from plane_state is that we
* do not want the enable hysteresis to less than the wakeup latency.
*
* igt/gem_exec_nop/sequential provides a rough estimate for the
* service latency, and puts it around 10us for Broadwell (and other
* big core) and around 40us for Broxton (and other low power cores).
* [Note that for legacy ringbuffer submission, this is less than 1us!]
* However, the wakeup latency on Broxton is closer to 100us. To be
* conservative, we have to factor in a context switch on top (due
* to ksoftirqd).
*/
I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
/* 3a: Enable RC6 */
I915_WRITE(GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN6_RC_CTL_RC6_ENABLE |
GEN6_RC_CTL_EI_MODE(1));
/* 3b: Enable Coarse Power Gating only when RC6 is enabled. */
I915_WRITE(GEN9_PG_ENABLE,
GEN9_RENDER_PG_ENABLE |
GEN9_MEDIA_PG_ENABLE |
GEN11_MEDIA_SAMPLER_PG_ENABLE);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 rc6_mode;
/* 1a: Software RC state - RC0 */
I915_WRITE(GEN6_RC_STATE, 0);
/* 1b: Get forcewake during program sequence. Although the driver
* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* 2a: Disable RC states. */
I915_WRITE(GEN6_RC_CONTROL, 0);
/* 2b: Program RC6 thresholds.*/
if (INTEL_GEN(dev_priv) >= 10) {
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
} else if (IS_SKYLAKE(dev_priv)) {
/*
* WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
* when CPG is enabled
*/
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
} else {
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
}
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
if (HAS_GT_UC(dev_priv))
I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
I915_WRITE(GEN6_RC_SLEEP, 0);
/*
* 2c: Program Coarse Power Gating Policies.
*
* Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
* use instead is a more conservative estimate for the maximum time
* it takes us to service a CS interrupt and submit a new ELSP - that
* is the time which the GPU is idle waiting for the CPU to select the
* next request to execute. If the idle hysteresis is less than that
* interrupt service latency, the hardware will automatically gate
* the power well and we will then incur the wake up cost on top of
* the service latency. A similar guide from plane_state is that we
* do not want the enable hysteresis to less than the wakeup latency.
*
* igt/gem_exec_nop/sequential provides a rough estimate for the
* service latency, and puts it around 10us for Broadwell (and other
* big core) and around 40us for Broxton (and other low power cores).
* [Note that for legacy ringbuffer submission, this is less than 1us!]
* However, the wakeup latency on Broxton is closer to 100us. To be
* conservative, we have to factor in a context switch on top (due
* to ksoftirqd).
*/
I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
/* 3a: Enable RC6 */
I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
/* WaRsUseTimeoutMode:cnl (pre-prod) */
if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
rc6_mode = GEN7_RC_CTL_TO_MODE;
else
rc6_mode = GEN6_RC_CTL_EI_MODE(1);
I915_WRITE(GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN6_RC_CTL_RC6_ENABLE |
rc6_mode);
/*
* 3b: Enable Coarse Power Gating only when RC6 is enabled.
* WaRsDisableCoarsePowerGating:skl,cnl - Render/Media PG need to be disabled with RC6.
*/
if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
I915_WRITE(GEN9_PG_ENABLE, 0);
else
I915_WRITE(GEN9_PG_ENABLE,
GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* 1a: Software RC state - RC0 */
I915_WRITE(GEN6_RC_STATE, 0);
/* 1b: Get forcewake during program sequence. Although the driver
* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* 2a: Disable RC states. */
I915_WRITE(GEN6_RC_CONTROL, 0);
/* 2b: Program RC6 thresholds.*/
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
I915_WRITE(GEN6_RC_SLEEP, 0);
I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
/* 3: Enable RC6 */
I915_WRITE(GEN6_RC_CONTROL,
GEN6_RC_CTL_HW_ENABLE |
GEN7_RC_CTL_TO_MODE |
GEN6_RC_CTL_RC6_ENABLE);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen8_enable_rps(struct drm_i915_private *dev_priv)
{
struct intel_rps *rps = &dev_priv->gt_pm.rps;
@ -7377,75 +7045,6 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 rc6vids, rc6_mask;
u32 gtfifodbg;
int ret;
I915_WRITE(GEN6_RC_STATE, 0);
/* Clear the DBG now so we don't confuse earlier errors */
gtfifodbg = I915_READ(GTFIFODBG);
if (gtfifodbg) {
DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
I915_WRITE(GTFIFODBG, gtfifodbg);
}
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* disable the counters and set deterministic thresholds */
I915_WRITE(GEN6_RC_CONTROL, 0);
I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
I915_WRITE(GEN6_RC_SLEEP, 0);
I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
if (IS_IVYBRIDGE(dev_priv))
I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
else
I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
/* We don't use those on Haswell */
rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
if (HAS_RC6p(dev_priv))
rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
if (HAS_RC6pp(dev_priv))
rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
I915_WRITE(GEN6_RC_CONTROL,
rc6_mask |
GEN6_RC_CTL_EI_MODE(1) |
GEN6_RC_CTL_HW_ENABLE);
rc6vids = 0;
ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS,
&rc6vids, NULL);
if (IS_GEN(dev_priv, 6) && ret) {
DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
} else if (IS_GEN(dev_priv, 6) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
rc6vids &= 0xffff00;
rc6vids |= GEN6_ENCODE_RC6_VID(450);
ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
if (ret)
DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
}
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void gen6_enable_rps(struct drm_i915_private *dev_priv)
{
/* Here begins a magic sequence of register writes to enable
@ -7662,100 +7261,6 @@ static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
return max_t(u32, val, 0xc0);
}
/* Check that the pctx buffer wasn't move under us. */
static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
{
unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
WARN_ON(pctx_addr != dev_priv->dsm.start +
dev_priv->vlv_pctx->stolen->start);
}
/* Check that the pcbr address is not empty. */
static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
{
unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
}
static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
{
resource_size_t pctx_paddr, paddr;
resource_size_t pctx_size = 32*1024;
u32 pcbr;
pcbr = I915_READ(VLV_PCBR);
if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
paddr = dev_priv->dsm.end + 1 - pctx_size;
GEM_BUG_ON(paddr > U32_MAX);
pctx_paddr = (paddr & (~4095));
I915_WRITE(VLV_PCBR, pctx_paddr);
}
DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
}
static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
{
struct drm_i915_gem_object *pctx;
resource_size_t pctx_paddr;
resource_size_t pctx_size = 24*1024;
u32 pcbr;
pcbr = I915_READ(VLV_PCBR);
if (pcbr) {
/* BIOS set it up already, grab the pre-alloc'd space */
resource_size_t pcbr_offset;
pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
pcbr_offset,
I915_GTT_OFFSET_NONE,
pctx_size);
goto out;
}
DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
/*
* From the Gunit register HAS:
* The Gfx driver is expected to program this register and ensure
* proper allocation within Gfx stolen memory. For example, this
* register should be programmed such than the PCBR range does not
* overlap with other ranges, such as the frame buffer, protected
* memory, or any other relevant ranges.
*/
pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
if (!pctx) {
DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
goto out;
}
GEM_BUG_ON(range_overflows_t(u64,
dev_priv->dsm.start,
pctx->stolen->start,
U32_MAX));
pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
I915_WRITE(VLV_PCBR, pctx_paddr);
out:
DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
dev_priv->vlv_pctx = pctx;
}
static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
{
struct drm_i915_gem_object *pctx;
pctx = fetch_and_zero(&dev_priv->vlv_pctx);
if (pctx)
i915_gem_object_put(pctx);
}
static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
{
dev_priv->gt_pm.rps.gpll_ref_freq =
@ -7772,8 +7277,6 @@ static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
struct intel_rps *rps = &dev_priv->gt_pm.rps;
u32 val;
valleyview_setup_pctx(dev_priv);
vlv_iosf_sb_get(dev_priv,
BIT(VLV_IOSF_SB_PUNIT) |
BIT(VLV_IOSF_SB_NC) |
@ -7828,8 +7331,6 @@ static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
struct intel_rps *rps = &dev_priv->gt_pm.rps;
u32 val;
cherryview_setup_pctx(dev_priv);
vlv_iosf_sb_get(dev_priv,
BIT(VLV_IOSF_SB_PUNIT) |
BIT(VLV_IOSF_SB_NC) |
@ -7880,64 +7381,6 @@ static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
"Odd GPU freq values\n");
}
static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
{
valleyview_cleanup_pctx(dev_priv);
}
static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 gtfifodbg, rc6_mode, pcbr;
gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
GT_FIFO_FREE_ENTRIES_CHV);
if (gtfifodbg) {
DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
gtfifodbg);
I915_WRITE(GTFIFODBG, gtfifodbg);
}
cherryview_check_pctx(dev_priv);
/* 1a & 1b: Get forcewake during program sequence. Although the driver
* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* Disable RC states. */
I915_WRITE(GEN6_RC_CONTROL, 0);
/* 2a: Program RC6 thresholds.*/
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
I915_WRITE(GEN6_RC_SLEEP, 0);
/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
/* Allows RC6 residency counter to work */
I915_WRITE(VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
VLV_MEDIA_RC6_COUNT_EN |
VLV_RENDER_RC6_COUNT_EN));
/* For now we assume BIOS is allocating and populating the PCBR */
pcbr = I915_READ(VLV_PCBR);
/* 3: Enable RC6 */
rc6_mode = 0;
if (pcbr >> VLV_PCBR_ADDR_SHIFT)
rc6_mode = GEN7_RC_CTL_TO_MODE;
I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
{
u32 val;
@ -7982,49 +7425,6 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
u32 gtfifodbg;
valleyview_check_pctx(dev_priv);
gtfifodbg = I915_READ(GTFIFODBG);
if (gtfifodbg) {
DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
gtfifodbg);
I915_WRITE(GTFIFODBG, gtfifodbg);
}
intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
/* Disable RC states. */
I915_WRITE(GEN6_RC_CONTROL, 0);
I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
for_each_engine(engine, dev_priv, id)
I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
/* Allows RC6 residency counter to work */
I915_WRITE(VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
VLV_MEDIA_RC0_COUNT_EN |
VLV_RENDER_RC0_COUNT_EN |
VLV_MEDIA_RC6_COUNT_EN |
VLV_RENDER_RC6_COUNT_EN));
I915_WRITE(GEN6_RC_CONTROL,
GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
}
static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
{
u32 val;
@ -8551,14 +7951,9 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
{
struct intel_rps *rps = &dev_priv->gt_pm.rps;
/*
* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
* requirement.
*/
if (!sanitize_rc6(dev_priv)) {
DRM_INFO("RC6 disabled, disabling runtime PM support\n");
pm_runtime_get(&dev_priv->drm.pdev->dev);
}
/* Powersaving is controlled by the host when inside a VM */
if (intel_vgpu_active(dev_priv))
mkwrite_device_info(dev_priv)->has_rps = false;
/* Initialize RPS limits (for userspace) */
if (IS_CHERRYVIEW(dev_priv))
@ -8593,19 +7988,9 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
rps->cur_freq = rps->idle_freq;
}
void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
{
if (IS_VALLEYVIEW(dev_priv))
valleyview_cleanup_gt_powersave(dev_priv);
if (!HAS_RC6(dev_priv))
pm_runtime_put(&dev_priv->drm.pdev->dev);
}
void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
{
dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
intel_disable_gt_powersave(dev_priv);
if (INTEL_GEN(dev_priv) >= 11)
@ -8626,25 +8011,6 @@ static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
i915->gt_pm.llc_pstate.enabled = false;
}
static void intel_disable_rc6(struct drm_i915_private *dev_priv)
{
lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
if (!dev_priv->gt_pm.rc6.enabled)
return;
if (INTEL_GEN(dev_priv) >= 9)
gen9_disable_rc6(dev_priv);
else if (IS_CHERRYVIEW(dev_priv))
cherryview_disable_rc6(dev_priv);
else if (IS_VALLEYVIEW(dev_priv))
valleyview_disable_rc6(dev_priv);
else if (INTEL_GEN(dev_priv) >= 6)
gen6_disable_rc6(dev_priv);
dev_priv->gt_pm.rc6.enabled = false;
}
static void intel_disable_rps(struct drm_i915_private *dev_priv)
{
lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
@ -8670,8 +8036,6 @@ void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
{
mutex_lock(&dev_priv->gt_pm.rps.lock);
intel_disable_rc6(dev_priv);
intel_disable_rps(dev_priv);
if (HAS_LLC(dev_priv))
intel_disable_llc_pstate(dev_priv);
@ -8691,29 +8055,6 @@ static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
i915->gt_pm.llc_pstate.enabled = true;
}
static void intel_enable_rc6(struct drm_i915_private *dev_priv)
{
lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
if (dev_priv->gt_pm.rc6.enabled)
return;
if (IS_CHERRYVIEW(dev_priv))
cherryview_enable_rc6(dev_priv);
else if (IS_VALLEYVIEW(dev_priv))
valleyview_enable_rc6(dev_priv);
else if (INTEL_GEN(dev_priv) >= 11)
gen11_enable_rc6(dev_priv);
else if (INTEL_GEN(dev_priv) >= 9)
gen9_enable_rc6(dev_priv);
else if (IS_BROADWELL(dev_priv))
gen8_enable_rc6(dev_priv);
else if (INTEL_GEN(dev_priv) >= 6)
gen6_enable_rc6(dev_priv);
dev_priv->gt_pm.rc6.enabled = true;
}
static void intel_enable_rps(struct drm_i915_private *dev_priv)
{
struct intel_rps *rps = &dev_priv->gt_pm.rps;
@ -8755,8 +8096,6 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
mutex_lock(&dev_priv->gt_pm.rps.lock);
if (HAS_RC6(dev_priv))
intel_enable_rc6(dev_priv);
if (HAS_RPS(dev_priv))
intel_enable_rps(dev_priv);
if (HAS_LLC(dev_priv))
@ -9818,133 +9157,6 @@ void intel_pm_setup(struct drm_i915_private *dev_priv)
atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
}
static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
const i915_reg_t reg)
{
u32 lower, upper, tmp;
int loop = 2;
/*
* The register accessed do not need forcewake. We borrow
* uncore lock to prevent concurrent access to range reg.
*/
lockdep_assert_held(&dev_priv->uncore.lock);
/*
* vlv and chv residency counters are 40 bits in width.
* With a control bit, we can choose between upper or lower
* 32bit window into this counter.
*
* Although we always use the counter in high-range mode elsewhere,
* userspace may attempt to read the value before rc6 is initialised,
* before we have set the default VLV_COUNTER_CONTROL value. So always
* set the high bit to be safe.
*/
I915_WRITE_FW(VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
upper = I915_READ_FW(reg);
do {
tmp = upper;
I915_WRITE_FW(VLV_COUNTER_CONTROL,
_MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
lower = I915_READ_FW(reg);
I915_WRITE_FW(VLV_COUNTER_CONTROL,
_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
upper = I915_READ_FW(reg);
} while (upper != tmp && --loop);
/*
* Everywhere else we always use VLV_COUNTER_CONTROL with the
* VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
* now.
*/
return lower | (u64)upper << 8;
}
u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
const i915_reg_t reg)
{
struct intel_uncore *uncore = &dev_priv->uncore;
u64 time_hw, prev_hw, overflow_hw;
unsigned int fw_domains;
unsigned long flags;
unsigned int i;
u32 mul, div;
if (!HAS_RC6(dev_priv))
return 0;
/*
* Store previous hw counter values for counter wrap-around handling.
*
* There are only four interesting registers and they live next to each
* other so we can use the relative address, compared to the smallest
* one as the index into driver storage.
*/
i = (i915_mmio_reg_offset(reg) -
i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
return 0;
fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
spin_lock_irqsave(&uncore->lock, flags);
intel_uncore_forcewake_get__locked(uncore, fw_domains);
/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
mul = 1000000;
div = dev_priv->czclk_freq;
overflow_hw = BIT_ULL(40);
time_hw = vlv_residency_raw(dev_priv, reg);
} else {
/* 833.33ns units on Gen9LP, 1.28us elsewhere. */
if (IS_GEN9_LP(dev_priv)) {
mul = 10000;
div = 12;
} else {
mul = 1280;
div = 1;
}
overflow_hw = BIT_ULL(32);
time_hw = intel_uncore_read_fw(uncore, reg);
}
/*
* Counter wrap handling.
*
* But relying on a sufficient frequency of queries otherwise counters
* can still wrap.
*/
prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
/* RC6 delta from last sample. */
if (time_hw >= prev_hw)
time_hw -= prev_hw;
else
time_hw += overflow_hw - prev_hw;
/* Add delta to RC6 extended raw driver copy. */
time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
intel_uncore_forcewake_put__locked(uncore, fw_domains);
spin_unlock_irqrestore(&uncore->lock, flags);
return mul_u64_u32_div(time_hw, mul, div);
}
u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
i915_reg_t reg)
{
return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000);
}
u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
{
u32 cagf;

3
drivers/gpu/drm/i915/intel_pm.h
View File

@ -32,7 +32,6 @@ void intel_pm_setup(struct drm_i915_private *dev_priv);
void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
void intel_gpu_ips_teardown(void);
void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
@ -72,8 +71,6 @@ void intel_enable_ipc(struct drm_i915_private *dev_priv);
int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv, i915_reg_t reg);
u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv, i915_reg_t reg);
u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1);

1
drivers/gpu/drm/i915/selftests/i915_live_selftests.h
View File

@ -16,6 +16,7 @@ selftest(gt_engines, intel_engine_live_selftests)
selftest(gt_timelines, intel_timeline_live_selftests)
selftest(gt_contexts, intel_context_live_selftests)
selftest(gt_lrc, intel_lrc_live_selftests)
selftest(gt_pm, intel_gt_pm_live_selftests)
selftest(requests, i915_request_live_selftests)
selftest(active, i915_active_live_selftests)
selftest(objects, i915_gem_object_live_selftests)