2007-07-09 23:52:00 +07:00
|
|
|
/*
|
2011-11-15 23:14:39 +07:00
|
|
|
* kernel/sched/debug.c
|
2007-07-09 23:52:00 +07:00
|
|
|
*
|
|
|
|
* Print the CFS rbtree
|
|
|
|
*
|
|
|
|
* Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
#include <linux/utsname.h>
|
2013-10-07 17:29:30 +07:00
|
|
|
#include <linux/mempolicy.h>
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2011-10-25 15:00:11 +07:00
|
|
|
#include "sched.h"
|
|
|
|
|
2011-01-11 17:11:54 +07:00
|
|
|
static DEFINE_SPINLOCK(sched_debug_lock);
|
|
|
|
|
2007-07-09 23:52:00 +07:00
|
|
|
/*
|
|
|
|
* This allows printing both to /proc/sched_debug and
|
|
|
|
* to the console
|
|
|
|
*/
|
|
|
|
#define SEQ_printf(m, x...) \
|
|
|
|
do { \
|
|
|
|
if (m) \
|
|
|
|
seq_printf(m, x); \
|
|
|
|
else \
|
|
|
|
printk(x); \
|
|
|
|
} while (0)
|
|
|
|
|
2007-10-15 22:00:08 +07:00
|
|
|
/*
|
|
|
|
* Ease the printing of nsec fields:
|
|
|
|
*/
|
2007-12-30 23:24:35 +07:00
|
|
|
static long long nsec_high(unsigned long long nsec)
|
2007-10-15 22:00:08 +07:00
|
|
|
{
|
2007-12-30 23:24:35 +07:00
|
|
|
if ((long long)nsec < 0) {
|
2007-10-15 22:00:08 +07:00
|
|
|
nsec = -nsec;
|
|
|
|
do_div(nsec, 1000000);
|
|
|
|
return -nsec;
|
|
|
|
}
|
|
|
|
do_div(nsec, 1000000);
|
|
|
|
|
|
|
|
return nsec;
|
|
|
|
}
|
|
|
|
|
2007-12-30 23:24:35 +07:00
|
|
|
static unsigned long nsec_low(unsigned long long nsec)
|
2007-10-15 22:00:08 +07:00
|
|
|
{
|
2007-12-30 23:24:35 +07:00
|
|
|
if ((long long)nsec < 0)
|
2007-10-15 22:00:08 +07:00
|
|
|
nsec = -nsec;
|
|
|
|
|
|
|
|
return do_div(nsec, 1000000);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define SPLIT_NS(x) nsec_high(x), nsec_low(x)
|
|
|
|
|
2008-11-10 23:04:09 +07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched: Add 'autogroup' scheduling feature: automated per session task groups
A recurring complaint from CFS users is that parallel kbuild has
a negative impact on desktop interactivity. This patch
implements an idea from Linus, to automatically create task
groups. Currently, only per session autogroups are implemented,
but the patch leaves the way open for enhancement.
Implementation: each task's signal struct contains an inherited
pointer to a refcounted autogroup struct containing a task group
pointer, the default for all tasks pointing to the
init_task_group. When a task calls setsid(), a new task group
is created, the process is moved into the new task group, and a
reference to the preveious task group is dropped. Child
processes inherit this task group thereafter, and increase it's
refcount. When the last thread of a process exits, the
process's reference is dropped, such that when the last process
referencing an autogroup exits, the autogroup is destroyed.
At runqueue selection time, IFF a task has no cgroup assignment,
its current autogroup is used.
Autogroup bandwidth is controllable via setting it's nice level
through the proc filesystem:
cat /proc/<pid>/autogroup
Displays the task's group and the group's nice level.
echo <nice level> > /proc/<pid>/autogroup
Sets the task group's shares to the weight of nice <level> task.
Setting nice level is rate limited for !admin users due to the
abuse risk of task group locking.
The feature is enabled from boot by default if
CONFIG_SCHED_AUTOGROUP=y is selected, but can be disabled via
the boot option noautogroup, and can also be turned on/off on
the fly via:
echo [01] > /proc/sys/kernel/sched_autogroup_enabled
... which will automatically move tasks to/from the root task group.
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Markus Trippelsdorf <markus@trippelsdorf.de>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Paul Turner <pjt@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
[ Removed the task_group_path() debug code, and fixed !EVENTFD build failure. ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
LKML-Reference: <1290281700.28711.9.camel@maggy.simson.net>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-11-30 20:18:03 +07:00
|
|
|
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
|
2008-11-10 23:04:09 +07:00
|
|
|
{
|
|
|
|
struct sched_entity *se = tg->se[cpu];
|
|
|
|
|
|
|
|
#define P(F) \
|
|
|
|
SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
|
|
|
|
#define PN(F) \
|
|
|
|
SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
|
|
|
|
|
2015-07-15 07:04:36 +07:00
|
|
|
if (!se)
|
2012-10-04 17:51:20 +07:00
|
|
|
return;
|
|
|
|
|
2008-11-10 23:04:09 +07:00
|
|
|
PN(se->exec_start);
|
|
|
|
PN(se->vruntime);
|
|
|
|
PN(se->sum_exec_runtime);
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2010-03-11 09:37:45 +07:00
|
|
|
PN(se->statistics.wait_start);
|
|
|
|
PN(se->statistics.sleep_start);
|
|
|
|
PN(se->statistics.block_start);
|
|
|
|
PN(se->statistics.sleep_max);
|
|
|
|
PN(se->statistics.block_max);
|
|
|
|
PN(se->statistics.exec_max);
|
|
|
|
PN(se->statistics.slice_max);
|
|
|
|
PN(se->statistics.wait_max);
|
|
|
|
PN(se->statistics.wait_sum);
|
|
|
|
P(se->statistics.wait_count);
|
2008-11-10 23:04:09 +07:00
|
|
|
#endif
|
|
|
|
P(se->load.weight);
|
2012-10-04 18:18:29 +07:00
|
|
|
#ifdef CONFIG_SMP
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-15 07:04:37 +07:00
|
|
|
P(se->avg.load_avg);
|
|
|
|
P(se->avg.util_avg);
|
2012-10-04 18:18:29 +07:00
|
|
|
#endif
|
2008-11-10 23:04:09 +07:00
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2011-01-11 17:11:54 +07:00
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
|
|
static char group_path[PATH_MAX];
|
|
|
|
|
|
|
|
static char *task_group_path(struct task_group *tg)
|
|
|
|
{
|
2011-01-11 17:12:57 +07:00
|
|
|
if (autogroup_path(tg, group_path, PATH_MAX))
|
|
|
|
return group_path;
|
|
|
|
|
2014-02-12 21:29:50 +07:00
|
|
|
return cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
|
2011-01-11 17:11:54 +07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-07-09 23:52:00 +07:00
|
|
|
static void
|
2007-08-09 16:16:51 +07:00
|
|
|
print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
|
|
|
if (rq->curr == p)
|
|
|
|
SEQ_printf(m, "R");
|
|
|
|
else
|
|
|
|
SEQ_printf(m, " ");
|
|
|
|
|
2007-10-15 22:00:08 +07:00
|
|
|
SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ",
|
2013-09-09 18:01:41 +07:00
|
|
|
p->comm, task_pid_nr(p),
|
2007-10-15 22:00:08 +07:00
|
|
|
SPLIT_NS(p->se.vruntime),
|
2007-07-09 23:52:00 +07:00
|
|
|
(long long)(p->nvcsw + p->nivcsw),
|
2007-08-06 10:26:59 +07:00
|
|
|
p->prio);
|
2007-08-02 22:41:40 +07:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2008-04-20 00:45:00 +07:00
|
|
|
SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
|
2015-06-08 15:10:40 +07:00
|
|
|
SPLIT_NS(p->se.statistics.wait_sum),
|
2007-10-15 22:00:08 +07:00
|
|
|
SPLIT_NS(p->se.sum_exec_runtime),
|
2010-03-11 09:37:45 +07:00
|
|
|
SPLIT_NS(p->se.statistics.sum_sleep_runtime));
|
2007-08-02 22:41:40 +07:00
|
|
|
#else
|
2015-06-08 15:10:39 +07:00
|
|
|
SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
|
|
|
|
0LL, 0L,
|
|
|
|
SPLIT_NS(p->se.sum_exec_runtime),
|
|
|
|
0LL, 0L);
|
2007-08-02 22:41:40 +07:00
|
|
|
#endif
|
2013-10-07 17:29:30 +07:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
2015-06-26 00:21:42 +07:00
|
|
|
SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
|
2013-10-07 17:29:30 +07:00
|
|
|
#endif
|
2011-01-11 17:11:54 +07:00
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
|
|
SEQ_printf(m, " %s", task_group_path(task_group(p)));
|
|
|
|
#endif
|
2008-04-20 00:45:00 +07:00
|
|
|
|
|
|
|
SEQ_printf(m, "\n");
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
2007-08-09 16:16:51 +07:00
|
|
|
static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
|
|
|
struct task_struct *g, *p;
|
|
|
|
|
|
|
|
SEQ_printf(m,
|
|
|
|
"\nrunnable tasks:\n"
|
2007-10-15 22:00:08 +07:00
|
|
|
" task PID tree-key switches prio"
|
2015-06-08 15:10:40 +07:00
|
|
|
" wait-time sum-exec sum-sleep\n"
|
2007-10-15 22:00:08 +07:00
|
|
|
"------------------------------------------------------"
|
2007-10-15 22:00:08 +07:00
|
|
|
"----------------------------------------------------\n");
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2014-09-22 02:33:41 +07:00
|
|
|
rcu_read_lock();
|
2014-08-14 02:19:56 +07:00
|
|
|
for_each_process_thread(g, p) {
|
2013-10-07 17:29:30 +07:00
|
|
|
if (task_cpu(p) != rq_cpu)
|
2007-07-09 23:52:00 +07:00
|
|
|
continue;
|
|
|
|
|
2007-08-09 16:16:51 +07:00
|
|
|
print_task(m, rq, p);
|
2014-08-14 02:19:56 +07:00
|
|
|
}
|
2014-09-22 02:33:41 +07:00
|
|
|
rcu_read_unlock();
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
2007-08-09 16:16:47 +07:00
|
|
|
void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
2007-10-15 22:00:06 +07:00
|
|
|
s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
|
|
|
|
spread, rq0_min_vruntime, spread0;
|
2009-06-17 20:20:55 +07:00
|
|
|
struct rq *rq = cpu_rq(cpu);
|
2007-10-15 22:00:05 +07:00
|
|
|
struct sched_entity *last;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2011-01-11 17:11:54 +07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
|
SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg));
|
|
|
|
#else
|
2008-06-19 19:22:24 +07:00
|
|
|
SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
|
2011-01-11 17:11:54 +07:00
|
|
|
#endif
|
2007-10-15 22:00:08 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
|
|
|
|
SPLIT_NS(cfs_rq->exec_clock));
|
2007-10-15 22:00:05 +07:00
|
|
|
|
2009-11-17 20:28:38 +07:00
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
2007-10-15 22:00:05 +07:00
|
|
|
if (cfs_rq->rb_leftmost)
|
2011-02-01 21:51:03 +07:00
|
|
|
MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;
|
2007-10-15 22:00:05 +07:00
|
|
|
last = __pick_last_entity(cfs_rq);
|
|
|
|
if (last)
|
|
|
|
max_vruntime = last->vruntime;
|
2008-11-10 16:46:32 +07:00
|
|
|
min_vruntime = cfs_rq->min_vruntime;
|
2009-06-17 20:20:55 +07:00
|
|
|
rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
|
2009-11-17 20:28:38 +07:00
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
2007-10-15 22:00:08 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime",
|
|
|
|
SPLIT_NS(MIN_vruntime));
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime",
|
|
|
|
SPLIT_NS(min_vruntime));
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime",
|
|
|
|
SPLIT_NS(max_vruntime));
|
2007-10-15 22:00:05 +07:00
|
|
|
spread = max_vruntime - MIN_vruntime;
|
2007-10-15 22:00:08 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread",
|
|
|
|
SPLIT_NS(spread));
|
2007-10-15 22:00:06 +07:00
|
|
|
spread0 = min_vruntime - rq0_min_vruntime;
|
2007-10-15 22:00:08 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0",
|
|
|
|
SPLIT_NS(spread0));
|
2008-11-10 16:46:32 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
|
2007-10-15 22:00:10 +07:00
|
|
|
cfs_rq->nr_spread_over);
|
2012-04-26 18:12:27 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
|
2010-11-16 06:47:00 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
|
2008-06-27 18:41:14 +07:00
|
|
|
#ifdef CONFIG_SMP
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-15 07:04:37 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
|
|
|
|
cfs_rq->avg.load_avg);
|
2015-07-15 07:04:41 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "runnable_load_avg",
|
|
|
|
cfs_rq->runnable_load_avg);
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-15 07:04:37 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
|
|
|
|
cfs_rq->avg.util_avg);
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "removed_load_avg",
|
|
|
|
atomic_long_read(&cfs_rq->removed_load_avg));
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "removed_util_avg",
|
|
|
|
atomic_long_read(&cfs_rq->removed_util_avg));
|
2013-06-28 18:10:35 +07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-15 07:04:37 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib",
|
|
|
|
cfs_rq->tg_load_avg_contrib);
|
2013-06-28 18:10:35 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg",
|
|
|
|
atomic_long_read(&cfs_rq->tg->load_avg));
|
2008-06-27 18:41:14 +07:00
|
|
|
#endif
|
2013-06-28 18:10:35 +07:00
|
|
|
#endif
|
2013-10-17 01:16:32 +07:00
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "throttled",
|
|
|
|
cfs_rq->throttled);
|
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
|
|
|
|
cfs_rq->throttle_count);
|
|
|
|
#endif
|
2010-11-16 06:47:00 +07:00
|
|
|
|
2013-06-28 18:10:35 +07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2008-11-10 23:04:09 +07:00
|
|
|
print_cfs_group_stats(m, cpu, cfs_rq->tg);
|
2008-06-27 18:41:14 +07:00
|
|
|
#endif
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
2008-06-19 19:22:24 +07:00
|
|
|
void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
|
|
|
|
{
|
2011-01-11 17:11:54 +07:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
|
SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, task_group_path(rt_rq->tg));
|
|
|
|
#else
|
2008-06-19 19:22:24 +07:00
|
|
|
SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
|
2011-01-11 17:11:54 +07:00
|
|
|
#endif
|
2008-06-19 19:22:24 +07:00
|
|
|
|
|
|
|
#define P(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
|
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
|
|
|
|
|
|
|
|
P(rt_nr_running);
|
|
|
|
P(rt_throttled);
|
|
|
|
PN(rt_time);
|
|
|
|
PN(rt_runtime);
|
|
|
|
|
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
}
|
|
|
|
|
2014-10-31 05:39:33 +07:00
|
|
|
void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
|
|
|
|
{
|
|
|
|
SEQ_printf(m, "\ndl_rq[%d]:\n", cpu);
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "dl_nr_running", dl_rq->dl_nr_running);
|
|
|
|
}
|
|
|
|
|
2010-11-20 03:11:09 +07:00
|
|
|
extern __read_mostly int sched_clock_running;
|
|
|
|
|
2007-08-09 16:16:51 +07:00
|
|
|
static void print_cpu(struct seq_file *m, int cpu)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
2009-06-17 20:20:55 +07:00
|
|
|
struct rq *rq = cpu_rq(cpu);
|
2011-01-11 17:11:54 +07:00
|
|
|
unsigned long flags;
|
2007-07-09 23:52:00 +07:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86
|
|
|
|
{
|
|
|
|
unsigned int freq = cpu_khz ? : 1;
|
|
|
|
|
2013-02-22 06:15:09 +07:00
|
|
|
SEQ_printf(m, "cpu#%d, %u.%03u MHz\n",
|
2007-07-09 23:52:00 +07:00
|
|
|
cpu, freq / 1000, (freq % 1000));
|
|
|
|
}
|
|
|
|
#else
|
2013-02-22 06:15:09 +07:00
|
|
|
SEQ_printf(m, "cpu#%d\n", cpu);
|
2007-07-09 23:52:00 +07:00
|
|
|
#endif
|
|
|
|
|
2012-05-14 19:34:00 +07:00
|
|
|
#define P(x) \
|
|
|
|
do { \
|
|
|
|
if (sizeof(rq->x) == 4) \
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \
|
|
|
|
else \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\
|
|
|
|
} while (0)
|
|
|
|
|
2007-10-15 22:00:08 +07:00
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x))
|
2007-07-09 23:52:00 +07:00
|
|
|
|
|
|
|
P(nr_running);
|
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "load",
|
2007-10-15 22:00:06 +07:00
|
|
|
rq->load.weight);
|
2007-07-09 23:52:00 +07:00
|
|
|
P(nr_switches);
|
|
|
|
P(nr_load_updates);
|
|
|
|
P(nr_uninterruptible);
|
2007-10-15 22:00:08 +07:00
|
|
|
PN(next_balance);
|
2013-09-09 18:01:41 +07:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
|
2007-10-15 22:00:08 +07:00
|
|
|
PN(clock);
|
2015-01-05 17:18:12 +07:00
|
|
|
PN(clock_task);
|
2007-07-09 23:52:00 +07:00
|
|
|
P(cpu_load[0]);
|
|
|
|
P(cpu_load[1]);
|
|
|
|
P(cpu_load[2]);
|
|
|
|
P(cpu_load[3]);
|
|
|
|
P(cpu_load[4]);
|
|
|
|
#undef P
|
2007-10-15 22:00:08 +07:00
|
|
|
#undef PN
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2008-11-10 16:46:32 +07:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
#define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n);
|
2009-11-04 23:53:50 +07:00
|
|
|
#define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n);
|
2008-11-10 16:46:32 +07:00
|
|
|
|
|
|
|
P(yld_count);
|
|
|
|
|
|
|
|
P(sched_count);
|
|
|
|
P(sched_goidle);
|
2009-11-04 23:53:50 +07:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
P64(avg_idle);
|
2014-01-23 17:39:54 +07:00
|
|
|
P64(max_idle_balance_cost);
|
2009-11-04 23:53:50 +07:00
|
|
|
#endif
|
2008-11-10 16:46:32 +07:00
|
|
|
|
|
|
|
P(ttwu_count);
|
|
|
|
P(ttwu_local);
|
|
|
|
|
|
|
|
#undef P
|
2011-01-14 14:57:39 +07:00
|
|
|
#undef P64
|
2008-11-10 16:46:32 +07:00
|
|
|
#endif
|
2011-01-11 17:11:54 +07:00
|
|
|
spin_lock_irqsave(&sched_debug_lock, flags);
|
2007-08-09 16:16:47 +07:00
|
|
|
print_cfs_stats(m, cpu);
|
2008-06-19 19:22:24 +07:00
|
|
|
print_rt_stats(m, cpu);
|
2014-10-31 05:39:33 +07:00
|
|
|
print_dl_stats(m, cpu);
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2007-08-09 16:16:51 +07:00
|
|
|
print_rq(m, rq, cpu);
|
2011-01-11 17:11:54 +07:00
|
|
|
spin_unlock_irqrestore(&sched_debug_lock, flags);
|
2013-02-22 06:15:09 +07:00
|
|
|
SEQ_printf(m, "\n");
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
2009-11-30 18:16:47 +07:00
|
|
|
static const char *sched_tunable_scaling_names[] = {
|
|
|
|
"none",
|
|
|
|
"logaritmic",
|
|
|
|
"linear"
|
|
|
|
};
|
|
|
|
|
2013-02-22 06:15:09 +07:00
|
|
|
static void sched_debug_header(struct seq_file *m)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
2010-11-20 03:11:09 +07:00
|
|
|
u64 ktime, sched_clk, cpu_clk;
|
|
|
|
unsigned long flags;
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2010-11-20 03:11:09 +07:00
|
|
|
local_irq_save(flags);
|
|
|
|
ktime = ktime_to_ns(ktime_get());
|
|
|
|
sched_clk = sched_clock();
|
|
|
|
cpu_clk = local_clock();
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
2013-10-07 17:29:30 +07:00
|
|
|
SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
|
2007-07-09 23:52:00 +07:00
|
|
|
init_utsname()->release,
|
|
|
|
(int)strcspn(init_utsname()->version, " "),
|
|
|
|
init_utsname()->version);
|
|
|
|
|
2010-11-20 03:11:09 +07:00
|
|
|
#define P(x) \
|
|
|
|
SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x))
|
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
|
|
|
|
PN(ktime);
|
|
|
|
PN(sched_clk);
|
|
|
|
PN(cpu_clk);
|
|
|
|
P(jiffies);
|
|
|
|
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
2013-11-29 01:38:42 +07:00
|
|
|
P(sched_clock_stable());
|
2010-11-20 03:11:09 +07:00
|
|
|
#endif
|
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
|
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "sysctl_sched\n");
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2007-10-15 22:00:10 +07:00
|
|
|
#define P(x) \
|
2007-10-15 22:00:10 +07:00
|
|
|
SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x))
|
2007-10-15 22:00:10 +07:00
|
|
|
#define PN(x) \
|
2007-10-15 22:00:10 +07:00
|
|
|
SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
|
2007-10-15 22:00:10 +07:00
|
|
|
PN(sysctl_sched_latency);
|
2007-11-10 04:39:37 +07:00
|
|
|
PN(sysctl_sched_min_granularity);
|
2007-10-15 22:00:10 +07:00
|
|
|
PN(sysctl_sched_wakeup_granularity);
|
2010-07-20 02:31:16 +07:00
|
|
|
P(sysctl_sched_child_runs_first);
|
2007-10-15 22:00:10 +07:00
|
|
|
P(sysctl_sched_features);
|
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
|
2013-02-22 06:15:09 +07:00
|
|
|
SEQ_printf(m, " .%-40s: %d (%s)\n",
|
|
|
|
"sysctl_sched_tunable_scaling",
|
2009-11-30 18:16:47 +07:00
|
|
|
sysctl_sched_tunable_scaling,
|
|
|
|
sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
|
2013-02-22 06:15:09 +07:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
}
|
2009-11-30 18:16:47 +07:00
|
|
|
|
2013-02-22 06:15:09 +07:00
|
|
|
static int sched_debug_show(struct seq_file *m, void *v)
|
|
|
|
{
|
|
|
|
int cpu = (unsigned long)(v - 2);
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2013-02-22 06:15:09 +07:00
|
|
|
if (cpu != -1)
|
|
|
|
print_cpu(m, cpu);
|
|
|
|
else
|
|
|
|
sched_debug_header(m);
|
2007-07-09 23:52:00 +07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-10-25 15:00:11 +07:00
|
|
|
void sysrq_sched_debug_show(void)
|
2007-07-09 23:52:00 +07:00
|
|
|
{
|
2013-02-22 06:15:09 +07:00
|
|
|
int cpu;
|
|
|
|
|
|
|
|
sched_debug_header(NULL);
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
print_cpu(NULL, cpu);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This itererator needs some explanation.
|
|
|
|
* It returns 1 for the header position.
|
|
|
|
* This means 2 is cpu 0.
|
|
|
|
* In a hotplugged system some cpus, including cpu 0, may be missing so we have
|
|
|
|
* to use cpumask_* to iterate over the cpus.
|
|
|
|
*/
|
|
|
|
static void *sched_debug_start(struct seq_file *file, loff_t *offset)
|
|
|
|
{
|
|
|
|
unsigned long n = *offset;
|
|
|
|
|
|
|
|
if (n == 0)
|
|
|
|
return (void *) 1;
|
|
|
|
|
|
|
|
n--;
|
|
|
|
|
|
|
|
if (n > 0)
|
|
|
|
n = cpumask_next(n - 1, cpu_online_mask);
|
|
|
|
else
|
|
|
|
n = cpumask_first(cpu_online_mask);
|
|
|
|
|
|
|
|
*offset = n + 1;
|
|
|
|
|
|
|
|
if (n < nr_cpu_ids)
|
|
|
|
return (void *)(unsigned long)(n + 2);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset)
|
|
|
|
{
|
|
|
|
(*offset)++;
|
|
|
|
return sched_debug_start(file, offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sched_debug_stop(struct seq_file *file, void *data)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct seq_operations sched_debug_sops = {
|
|
|
|
.start = sched_debug_start,
|
|
|
|
.next = sched_debug_next,
|
|
|
|
.stop = sched_debug_stop,
|
|
|
|
.show = sched_debug_show,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int sched_debug_release(struct inode *inode, struct file *file)
|
|
|
|
{
|
|
|
|
seq_release(inode, file);
|
|
|
|
|
|
|
|
return 0;
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int sched_debug_open(struct inode *inode, struct file *filp)
|
|
|
|
{
|
2013-02-22 06:15:09 +07:00
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = seq_open(filp, &sched_debug_sops);
|
|
|
|
|
|
|
|
return ret;
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
2007-10-15 22:00:19 +07:00
|
|
|
static const struct file_operations sched_debug_fops = {
|
2007-07-09 23:52:00 +07:00
|
|
|
.open = sched_debug_open,
|
|
|
|
.read = seq_read,
|
|
|
|
.llseek = seq_lseek,
|
2013-02-22 06:15:09 +07:00
|
|
|
.release = sched_debug_release,
|
2007-07-09 23:52:00 +07:00
|
|
|
};
|
|
|
|
|
|
|
|
static int __init init_sched_debug_procfs(void)
|
|
|
|
{
|
|
|
|
struct proc_dir_entry *pe;
|
|
|
|
|
2008-10-30 14:23:34 +07:00
|
|
|
pe = proc_create("sched_debug", 0444, NULL, &sched_debug_fops);
|
2007-07-09 23:52:00 +07:00
|
|
|
if (!pe)
|
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
__initcall(init_sched_debug_procfs);
|
|
|
|
|
2013-10-07 17:29:30 +07:00
|
|
|
#define __P(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
|
|
|
#define P(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
|
|
|
#define __PN(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
|
|
|
#define PN(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
|
|
|
|
|
|
|
|
2015-06-26 00:21:43 +07:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
|
|
|
|
unsigned long tpf, unsigned long gsf, unsigned long gpf)
|
|
|
|
{
|
|
|
|
SEQ_printf(m, "numa_faults node=%d ", node);
|
|
|
|
SEQ_printf(m, "task_private=%lu task_shared=%lu ", tsf, tpf);
|
|
|
|
SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gsf, gpf);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
2013-10-07 17:29:30 +07:00
|
|
|
static void sched_show_numa(struct task_struct *p, struct seq_file *m)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
struct mempolicy *pol;
|
|
|
|
|
|
|
|
if (p->mm)
|
|
|
|
P(mm->numa_scan_seq);
|
|
|
|
|
|
|
|
task_lock(p);
|
|
|
|
pol = p->mempolicy;
|
|
|
|
if (pol && !(pol->flags & MPOL_F_MORON))
|
|
|
|
pol = NULL;
|
|
|
|
mpol_get(pol);
|
|
|
|
task_unlock(p);
|
|
|
|
|
2015-06-26 00:21:43 +07:00
|
|
|
P(numa_pages_migrated);
|
|
|
|
P(numa_preferred_nid);
|
|
|
|
P(total_numa_faults);
|
|
|
|
SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
|
|
|
|
task_node(p), task_numa_group_id(p));
|
|
|
|
show_numa_stats(p, m);
|
2013-10-07 17:29:30 +07:00
|
|
|
mpol_put(pol);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-07-09 23:52:00 +07:00
|
|
|
void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
|
|
|
|
{
|
2007-10-15 22:00:18 +07:00
|
|
|
unsigned long nr_switches;
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2013-09-09 18:01:41 +07:00
|
|
|
SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p),
|
2010-05-27 04:43:22 +07:00
|
|
|
get_nr_threads(p));
|
2007-10-15 22:00:18 +07:00
|
|
|
SEQ_printf(m,
|
2013-06-27 23:50:05 +07:00
|
|
|
"---------------------------------------------------------"
|
|
|
|
"----------\n");
|
2007-10-15 22:00:18 +07:00
|
|
|
#define __P(F) \
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
2007-07-09 23:52:00 +07:00
|
|
|
#define P(F) \
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
2007-10-15 22:00:18 +07:00
|
|
|
#define __PN(F) \
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
2007-10-15 22:00:08 +07:00
|
|
|
#define PN(F) \
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
2007-07-09 23:52:00 +07:00
|
|
|
|
2007-10-15 22:00:08 +07:00
|
|
|
PN(se.exec_start);
|
|
|
|
PN(se.vruntime);
|
|
|
|
PN(se.sum_exec_runtime);
|
2007-08-02 22:41:40 +07:00
|
|
|
|
2007-10-15 22:00:18 +07:00
|
|
|
nr_switches = p->nvcsw + p->nivcsw;
|
|
|
|
|
2007-08-02 22:41:40 +07:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2015-06-08 15:10:41 +07:00
|
|
|
PN(se.statistics.sum_sleep_runtime);
|
2010-03-11 09:37:45 +07:00
|
|
|
PN(se.statistics.wait_start);
|
|
|
|
PN(se.statistics.sleep_start);
|
|
|
|
PN(se.statistics.block_start);
|
|
|
|
PN(se.statistics.sleep_max);
|
|
|
|
PN(se.statistics.block_max);
|
|
|
|
PN(se.statistics.exec_max);
|
|
|
|
PN(se.statistics.slice_max);
|
|
|
|
PN(se.statistics.wait_max);
|
|
|
|
PN(se.statistics.wait_sum);
|
|
|
|
P(se.statistics.wait_count);
|
|
|
|
PN(se.statistics.iowait_sum);
|
|
|
|
P(se.statistics.iowait_count);
|
2007-10-15 22:00:18 +07:00
|
|
|
P(se.nr_migrations);
|
2010-03-11 09:37:45 +07:00
|
|
|
P(se.statistics.nr_migrations_cold);
|
|
|
|
P(se.statistics.nr_failed_migrations_affine);
|
|
|
|
P(se.statistics.nr_failed_migrations_running);
|
|
|
|
P(se.statistics.nr_failed_migrations_hot);
|
|
|
|
P(se.statistics.nr_forced_migrations);
|
|
|
|
P(se.statistics.nr_wakeups);
|
|
|
|
P(se.statistics.nr_wakeups_sync);
|
|
|
|
P(se.statistics.nr_wakeups_migrate);
|
|
|
|
P(se.statistics.nr_wakeups_local);
|
|
|
|
P(se.statistics.nr_wakeups_remote);
|
|
|
|
P(se.statistics.nr_wakeups_affine);
|
|
|
|
P(se.statistics.nr_wakeups_affine_attempts);
|
|
|
|
P(se.statistics.nr_wakeups_passive);
|
|
|
|
P(se.statistics.nr_wakeups_idle);
|
2007-10-15 22:00:18 +07:00
|
|
|
|
|
|
|
{
|
|
|
|
u64 avg_atom, avg_per_cpu;
|
|
|
|
|
|
|
|
avg_atom = p->se.sum_exec_runtime;
|
|
|
|
if (nr_switches)
|
2014-06-14 20:00:09 +07:00
|
|
|
avg_atom = div64_ul(avg_atom, nr_switches);
|
2007-10-15 22:00:18 +07:00
|
|
|
else
|
|
|
|
avg_atom = -1LL;
|
|
|
|
|
|
|
|
avg_per_cpu = p->se.sum_exec_runtime;
|
2007-11-28 21:52:56 +07:00
|
|
|
if (p->se.nr_migrations) {
|
2008-05-01 18:34:28 +07:00
|
|
|
avg_per_cpu = div64_u64(avg_per_cpu,
|
|
|
|
p->se.nr_migrations);
|
2007-11-28 21:52:56 +07:00
|
|
|
} else {
|
2007-10-15 22:00:18 +07:00
|
|
|
avg_per_cpu = -1LL;
|
2007-11-28 21:52:56 +07:00
|
|
|
}
|
2007-10-15 22:00:18 +07:00
|
|
|
|
|
|
|
__PN(avg_atom);
|
|
|
|
__PN(avg_per_cpu);
|
|
|
|
}
|
2007-08-02 22:41:40 +07:00
|
|
|
#endif
|
2007-10-15 22:00:18 +07:00
|
|
|
__P(nr_switches);
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-10-15 22:00:18 +07:00
|
|
|
"nr_voluntary_switches", (long long)p->nvcsw);
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-10-15 22:00:18 +07:00
|
|
|
"nr_involuntary_switches", (long long)p->nivcsw);
|
|
|
|
|
2007-07-09 23:52:00 +07:00
|
|
|
P(se.load.weight);
|
2013-06-28 18:10:35 +07:00
|
|
|
#ifdef CONFIG_SMP
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-15 07:04:37 +07:00
|
|
|
P(se.avg.load_sum);
|
|
|
|
P(se.avg.util_sum);
|
|
|
|
P(se.avg.load_avg);
|
|
|
|
P(se.avg.util_avg);
|
|
|
|
P(se.avg.last_update_time);
|
2013-06-25 15:03:36 +07:00
|
|
|
#endif
|
2007-07-09 23:52:00 +07:00
|
|
|
P(policy);
|
|
|
|
P(prio);
|
2007-10-15 22:00:08 +07:00
|
|
|
#undef PN
|
2007-10-15 22:00:18 +07:00
|
|
|
#undef __PN
|
|
|
|
#undef P
|
|
|
|
#undef __P
|
2007-07-09 23:52:00 +07:00
|
|
|
|
|
|
|
{
|
2008-11-16 14:07:15 +07:00
|
|
|
unsigned int this_cpu = raw_smp_processor_id();
|
2007-07-09 23:52:00 +07:00
|
|
|
u64 t0, t1;
|
|
|
|
|
2008-11-16 14:07:15 +07:00
|
|
|
t0 = cpu_clock(this_cpu);
|
|
|
|
t1 = cpu_clock(this_cpu);
|
2013-06-27 23:50:05 +07:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-07-09 23:52:00 +07:00
|
|
|
"clock-delta", (long long)(t1-t0));
|
|
|
|
}
|
2013-10-07 17:29:30 +07:00
|
|
|
|
|
|
|
sched_show_numa(p, m);
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
void proc_sched_set_task(struct task_struct *p)
|
|
|
|
{
|
2007-08-02 22:41:40 +07:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2010-03-11 09:37:45 +07:00
|
|
|
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
|
2007-08-02 22:41:40 +07:00
|
|
|
#endif
|
2007-07-09 23:52:00 +07:00
|
|
|
}
|