mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-24 21:30:53 +07:00
sched: revert revert of: fair-group: SMP-nice for group scheduling
Try again.. Initial commit:18d95a2832Revert:6363ca57c7Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> Cc: Mike Galbraith <efault@gmx.de> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Peter Zijlstra
Ingo Molnar
parent
ced8aa16e1
commit
c09595f63b
@ -765,6 +765,7 @@ struct sched_domain {
|
||||
struct sched_domain *child; /* bottom domain must be null terminated */
|
||||
struct sched_group *groups; /* the balancing groups of the domain */
|
||||
cpumask_t span; /* span of all CPUs in this domain */
|
||||
int first_cpu; /* cache of the first cpu in this domain */
|
||||
unsigned long min_interval; /* Minimum balance interval ms */
|
||||
unsigned long max_interval; /* Maximum balance interval ms */
|
||||
unsigned int busy_factor; /* less balancing by factor if busy */
|
||||
|
||||
430
kernel/sched.c
430
kernel/sched.c
@ -403,6 +403,43 @@ struct cfs_rq {
|
||||
*/
|
||||
struct list_head leaf_cfs_rq_list;
|
||||
struct task_group *tg; /* group that "owns" this runqueue */
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
unsigned long task_weight;
|
||||
unsigned long shares;
|
||||
/*
|
||||
* We need space to build a sched_domain wide view of the full task
|
||||
* group tree, in order to avoid depending on dynamic memory allocation
|
||||
* during the load balancing we place this in the per cpu task group
|
||||
* hierarchy. This limits the load balancing to one instance per cpu,
|
||||
* but more should not be needed anyway.
|
||||
*/
|
||||
struct aggregate_struct {
|
||||
/*
|
||||
* load = weight(cpus) * f(tg)
|
||||
*
|
||||
* Where f(tg) is the recursive weight fraction assigned to
|
||||
* this group.
|
||||
*/
|
||||
unsigned long load;
|
||||
|
||||
/*
|
||||
* part of the group weight distributed to this span.
|
||||
*/
|
||||
unsigned long shares;
|
||||
|
||||
/*
|
||||
* The sum of all runqueue weights within this span.
|
||||
*/
|
||||
unsigned long rq_weight;
|
||||
|
||||
/*
|
||||
* Weight contributed by tasks; this is the part we can
|
||||
* influence by moving tasks around.
|
||||
*/
|
||||
unsigned long task_weight;
|
||||
} aggregate;
|
||||
#endif
|
||||
#endif
|
||||
};
|
||||
|
||||
@ -1484,6 +1521,326 @@ static unsigned long source_load(int cpu, int type);
|
||||
static unsigned long target_load(int cpu, int type);
|
||||
static unsigned long cpu_avg_load_per_task(int cpu);
|
||||
static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
|
||||
/*
|
||||
* Group load balancing.
|
||||
*
|
||||
* We calculate a few balance domain wide aggregate numbers; load and weight.
|
||||
* Given the pictures below, and assuming each item has equal weight:
|
||||
*
|
||||
* root 1 - thread
|
||||
* / | \ A - group
|
||||
* A 1 B
|
||||
* /|\ / \
|
||||
* C 2 D 3 4
|
||||
* | |
|
||||
* 5 6
|
||||
*
|
||||
* load:
|
||||
* A and B get 1/3-rd of the total load. C and D get 1/3-rd of A's 1/3-rd,
|
||||
* which equals 1/9-th of the total load.
|
||||
*
|
||||
* shares:
|
||||
* The weight of this group on the selected cpus.
|
||||
*
|
||||
* rq_weight:
|
||||
* Direct sum of all the cpu's their rq weight, e.g. A would get 3 while
|
||||
* B would get 2.
|
||||
*
|
||||
* task_weight:
|
||||
* Part of the rq_weight contributed by tasks; all groups except B would
|
||||
* get 1, B gets 2.
|
||||
*/
|
||||
|
||||
static inline struct aggregate_struct *
|
||||
aggregate(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
return &tg->cfs_rq[sd->first_cpu]->aggregate;
|
||||
}
|
||||
|
||||
typedef void (*aggregate_func)(struct task_group *, struct sched_domain *);
|
||||
|
||||
/*
|
||||
* Iterate the full tree, calling @down when first entering a node and @up when
|
||||
* leaving it for the final time.
|
||||
*/
|
||||
static
|
||||
void aggregate_walk_tree(aggregate_func down, aggregate_func up,
|
||||
struct sched_domain *sd)
|
||||
{
|
||||
struct task_group *parent, *child;
|
||||
|
||||
rcu_read_lock();
|
||||
parent = &root_task_group;
|
||||
down:
|
||||
(*down)(parent, sd);
|
||||
list_for_each_entry_rcu(child, &parent->children, siblings) {
|
||||
parent = child;
|
||||
goto down;
|
||||
|
||||
up:
|
||||
continue;
|
||||
}
|
||||
(*up)(parent, sd);
|
||||
|
||||
child = parent;
|
||||
parent = parent->parent;
|
||||
if (parent)
|
||||
goto up;
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate the aggregate runqueue weight.
|
||||
*/
|
||||
static
|
||||
void aggregate_group_weight(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
unsigned long rq_weight = 0;
|
||||
unsigned long task_weight = 0;
|
||||
int i;
|
||||
|
||||
for_each_cpu_mask(i, sd->span) {
|
||||
rq_weight += tg->cfs_rq[i]->load.weight;
|
||||
task_weight += tg->cfs_rq[i]->task_weight;
|
||||
}
|
||||
|
||||
aggregate(tg, sd)->rq_weight = rq_weight;
|
||||
aggregate(tg, sd)->task_weight = task_weight;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compute the weight of this group on the given cpus.
|
||||
*/
|
||||
static
|
||||
void aggregate_group_shares(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
unsigned long shares = 0;
|
||||
int i;
|
||||
|
||||
for_each_cpu_mask(i, sd->span)
|
||||
shares += tg->cfs_rq[i]->shares;
|
||||
|
||||
if ((!shares && aggregate(tg, sd)->rq_weight) || shares > tg->shares)
|
||||
shares = tg->shares;
|
||||
|
||||
aggregate(tg, sd)->shares = shares;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compute the load fraction assigned to this group, relies on the aggregate
|
||||
* weight and this group's parent's load, i.e. top-down.
|
||||
*/
|
||||
static
|
||||
void aggregate_group_load(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
unsigned long load;
|
||||
|
||||
if (!tg->parent) {
|
||||
int i;
|
||||
|
||||
load = 0;
|
||||
for_each_cpu_mask(i, sd->span)
|
||||
load += cpu_rq(i)->load.weight;
|
||||
|
||||
} else {
|
||||
load = aggregate(tg->parent, sd)->load;
|
||||
|
||||
/*
|
||||
* shares is our weight in the parent's rq so
|
||||
* shares/parent->rq_weight gives our fraction of the load
|
||||
*/
|
||||
load *= aggregate(tg, sd)->shares;
|
||||
load /= aggregate(tg->parent, sd)->rq_weight + 1;
|
||||
}
|
||||
|
||||
aggregate(tg, sd)->load = load;
|
||||
}
|
||||
|
||||
static void __set_se_shares(struct sched_entity *se, unsigned long shares);
|
||||
|
||||
/*
|
||||
* Calculate and set the cpu's group shares.
|
||||
*/
|
||||
static void
|
||||
__update_group_shares_cpu(struct task_group *tg, struct sched_domain *sd,
|
||||
int tcpu)
|
||||
{
|
||||
int boost = 0;
|
||||
unsigned long shares;
|
||||
unsigned long rq_weight;
|
||||
|
||||
if (!tg->se[tcpu])
|
||||
return;
|
||||
|
||||
rq_weight = tg->cfs_rq[tcpu]->load.weight;
|
||||
|
||||
/*
|
||||
* If there are currently no tasks on the cpu pretend there is one of
|
||||
* average load so that when a new task gets to run here it will not
|
||||
* get delayed by group starvation.
|
||||
*/
|
||||
if (!rq_weight) {
|
||||
boost = 1;
|
||||
rq_weight = NICE_0_LOAD;
|
||||
}
|
||||
|
||||
/*
|
||||
* \Sum shares * rq_weight
|
||||
* shares = -----------------------
|
||||
* \Sum rq_weight
|
||||
*
|
||||
*/
|
||||
shares = aggregate(tg, sd)->shares * rq_weight;
|
||||
shares /= aggregate(tg, sd)->rq_weight + 1;
|
||||
|
||||
/*
|
||||
* record the actual number of shares, not the boosted amount.
|
||||
*/
|
||||
tg->cfs_rq[tcpu]->shares = boost ? 0 : shares;
|
||||
|
||||
if (shares < MIN_SHARES)
|
||||
shares = MIN_SHARES;
|
||||
else if (shares > MAX_SHARES)
|
||||
shares = MAX_SHARES;
|
||||
|
||||
__set_se_shares(tg->se[tcpu], shares);
|
||||
}
|
||||
|
||||
/*
|
||||
* Re-adjust the weights on the cpu the task came from and on the cpu the
|
||||
* task went to.
|
||||
*/
|
||||
static void
|
||||
__move_group_shares(struct task_group *tg, struct sched_domain *sd,
|
||||
int scpu, int dcpu)
|
||||
{
|
||||
unsigned long shares;
|
||||
|
||||
shares = tg->cfs_rq[scpu]->shares + tg->cfs_rq[dcpu]->shares;
|
||||
|
||||
__update_group_shares_cpu(tg, sd, scpu);
|
||||
__update_group_shares_cpu(tg, sd, dcpu);
|
||||
|
||||
/*
|
||||
* ensure we never loose shares due to rounding errors in the
|
||||
* above redistribution.
|
||||
*/
|
||||
shares -= tg->cfs_rq[scpu]->shares + tg->cfs_rq[dcpu]->shares;
|
||||
if (shares)
|
||||
tg->cfs_rq[dcpu]->shares += shares;
|
||||
}
|
||||
|
||||
/*
|
||||
* Because changing a group's shares changes the weight of the super-group
|
||||
* we need to walk up the tree and change all shares until we hit the root.
|
||||
*/
|
||||
static void
|
||||
move_group_shares(struct task_group *tg, struct sched_domain *sd,
|
||||
int scpu, int dcpu)
|
||||
{
|
||||
while (tg) {
|
||||
__move_group_shares(tg, sd, scpu, dcpu);
|
||||
tg = tg->parent;
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void aggregate_group_set_shares(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
unsigned long shares = aggregate(tg, sd)->shares;
|
||||
int i;
|
||||
|
||||
for_each_cpu_mask(i, sd->span) {
|
||||
struct rq *rq = cpu_rq(i);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&rq->lock, flags);
|
||||
__update_group_shares_cpu(tg, sd, i);
|
||||
spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
aggregate_group_shares(tg, sd);
|
||||
|
||||
/*
|
||||
* ensure we never loose shares due to rounding errors in the
|
||||
* above redistribution.
|
||||
*/
|
||||
shares -= aggregate(tg, sd)->shares;
|
||||
if (shares) {
|
||||
tg->cfs_rq[sd->first_cpu]->shares += shares;
|
||||
aggregate(tg, sd)->shares += shares;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate the accumulative weight and recursive load of each task group
|
||||
* while walking down the tree.
|
||||
*/
|
||||
static
|
||||
void aggregate_get_down(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
aggregate_group_weight(tg, sd);
|
||||
aggregate_group_shares(tg, sd);
|
||||
aggregate_group_load(tg, sd);
|
||||
}
|
||||
|
||||
/*
|
||||
* Rebalance the cpu shares while walking back up the tree.
|
||||
*/
|
||||
static
|
||||
void aggregate_get_up(struct task_group *tg, struct sched_domain *sd)
|
||||
{
|
||||
aggregate_group_set_shares(tg, sd);
|
||||
}
|
||||
|
||||
static DEFINE_PER_CPU(spinlock_t, aggregate_lock);
|
||||
|
||||
static void __init init_aggregate(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
for_each_possible_cpu(i)
|
||||
spin_lock_init(&per_cpu(aggregate_lock, i));
|
||||
}
|
||||
|
||||
static int get_aggregate(struct sched_domain *sd)
|
||||
{
|
||||
if (!spin_trylock(&per_cpu(aggregate_lock, sd->first_cpu)))
|
||||
return 0;
|
||||
|
||||
aggregate_walk_tree(aggregate_get_down, aggregate_get_up, sd);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void put_aggregate(struct sched_domain *sd)
|
||||
{
|
||||
spin_unlock(&per_cpu(aggregate_lock, sd->first_cpu));
|
||||
}
|
||||
|
||||
static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
|
||||
{
|
||||
cfs_rq->shares = shares;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static inline void init_aggregate(void)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int get_aggregate(struct sched_domain *sd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void put_aggregate(struct sched_domain *sd)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#include "sched_stats.h"
|
||||
@ -1498,26 +1855,14 @@ static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
|
||||
#define for_each_class(class) \
|
||||
for (class = sched_class_highest; class; class = class->next)
|
||||
|
||||
static inline void inc_load(struct rq *rq, const struct task_struct *p)
|
||||
{
|
||||
update_load_add(&rq->load, p->se.load.weight);
|
||||
}
|
||||
|
||||
static inline void dec_load(struct rq *rq, const struct task_struct *p)
|
||||
{
|
||||
update_load_sub(&rq->load, p->se.load.weight);
|
||||
}
|
||||
|
||||
static void inc_nr_running(struct task_struct *p, struct rq *rq)
|
||||
static void inc_nr_running(struct rq *rq)
|
||||
{
|
||||
rq->nr_running++;
|
||||
inc_load(rq, p);
|
||||
}
|
||||
|
||||
static void dec_nr_running(struct task_struct *p, struct rq *rq)
|
||||
static void dec_nr_running(struct rq *rq)
|
||||
{
|
||||
rq->nr_running--;
|
||||
dec_load(rq, p);
|
||||
}
|
||||
|
||||
static void set_load_weight(struct task_struct *p)
|
||||
@ -1609,7 +1954,7 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
|
||||
rq->nr_uninterruptible--;
|
||||
|
||||
enqueue_task(rq, p, wakeup);
|
||||
inc_nr_running(p, rq);
|
||||
inc_nr_running(rq);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1621,7 +1966,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
|
||||
rq->nr_uninterruptible++;
|
||||
|
||||
dequeue_task(rq, p, sleep);
|
||||
dec_nr_running(p, rq);
|
||||
dec_nr_running(rq);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2274,7 +2619,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
|
||||
* management (if any):
|
||||
*/
|
||||
p->sched_class->task_new(rq, p);
|
||||
inc_nr_running(p, rq);
|
||||
inc_nr_running(rq);
|
||||
}
|
||||
check_preempt_curr(rq, p);
|
||||
#ifdef CONFIG_SMP
|
||||
@ -3265,9 +3610,12 @@ static int load_balance(int this_cpu, struct rq *this_rq,
|
||||
unsigned long imbalance;
|
||||
struct rq *busiest;
|
||||
unsigned long flags;
|
||||
int unlock_aggregate;
|
||||
|
||||
cpus_setall(*cpus);
|
||||
|
||||
unlock_aggregate = get_aggregate(sd);
|
||||
|
||||
/*
|
||||
* When power savings policy is enabled for the parent domain, idle
|
||||
* sibling can pick up load irrespective of busy siblings. In this case,
|
||||
@ -3383,8 +3731,9 @@ static int load_balance(int this_cpu, struct rq *this_rq,
|
||||
|
||||
if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
|
||||
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
||||
return -1;
|
||||
return ld_moved;
|
||||
ld_moved = -1;
|
||||
|
||||
goto out;
|
||||
|
||||
out_balanced:
|
||||
schedstat_inc(sd, lb_balanced[idle]);
|
||||
@ -3399,8 +3748,13 @@ static int load_balance(int this_cpu, struct rq *this_rq,
|
||||
|
||||
if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
|
||||
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
||||
return -1;
|
||||
return 0;
|
||||
ld_moved = -1;
|
||||
else
|
||||
ld_moved = 0;
|
||||
out:
|
||||
if (unlock_aggregate)
|
||||
put_aggregate(sd);
|
||||
return ld_moved;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -4588,10 +4942,8 @@ void set_user_nice(struct task_struct *p, long nice)
|
||||
goto out_unlock;
|
||||
}
|
||||
on_rq = p->se.on_rq;
|
||||
if (on_rq) {
|
||||
if (on_rq)
|
||||
dequeue_task(rq, p, 0);
|
||||
dec_load(rq, p);
|
||||
}
|
||||
|
||||
p->static_prio = NICE_TO_PRIO(nice);
|
||||
set_load_weight(p);
|
||||
@ -4601,7 +4953,6 @@ void set_user_nice(struct task_struct *p, long nice)
|
||||
|
||||
if (on_rq) {
|
||||
enqueue_task(rq, p, 0);
|
||||
inc_load(rq, p);
|
||||
/*
|
||||
* If the task increased its priority or is running and
|
||||
* lowered its priority, then reschedule its CPU:
|
||||
@ -7016,6 +7367,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
|
||||
SD_INIT(sd, ALLNODES);
|
||||
set_domain_attribute(sd, attr);
|
||||
sd->span = *cpu_map;
|
||||
sd->first_cpu = first_cpu(sd->span);
|
||||
cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
|
||||
p = sd;
|
||||
sd_allnodes = 1;
|
||||
@ -7026,6 +7378,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
|
||||
SD_INIT(sd, NODE);
|
||||
set_domain_attribute(sd, attr);
|
||||
sched_domain_node_span(cpu_to_node(i), &sd->span);
|
||||
sd->first_cpu = first_cpu(sd->span);
|
||||
sd->parent = p;
|
||||
if (p)
|
||||
p->child = sd;
|
||||
@ -7037,6 +7390,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
|
||||
SD_INIT(sd, CPU);
|
||||
set_domain_attribute(sd, attr);
|
||||
sd->span = *nodemask;
|
||||
sd->first_cpu = first_cpu(sd->span);
|
||||
sd->parent = p;
|
||||
if (p)
|
||||
p->child = sd;
|
||||
@ -7048,6 +7402,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
|
||||
SD_INIT(sd, MC);
|
||||
set_domain_attribute(sd, attr);
|
||||
sd->span = cpu_coregroup_map(i);
|
||||
sd->first_cpu = first_cpu(sd->span);
|
||||
cpus_and(sd->span, sd->span, *cpu_map);
|
||||
sd->parent = p;
|
||||
p->child = sd;
|
||||
@ -7060,6 +7415,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
|
||||
SD_INIT(sd, SIBLING);
|
||||
set_domain_attribute(sd, attr);
|
||||
sd->span = per_cpu(cpu_sibling_map, i);
|
||||
sd->first_cpu = first_cpu(sd->span);
|
||||
cpus_and(sd->span, sd->span, *cpu_map);
|
||||
sd->parent = p;
|
||||
p->child = sd;
|
||||
@ -7757,6 +8113,7 @@ void __init sched_init(void)
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
init_aggregate();
|
||||
init_defrootdomain();
|
||||
#endif
|
||||
|
||||
@ -8322,14 +8679,11 @@ void sched_move_task(struct task_struct *tsk)
|
||||
#endif /* CONFIG_GROUP_SCHED */
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
static void set_se_shares(struct sched_entity *se, unsigned long shares)
|
||||
static void __set_se_shares(struct sched_entity *se, unsigned long shares)
|
||||
{
|
||||
struct cfs_rq *cfs_rq = se->cfs_rq;
|
||||
struct rq *rq = cfs_rq->rq;
|
||||
int on_rq;
|
||||
|
||||
spin_lock_irq(&rq->lock);
|
||||
|
||||
on_rq = se->on_rq;
|
||||
if (on_rq)
|
||||
dequeue_entity(cfs_rq, se, 0);
|
||||
@ -8339,8 +8693,17 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
|
||||
|
||||
if (on_rq)
|
||||
enqueue_entity(cfs_rq, se, 0);
|
||||
}
|
||||
|
||||
spin_unlock_irq(&rq->lock);
|
||||
static void set_se_shares(struct sched_entity *se, unsigned long shares)
|
||||
{
|
||||
struct cfs_rq *cfs_rq = se->cfs_rq;
|
||||
struct rq *rq = cfs_rq->rq;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&rq->lock, flags);
|
||||
__set_se_shares(se, shares);
|
||||
spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
static DEFINE_MUTEX(shares_mutex);
|
||||
@ -8379,8 +8742,13 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
|
||||
* w/o tripping rebalance_share or load_balance_fair.
|
||||
*/
|
||||
tg->shares = shares;
|
||||
for_each_possible_cpu(i)
|
||||
for_each_possible_cpu(i) {
|
||||
/*
|
||||
* force a rebalance
|
||||
*/
|
||||
cfs_rq_set_shares(tg->cfs_rq[i], 0);
|
||||
set_se_shares(tg->se[i], shares);
|
||||
}
|
||||
|
||||
/*
|
||||
* Enable load balance activity on this group, by inserting it back on
|
||||
|
||||
@ -167,6 +167,11 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
|
||||
#endif
|
||||
SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over",
|
||||
cfs_rq->nr_spread_over);
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
#ifdef CONFIG_SMP
|
||||
SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares);
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
|
||||
|
||||
@ -567,10 +567,27 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
||||
* Scheduling class queueing methods:
|
||||
*/
|
||||
|
||||
#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
|
||||
static void
|
||||
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
||||
{
|
||||
cfs_rq->task_weight += weight;
|
||||
}
|
||||
#else
|
||||
static inline void
|
||||
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static void
|
||||
account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
||||
{
|
||||
update_load_add(&cfs_rq->load, se->load.weight);
|
||||
if (!parent_entity(se))
|
||||
inc_cpu_load(rq_of(cfs_rq), se->load.weight);
|
||||
if (entity_is_task(se))
|
||||
add_cfs_task_weight(cfs_rq, se->load.weight);
|
||||
cfs_rq->nr_running++;
|
||||
se->on_rq = 1;
|
||||
list_add(&se->group_node, &cfs_rq->tasks);
|
||||
@ -580,6 +597,10 @@ static void
|
||||
account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
||||
{
|
||||
update_load_sub(&cfs_rq->load, se->load.weight);
|
||||
if (!parent_entity(se))
|
||||
dec_cpu_load(rq_of(cfs_rq), se->load.weight);
|
||||
if (entity_is_task(se))
|
||||
add_cfs_task_weight(cfs_rq, -se->load.weight);
|
||||
cfs_rq->nr_running--;
|
||||
se->on_rq = 0;
|
||||
list_del_init(&se->group_node);
|
||||
@ -1372,75 +1393,90 @@ static struct task_struct *load_balance_next_fair(void *arg)
|
||||
return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
|
||||
static unsigned long
|
||||
__load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
||||
unsigned long max_load_move, struct sched_domain *sd,
|
||||
enum cpu_idle_type idle, int *all_pinned, int *this_best_prio,
|
||||
struct cfs_rq *cfs_rq)
|
||||
{
|
||||
struct sched_entity *curr;
|
||||
struct task_struct *p;
|
||||
struct rq_iterator cfs_rq_iterator;
|
||||
|
||||
if (!cfs_rq->nr_running || !first_fair(cfs_rq))
|
||||
return MAX_PRIO;
|
||||
cfs_rq_iterator.start = load_balance_start_fair;
|
||||
cfs_rq_iterator.next = load_balance_next_fair;
|
||||
cfs_rq_iterator.arg = cfs_rq;
|
||||
|
||||
curr = cfs_rq->curr;
|
||||
if (!curr)
|
||||
curr = __pick_next_entity(cfs_rq);
|
||||
|
||||
p = task_of(curr);
|
||||
|
||||
return p->prio;
|
||||
return balance_tasks(this_rq, this_cpu, busiest,
|
||||
max_load_move, sd, idle, all_pinned,
|
||||
this_best_prio, &cfs_rq_iterator);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
static unsigned long
|
||||
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
||||
unsigned long max_load_move,
|
||||
struct sched_domain *sd, enum cpu_idle_type idle,
|
||||
int *all_pinned, int *this_best_prio)
|
||||
{
|
||||
struct cfs_rq *busy_cfs_rq;
|
||||
long rem_load_move = max_load_move;
|
||||
struct rq_iterator cfs_rq_iterator;
|
||||
int busiest_cpu = cpu_of(busiest);
|
||||
struct task_group *tg;
|
||||
|
||||
cfs_rq_iterator.start = load_balance_start_fair;
|
||||
cfs_rq_iterator.next = load_balance_next_fair;
|
||||
|
||||
for_each_leaf_cfs_rq(busiest, busy_cfs_rq) {
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
struct cfs_rq *this_cfs_rq;
|
||||
rcu_read_lock();
|
||||
list_for_each_entry(tg, &task_groups, list) {
|
||||
long imbalance;
|
||||
unsigned long maxload;
|
||||
unsigned long this_weight, busiest_weight;
|
||||
long rem_load, max_load, moved_load;
|
||||
|
||||
this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu);
|
||||
|
||||
imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight;
|
||||
/* Don't pull if this_cfs_rq has more load than busy_cfs_rq */
|
||||
if (imbalance <= 0)
|
||||
/*
|
||||
* empty group
|
||||
*/
|
||||
if (!aggregate(tg, sd)->task_weight)
|
||||
continue;
|
||||
|
||||
/* Don't pull more than imbalance/2 */
|
||||
imbalance /= 2;
|
||||
maxload = min(rem_load_move, imbalance);
|
||||
rem_load = rem_load_move * aggregate(tg, sd)->rq_weight;
|
||||
rem_load /= aggregate(tg, sd)->load + 1;
|
||||
|
||||
*this_best_prio = cfs_rq_best_prio(this_cfs_rq);
|
||||
#else
|
||||
# define maxload rem_load_move
|
||||
#endif
|
||||
/*
|
||||
* pass busy_cfs_rq argument into
|
||||
* load_balance_[start|next]_fair iterators
|
||||
*/
|
||||
cfs_rq_iterator.arg = busy_cfs_rq;
|
||||
rem_load_move -= balance_tasks(this_rq, this_cpu, busiest,
|
||||
maxload, sd, idle, all_pinned,
|
||||
this_best_prio,
|
||||
&cfs_rq_iterator);
|
||||
this_weight = tg->cfs_rq[this_cpu]->task_weight;
|
||||
busiest_weight = tg->cfs_rq[busiest_cpu]->task_weight;
|
||||
|
||||
if (rem_load_move <= 0)
|
||||
imbalance = (busiest_weight - this_weight) / 2;
|
||||
|
||||
if (imbalance < 0)
|
||||
imbalance = busiest_weight;
|
||||
|
||||
max_load = max(rem_load, imbalance);
|
||||
moved_load = __load_balance_fair(this_rq, this_cpu, busiest,
|
||||
max_load, sd, idle, all_pinned, this_best_prio,
|
||||
tg->cfs_rq[busiest_cpu]);
|
||||
|
||||
if (!moved_load)
|
||||
continue;
|
||||
|
||||
move_group_shares(tg, sd, busiest_cpu, this_cpu);
|
||||
|
||||
moved_load *= aggregate(tg, sd)->load;
|
||||
moved_load /= aggregate(tg, sd)->rq_weight + 1;
|
||||
|
||||
rem_load_move -= moved_load;
|
||||
if (rem_load_move < 0)
|
||||
break;
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
return max_load_move - rem_load_move;
|
||||
}
|
||||
#else
|
||||
static unsigned long
|
||||
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
||||
unsigned long max_load_move,
|
||||
struct sched_domain *sd, enum cpu_idle_type idle,
|
||||
int *all_pinned, int *this_best_prio)
|
||||
{
|
||||
return __load_balance_fair(this_rq, this_cpu, busiest,
|
||||
max_load_move, sd, idle, all_pinned,
|
||||
this_best_prio, &busiest->cfs);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
||||
|
||||
@ -670,6 +670,8 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
|
||||
rt_se->timeout = 0;
|
||||
|
||||
enqueue_rt_entity(rt_se);
|
||||
|
||||
inc_cpu_load(rq, p->se.load.weight);
|
||||
}
|
||||
|
||||
static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
|
||||
@ -678,6 +680,8 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
|
||||
|
||||
update_curr_rt(rq);
|
||||
dequeue_rt_entity(rt_se);
|
||||
|
||||
dec_cpu_load(rq, p->se.load.weight);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user