diff options
| -rw-r--r-- | include/linux/sched.h | 1 | ||||
| -rw-r--r-- | kernel/sched.c | 1 | ||||
| -rw-r--r-- | kernel/sched_fair.c | 81 |
3 files changed, 50 insertions, 33 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h index 3c38a5040e8..5e5c457fba8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -902,6 +902,7 @@ struct sched_entity { u64 exec_start; u64 sum_exec_runtime; + u64 vruntime; u64 prev_sum_exec_runtime; u64 wait_start_fair; u64 sleep_start_fair; diff --git a/kernel/sched.c b/kernel/sched.c index 992a1fae72a..8f80ebafacc 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -182,6 +182,7 @@ struct cfs_rq { s64 fair_clock; u64 exec_clock; + u64 min_vruntime; s64 wait_runtime; u64 sleeper_bonus; unsigned long wait_runtime_overruns, wait_runtime_underruns; diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index b46f8078e78..a2af09cb6a7 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -92,14 +92,16 @@ unsigned int sysctl_sched_runtime_limit __read_mostly; */ enum { SCHED_FEAT_FAIR_SLEEPERS = 1, - SCHED_FEAT_SLEEPER_AVG = 2, - SCHED_FEAT_SLEEPER_LOAD_AVG = 4, - SCHED_FEAT_START_DEBIT = 8, - SCHED_FEAT_SKIP_INITIAL = 16, + SCHED_FEAT_NEW_FAIR_SLEEPERS = 2, + SCHED_FEAT_SLEEPER_AVG = 4, + SCHED_FEAT_SLEEPER_LOAD_AVG = 8, + SCHED_FEAT_START_DEBIT = 16, + SCHED_FEAT_SKIP_INITIAL = 32, }; const_debug unsigned int sysctl_sched_features = - SCHED_FEAT_FAIR_SLEEPERS *1 | + SCHED_FEAT_FAIR_SLEEPERS *0 | + SCHED_FEAT_NEW_FAIR_SLEEPERS *1 | SCHED_FEAT_SLEEPER_AVG *0 | SCHED_FEAT_SLEEPER_LOAD_AVG *1 | SCHED_FEAT_START_DEBIT *1 | @@ -145,6 +147,19 @@ static inline struct task_struct *task_of(struct sched_entity *se) * Scheduling class tree data structure manipulation methods: */ +static inline void +set_leftmost(struct cfs_rq *cfs_rq, struct rb_node *leftmost) +{ + struct sched_entity *se; + + cfs_rq->rb_leftmost = leftmost; + if (leftmost) { + se = rb_entry(leftmost, struct sched_entity, run_node); + cfs_rq->min_vruntime = max(se->vruntime, + cfs_rq->min_vruntime); + } +} + /* * Enqueue an entity into the rb-tree: */ @@ -180,7 +195,7 @@ __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) * used): */ if (leftmost) - cfs_rq->rb_leftmost = &se->run_node; + set_leftmost(cfs_rq, &se->run_node); rb_link_node(&se->run_node, parent, link); rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); @@ -195,7 +210,8 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->rb_leftmost == &se->run_node) - cfs_rq->rb_leftmost = rb_next(&se->run_node); + set_leftmost(cfs_rq, rb_next(&se->run_node)); + rb_erase(&se->run_node, &cfs_rq->tasks_timeline); update_load_sub(&cfs_rq->load, se->load.weight); cfs_rq->nr_running--; @@ -336,7 +352,7 @@ static inline void __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, unsigned long delta_exec) { - unsigned long delta, delta_fair, delta_mine; + unsigned long delta, delta_fair, delta_mine, delta_exec_weighted; struct load_weight *lw = &cfs_rq->load; unsigned long load = lw->weight; @@ -344,6 +360,12 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, curr->sum_exec_runtime += delta_exec; cfs_rq->exec_clock += delta_exec; + delta_exec_weighted = delta_exec; + if (unlikely(curr->load.weight != NICE_0_LOAD)) { + delta_exec_weighted = calc_delta_fair(delta_exec_weighted, + &curr->load); + } + curr->vruntime += delta_exec_weighted; if (unlikely(!load)) return; @@ -413,8 +435,6 @@ calc_weighted(unsigned long delta, struct sched_entity *se) */ static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - s64 key; - /* * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) @@ -424,28 +444,7 @@ static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) /* * Update the key: */ - key = cfs_rq->fair_clock; - - /* - * Optimize the common nice 0 case: - */ - if (likely(se->load.weight == NICE_0_LOAD)) { - key -= se->wait_runtime; - } else { - u64 tmp; - - if (se->wait_runtime < 0) { - tmp = -se->wait_runtime; - key += (tmp * se->load.inv_weight) >> - (WMULT_SHIFT - NICE_0_SHIFT); - } else { - tmp = se->wait_runtime; - key -= (tmp * se->load.inv_weight) >> - (WMULT_SHIFT - NICE_0_SHIFT); - } - } - - se->fair_key = key; + se->fair_key = se->vruntime; } /* @@ -615,8 +614,22 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) */ update_curr(cfs_rq); - if (wakeup) + if (wakeup) { + u64 min_runtime, latency; + + min_runtime = cfs_rq->min_vruntime; + min_runtime += sysctl_sched_latency/2; + + if (sched_feat(NEW_FAIR_SLEEPERS)) { + latency = calc_weighted(sysctl_sched_latency, se); + if (min_runtime > latency) + min_runtime -= latency; + } + + se->vruntime = max(se->vruntime, min_runtime); + enqueue_sleeper(cfs_rq, se); + } update_stats_enqueue(cfs_rq, se); __enqueue_entity(cfs_rq, se); @@ -1155,6 +1168,8 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) if (sched_feat(START_DEBIT)) se->wait_runtime = -(sched_granularity(cfs_rq) / 2); + se->vruntime = cfs_rq->min_vruntime; + update_stats_enqueue(cfs_rq, se); __enqueue_entity(cfs_rq, se); resched_task(rq->curr); } |