From 7b456d0f2c0eba48c737a7fac1587ddf9cd72803 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Tue, 17 Sep 2019 09:23:19 -0700 Subject: [PATCH] sched: Add snapshot of task boost feature This snapshot is taken from msm-4.19 as of commit 5debecbe7195 ("trace: filter out spurious preemption and IRQs disable traces"). Change-Id: I3c9663da1fd89e9e942831fda00a47b4a29ea4e3 Signed-off-by: Satya Durga Srinivasu Prabhala --- fs/proc/base.c | 117 +++++++++++++++++++++++++++++++++++ include/linux/sched.h | 15 +++++ include/trace/events/sched.h | 10 ++- kernel/sched/core.c | 23 +++++++ kernel/sched/fair.c | 49 ++++++++++++--- kernel/sched/sched.h | 19 ++++++ 6 files changed, 222 insertions(+), 11 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index ed38e70f475a..b057ccdbd9f9 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2874,6 +2874,121 @@ static int proc_tgid_io_accounting(struct seq_file *m, struct pid_namespace *ns, } #endif /* CONFIG_TASK_IO_ACCOUNTING */ +#ifdef CONFIG_SCHED_WALT +static ssize_t proc_sched_task_boost_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + int sched_boost; + size_t len; + + if (!task) + return -ESRCH; + sched_boost = task->boost; + put_task_struct(task); + len = scnprintf(buffer, sizeof(buffer), "%d\n", sched_boost); + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static ssize_t proc_sched_task_boost_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + int sched_boost; + int err; + + if (!task) + return -ESRCH; + memset(buffer, 0, sizeof(buffer)); + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) { + err = -EFAULT; + goto out; + } + + err = kstrtoint(strstrip(buffer), 0, &sched_boost); + if (err) + goto out; + if (sched_boost < TASK_BOOST_NONE || sched_boost >= TASK_BOOST_END) { + err = -EINVAL; + goto out; + } + + task->boost = sched_boost; + if (sched_boost == 0) + task->boost_period = 0; +out: + put_task_struct(task); + return err < 0 ? err : count; +} + +static ssize_t proc_sched_task_boost_period_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + u64 sched_boost_period_ms = 0; + size_t len; + + if (!task) + return -ESRCH; + sched_boost_period_ms = div64_ul(task->boost_period, 1000000UL); + put_task_struct(task); + len = snprintf(buffer, sizeof(buffer), "%llu\n", sched_boost_period_ms); + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static ssize_t proc_sched_task_boost_period_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + unsigned int sched_boost_period; + int err; + + if (!task) + return -ESRCH; + + memset(buffer, 0, sizeof(buffer)); + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) { + err = -EFAULT; + goto out; + } + + err = kstrtouint(strstrip(buffer), 0, &sched_boost_period); + if (err) + goto out; + if (task->boost == 0 && sched_boost_period) { + /* setting boost period without boost is invalid */ + err = -EINVAL; + goto out; + } + + task->boost_period = (u64)sched_boost_period * 1000 * 1000; + task->boost_expires = sched_clock() + task->boost_period; +out: + put_task_struct(task); + return err < 0 ? err : count; +} + +static const struct file_operations proc_task_boost_enabled_operations = { + .read = proc_sched_task_boost_read, + .write = proc_sched_task_boost_write, + .llseek = generic_file_llseek, +}; + +static const struct file_operations proc_task_boost_period_operations = { + .read = proc_sched_task_boost_period_read, + .write = proc_sched_task_boost_period_write, + .llseek = generic_file_llseek, +}; +#endif /* CONFIG_SCHED_WALT */ + #ifdef CONFIG_USER_NS static int proc_id_map_open(struct inode *inode, struct file *file, const struct seq_operations *seq_ops) @@ -3067,6 +3182,8 @@ static const struct pid_entry tgid_base_stuff[] = { REG("sched_init_task_load", 00644, proc_pid_sched_init_task_load_operations), REG("sched_group_id", 00666, proc_pid_sched_group_id_operations), + REG("sched_boost", 0666, proc_task_boost_enabled_operations), + REG("sched_boost_period_ms", 0666, proc_task_boost_period_operations), #endif #ifdef CONFIG_SCHED_DEBUG REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), diff --git a/include/linux/sched.h b/include/linux/sched.h index 19a379613a94..0e92ee6464e5 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -129,6 +129,14 @@ enum fps { FPS120 = 120, }; +enum task_boost_type { + TASK_BOOST_NONE = 0, + TASK_BOOST_ON_MID, + TASK_BOOST_ON_MAX, + TASK_BOOST_STRICT_MAX, + TASK_BOOST_END, +}; + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP /* @@ -540,6 +548,7 @@ extern void __weak sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax); extern void __weak free_task_load_ptrs(struct task_struct *p); extern void __weak sched_set_refresh_rate(enum fps fps); +extern int set_task_boost(int boost, u64 period); #define RAVG_HIST_SIZE_MAX 5 #define NUM_BUSY_BUCKETS 10 @@ -607,6 +616,8 @@ static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax) { } static inline void sched_set_refresh_rate(enum fps fps) { } + +static inline void set_task_boost(int boost, u64 period) { } #endif /* CONFIG_SCHED_WALT */ struct sched_rt_entity { @@ -806,7 +817,11 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; + #ifdef CONFIG_SCHED_WALT + int boost; + u64 boost_period; + u64 boost_expires; u64 last_sleep_ts; bool wake_up_idle; struct ravg ravg; diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 756a1efb89b7..0a1ec7cc06fd 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -1017,6 +1017,7 @@ TRACE_EVENT(sched_task_util, __field(int, start_cpu) __field(int, unfilter) __field(unsigned long, cpus_allowed) + __field(int, task_boost) ), TP_fast_assign( @@ -1042,15 +1043,20 @@ TRACE_EVENT(sched_task_util, #endif __entry->cpus_allowed = cpumask_bits(&p->cpus_mask)[0]; +#ifdef CONFIG_SCHED_WALT + __entry->task_boost = per_task_boost(p); +#else + __entry->task_boost = 0; +#endif ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%d affinity=%lx", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%d affinity=%lx task_boost=%d", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->candidates, __entry->best_energy_cpu, __entry->sync, __entry->need_idle, __entry->fastpath, __entry->placement_boost, __entry->latency, __entry->stune_boosted, __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, - __entry->unfilter, __entry->cpus_allowed) + __entry->unfilter, __entry->cpus_allowed, __entry->task_boost) ) /* diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2319d36a38e7..ba94863d96aa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2751,6 +2751,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #ifdef CONFIG_SCHED_WALT p->last_sleep_ts = 0; p->wake_up_idle = false; + p->boost = 0; + p->boost_expires = 0; + p->boost_period = 0; #endif INIT_LIST_HEAD(&p->se.group_node); @@ -8392,6 +8395,26 @@ void dequeue_task_core(struct rq *rq, struct task_struct *p, int flags) } #ifdef CONFIG_SCHED_WALT +/* + *@boost:should be 0,1,2. + *@period:boost time based on ms units. + */ +int set_task_boost(int boost, u64 period) +{ + if (boost < TASK_BOOST_NONE || boost >= TASK_BOOST_END) + return -EINVAL; + if (boost) { + current->boost = boost; + current->boost_period = (u64)period * 1000 * 1000; + current->boost_expires = sched_clock() + current->boost_period; + } else { + current->boost = 0; + current->boost_expires = 0; + current->boost_period = 0; + } + return 0; +} + void sched_account_irqtime(int cpu, struct task_struct *curr, u64 delta, u64 wallclock) { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6cd50bb67b7d..6fe3dd16d027 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3848,7 +3848,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) { unsigned long capacity = capacity_orig_of(cpu); unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity.val; - unsigned long task_boost = 0; + unsigned long task_boost = per_task_boost(p); if (capacity == max_capacity) return true; @@ -3858,7 +3858,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) task_boost > 0) return false; } else { /* mid cap cpu */ - if (task_boost > 1) + if (task_boost > TASK_BOOST_ON_MID) return false; } @@ -3883,6 +3883,7 @@ struct find_best_target_env { bool boosted; int fastpath; int start_cpu; + bool strict_max; }; static inline void adjust_cpus_for_packing(struct task_struct *p, @@ -6341,8 +6342,9 @@ static int get_start_cpu(struct task_struct *p) #ifdef CONFIG_SCHED_WALT struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int start_cpu = rd->min_cap_orig_cpu; - int task_boost = 0; - bool boosted = task_boost_policy(p) == SCHED_BOOST_ON_BIG; + int task_boost = per_task_boost(p); + bool boosted = task_boost_policy(p) == SCHED_BOOST_ON_BIG || + task_boost == TASK_BOOST_ON_MID; bool task_skip_min = task_skip_min_cpu(p); /* @@ -6350,12 +6352,12 @@ static int get_start_cpu(struct task_struct *p) * or just mid will be -1, there never be any other combinations of -1s * beyond these */ - if (task_skip_min || boosted || task_boost == 1) { + if (task_skip_min || boosted) { start_cpu = rd->mid_cap_orig_cpu == -1 ? rd->max_cap_orig_cpu : rd->mid_cap_orig_cpu; } - if (task_boost == 2) { + if (task_boost > TASK_BOOST_ON_MID) { start_cpu = rd->max_cap_orig_cpu; return start_cpu; } @@ -6420,6 +6422,9 @@ static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, if (boosted) target_capacity = 0; + if (fbt_env->strict_max) + most_spare_wake_cap = LONG_MIN; + /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; /* Find SD for the start CPU */ @@ -6482,6 +6487,9 @@ static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, most_spare_cap_cpu = i; } + if (per_task_boost(cpu_rq(i)->curr) == + TASK_BOOST_STRICT_MAX) + continue; /* * Cumulative demand may already be accounting for the * task. If so, add just the boost-utilization to @@ -6627,7 +6635,8 @@ static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, * unless the task can't be accommodated in the higher * capacity CPUs. */ - if (boosted && (best_idle_cpu != -1 || target_cpu != -1)) { + if (boosted && (best_idle_cpu != -1 || target_cpu != -1 || + (fbt_env->strict_max && most_spare_cap_cpu != -1))) { if (boosted) { if (!next_group_higher_cap) break; @@ -6949,7 +6958,8 @@ int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync) int placement_boost = task_boost_policy(p); u64 start_t = 0; int delta = 0; - bool boosted = uclamp_boosted(p); + int task_boost = per_task_boost(p); + bool boosted = uclamp_boosted(p) || (task_boost > 0); int start_cpu = get_start_cpu(p); if (start_cpu < 0) @@ -7000,6 +7010,8 @@ int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync) fbt_env.need_idle = need_idle; fbt_env.start_cpu = start_cpu; fbt_env.boosted = boosted; + fbt_env.strict_max = is_rtg && + (task_boost == TASK_BOOST_STRICT_MAX); find_best_target(NULL, candidates, p, &fbt_env); @@ -7962,6 +7974,16 @@ static inline int migrate_degrades_locality(struct task_struct *p, } #endif +static inline bool can_migrate_boosted_task(struct task_struct *p, + int src_cpu, int dst_cpu) +{ + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && + task_in_related_thread_group(p) && + (capacity_orig_of(dst_cpu) < capacity_orig_of(src_cpu))) + return false; + return true; +} + /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ @@ -7982,6 +8004,12 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) return 0; + /* + * don't allow pull boost task to smaller cores. + */ + if (!can_migrate_boosted_task(p, env->src_cpu, env->dst_cpu)) + return 0; + if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { int cpu; @@ -10041,7 +10069,10 @@ no_move: * if the curr task on busiest CPU can't be * moved to this_cpu: */ - if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { + if (!cpumask_test_cpu(this_cpu, + busiest->curr->cpus_ptr) || + !can_migrate_boosted_task(busiest->curr, + cpu_of(busiest), this_cpu)) { raw_spin_unlock_irqrestore(&busiest->lock, flags); env.flags |= LBF_ALL_PINNED; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4e3dd5190b26..a9a9223eee7d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2205,6 +2205,25 @@ static inline unsigned long capacity_orig_of(int cpu) return cpu_rq(cpu)->cpu_capacity_orig; } +#ifdef CONFIG_SCHED_WALT +static inline int per_task_boost(struct task_struct *p) +{ + if (p->boost_period) { + if (sched_clock() > p->boost_expires) { + p->boost_period = 0; + p->boost_expires = 0; + p->boost = 0; + } + } + return p->boost; +} +#else +static inline int per_task_boost(struct task_struct *p) +{ + return 0; +} +#endif + static inline unsigned long task_util(struct task_struct *p) { #ifdef CONFIG_SCHED_WALT