diff --git a/fs/proc/base.c b/fs/proc/base.c index cc32c775a5cf..3deb937f8901 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1488,6 +1488,58 @@ static const struct file_operations proc_pid_sched_group_id_operations = { .release = single_release, }; +static int sched_low_latency_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + bool low_latency; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + low_latency = p->wts.low_latency; + seq_printf(m, "%d\n", low_latency); + + put_task_struct(p); + + return 0; +} + +static ssize_t +sched_low_latency_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct task_struct *p = get_proc_task(file_inode(file)); + bool low_latency; + int err; + + if (!p) + return -ESRCH; + + err = kstrtobool_from_user(buf, count, &low_latency); + if (err) + goto out; + + p->wts.low_latency = low_latency; +out: + put_task_struct(p); + return err < 0 ? err : count; +} + +static int sched_low_latency_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_low_latency_show, inode); +} + +static const struct file_operations proc_pid_sched_low_latency_operations = { + .open = sched_low_latency_open, + .read = seq_read, + .write = sched_low_latency_write, + .llseek = seq_lseek, + .release = single_release, +}; + #endif /* CONFIG_SCHED_WALT */ #ifdef CONFIG_SCHED_AUTOGROUP @@ -3212,6 +3264,7 @@ static const struct pid_entry tgid_base_stuff[] = { REG("sched_group_id", 00666, proc_pid_sched_group_id_operations), REG("sched_boost", 0666, proc_task_boost_enabled_operations), REG("sched_boost_period_ms", 0666, proc_task_boost_period_operations), + REG("sched_low_latency", 00666, proc_pid_sched_low_latency_operations), #endif #ifdef CONFIG_SCHED_DEBUG REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), diff --git a/include/linux/sched.h b/include/linux/sched.h index 35cb8f4f6006..96ba8aaaff47 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -618,6 +618,7 @@ struct walt_task_struct { int boost; bool wake_up_idle; bool misfit; + bool low_latency; u64 boost_period; u64 boost_expires; u64 last_sleep_ts; diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 73a5fa7a6368..dd64373b2d37 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -451,48 +451,6 @@ TRACE_EVENT(sched_load_balance_stats, #endif /* NR_CPUS > BITS_PER_LONG */ #endif /* CONFIG_SMP */ -#ifdef CONFIG_SCHED_WALT -TRACE_EVENT(sched_load_balance_skip_tasks, - - TP_PROTO(int scpu, int dcpu, int grp_type, int pid, - unsigned long h_load, unsigned long task_util, - unsigned long affinity), - - TP_ARGS(scpu, dcpu, grp_type, pid, h_load, task_util, affinity), - - TP_STRUCT__entry( - __field(int, scpu) - __field(unsigned long, src_util_cum) - __field(int, grp_type) - __field(int, dcpu) - __field(unsigned long, dst_util_cum) - __field(int, pid) - __field(unsigned long, affinity) - __field(unsigned long, task_util) - __field(unsigned long, h_load) - ), - - TP_fast_assign( - __entry->scpu = scpu; - __entry->src_util_cum = - cpu_rq(scpu)->wrq.cum_window_demand_scaled; - __entry->grp_type = grp_type; - __entry->dcpu = dcpu; - __entry->dst_util_cum = - cpu_rq(dcpu)->wrq.cum_window_demand_scaled; - __entry->pid = pid; - __entry->affinity = affinity; - __entry->task_util = task_util; - __entry->h_load = h_load; - ), - - TP_printk("source_cpu=%d util_cum=%lu group_type=%d dest_cpu=%d util_cum=%lu pid=%d affinity=%#lx task_util=%lu task_h_load=%lu", - __entry->scpu, __entry->src_util_cum, __entry->grp_type, - __entry->dcpu, __entry->dst_util_cum, __entry->pid, - __entry->affinity, __entry->task_util, __entry->h_load) -); -#endif - DECLARE_EVENT_CLASS(sched_process_template, TP_PROTO(struct task_struct *p), @@ -905,7 +863,47 @@ DECLARE_TRACE(sched_overutilized_tp, TP_PROTO(struct root_domain *rd, bool overutilized), TP_ARGS(rd, overutilized)); -#ifdef CONFIG_SMP +#ifdef CONFIG_SCHED_WALT +TRACE_EVENT(sched_load_balance_skip_tasks, + + TP_PROTO(int scpu, int dcpu, int grp_type, int pid, + unsigned long h_load, unsigned long task_util, + unsigned long affinity), + + TP_ARGS(scpu, dcpu, grp_type, pid, h_load, task_util, affinity), + + TP_STRUCT__entry( + __field(int, scpu) + __field(unsigned long, src_util_cum) + __field(int, grp_type) + __field(int, dcpu) + __field(unsigned long, dst_util_cum) + __field(int, pid) + __field(unsigned long, affinity) + __field(unsigned long, task_util) + __field(unsigned long, h_load) + ), + + TP_fast_assign( + __entry->scpu = scpu; + __entry->src_util_cum = + cpu_rq(scpu)->wrq.cum_window_demand_scaled; + __entry->grp_type = grp_type; + __entry->dcpu = dcpu; + __entry->dst_util_cum = + cpu_rq(dcpu)->wrq.cum_window_demand_scaled; + __entry->pid = pid; + __entry->affinity = affinity; + __entry->task_util = task_util; + __entry->h_load = h_load; + ), + + TP_printk("source_cpu=%d util_cum=%lu group_type=%d dest_cpu=%d util_cum=%lu pid=%d affinity=%#lx task_util=%lu task_h_load=%lu", + __entry->scpu, __entry->src_util_cum, __entry->grp_type, + __entry->dcpu, __entry->dst_util_cum, __entry->pid, + __entry->affinity, __entry->task_util, __entry->h_load) +); + TRACE_EVENT(sched_cpu_util, TP_PROTO(int cpu), @@ -1025,6 +1023,7 @@ TRACE_EVENT(sched_task_util, __field(u32, unfilter) __field(unsigned long, cpus_allowed) __field(int, task_boost) + __field(bool, low_latency) ), TP_fast_assign( @@ -1043,28 +1042,22 @@ TRACE_EVENT(sched_task_util, __entry->is_rtg = is_rtg; __entry->rtg_skip_min = rtg_skip_min; __entry->start_cpu = start_cpu; -#ifdef CONFIG_SCHED_WALT __entry->unfilter = p->wts.unfilter; -#else - __entry->unfilter = 0; -#endif __entry->cpus_allowed = cpumask_bits(&p->cpus_mask)[0]; -#ifdef CONFIG_SCHED_WALT __entry->task_boost = per_task_boost(p); -#else - __entry->task_boost = 0; -#endif + __entry->low_latency = p->wts.low_latency; ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->candidates, __entry->best_energy_cpu, __entry->sync, __entry->need_idle, __entry->fastpath, __entry->placement_boost, __entry->latency, __entry->uclamp_boosted, __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, - __entry->unfilter, __entry->cpus_allowed, __entry->task_boost) -) + __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, + __entry->low_latency) +); /* * Tracepoint for find_best_target @@ -1120,12 +1113,10 @@ TRACE_EVENT(sched_find_best_target, __entry->skip, __entry->running) ); -#endif /* CONFIG_SMP */ /* * Tracepoint for system overutilized flag */ -#ifdef CONFIG_SCHED_WALT struct sched_domain; TRACE_EVENT_CONDITION(sched_overutilized, @@ -1148,7 +1139,7 @@ TRACE_EVENT_CONDITION(sched_overutilized, TP_printk("overutilized=%d sd_span=%s", __entry->overutilized ? 1 : 0, __entry->cpulist) ); -#endif +#endif /* CONFIG_SCHED_WALT */ #endif /* _TRACE_SCHED_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4cd0c831da1e..8178439f2d9f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2800,10 +2800,11 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.vruntime = 0; #ifdef CONFIG_SCHED_WALT p->wts.last_sleep_ts = 0; - p->wts.wake_up_idle = false; + p->wts.wake_up_idle = false; p->wts.boost = 0; p->wts.boost_expires = 0; - p->wts.boost_period = 0; + p->wts.boost_period = 0; + p->wts.low_latency = 0; #endif INIT_LIST_HEAD(&p->se.group_node); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d2aec043fe62..8cb1cb2906d6 100755 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4055,9 +4055,20 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) thresh >>= 1; vruntime -= thresh; - if (entity_is_task(se) && per_task_boost(task_of(se)) == - TASK_BOOST_STRICT_MAX) - vruntime -= sysctl_sched_latency; + if (entity_is_task(se)) { + if (per_task_boost(task_of(se)) == + TASK_BOOST_STRICT_MAX) + vruntime -= sysctl_sched_latency; +#ifdef CONFIG_SCHED_WALT + else if (unlikely(task_of(se)->wts.low_latency)) { + vruntime -= sysctl_sched_latency; + vruntime -= thresh; + se->vruntime = min_vruntime(vruntime, + se->vruntime); + return; + } +#endif + } } /* ensure we never gain time by being placed backwards. */