From 6de989b9467504cea2bc4638f71d430b12443486 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 28 Aug 2020 09:34:57 -0700 Subject: [PATCH] sched/preemptirq: Use restricted trace hooks Currently the software implements preemptirq notation points as unrestricted hooks. This causes unacceptable delays as those *types* of hooks perform rcu_lock() for a long period of time (uS, but 1000s of these per s). This in turn reduces system performance, and impacts our metrics. Fix by re-doing the preempt irq trace-points as restricted hooks, based upon vendor_hooks. Change-Id: Ib3022ceb41407f93543c30e695880f9070aae78a Signed-off-by: Shaleen Agrawal Signed-off-by: Stephen Dickey --- include/trace/events/preemptirq.h | 3 ++ include/trace/events/preemptirq_long.h | 3 ++ include/trace/hooks/restricted_preemptirq.h | 56 +++++++++++++++++++++ kernel/trace/preemptirq_long.c | 16 +++--- kernel/trace/trace_preemptirq.c | 27 +++++++--- 5 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 include/trace/hooks/restricted_preemptirq.h diff --git a/include/trace/events/preemptirq.h b/include/trace/events/preemptirq.h index 3f249e150c0c..9718136488e9 100644 --- a/include/trace/events/preemptirq.h +++ b/include/trace/events/preemptirq.h @@ -3,6 +3,9 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM preemptirq +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/events + #if !defined(_TRACE_PREEMPTIRQ_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_PREEMPTIRQ_H diff --git a/include/trace/events/preemptirq_long.h b/include/trace/events/preemptirq_long.h index fa5d27cc8e8a..a486134efdb9 100644 --- a/include/trace/events/preemptirq_long.h +++ b/include/trace/events/preemptirq_long.h @@ -6,6 +6,9 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM preemptirq_long +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/events + #if !defined(_TRACE_PREEMPTIRQ_LONG_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_PREEMPTIRQ_LONG_H diff --git a/include/trace/hooks/restricted_preemptirq.h b/include/trace/hooks/restricted_preemptirq.h new file mode 100644 index 000000000000..238c3e665136 --- /dev/null +++ b/include/trace/hooks/restricted_preemptirq.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ +#if !defined(_TRACE_RESTRICTED_PREEMPTIRQ_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RESTRICTED_PREEMPTIRQ_H + +#ifdef CONFIG_PREEMPTIRQ_TRACEPOINTS + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM restricted_preemptirq + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/hooks + +#include +#include + +#ifdef CONFIG_TRACE_IRQFLAGS +DECLARE_RESTRICTED_HOOK(restricted_irq_disable, + TP_PROTO(unsigned long ip, unsigned long parent_ip), + TP_ARGS(ip, parent_ip), 1); + +DECLARE_RESTRICTED_HOOK(restricted_irq_enable, + TP_PROTO(unsigned long ip, unsigned long parent_ip), + TP_ARGS(ip, parent_ip), 1); + +#else +#define trace_restricted_irq_enable(ip, parent_ip) +#define trace_restricted_irq_disable(ip, parent_ip) +#endif /* CONFIG_TRACE_IRQFLAGS */ + +#ifdef CONFIG_TRACE_PREEMPT_TOGGLE +DECLARE_RESTRICTED_HOOK(restricted_preempt_disable, + TP_PROTO(unsigned long ip, unsigned long parent_ip), + TP_ARGS(ip, parent_ip), 1); + +DECLARE_RESTRICTED_HOOK(restricted_preempt_enable, + TP_PROTO(unsigned long ip, unsigned long parent_ip), + TP_ARGS(ip, parent_ip), 1); + +#else +#define trace_restricted_preempt_enable(ip, parent_ip) +#define trace_restricted_preempt_disable(ip, parent_ip) +#endif /* CONFIG_TRACE_PREEMPT_TOGGLE */ + +#include + +#else /* ! CONFIG_PREEMPTIRQ_TRACEPOINTS */ +#define trace_restricted_irq_enable(...) +#define trace_restricted_irq_disable(...) +#define trace_restricted_preempt_enable(...) +#define trace_restricted_preempt_disable(...) +#endif /* ! CONFIG_PREEMPTIRQ_TRACEPOINTS */ + +#endif /* TRACE_RESTRICTED_PREEMPTIRQ_H || TRACE_HEADER_MULTI_READ */ diff --git a/kernel/trace/preemptirq_long.c b/kernel/trace/preemptirq_long.c index 6fa3df7a93c4..e7c02b42edb7 100644 --- a/kernel/trace/preemptirq_long.c +++ b/kernel/trace/preemptirq_long.c @@ -9,11 +9,9 @@ #include #include #include - -#include #include - #define CREATE_TRACE_POINTS +#include #include #define IRQSOFF_SENTINEL 0x0fffDEAD @@ -151,13 +149,13 @@ static int preemptirq_long_init(void) return -EPERM; } - register_trace_irq_disable(note_irq_disable, NULL); - register_trace_irq_enable(test_irq_disable_long, NULL); - - register_trace_preempt_disable(note_preempt_disable, NULL); - register_trace_preempt_enable(test_preempt_disable_long, NULL); - + register_trace_restricted_irq_disable(note_irq_disable, NULL); + register_trace_restricted_irq_enable(test_irq_disable_long, NULL); + register_trace_restricted_preempt_disable(note_preempt_disable, NULL); + register_trace_restricted_preempt_enable(test_preempt_disable_long, + NULL); register_trace_sched_switch(note_context_switch, NULL); + return 0; } diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index 4d8e99fdbbbe..105cae773c31 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -11,7 +11,7 @@ #include #include #include "trace.h" - +#include #define CREATE_TRACE_POINTS #include @@ -22,8 +22,10 @@ static DEFINE_PER_CPU(int, tracing_irq_cpu); void trace_hardirqs_on(void) { if (this_cpu_read(tracing_irq_cpu)) { - if (!in_nmi()) + if (!in_nmi()) { trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); + trace_restricted_irq_enable(CALLER_ADDR0, CALLER_ADDR1); + } tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); this_cpu_write(tracing_irq_cpu, 0); } @@ -38,8 +40,11 @@ void trace_hardirqs_off(void) if (!this_cpu_read(tracing_irq_cpu)) { this_cpu_write(tracing_irq_cpu, 1); tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); - if (!in_nmi()) + if (!in_nmi()) { trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); + trace_restricted_irq_disable(CALLER_ADDR0, + CALLER_ADDR1); + } } lockdep_hardirqs_off(CALLER_ADDR0); @@ -50,8 +55,10 @@ NOKPROBE_SYMBOL(trace_hardirqs_off); __visible void trace_hardirqs_on_caller(unsigned long caller_addr) { if (this_cpu_read(tracing_irq_cpu)) { - if (!in_nmi()) + if (!in_nmi()) { trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); + trace_restricted_irq_enable(CALLER_ADDR0, caller_addr); + } tracer_hardirqs_on(CALLER_ADDR0, caller_addr); this_cpu_write(tracing_irq_cpu, 0); } @@ -66,8 +73,10 @@ __visible void trace_hardirqs_off_caller(unsigned long caller_addr) if (!this_cpu_read(tracing_irq_cpu)) { this_cpu_write(tracing_irq_cpu, 1); tracer_hardirqs_off(CALLER_ADDR0, caller_addr); - if (!in_nmi()) + if (!in_nmi()) { trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); + trace_restricted_irq_disable(CALLER_ADDR0, caller_addr); + } } lockdep_hardirqs_off(CALLER_ADDR0); @@ -80,15 +89,19 @@ NOKPROBE_SYMBOL(trace_hardirqs_off_caller); void trace_preempt_on(unsigned long a0, unsigned long a1) { - if (!in_nmi()) + if (!in_nmi()) { trace_preempt_enable_rcuidle(a0, a1); + trace_restricted_preempt_enable(a0, a1); + } tracer_preempt_on(a0, a1); } void trace_preempt_off(unsigned long a0, unsigned long a1) { - if (!in_nmi()) + if (!in_nmi()) { trace_preempt_disable_rcuidle(a0, a1); + trace_restricted_preempt_disable(a0, a1); + } tracer_preempt_off(a0, a1); } #endif