A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-26 06:33:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

irqchip updates for 4.7-rc1:

Browse source 
- A number of embarassing buglets (GICv3, PIC32)
 - A more substential errata workaround for Cavium's GICv3 ITS
   (kept for post-rc1 due to its dependency on NUMA)
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
 iQIcBAABAgAGBQJXUXcjAAoJECPQ0LrRPXpDGFcQAICkj5cIsQghW2dgR0eo2d7S
 +ieyxr55tz3A0c1Cisw09ESHz6wCrQ+PmvXkKISIG1l2AHv9UOUZVrsB1cl2CvBg
 C9ClvUyMafiZ+FlhxMO1QM1Vfa3EUV2EPIx3mh5klUp8ph/cT+aArJe+WmJApS25
 nlYiobi2AE0+m2V5ekikMtVbM5xXWKHPRgzYqZUlNBV74k/FGgRlBk/bw1AWqnsd
 TfUF+QZpEd/4GPglbQLvwJwjQg+qanl59CJqi403U00emLuvRqdqTeMoqPEfw5id
 MgVPMtUF+N7fgtygo20oPFBriFBHFUTj+c5Oafd4ahgp6eU02HYX8A7w/jj84tTP
 cPa9bcoyKyec5vpO1mbU2a/VzqXPDNL17Dg9tRaf0NpksMeLvBh14jXWp5v8vEqU
 Qm4mFlmEYKivWTJhz6pGJmxFX/X5vMa2wrFY7xvOVYby5mSuEGD7+puuKuVNNBEa
 THaElOYM4ogTrUBM39dzfzXxSEQN/bcLHXNd2IDuUUK49NvNFjnke3PvLxesiDuT
 Pxk4mO912+/Ldk7K1LVVVWltVtOHFNG+I7a3R75gwftHXMYONuOpEiflA4QorxVk
 9Rq9yUI4h/69V5fIthBN8BUYU5hxaTtLi0DI1fWSweugZb5PUXiagKnXKaIOLQ/4
 A3pvoYEHdynDVO7nJ+Kd
 =E+j+
 -----END PGP SIGNATURE-----

Merge tag 'irqchip-4.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms into irq/urgent

Merge irqchip updates from Marc Zyngier:

- A number of embarassing buglets (GICv3, PIC32)
- A more substential errata workaround for Cavium's GICv3 ITS
  (kept for post-rc1 due to its dependency on NUMA)
This commit is contained in:
Thomas Gleixner 2016-06-03 15:05:51 +02:00
commit 2eec3707a3
8139 changed files with 426722 additions and 163917 deletions
Download patch file Download diff file Expand all files Collapse all files

4
kernel/Makefile
View file

@ -91,9 +91,7 @@ obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
obj-$(CONFIG_TRACEPOINTS) += tracepoint.o
obj-$(CONFIG_LATENCYTOP) += latencytop.o
obj-$(CONFIG_BINFMT_ELF) += elfcore.o
obj-$(CONFIG_COMPAT_BINFMT_ELF) += elfcore.o
obj-$(CONFIG_BINFMT_ELF_FDPIC) += elfcore.o
obj-$(CONFIG_ELFCORE) += elfcore.o
obj-$(CONFIG_FUNCTION_TRACER) += trace/
obj-$(CONFIG_TRACING) += trace/
obj-$(CONFIG_TRACE_CLOCK) += trace/

30
kernel/audit.c
View file

@ -64,7 +64,6 @@
#include <linux/security.h>
#endif
#include <linux/freezer.h>
#include <linux/tty.h>
#include <linux/pid_namespace.h>
#include <net/netns/generic.h>
@ -430,7 +429,6 @@ restart:
attempts, audit_pid);
set_current_state(TASK_INTERRUPTIBLE);
schedule();
__set_current_state(TASK_RUNNING);
goto restart;
}
}
@ -1341,15 +1339,14 @@ static inline void audit_get_stamp(struct audit_context *ctx,
static long wait_for_auditd(long sleep_time)
{
DECLARE_WAITQUEUE(wait, current);
set_current_state(TASK_UNINTERRUPTIBLE);
add_wait_queue_exclusive(&audit_backlog_wait, &wait);
if (audit_backlog_limit &&
skb_queue_len(&audit_skb_queue) > audit_backlog_limit)
skb_queue_len(&audit_skb_queue) > audit_backlog_limit) {
add_wait_queue_exclusive(&audit_backlog_wait, &wait);
set_current_state(TASK_UNINTERRUPTIBLE);
sleep_time = schedule_timeout(sleep_time);
__set_current_state(TASK_RUNNING);
remove_wait_queue(&audit_backlog_wait, &wait);
remove_wait_queue(&audit_backlog_wait, &wait);
}
return sleep_time;
}
@ -1890,21 +1887,14 @@ void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
{
const struct cred *cred;
char comm[sizeof(tsk->comm)];
char *tty;
struct tty_struct *tty;
if (!ab)
return;
/* tsk == current */
cred = current_cred();
spin_lock_irq(&tsk->sighand->siglock);
if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name)
tty = tsk->signal->tty->name;
else
tty = "(none)";
spin_unlock_irq(&tsk->sighand->siglock);
tty = audit_get_tty(tsk);
audit_log_format(ab,
" ppid=%d pid=%d auid=%u uid=%u gid=%u"
" euid=%u suid=%u fsuid=%u"
@ -1920,11 +1910,11 @@ void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
from_kgid(&init_user_ns, cred->egid),
from_kgid(&init_user_ns, cred->sgid),
from_kgid(&init_user_ns, cred->fsgid),
tty, audit_get_sessionid(tsk));
tty ? tty_name(tty) : "(none)",
audit_get_sessionid(tsk));
audit_put_tty(tty);
audit_log_format(ab, " comm=");
audit_log_untrustedstring(ab, get_task_comm(comm, tsk));
audit_log_d_path_exe(ab, tsk->mm);
audit_log_task_context(ab);
}

12
kernel/audit_tree.c
View file

@ -661,10 +661,10 @@ static int tag_mount(struct vfsmount *mnt, void *arg)
static int prune_tree_thread(void *unused)
{
for (;;) {
set_current_state(TASK_INTERRUPTIBLE);
if (list_empty(&prune_list))
if (list_empty(&prune_list)) {
set_current_state(TASK_INTERRUPTIBLE);
schedule();
__set_current_state(TASK_RUNNING);
}
mutex_lock(&audit_cmd_mutex);
mutex_lock(&audit_filter_mutex);
@ -693,16 +693,14 @@ static int audit_launch_prune(void)
{
if (prune_thread)
return 0;
prune_thread = kthread_create(prune_tree_thread, NULL,
prune_thread = kthread_run(prune_tree_thread, NULL,
"audit_prune_tree");
if (IS_ERR(prune_thread)) {
pr_err("cannot start thread audit_prune_tree");
prune_thread = NULL;
return -ENOMEM;
} else {
wake_up_process(prune_thread);
return 0;
}
return 0;
}
/* called with audit_filter_mutex */

2
kernel/audit_watch.c
View file

@ -367,7 +367,7 @@ static int audit_get_nd(struct audit_watch *watch, struct path *parent)
inode_unlock(d_backing_inode(parent->dentry));
if (d_is_positive(d)) {
/* update watch filter fields */
watch->dev = d_backing_inode(d)->i_sb->s_dev;
watch->dev = d->d_sb->s_dev;
watch->ino = d_backing_inode(d)->i_ino;
}
dput(d);

8
kernel/auditsc.c
View file

@ -1980,6 +1980,7 @@ static void audit_log_set_loginuid(kuid_t koldloginuid, kuid_t kloginuid,
{
struct audit_buffer *ab;
uid_t uid, oldloginuid, loginuid;
struct tty_struct *tty;
if (!audit_enabled)
return;
@ -1987,14 +1988,17 @@ static void audit_log_set_loginuid(kuid_t koldloginuid, kuid_t kloginuid,
uid = from_kuid(&init_user_ns, task_uid(current));
oldloginuid = from_kuid(&init_user_ns, koldloginuid);
loginuid = from_kuid(&init_user_ns, kloginuid),
tty = audit_get_tty(current);
ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN);
if (!ab)
return;
audit_log_format(ab, "pid=%d uid=%u", task_pid_nr(current), uid);
audit_log_task_context(ab);
audit_log_format(ab, " old-auid=%u auid=%u old-ses=%u ses=%u res=%d",
oldloginuid, loginuid, oldsessionid, sessionid, !rc);
audit_log_format(ab, " old-auid=%u auid=%u tty=%s old-ses=%u ses=%u res=%d",
oldloginuid, loginuid, tty ? tty_name(tty) : "(none)",
oldsessionid, sessionid, !rc);
audit_put_tty(tty);
audit_log_end(ab);
}

308
kernel/bpf/core.c
View file

@ -129,14 +129,83 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
return fp;
}
EXPORT_SYMBOL_GPL(bpf_prog_realloc);
void __bpf_prog_free(struct bpf_prog *fp)
{
kfree(fp->aux);
vfree(fp);
}
EXPORT_SYMBOL_GPL(__bpf_prog_free);
static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_JMP &&
/* Call and Exit are both special jumps with no
* target inside the BPF instruction image.
*/
BPF_OP(insn->code) != BPF_CALL &&
BPF_OP(insn->code) != BPF_EXIT;
}
static void bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta)
{
struct bpf_insn *insn = prog->insnsi;
u32 i, insn_cnt = prog->len;
for (i = 0; i < insn_cnt; i++, insn++) {
if (!bpf_is_jmp_and_has_target(insn))
continue;
/* Adjust offset of jmps if we cross boundaries. */
if (i < pos && i + insn->off + 1 > pos)
insn->off += delta;
else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
insn->off -= delta;
}
}
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
const struct bpf_insn *patch, u32 len)
{
u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
struct bpf_prog *prog_adj;
/* Since our patchlet doesn't expand the image, we're done. */
if (insn_delta == 0) {
memcpy(prog->insnsi + off, patch, sizeof(*patch));
return prog;
}
insn_adj_cnt = prog->len + insn_delta;
/* Several new instructions need to be inserted. Make room
* for them. Likely, there's no need for a new allocation as
* last page could have large enough tailroom.
*/
prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
GFP_USER);
if (!prog_adj)
return NULL;
prog_adj->len = insn_adj_cnt;
/* Patching happens in 3 steps:
*
* 1) Move over tail of insnsi from next instruction onwards,
* so we can patch the single target insn with one or more
* new ones (patching is always from 1 to n insns, n > 0).
* 2) Inject new instructions at the target location.
* 3) Adjust branch offsets if necessary.
*/
insn_rest = insn_adj_cnt - off - len;
memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
sizeof(*patch) * insn_rest);
memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);
bpf_adj_branches(prog_adj, off, insn_delta);
return prog_adj;
}
#ifdef CONFIG_BPF_JIT
struct bpf_binary_header *
@ -162,7 +231,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
hdr->pages = size / PAGE_SIZE;
hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
PAGE_SIZE - sizeof(*hdr));
start = (prandom_u32() % hole) & ~(alignment - 1);
start = (get_random_int() % hole) & ~(alignment - 1);
/* Leave a random number of instructions before BPF code. */
*image_ptr = &hdr->image[start];
@ -174,6 +243,209 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr)
{
module_memfree(hdr);
}
int bpf_jit_harden __read_mostly;
static int bpf_jit_blind_insn(const struct bpf_insn *from,
const struct bpf_insn *aux,
struct bpf_insn *to_buff)
{
struct bpf_insn *to = to_buff;
u32 imm_rnd = get_random_int();
s16 off;
BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG);
if (from->imm == 0 &&
(from->code == (BPF_ALU | BPF_MOV | BPF_K) ||
from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) {
*to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg);
goto out;
}
switch (from->code) {
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU | BPF_MOV | BPF_K:
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU | BPF_MOD | BPF_K:
*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX);
break;
case BPF_ALU64 | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
case BPF_ALU64 | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_MOV | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_K:
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX);
break;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSET | BPF_K:
/* Accommodate for extra offset in case of a backjump. */
off = from->off;
if (off < 0)
off -= 2;
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
break;
case BPF_LD | BPF_ABS | BPF_W:
case BPF_LD | BPF_ABS | BPF_H:
case BPF_LD | BPF_ABS | BPF_B:
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_LD_IND(from->code, BPF_REG_AX, 0);
break;
case BPF_LD | BPF_IND | BPF_W:
case BPF_LD | BPF_IND | BPF_H:
case BPF_LD | BPF_IND | BPF_B:
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_ALU32_REG(BPF_ADD, BPF_REG_AX, from->src_reg);
*to++ = BPF_LD_IND(from->code, BPF_REG_AX, 0);
break;
case BPF_LD | BPF_IMM | BPF_DW:
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
*to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);
break;
case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */
*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);
*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX);
break;
case BPF_ST | BPF_MEM | BPF_DW:
case BPF_ST | BPF_MEM | BPF_W:
case BPF_ST | BPF_MEM | BPF_H:
case BPF_ST | BPF_MEM | BPF_B:
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off);
break;
}
out:
return to - to_buff;
}
static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,
gfp_t gfp_extra_flags)
{
gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO |
gfp_extra_flags;
struct bpf_prog *fp;
fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL);
if (fp != NULL) {
kmemcheck_annotate_bitfield(fp, meta);
/* aux->prog still points to the fp_other one, so
* when promoting the clone to the real program,
* this still needs to be adapted.
*/
memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE);
}
return fp;
}
static void bpf_prog_clone_free(struct bpf_prog *fp)
{
/* aux was stolen by the other clone, so we cannot free
* it from this path! It will be freed eventually by the
* other program on release.
*
* At this point, we don't need a deferred release since
* clone is guaranteed to not be locked.
*/
fp->aux = NULL;
__bpf_prog_free(fp);
}
void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
{
/* We have to repoint aux->prog to self, as we don't
* know whether fp here is the clone or the original.
*/
fp->aux->prog = fp;
bpf_prog_clone_free(fp_other);
}
struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
{
struct bpf_insn insn_buff[16], aux[2];
struct bpf_prog *clone, *tmp;
int insn_delta, insn_cnt;
struct bpf_insn *insn;
int i, rewritten;
if (!bpf_jit_blinding_enabled())
return prog;
clone = bpf_prog_clone_create(prog, GFP_USER);
if (!clone)
return ERR_PTR(-ENOMEM);
insn_cnt = clone->len;
insn = clone->insnsi;
for (i = 0; i < insn_cnt; i++, insn++) {
/* We temporarily need to hold the original ld64 insn
* so that we can still access the first part in the
* second blinding run.
*/
if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) &&
insn[1].code == 0)
memcpy(aux, insn, sizeof(aux));
rewritten = bpf_jit_blind_insn(insn, aux, insn_buff);
if (!rewritten)
continue;
tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
if (!tmp) {
/* Patching may have repointed aux->prog during
* realloc from the original one, so we need to
* fix it up here on error.
*/
bpf_jit_prog_release_other(prog, clone);
return ERR_PTR(-ENOMEM);
}
clone = tmp;
insn_delta = rewritten - 1;
/* Walk new program and skip insns we just inserted. */
insn = clone->insnsi + i + insn_delta;
insn_cnt += insn_delta;
i += insn_delta;
}
return clone;
}
#endif /* CONFIG_BPF_JIT */
/* Base function for offset calculation. Needs to go into .text section,
@ -692,15 +964,22 @@ static int bpf_check_tail_call(const struct bpf_prog *fp)
/**
* bpf_prog_select_runtime - select exec runtime for BPF program
* @fp: bpf_prog populated with internal BPF program
* @err: pointer to error variable
*
* Try to JIT eBPF program, if JIT is not available, use interpreter.
* The BPF program will be executed via BPF_PROG_RUN() macro.
*/
int bpf_prog_select_runtime(struct bpf_prog *fp)
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
{
fp->bpf_func = (void *) __bpf_prog_run;
bpf_int_jit_compile(fp);
/* eBPF JITs can rewrite the program in case constant
* blinding is active. However, in case of error during
* blinding, bpf_int_jit_compile() must always return a
* valid program, which in this case would simply not
* be JITed, but falls back to the interpreter.
*/
fp = bpf_int_jit_compile(fp);
bpf_prog_lock_ro(fp);
/* The tail call compatibility check can only be done at
@ -708,7 +987,9 @@ int bpf_prog_select_runtime(struct bpf_prog *fp)
* with JITed or non JITed program concatenations and not
* all eBPF JITs might immediately support all features.
*/
return bpf_check_tail_call(fp);
*err = bpf_check_tail_call(fp);
return fp;
}
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
@ -764,14 +1045,21 @@ const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
const struct bpf_func_proto bpf_get_current_comm_proto __weak;
const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
{
return NULL;
}
const struct bpf_func_proto * __weak bpf_get_event_output_proto(void)
{
return NULL;
}
/* Always built-in helper functions. */
const struct bpf_func_proto bpf_tail_call_proto = {
.func = NULL,
@ -783,8 +1071,14 @@ const struct bpf_func_proto bpf_tail_call_proto = {
};
/* For classic BPF JITs that don't implement bpf_int_jit_compile(). */
void __weak bpf_int_jit_compile(struct bpf_prog *prog)
struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
{
return prog;
}
bool __weak bpf_helper_changes_skb_data(void *func)
{
return false;
}
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call

17
kernel/bpf/helpers.c
View file

@ -163,17 +163,26 @@ static u64 bpf_get_current_comm(u64 r1, u64 size, u64 r3, u64 r4, u64 r5)
struct task_struct *task = current;
char *buf = (char *) (long) r1;
if (!task)
return -EINVAL;
if (unlikely(!task))
goto err_clear;
strlcpy(buf, task->comm, min_t(size_t, size, sizeof(task->comm)));
strncpy(buf, task->comm, size);
/* Verifier guarantees that size > 0. For task->comm exceeding
* size, guarantee that buf is %NUL-terminated. Unconditionally
* done here to save the size test.
*/
buf[size - 1] = 0;
return 0;
err_clear:
memset(buf, 0, size);
return -EINVAL;
}
const struct bpf_func_proto bpf_get_current_comm_proto = {
.func = bpf_get_current_comm,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_STACK,
.arg1_type = ARG_PTR_TO_RAW_STACK,
.arg2_type = ARG_CONST_STACK_SIZE,
};

39
kernel/bpf/inode.c
View file

@ -119,18 +119,10 @@ static int bpf_inode_type(const struct inode *inode, enum bpf_type *type)
return 0;
}
static bool bpf_dname_reserved(const struct dentry *dentry)
{
return strchr(dentry->d_name.name, '.');
}
static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
{
struct inode *inode;
if (bpf_dname_reserved(dentry))
return -EPERM;
inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR);
if (IS_ERR(inode))
return PTR_ERR(inode);
@ -152,9 +144,6 @@ static int bpf_mkobj_ops(struct inode *dir, struct dentry *dentry,
{
struct inode *inode;
if (bpf_dname_reserved(dentry))
return -EPERM;
inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFREG);
if (IS_ERR(inode))
return PTR_ERR(inode);
@ -187,31 +176,21 @@ static int bpf_mkobj(struct inode *dir, struct dentry *dentry, umode_t mode,
}
}
static int bpf_link(struct dentry *old_dentry, struct inode *dir,
struct dentry *new_dentry)
static struct dentry *
bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags)
{
if (bpf_dname_reserved(new_dentry))
return -EPERM;
return simple_link(old_dentry, dir, new_dentry);
}
static int bpf_rename(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry)
{
if (bpf_dname_reserved(new_dentry))
return -EPERM;
return simple_rename(old_dir, old_dentry, new_dir, new_dentry);
if (strchr(dentry->d_name.name, '.'))
return ERR_PTR(-EPERM);
return simple_lookup(dir, dentry, flags);
}
static const struct inode_operations bpf_dir_iops = {
.lookup = simple_lookup,
.lookup = bpf_lookup,
.mknod = bpf_mkobj,
.mkdir = bpf_mkdir,
.rmdir = simple_rmdir,
.rename = bpf_rename,
.link = bpf_link,
.rename = simple_rename,
.link = simple_link,
.unlink = simple_unlink,
};
@ -378,7 +357,7 @@ static int bpf_fill_super(struct super_block *sb, void *data, int silent)
static struct dentry *bpf_mount(struct file_system_type *type, int flags,
const char *dev_name, void *data)
{
return mount_ns(type, flags, current->nsproxy->mnt_ns, bpf_fill_super);
return mount_nodev(type, flags, data, bpf_fill_super);
}
static struct file_system_type bpf_fs_type = {

5
kernel/bpf/stackmap.c
View file

@ -116,7 +116,7 @@ free_smap:
return ERR_PTR(err);
}
static u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5)
u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5)
{
struct pt_regs *regs = (struct pt_regs *) (long) r1;
struct bpf_map *map = (struct bpf_map *) (long) r2;
@ -136,7 +136,8 @@ static u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5)
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
return -EINVAL;
trace = get_perf_callchain(regs, init_nr, kernel, user, false, false);
trace = get_perf_callchain(regs, init_nr, kernel, user,
sysctl_perf_event_max_stack, false, false);
if (unlikely(!trace))
/* couldn't fetch the stack trace */

2
kernel/bpf/syscall.c
View file

@ -762,7 +762,7 @@ static int bpf_prog_load(union bpf_attr *attr)
fixup_bpf_calls(prog);
/* eBPF program is ready to be JITed */
err = bpf_prog_select_runtime(prog);
prog = bpf_prog_select_runtime(prog, &err);
if (err < 0)
goto free_used_maps;

741
kernel/bpf/verifier.c
View file

File diff suppressed because it is too large Load diff

22
kernel/cpuset.c
View file

@ -61,7 +61,7 @@
#include <linux/cgroup.h>
#include <linux/wait.h>
struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;
DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
/* See "Frequency meter" comments, below. */
@ -2528,27 +2528,27 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
* GFP_KERNEL - any node in enclosing hardwalled cpuset ok
* GFP_USER - only nodes in current tasks mems allowed ok.
*/
int __cpuset_node_allowed(int node, gfp_t gfp_mask)
bool __cpuset_node_allowed(int node, gfp_t gfp_mask)
{
struct cpuset *cs; /* current cpuset ancestors */
int allowed; /* is allocation in zone z allowed? */
unsigned long flags;
if (in_interrupt())
return 1;
return true;
if (node_isset(node, current->mems_allowed))
return 1;
return true;
/*
* Allow tasks that have access to memory reserves because they have
* been OOM killed to get memory anywhere.
*/
if (unlikely(test_thread_flag(TIF_MEMDIE)))
return 1;
return true;
if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
return 0;
return false;
if (current->flags & PF_EXITING) /* Let dying task have memory */
return 1;
return true;
/* Not hardwall and node outside mems_allowed: scan up cpusets */
spin_lock_irqsave(&callback_lock, flags);
@ -2591,13 +2591,7 @@ int __cpuset_node_allowed(int node, gfp_t gfp_mask)
static int cpuset_spread_node(int *rotor)
{
int node;
node = next_node(*rotor, current->mems_allowed);
if (node == MAX_NUMNODES)
node = first_node(current->mems_allowed);
*rotor = node;
return node;
return *rotor = next_node_in(*rotor, current->mems_allowed);
}
int cpuset_mem_spread_node(void)

36
kernel/events/callchain.c
View file

@ -19,11 +19,13 @@ struct callchain_cpus_entries {
};
int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
static inline size_t perf_callchain_entry__sizeof(void)
{
return (sizeof(struct perf_callchain_entry) +
sizeof(__u64) * sysctl_perf_event_max_stack);
sizeof(__u64) * (sysctl_perf_event_max_stack +
sysctl_perf_event_max_contexts_per_stack));
}
static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
@ -32,12 +34,12 @@ static DEFINE_MUTEX(callchain_mutex);
static struct callchain_cpus_entries *callchain_cpus_entries;
__weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
struct pt_regs *regs)
{
}
__weak void perf_callchain_user(struct perf_callchain_entry *entry,
__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
struct pt_regs *regs)
{
}
@ -176,14 +178,15 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
if (!kernel && !user)
return NULL;
return get_perf_callchain(regs, 0, kernel, user, crosstask, true);
return get_perf_callchain(regs, 0, kernel, user, sysctl_perf_event_max_stack, crosstask, true);
}
struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
bool crosstask, bool add_mark)
u32 max_stack, bool crosstask, bool add_mark)
{
struct perf_callchain_entry *entry;
struct perf_callchain_entry_ctx ctx;
int rctx;
entry = get_callchain_entry(&rctx);
@ -193,12 +196,16 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
if (!entry)
goto exit_put;
entry->nr = init_nr;
ctx.entry = entry;
ctx.max_stack = max_stack;
ctx.nr = entry->nr = init_nr;
ctx.contexts = 0;
ctx.contexts_maxed = false;
if (kernel && !user_mode(regs)) {
if (add_mark)
perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
perf_callchain_kernel(entry, regs);
perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
perf_callchain_kernel(&ctx, regs);
}
if (user) {
@ -214,8 +221,8 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
goto exit_put;
if (add_mark)
perf_callchain_store(entry, PERF_CONTEXT_USER);
perf_callchain_user(entry, regs);
perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
perf_callchain_user(&ctx, regs);
}
}
@ -225,10 +232,15 @@ exit_put:
return entry;
}
/*
* Used for sysctl_perf_event_max_stack and
* sysctl_perf_event_max_contexts_per_stack.
*/
int perf_event_max_stack_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int new_value = sysctl_perf_event_max_stack, ret;
int *value = table->data;
int new_value = *value, ret;
struct ctl_table new_table = *table;
new_table.data = &new_value;
@ -240,7 +252,7 @@ int perf_event_max_stack_handler(struct ctl_table *table, int write,
if (atomic_read(&nr_callchain_events))
ret = -EBUSY;
else
sysctl_perf_event_max_stack = new_value;
*value = new_value;
mutex_unlock(&callchain_mutex);

45
kernel/events/core.c
View file

@ -7100,7 +7100,7 @@ int perf_swevent_get_recursion_context(void)
}
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
inline void perf_swevent_put_recursion_context(int rctx)
void perf_swevent_put_recursion_context(int rctx)
{
struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
@ -7362,7 +7362,26 @@ static int perf_tp_event_match(struct perf_event *event,
return 1;
}
void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
struct trace_event_call *call, u64 count,
struct pt_regs *regs, struct hlist_head *head,
struct task_struct *task)
{
struct bpf_prog *prog = call->prog;
if (prog) {
*(struct pt_regs **)raw_data = regs;
if (!trace_call_bpf(prog, raw_data) || hlist_empty(head)) {
perf_swevent_put_recursion_context(rctx);
return;
}
}
perf_tp_event(call->event.type, count, raw_data, size, regs, head,
rctx, task);
}
EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
struct pt_regs *regs, struct hlist_head *head, int rctx,
struct task_struct *task)
{
@ -7374,9 +7393,11 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
.data = record,
};
perf_sample_data_init(&data, addr, 0);
perf_sample_data_init(&data, 0, 0);
data.raw = &raw;
perf_trace_buf_update(record, event_type);
hlist_for_each_entry_rcu(event, head, hlist_entry) {
if (perf_tp_event_match(event, &data, regs))
perf_swevent_event(event, count, &data, regs);
@ -7461,6 +7482,7 @@ static void perf_event_free_filter(struct perf_event *event)
static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
{
bool is_kprobe, is_tracepoint;
struct bpf_prog *prog;
if (event->attr.type != PERF_TYPE_TRACEPOINT)
@ -7469,20 +7491,31 @@ static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
if (event->tp_event->prog)
return -EEXIST;
if (!(event->tp_event->flags & TRACE_EVENT_FL_UKPROBE))
/* bpf programs can only be attached to u/kprobes */
is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
if (!is_kprobe && !is_tracepoint)
/* bpf programs can only be attached to u/kprobe or tracepoint */
return -EINVAL;
prog = bpf_prog_get(prog_fd);
if (IS_ERR(prog))
return PTR_ERR(prog);
if (prog->type != BPF_PROG_TYPE_KPROBE) {
if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) ||
(is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT)) {
/* valid fd, but invalid bpf program type */
bpf_prog_put(prog);
return -EINVAL;
}
if (is_tracepoint) {
int off = trace_event_get_offsets(event->tp_event);
if (prog->aux->max_ctx_offset > off) {
bpf_prog_put(prog);
return -EACCES;
}
}
event->tp_event->prog = prog;
return 0;

10
kernel/events/uprobes.c
View file

@ -1130,7 +1130,9 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
struct vm_area_struct *vma;
int ret;
down_write(&mm->mmap_sem);
if (down_write_killable(&mm->mmap_sem))
return -EINTR;
if (mm->uprobes_state.xol_area) {
ret = -EALREADY;
goto fail;
@ -1469,7 +1471,8 @@ static void dup_xol_work(struct callback_head *work)
if (current->flags & PF_EXITING)
return;
if (!__create_xol_area(current->utask->dup_xol_addr))
if (!__create_xol_area(current->utask->dup_xol_addr) &&
!fatal_signal_pending(current))
uprobe_warn(current, "dup xol area");
}
@ -1694,8 +1697,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
int result;
pagefault_disable();
result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,
sizeof(opcode));
result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
pagefault_enable();
if (likely(result == 0))

34
kernel/exit.c
View file

@ -746,7 +746,7 @@ void do_exit(long code)
disassociate_ctty(1);
exit_task_namespaces(tsk);
exit_task_work(tsk);
exit_thread();
exit_thread(tsk);
/*
* Flush inherited counters to the parent - before the parent
@ -918,17 +918,28 @@ static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
task_pid_type(p, wo->wo_type) == wo->wo_pid;
}
static int eligible_child(struct wait_opts *wo, struct task_struct *p)
static int
eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
{
if (!eligible_pid(wo, p))
return 0;
/* Wait for all children (clone and not) if __WALL is set;
* otherwise, wait for clone children *only* if __WCLONE is
* set; otherwise, wait for non-clone children *only*. (Note:
* A "clone" child here is one that reports to its parent
* using a signal other than SIGCHLD.) */
if (((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
&& !(wo->wo_flags & __WALL))
/*
* Wait for all children (clone and not) if __WALL is set or
* if it is traced by us.
*/
if (ptrace || (wo->wo_flags & __WALL))
return 1;
/*
* Otherwise, wait for clone children *only* if __WCLONE is set;
* otherwise, wait for non-clone children *only*.
*
* Note: a "clone" child here is one that reports to its parent
* using a signal other than SIGCHLD, or a non-leader thread which
* we can only see if it is traced by us.
*/
if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
return 0;
return 1;
@ -1300,7 +1311,7 @@ static int wait_consider_task(struct wait_opts *wo, int ptrace,
if (unlikely(exit_state == EXIT_DEAD))
return 0;
ret = eligible_child(wo, p);
ret = eligible_child(wo, ptrace, p);
if (!ret)
return ret;
@ -1524,7 +1535,8 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
enum pid_type type;
long ret;
if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED))
if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
__WNOTHREAD|__WCLONE|__WALL))
return -EINVAL;
if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
return -EINVAL;

79
kernel/fork.c
View file

@ -340,13 +340,14 @@ void set_task_stack_end_magic(struct task_struct *tsk)
*stackend = STACK_END_MAGIC; /* for overflow detection */
}
static struct task_struct *dup_task_struct(struct task_struct *orig)
static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
{
struct task_struct *tsk;
struct thread_info *ti;
int node = tsk_fork_get_node(orig);
int err;
if (node == NUMA_NO_NODE)
node = tsk_fork_get_node(orig);
tsk = alloc_task_struct_node(node);
if (!tsk)
return NULL;
@ -413,7 +414,10 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
unsigned long charge;
uprobe_start_dup_mmap();
down_write(&oldmm->mmap_sem);
if (down_write_killable(&oldmm->mmap_sem)) {
retval = -EINTR;
goto fail_uprobe_end;
}
flush_cache_dup_mm(oldmm);
uprobe_dup_mmap(oldmm, mm);
/*
@ -525,6 +529,7 @@ out:
up_write(&mm->mmap_sem);
flush_tlb_mm(oldmm);
up_write(&oldmm->mmap_sem);
fail_uprobe_end:
uprobe_end_dup_mmap();
return retval;
fail_nomem_anon_vma_fork:
@ -699,6 +704,26 @@ void __mmdrop(struct mm_struct *mm)
}
EXPORT_SYMBOL_GPL(__mmdrop);
static inline void __mmput(struct mm_struct *mm)
{
VM_BUG_ON(atomic_read(&mm->mm_users));
uprobe_clear_state(mm);
exit_aio(mm);
ksm_exit(mm);
khugepaged_exit(mm); /* must run before exit_mmap */
exit_mmap(mm);
set_mm_exe_file(mm, NULL);
if (!list_empty(&mm->mmlist)) {
spin_lock(&mmlist_lock);
list_del(&mm->mmlist);
spin_unlock(&mmlist_lock);
}
if (mm->binfmt)
module_put(mm->binfmt->module);
mmdrop(mm);
}
/*
* Decrement the use count and release all resources for an mm.
*/
@ -706,25 +731,27 @@ void mmput(struct mm_struct *mm)
{
might_sleep();
if (atomic_dec_and_test(&mm->mm_users)) {
uprobe_clear_state(mm);
exit_aio(mm);
ksm_exit(mm);
khugepaged_exit(mm); /* must run before exit_mmap */
exit_mmap(mm);
set_mm_exe_file(mm, NULL);
if (!list_empty(&mm->mmlist)) {
spin_lock(&mmlist_lock);
list_del(&mm->mmlist);
spin_unlock(&mmlist_lock);
}
if (mm->binfmt)
module_put(mm->binfmt->module);
mmdrop(mm);
}
if (atomic_dec_and_test(&mm->mm_users))
__mmput(mm);
}
EXPORT_SYMBOL_GPL(mmput);
#ifdef CONFIG_MMU
static void mmput_async_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
__mmput(mm);
}
void mmput_async(struct mm_struct *mm)
{
if (atomic_dec_and_test(&mm->mm_users)) {
INIT_WORK(&mm->async_put_work, mmput_async_fn);
schedule_work(&mm->async_put_work);
}
}
#endif
/**
* set_mm_exe_file - change a reference to the mm's executable file
*
@ -1256,7 +1283,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
int __user *child_tidptr,
struct pid *pid,
int trace,
unsigned long tls)
unsigned long tls,
int node)
{
int retval;
struct task_struct *p;
@ -1308,7 +1336,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
goto fork_out;
retval = -ENOMEM;
p = dup_task_struct(current);
p = dup_task_struct(current, node);
if (!p)
goto fork_out;
@ -1470,7 +1498,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
pid = alloc_pid(p->nsproxy->pid_ns_for_children);
if (IS_ERR(pid)) {
retval = PTR_ERR(pid);
goto bad_fork_cleanup_io;
goto bad_fork_cleanup_thread;
}
}
@ -1632,6 +1660,8 @@ bad_fork_cancel_cgroup:
bad_fork_free_pid:
if (pid != &init_struct_pid)
free_pid(pid);
bad_fork_cleanup_thread:
exit_thread(p);
bad_fork_cleanup_io:
if (p->io_context)
exit_io_context(p);
@ -1684,7 +1714,8 @@ static inline void init_idle_pids(struct pid_link *links)
struct task_struct *fork_idle(int cpu)
{
struct task_struct *task;
task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0);
task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0,
cpu_to_node(cpu));
if (!IS_ERR(task)) {
init_idle_pids(task->pids);
init_idle(task, cpu);
@ -1729,7 +1760,7 @@ long _do_fork(unsigned long clone_flags,
}
p = copy_process(clone_flags, stack_start, stack_size,
child_tidptr, NULL, trace, tls);
child_tidptr, NULL, trace, tls, NUMA_NO_NODE);
/*
* Do this prior waking up the new thread - the thread pointer
* might get invalid after that point, if the thread exits quickly.

2
kernel/futex.c
View file

@ -729,7 +729,7 @@ static int get_futex_value_locked(u32 *dest, u32 __user *from)
int ret;
pagefault_disable();
ret = __copy_from_user_inatomic(dest, from, sizeof(u32));
ret = __get_user(*dest, from);
pagefault_enable();
return ret ? -EFAULT : 0;

1
kernel/gcov/Kconfig
View file

@ -37,6 +37,7 @@ config ARCH_HAS_GCOV_PROFILE_ALL
config GCOV_PROFILE_ALL
bool "Profile entire Kernel"
depends on !COMPILE_TEST
depends on GCOV_KERNEL
depends on ARCH_HAS_GCOV_PROFILE_ALL
default n

8
kernel/irq/irqdomain.c
View file

@ -139,12 +139,7 @@ void irq_domain_remove(struct irq_domain *domain)
{
mutex_lock(&irq_domain_mutex);
/*
* radix_tree_delete() takes care of destroying the root
* node when all entries are removed. Shout if there are
* any mappings left.
*/
WARN_ON(domain->revmap_tree.height);
WARN_ON(!radix_tree_empty(&domain->revmap_tree));
list_del(&domain->link);
@ -1100,6 +1095,7 @@ void irq_domain_free_irqs_common(struct irq_domain *domain, unsigned int virq,
}
irq_domain_free_irqs_parent(domain, virq, nr_irqs);
}
EXPORT_SYMBOL_GPL(irq_domain_free_irqs_common);
/**
* irq_domain_free_irqs_top - Clear handler and handler data, clear irqdata and free parent

109
kernel/kexec.c
View file

@ -103,6 +103,65 @@ out_free_image:
return ret;
}
static int do_kexec_load(unsigned long entry, unsigned long nr_segments,
struct kexec_segment __user *segments, unsigned long flags)
{
struct kimage **dest_image, *image;
unsigned long i;
int ret;
if (flags & KEXEC_ON_CRASH) {
dest_image = &kexec_crash_image;
if (kexec_crash_image)
arch_kexec_unprotect_crashkres();
} else {
dest_image = &kexec_image;
}
if (nr_segments == 0) {
/* Uninstall image */
kimage_free(xchg(dest_image, NULL));
return 0;
}
if (flags & KEXEC_ON_CRASH) {
/*
* Loading another kernel to switch to if this one
* crashes. Free any current crash dump kernel before
* we corrupt it.
*/
kimage_free(xchg(&kexec_crash_image, NULL));
}
ret = kimage_alloc_init(&image, entry, nr_segments, segments, flags);
if (ret)
return ret;
if (flags & KEXEC_PRESERVE_CONTEXT)
image->preserve_context = 1;
ret = machine_kexec_prepare(image);
if (ret)
goto out;
for (i = 0; i < nr_segments; i++) {
ret = kimage_load_segment(image, &image->segment[i]);
if (ret)
goto out;
}
kimage_terminate(image);
/* Install the new kernel and uninstall the old */
image = xchg(dest_image, image);
out:
if ((flags & KEXEC_ON_CRASH) && kexec_crash_image)
arch_kexec_protect_crashkres();
kimage_free(image);
return ret;
}
/*
* Exec Kernel system call: for obvious reasons only root may call it.
*
@ -127,7 +186,6 @@ out_free_image:
SYSCALL_DEFINE4(kexec_load, unsigned long, entry, unsigned long, nr_segments,
struct kexec_segment __user *, segments, unsigned long, flags)
{
struct kimage **dest_image, *image;
int result;
/* We only trust the superuser with rebooting the system. */
@ -152,9 +210,6 @@ SYSCALL_DEFINE4(kexec_load, unsigned long, entry, unsigned long, nr_segments,
if (nr_segments > KEXEC_SEGMENT_MAX)
return -EINVAL;
image = NULL;
result = 0;
/* Because we write directly to the reserved memory
* region when loading crash kernels we need a mutex here to
* prevent multiple crash kernels from attempting to load
@ -166,53 +221,9 @@ SYSCALL_DEFINE4(kexec_load, unsigned long, entry, unsigned long, nr_segments,
if (!mutex_trylock(&kexec_mutex))
return -EBUSY;
dest_image = &kexec_image;
if (flags & KEXEC_ON_CRASH)
dest_image = &kexec_crash_image;
if (nr_segments > 0) {
unsigned long i;
result = do_kexec_load(entry, nr_segments, segments, flags);
if (flags & KEXEC_ON_CRASH) {
/*
* Loading another kernel to switch to if this one
* crashes. Free any current crash dump kernel before
* we corrupt it.
*/
kimage_free(xchg(&kexec_crash_image, NULL));
result = kimage_alloc_init(&image, entry, nr_segments,
segments, flags);
crash_map_reserved_pages();
} else {
/* Loading another kernel to reboot into. */
result = kimage_alloc_init(&image, entry, nr_segments,
segments, flags);
}
if (result)
goto out;
if (flags & KEXEC_PRESERVE_CONTEXT)
image->preserve_context = 1;
result = machine_kexec_prepare(image);
if (result)
goto out;
for (i = 0; i < nr_segments; i++) {
result = kimage_load_segment(image, &image->segment[i]);
if (result)
goto out;
}
kimage_terminate(image);
if (flags & KEXEC_ON_CRASH)
crash_unmap_reserved_pages();
}
/* Install the new kernel, and Uninstall the old */
image = xchg(dest_image, image);
out:
mutex_unlock(&kexec_mutex);
kimage_free(image);
return result;
}

12
kernel/kexec_core.c
View file

@ -893,6 +893,7 @@ void crash_kexec(struct pt_regs *regs)
old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
if (old_cpu == PANIC_CPU_INVALID) {
/* This is the 1st CPU which comes here, so go ahead. */
printk_nmi_flush_on_panic();
__crash_kexec(regs);
/*
@ -953,7 +954,6 @@ int crash_shrink_memory(unsigned long new_size)
start = roundup(start, KEXEC_CRASH_MEM_ALIGN);
end = roundup(start + new_size, KEXEC_CRASH_MEM_ALIGN);
crash_map_reserved_pages();
crash_free_reserved_phys_range(end, crashk_res.end);
if ((start == end) && (crashk_res.parent != NULL))
@ -967,7 +967,6 @@ int crash_shrink_memory(unsigned long new_size)
crashk_res.end = end - 1;
insert_resource(&iomem_resource, ram_res);
crash_unmap_reserved_pages();
unlock:
mutex_unlock(&kexec_mutex);
@ -1410,7 +1409,7 @@ static int __init crash_save_vmcoreinfo_init(void)
VMCOREINFO_STRUCT_SIZE(list_head);
VMCOREINFO_SIZE(nodemask_t);
VMCOREINFO_OFFSET(page, flags);
VMCOREINFO_OFFSET(page, _count);
VMCOREINFO_OFFSET(page, _refcount);
VMCOREINFO_OFFSET(page, mapping);
VMCOREINFO_OFFSET(page, lru);
VMCOREINFO_OFFSET(page, _mapcount);
@ -1552,13 +1551,14 @@ int kernel_kexec(void)
}
/*
* Add and remove page tables for crashkernel memory
* Protection mechanism for crashkernel reserved memory after
* the kdump kernel is loaded.
*
* Provide an empty default implementation here -- architecture
* code may override this
*/
void __weak crash_map_reserved_pages(void)
void __weak arch_kexec_protect_crashkres(void)
{}
void __weak crash_unmap_reserved_pages(void)
void __weak arch_kexec_unprotect_crashkres(void)
{}

8
kernel/kexec_file.c
View file

@ -274,8 +274,11 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
return -EBUSY;
dest_image = &kexec_image;
if (flags & KEXEC_FILE_ON_CRASH)
if (flags & KEXEC_FILE_ON_CRASH) {
dest_image = &kexec_crash_image;
if (kexec_crash_image)
arch_kexec_unprotect_crashkres();
}
if (flags & KEXEC_FILE_UNLOAD)
goto exchange;
@ -324,6 +327,9 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
exchange:
image = xchg(dest_image, image);
out:
if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
arch_kexec_protect_crashkres();
mutex_unlock(&kexec_mutex);
kimage_free(image);
return ret;

193
kernel/livepatch/core.c
View file

@ -28,6 +28,8 @@
#include <linux/list.h>
#include <linux/kallsyms.h>
#include <linux/livepatch.h>
#include <linux/elf.h>
#include <linux/moduleloader.h>
#include <asm/cacheflush.h>
/**
@ -204,75 +206,109 @@ static int klp_find_object_symbol(const char *objname, const char *name,
return -EINVAL;
}
/*
* external symbols are located outside the parent object (where the parent
* object is either vmlinux or the kmod being patched).
*/
static int klp_find_external_symbol(struct module *pmod, const char *name,
unsigned long *addr)
static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
{
const struct kernel_symbol *sym;
/* first, check if it's an exported symbol */
preempt_disable();
sym = find_symbol(name, NULL, NULL, true, true);
if (sym) {
*addr = sym->value;
preempt_enable();
return 0;
}
preempt_enable();
int i, cnt, vmlinux, ret;
char objname[MODULE_NAME_LEN];
char symname[KSYM_NAME_LEN];
char *strtab = pmod->core_kallsyms.strtab;
Elf_Rela *relas;
Elf_Sym *sym;
unsigned long sympos, addr;
/*
* Check if it's in another .o within the patch module. This also
* checks that the external symbol is unique.
* Since the field widths for objname and symname in the sscanf()
* call are hard-coded and correspond to MODULE_NAME_LEN and
* KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
* and KSYM_NAME_LEN have the values we expect them to have.
*
* Because the value of MODULE_NAME_LEN can differ among architectures,
* we use the smallest/strictest upper bound possible (56, based on
* the current definition of MODULE_NAME_LEN) to prevent overflows.
*/
return klp_find_object_symbol(pmod->name, name, 0, addr);
BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
relas = (Elf_Rela *) relasec->sh_addr;
/* For each rela in this klp relocation section */
for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
if (sym->st_shndx != SHN_LIVEPATCH) {
pr_err("symbol %s is not marked as a livepatch symbol",
strtab + sym->st_name);
return -EINVAL;
}
/* Format: .klp.sym.objname.symname,sympos */
cnt = sscanf(strtab + sym->st_name,
".klp.sym.%55[^.].%127[^,],%lu",
objname, symname, &sympos);
if (cnt != 3) {
pr_err("symbol %s has an incorrectly formatted name",
strtab + sym->st_name);
return -EINVAL;
}
/* klp_find_object_symbol() treats a NULL objname as vmlinux */
vmlinux = !strcmp(objname, "vmlinux");
ret = klp_find_object_symbol(vmlinux ? NULL : objname,
symname, sympos, &addr);
if (ret)
return ret;
sym->st_value = addr;
}
return 0;
}
static int klp_write_object_relocations(struct module *pmod,
struct klp_object *obj)
{
int ret = 0;
unsigned long val;
struct klp_reloc *reloc;
int i, cnt, ret = 0;
const char *objname, *secname;
char sec_objname[MODULE_NAME_LEN];
Elf_Shdr *sec;
if (WARN_ON(!klp_is_object_loaded(obj)))
return -EINVAL;
if (WARN_ON(!obj->relocs))
return -EINVAL;
objname = klp_is_module(obj) ? obj->name : "vmlinux";
module_disable_ro(pmod);
/* For each klp relocation section */
for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
sec = pmod->klp_info->sechdrs + i;
secname = pmod->klp_info->secstrings + sec->sh_name;
if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
continue;
for (reloc = obj->relocs; reloc->name; reloc++) {
/* discover the address of the referenced symbol */
if (reloc->external) {
if (reloc->sympos > 0) {
pr_err("non-zero sympos for external reloc symbol '%s' is not supported\n",
reloc->name);
ret = -EINVAL;
goto out;
}
ret = klp_find_external_symbol(pmod, reloc->name, &val);
} else
ret = klp_find_object_symbol(obj->name,
reloc->name,
reloc->sympos,
&val);
if (ret)
goto out;
ret = klp_write_module_reloc(pmod, reloc->type, reloc->loc,
val + reloc->addend);
if (ret) {
pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
reloc->name, val, ret);
goto out;
/*
* Format: .klp.rela.sec_objname.section_name
* See comment in klp_resolve_symbols() for an explanation
* of the selected field width value.
*/
cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
if (cnt != 1) {
pr_err("section %s has an incorrectly formatted name",
secname);
ret = -EINVAL;
break;
}
if (strcmp(objname, sec_objname))
continue;
ret = klp_resolve_symbols(sec, pmod);
if (ret)
break;
ret = apply_relocate_add(pmod->klp_info->sechdrs,
pmod->core_kallsyms.strtab,
pmod->klp_info->symndx, i, pmod);
if (ret)
break;
}
out:
module_enable_ro(pmod);
return ret;
}
@ -298,6 +334,19 @@ unlock:
rcu_read_unlock();
}
/*
* Convert a function address into the appropriate ftrace location.
*
* Usually this is just the address of the function, but on some architectures
* it's more complicated so allow them to provide a custom behaviour.
*/
#ifndef klp_get_ftrace_location
static unsigned long klp_get_ftrace_location(unsigned long faddr)
{
return faddr;
}
#endif
static void klp_disable_func(struct klp_func *func)
{
struct klp_ops *ops;
@ -312,8 +361,14 @@ static void klp_disable_func(struct klp_func *func)
return;
if (list_is_singular(&ops->func_stack)) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (WARN_ON(!ftrace_loc))
return;
WARN_ON(unregister_ftrace_function(&ops->fops));
WARN_ON(ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0));
WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
list_del_rcu(&func->stack_node);
list_del(&ops->node);
@ -338,6 +393,15 @@ static int klp_enable_func(struct klp_func *func)
ops = klp_find_ops(func->old_addr);
if (!ops) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (!ftrace_loc) {
pr_err("failed to find location for function '%s'\n",
func->old_name);
return -EINVAL;
}
ops = kzalloc(sizeof(*ops), GFP_KERNEL);
if (!ops)
return -ENOMEM;
@ -352,7 +416,7 @@ static int klp_enable_func(struct klp_func *func)
INIT_LIST_HEAD(&ops->func_stack);
list_add_rcu(&func->stack_node, &ops->func_stack);
ret = ftrace_set_filter_ip(&ops->fops, func->old_addr, 0, 0);
ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
if (ret) {
pr_err("failed to set ftrace filter for function '%s' (%d)\n",
func->old_name, ret);
@ -363,7 +427,7 @@ static int klp_enable_func(struct klp_func *func)
if (ret) {
pr_err("failed to register ftrace handler for function '%s' (%d)\n",
func->old_name, ret);
ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0);
ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
goto err;
}
@ -683,6 +747,9 @@ static void klp_free_patch(struct klp_patch *patch)
static int klp_init_func(struct klp_object *obj, struct klp_func *func)
{
if (!func->old_name || !func->new_func)
return -EINVAL;
INIT_LIST_HEAD(&func->stack_node);
func->state = KLP_DISABLED;
@ -703,11 +770,9 @@ static int klp_init_object_loaded(struct klp_patch *patch,
struct klp_func *func;
int ret;
if (obj->relocs) {
ret = klp_write_object_relocations(patch->mod, obj);
if (ret)
return ret;
}
ret = klp_write_object_relocations(patch->mod, obj);
if (ret)
return ret;
klp_for_each_func(obj, func) {
ret = klp_find_object_symbol(obj->name, func->old_name,
@ -842,12 +907,18 @@ int klp_register_patch(struct klp_patch *patch)
{
int ret;
if (!klp_initialized())
return -ENODEV;
if (!patch || !patch->mod)
return -EINVAL;
if (!is_livepatch_module(patch->mod)) {
pr_err("module %s is not marked as a livepatch module",
patch->mod->name);
return -EINVAL;
}
if (!klp_initialized())
return -ENODEV;
/*
* A reference is taken on the patch module to prevent it from being
* unloaded. Right now, we don't allow patch modules to unload since

1
kernel/locking/percpu-rwsem.c
View file

@ -37,6 +37,7 @@ void percpu_free_rwsem(struct percpu_rw_semaphore *brw)
free_percpu(brw->fast_read_ctr);
brw->fast_read_ctr = NULL; /* catch use after free bugs */
}
EXPORT_SYMBOL_GPL(percpu_free_rwsem);
/*
* This is the fast-path for down_read/up_read. If it succeeds we rely

16
kernel/locking/rwsem.c
View file

@ -173,6 +173,22 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_write_nested);
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
rwsem_release(&sem->dep_map, 1, _RET_IP_);
return -EINTR;
}
rwsem_set_owner(sem);
return 0;
}
EXPORT_SYMBOL(down_write_killable_nested);
void up_read_non_owner(struct rw_semaphore *sem)
{
__up_read(sem);

11
kernel/memremap.c
View file

@ -27,6 +27,13 @@ __weak void __iomem *ioremap_cache(resource_size_t offset, unsigned long size)
}
#endif
#ifndef arch_memremap_wb
static void *arch_memremap_wb(resource_size_t offset, unsigned long size)
{
return (__force void *)ioremap_cache(offset, size);
}
#endif
static void *try_ram_remap(resource_size_t offset, size_t size)
{
unsigned long pfn = PHYS_PFN(offset);
@ -34,7 +41,7 @@ static void *try_ram_remap(resource_size_t offset, size_t size)
/* In the simple case just return the existing linear address */
if (pfn_valid(pfn) && !PageHighMem(pfn_to_page(pfn)))
return __va(offset);
return NULL; /* fallback to ioremap_cache */
return NULL; /* fallback to arch_memremap_wb */
}
/**
@ -90,7 +97,7 @@ void *memremap(resource_size_t offset, size_t size, unsigned long flags)
if (is_ram == REGION_INTERSECTS)
addr = try_ram_remap(offset, size);
if (!addr)
addr = ioremap_cache(offset, size);
addr = arch_memremap_wb(offset, size);
}
/*

125
kernel/module.c
View file

@ -1973,6 +1973,83 @@ static void module_enable_nx(const struct module *mod) { }
static void module_disable_nx(const struct module *mod) { }
#endif
#ifdef CONFIG_LIVEPATCH
/*
* Persist Elf information about a module. Copy the Elf header,
* section header table, section string table, and symtab section
* index from info to mod->klp_info.
*/
static int copy_module_elf(struct module *mod, struct load_info *info)
{
unsigned int size, symndx;
int ret;
size = sizeof(*mod->klp_info);
mod->klp_info = kmalloc(size, GFP_KERNEL);
if (mod->klp_info == NULL)
return -ENOMEM;
/* Elf header */
size = sizeof(mod->klp_info->hdr);
memcpy(&mod->klp_info->hdr, info->hdr, size);
/* Elf section header table */
size = sizeof(*info->sechdrs) * info->hdr->e_shnum;
mod->klp_info->sechdrs = kmalloc(size, GFP_KERNEL);
if (mod->klp_info->sechdrs == NULL) {
ret = -ENOMEM;
goto free_info;
}
memcpy(mod->klp_info->sechdrs, info->sechdrs, size);
/* Elf section name string table */
size = info->sechdrs[info->hdr->e_shstrndx].sh_size;
mod->klp_info->secstrings = kmalloc(size, GFP_KERNEL);
if (mod->klp_info->secstrings == NULL) {
ret = -ENOMEM;
goto free_sechdrs;
}
memcpy(mod->klp_info->secstrings, info->secstrings, size);
/* Elf symbol section index */
symndx = info->index.sym;
mod->klp_info->symndx = symndx;
/*
* For livepatch modules, core_kallsyms.symtab is a complete
* copy of the original symbol table. Adjust sh_addr to point
* to core_kallsyms.symtab since the copy of the symtab in module
* init memory is freed at the end of do_init_module().
*/
mod->klp_info->sechdrs[symndx].sh_addr = \
(unsigned long) mod->core_kallsyms.symtab;
return 0;
free_sechdrs:
kfree(mod->klp_info->sechdrs);
free_info:
kfree(mod->klp_info);
return ret;
}
static void free_module_elf(struct module *mod)
{
kfree(mod->klp_info->sechdrs);
kfree(mod->klp_info->secstrings);
kfree(mod->klp_info);
}
#else /* !CONFIG_LIVEPATCH */
static int copy_module_elf(struct module *mod, struct load_info *info)
{
return 0;
}
static void free_module_elf(struct module *mod)
{
}
#endif /* CONFIG_LIVEPATCH */
void __weak module_memfree(void *module_region)
{
vfree(module_region);
@ -2011,6 +2088,9 @@ static void free_module(struct module *mod)
/* Free any allocated parameters. */
destroy_params(mod->kp, mod->num_kp);
if (is_livepatch_module(mod))
free_module_elf(mod);
/* Now we can delete it from the lists */
mutex_lock(&module_mutex);
/* Unlink carefully: kallsyms could be walking list. */
@ -2126,6 +2206,10 @@ static int simplify_symbols(struct module *mod, const struct load_info *info)
(long)sym[i].st_value);
break;
case SHN_LIVEPATCH:
/* Livepatch symbols are resolved by livepatch */
break;
case SHN_UNDEF:
ksym = resolve_symbol_wait(mod, info, name);
/* Ok if resolved. */
@ -2174,6 +2258,10 @@ static int apply_relocations(struct module *mod, const struct load_info *info)
if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
continue;
/* Livepatch relocation sections are applied by livepatch */
if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
continue;
if (info->sechdrs[i].sh_type == SHT_REL)
err = apply_relocate(info->sechdrs, info->strtab,
info->index.sym, i, mod);
@ -2469,7 +2557,7 @@ static void layout_symtab(struct module *mod, struct load_info *info)
/* Compute total space required for the core symbols' strtab. */
for (ndst = i = 0; i < nsrc; i++) {
if (i == 0 ||
if (i == 0 || is_livepatch_module(mod) ||
is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
info->index.pcpu)) {
strtab_size += strlen(&info->strtab[src[i].st_name])+1;
@ -2528,7 +2616,7 @@ static void add_kallsyms(struct module *mod, const struct load_info *info)
mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
src = mod->kallsyms->symtab;
for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
if (i == 0 ||
if (i == 0 || is_livepatch_module(mod) ||
is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
info->index.pcpu)) {
dst[ndst] = src[i];
@ -2667,6 +2755,26 @@ static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned l
return 0;
}
#ifdef CONFIG_LIVEPATCH
static int find_livepatch_modinfo(struct module *mod, struct load_info *info)
{
mod->klp = get_modinfo(info, "livepatch") ? true : false;
return 0;
}
#else /* !CONFIG_LIVEPATCH */
static int find_livepatch_modinfo(struct module *mod, struct load_info *info)
{
if (get_modinfo(info, "livepatch")) {
pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
mod->name);
return -ENOEXEC;
}
return 0;
}
#endif /* CONFIG_LIVEPATCH */
/* Sets info->hdr and info->len. */
static int copy_module_from_user(const void __user *umod, unsigned long len,
struct load_info *info)
@ -2821,6 +2929,10 @@ static int check_modinfo(struct module *mod, struct load_info *info, int flags)
"is unknown, you have been warned.\n", mod->name);
}
err = find_livepatch_modinfo(mod, info);
if (err)
return err;
/* Set up license info based on the info section */
set_license(mod, get_modinfo(info, "license"));
@ -3494,6 +3606,12 @@ static int load_module(struct load_info *info, const char __user *uargs,
if (err < 0)
goto coming_cleanup;
if (is_livepatch_module(mod)) {
err = copy_module_elf(mod, info);
if (err < 0)
goto sysfs_cleanup;
}
/* Get rid of temporary copy. */
free_copy(info);
@ -3502,11 +3620,12 @@ static int load_module(struct load_info *info, const char __user *uargs,
return do_init_module(mod);
sysfs_cleanup:
mod_sysfs_teardown(mod);
coming_cleanup:
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_GOING, mod);
klp_module_going(mod);
bug_cleanup:
/* module_bug_cleanup needs module_mutex protection */
mutex_lock(&module_mutex);

7
kernel/module_signing.c
View file

@ -12,7 +12,7 @@
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <keys/system_keyring.h>
#include <linux/verification.h>
#include <crypto/public_key.h>
#include "module-internal.h"
@ -80,6 +80,7 @@ int mod_verify_sig(const void *mod, unsigned long *_modlen)
return -EBADMSG;
}
return system_verify_data(mod, modlen, mod + modlen, sig_len,
VERIFYING_MODULE_SIGNATURE);
return verify_pkcs7_signature(mod, modlen, mod + modlen, sig_len,
NULL, VERIFYING_MODULE_SIGNATURE,
NULL, NULL);
}

138
kernel/padata.c
View file

@ -606,33 +606,6 @@ out_replace:
return 0;
}
/**
* padata_set_cpumasks - Set both parallel and serial cpumasks. The first
* one is used by parallel workers and the second one
* by the wokers doing serialization.
*
* @pinst: padata instance
* @pcpumask: the cpumask to use for parallel workers
* @cbcpumask: the cpumsak to use for serial workers
*/
int padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask,
cpumask_var_t cbcpumask)
{
int err;
mutex_lock(&pinst->lock);
get_online_cpus();
err = __padata_set_cpumasks(pinst, pcpumask, cbcpumask);
put_online_cpus();
mutex_unlock(&pinst->lock);
return err;
}
EXPORT_SYMBOL(padata_set_cpumasks);
/**
* padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value
* equivalent to @cpumask.
@ -674,6 +647,43 @@ out:
}
EXPORT_SYMBOL(padata_set_cpumask);
/**
* padata_start - start the parallel processing
*
* @pinst: padata instance to start
*/
int padata_start(struct padata_instance *pinst)
{
int err = 0;
mutex_lock(&pinst->lock);
if (pinst->flags & PADATA_INVALID)
err = -EINVAL;
__padata_start(pinst);
mutex_unlock(&pinst->lock);
return err;
}
EXPORT_SYMBOL(padata_start);
/**
* padata_stop - stop the parallel processing
*
* @pinst: padata instance to stop
*/
void padata_stop(struct padata_instance *pinst)
{
mutex_lock(&pinst->lock);
__padata_stop(pinst);
mutex_unlock(&pinst->lock);
}
EXPORT_SYMBOL(padata_stop);
#ifdef CONFIG_HOTPLUG_CPU
static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
{
struct parallel_data *pd;
@ -694,42 +704,6 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
return 0;
}
/**
* padata_add_cpu - add a cpu to one or both(parallel and serial)
* padata cpumasks.
*
* @pinst: padata instance
* @cpu: cpu to add
* @mask: bitmask of flags specifying to which cpumask @cpu shuld be added.
* The @mask may be any combination of the following flags:
* PADATA_CPU_SERIAL - serial cpumask
* PADATA_CPU_PARALLEL - parallel cpumask
*/
int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask)
{
int err;
if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL)))
return -EINVAL;
mutex_lock(&pinst->lock);
get_online_cpus();
if (mask & PADATA_CPU_SERIAL)
cpumask_set_cpu(cpu, pinst->cpumask.cbcpu);
if (mask & PADATA_CPU_PARALLEL)
cpumask_set_cpu(cpu, pinst->cpumask.pcpu);
err = __padata_add_cpu(pinst, cpu);
put_online_cpus();
mutex_unlock(&pinst->lock);
return err;
}
EXPORT_SYMBOL(padata_add_cpu);
static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
{
struct parallel_data *pd = NULL;
@ -789,43 +763,6 @@ int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask)
}
EXPORT_SYMBOL(padata_remove_cpu);
/**
* padata_start - start the parallel processing
*
* @pinst: padata instance to start
*/
int padata_start(struct padata_instance *pinst)
{
int err = 0;
mutex_lock(&pinst->lock);
if (pinst->flags & PADATA_INVALID)
err =-EINVAL;
__padata_start(pinst);
mutex_unlock(&pinst->lock);
return err;
}
EXPORT_SYMBOL(padata_start);
/**
* padata_stop - stop the parallel processing
*
* @pinst: padata instance to stop
*/
void padata_stop(struct padata_instance *pinst)
{
mutex_lock(&pinst->lock);
__padata_stop(pinst);
mutex_unlock(&pinst->lock);
}
EXPORT_SYMBOL(padata_stop);
#ifdef CONFIG_HOTPLUG_CPU
static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
{
return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
@ -1091,7 +1028,6 @@ err_free_inst:
err:
return NULL;
}
EXPORT_SYMBOL(padata_alloc);
/**
* padata_free - free a padata instance

6
kernel/panic.c
View file

@ -160,8 +160,10 @@ void panic(const char *fmt, ...)
*
* Bypass the panic_cpu check and call __crash_kexec directly.
*/
if (!crash_kexec_post_notifiers)
if (!crash_kexec_post_notifiers) {
printk_nmi_flush_on_panic();
__crash_kexec(NULL);
}
/*
* Note smp_send_stop is the usual smp shutdown function, which
@ -176,6 +178,8 @@ void panic(const char *fmt, ...)
*/
atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
/* Call flush even twice. It tries harder with a single online CPU */
printk_nmi_flush_on_panic();
kmsg_dump(KMSG_DUMP_PANIC);
/*

2
kernel/pid.c
View file

@ -311,7 +311,7 @@ struct pid *alloc_pid(struct pid_namespace *ns)
pid->level = ns->level;
for (i = ns->level; i >= 0; i--) {
nr = alloc_pidmap(tmp);
if (IS_ERR_VALUE(nr)) {
if (nr < 0) {
retval = nr;
goto out_free;
}

1
kernel/printk/Makefile
View file

@ -1,2 +1,3 @@
obj-y = printk.o
obj-$(CONFIG_PRINTK_NMI) += nmi.o
obj-$(CONFIG_A11Y_BRAILLE_CONSOLE) += braille.o

57
kernel/printk/internal.h Normal file
View file

@ -0,0 +1,57 @@
/*
* internal.h - printk internal definitions
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/percpu.h>
typedef __printf(1, 0) int (*printk_func_t)(const char *fmt, va_list args);
int __printf(1, 0) vprintk_default(const char *fmt, va_list args);
#ifdef CONFIG_PRINTK_NMI
extern raw_spinlock_t logbuf_lock;
/*
* printk() could not take logbuf_lock in NMI context. Instead,
* it temporary stores the strings into a per-CPU buffer.
* The alternative implementation is chosen transparently
* via per-CPU variable.
*/
DECLARE_PER_CPU(printk_func_t, printk_func);
static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
{
return this_cpu_read(printk_func)(fmt, args);
}
extern atomic_t nmi_message_lost;
static inline int get_nmi_message_lost(void)
{
return atomic_xchg(&nmi_message_lost, 0);
}
#else /* CONFIG_PRINTK_NMI */
static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
{
return vprintk_default(fmt, args);
}
static inline int get_nmi_message_lost(void)
{
return 0;
}
#endif /* CONFIG_PRINTK_NMI */

260
kernel/printk/nmi.c Normal file
View file

@ -0,0 +1,260 @@
/*
* nmi.c - Safe printk in NMI context
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/debug_locks.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
#include <linux/irq_work.h>
#include <linux/printk.h>
#include "internal.h"
/*
* printk() could not take logbuf_lock in NMI context. Instead,
* it uses an alternative implementation that temporary stores
* the strings into a per-CPU buffer. The content of the buffer
* is later flushed into the main ring buffer via IRQ work.
*
* The alternative implementation is chosen transparently
* via @printk_func per-CPU variable.
*
* The implementation allows to flush the strings also from another CPU.
* There are situations when we want to make sure that all buffers
* were handled or when IRQs are blocked.
*/
DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default;
static int printk_nmi_irq_ready;
atomic_t nmi_message_lost;
#define NMI_LOG_BUF_LEN ((1 << CONFIG_NMI_LOG_BUF_SHIFT) - \
sizeof(atomic_t) - sizeof(struct irq_work))
struct nmi_seq_buf {
atomic_t len; /* length of written data */
struct irq_work work; /* IRQ work that flushes the buffer */
unsigned char buffer[NMI_LOG_BUF_LEN];
};
static DEFINE_PER_CPU(struct nmi_seq_buf, nmi_print_seq);
/*
* Safe printk() for NMI context. It uses a per-CPU buffer to
* store the message. NMIs are not nested, so there is always only
* one writer running. But the buffer might get flushed from another
* CPU, so we need to be careful.
*/
static int vprintk_nmi(const char *fmt, va_list args)
{
struct nmi_seq_buf *s = this_cpu_ptr(&nmi_print_seq);
int add = 0;
size_t len;
again:
len = atomic_read(&s->len);
if (len >= sizeof(s->buffer)) {
atomic_inc(&nmi_message_lost);
return 0;
}
/*
* Make sure that all old data have been read before the buffer was
* reseted. This is not needed when we just append data.
*/
if (!len)
smp_rmb();
add = vsnprintf(s->buffer + len, sizeof(s->buffer) - len, fmt, args);
/*
* Do it once again if the buffer has been flushed in the meantime.
* Note that atomic_cmpxchg() is an implicit memory barrier that
* makes sure that the data were written before updating s->len.
*/
if (atomic_cmpxchg(&s->len, len, len + add) != len)
goto again;
/* Get flushed in a more safe context. */
if (add && printk_nmi_irq_ready) {
/* Make sure that IRQ work is really initialized. */
smp_rmb();
irq_work_queue(&s->work);
}
return add;
}
/*
* printk one line from the temporary buffer from @start index until
* and including the @end index.
*/
static void print_nmi_seq_line(struct nmi_seq_buf *s, int start, int end)
{
const char *buf = s->buffer + start;
/*
* The buffers are flushed in NMI only on panic. The messages must
* go only into the ring buffer at this stage. Consoles will get
* explicitly called later when a crashdump is not generated.
*/
if (in_nmi())
printk_deferred("%.*s", (end - start) + 1, buf);
else
printk("%.*s", (end - start) + 1, buf);
}
/*
* Flush data from the associated per_CPU buffer. The function
* can be called either via IRQ work or independently.
*/
static void __printk_nmi_flush(struct irq_work *work)
{
static raw_spinlock_t read_lock =
__RAW_SPIN_LOCK_INITIALIZER(read_lock);
struct nmi_seq_buf *s = container_of(work, struct nmi_seq_buf, work);
unsigned long flags;
size_t len, size;
int i, last_i;
/*
* The lock has two functions. First, one reader has to flush all
* available message to make the lockless synchronization with
* writers easier. Second, we do not want to mix messages from
* different CPUs. This is especially important when printing
* a backtrace.
*/
raw_spin_lock_irqsave(&read_lock, flags);
i = 0;
more:
len = atomic_read(&s->len);
/*
* This is just a paranoid check that nobody has manipulated
* the buffer an unexpected way. If we printed something then
* @len must only increase.
*/
if (i && i >= len)
pr_err("printk_nmi_flush: internal error: i=%d >= len=%zu\n",
i, len);
if (!len)
goto out; /* Someone else has already flushed the buffer. */
/* Make sure that data has been written up to the @len */
smp_rmb();
size = min(len, sizeof(s->buffer));
last_i = i;
/* Print line by line. */
for (; i < size; i++) {
if (s->buffer[i] == '\n') {
print_nmi_seq_line(s, last_i, i);
last_i = i + 1;
}
}
/* Check if there was a partial line. */
if (last_i < size) {
print_nmi_seq_line(s, last_i, size - 1);
pr_cont("\n");
}
/*
* Check that nothing has got added in the meantime and truncate
* the buffer. Note that atomic_cmpxchg() is an implicit memory
* barrier that makes sure that the data were copied before
* updating s->len.
*/
if (atomic_cmpxchg(&s->len, len, 0) != len)
goto more;
out:
raw_spin_unlock_irqrestore(&read_lock, flags);
}
/**
* printk_nmi_flush - flush all per-cpu nmi buffers.
*
* The buffers are flushed automatically via IRQ work. This function
* is useful only when someone wants to be sure that all buffers have
* been flushed at some point.
*/
void printk_nmi_flush(void)
{
int cpu;
for_each_possible_cpu(cpu)
__printk_nmi_flush(&per_cpu(nmi_print_seq, cpu).work);
}
/**
* printk_nmi_flush_on_panic - flush all per-cpu nmi buffers when the system
* goes down.
*
* Similar to printk_nmi_flush() but it can be called even in NMI context when
* the system goes down. It does the best effort to get NMI messages into
* the main ring buffer.
*
* Note that it could try harder when there is only one CPU online.
*/
void printk_nmi_flush_on_panic(void)
{
/*
* Make sure that we could access the main ring buffer.
* Do not risk a double release when more CPUs are up.
*/
if (in_nmi() && raw_spin_is_locked(&logbuf_lock)) {
if (num_online_cpus() > 1)
return;
debug_locks_off();
raw_spin_lock_init(&logbuf_lock);
}
printk_nmi_flush();
}
void __init printk_nmi_init(void)
{
int cpu;
for_each_possible_cpu(cpu) {
struct nmi_seq_buf *s = &per_cpu(nmi_print_seq, cpu);
init_irq_work(&s->work, __printk_nmi_flush);
}
/* Make sure that IRQ works are initialized before enabling. */
smp_wmb();
printk_nmi_irq_ready = 1;
/* Flush pending messages that did not have scheduled IRQ works. */
printk_nmi_flush();
}
void printk_nmi_enter(void)
{
this_cpu_write(printk_func, vprintk_nmi);
}
void printk_nmi_exit(void)
{
this_cpu_write(printk_func, vprintk_default);
}

31
kernel/printk/printk.c
View file

@ -55,6 +55,7 @@
#include "console_cmdline.h"
#include "braille.h"
#include "internal.h"
int console_printk[4] = {
CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */
@ -244,7 +245,7 @@ __packed __aligned(4)
* within the scheduler's rq lock. It must be released before calling
* console_unlock() or anything else that might wake up a process.
*/
static DEFINE_RAW_SPINLOCK(logbuf_lock);
DEFINE_RAW_SPINLOCK(logbuf_lock);
#ifdef CONFIG_PRINTK
DECLARE_WAIT_QUEUE_HEAD(log_wait);
@ -1616,6 +1617,7 @@ asmlinkage int vprintk_emit(int facility, int level,
unsigned long flags;
int this_cpu;
int printed_len = 0;
int nmi_message_lost;
bool in_sched = false;
/* cpu currently holding logbuf_lock in this function */
static unsigned int logbuf_cpu = UINT_MAX;
@ -1666,6 +1668,15 @@ asmlinkage int vprintk_emit(int facility, int level,
strlen(recursion_msg));
}
nmi_message_lost = get_nmi_message_lost();
if (unlikely(nmi_message_lost)) {
text_len = scnprintf(textbuf, sizeof(textbuf),
"BAD LUCK: lost %d message(s) from NMI context!",
nmi_message_lost);
printed_len += log_store(0, 2, LOG_PREFIX|LOG_NEWLINE, 0,
NULL, 0, textbuf, text_len);
}
/*
* The printf needs to come first; we need the syslog
* prefix which might be passed-in as a parameter.
@ -1807,14 +1818,6 @@ int vprintk_default(const char *fmt, va_list args)
}
EXPORT_SYMBOL_GPL(vprintk_default);
/*
* This allows printk to be diverted to another function per cpu.
* This is useful for calling printk functions from within NMI
* without worrying about race conditions that can lock up the
* box.
*/
DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default;
/**
* printk - print a kernel message
* @fmt: format string
@ -1838,21 +1841,11 @@ DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default;
*/
asmlinkage __visible int printk(const char *fmt, ...)
{
printk_func_t vprintk_func;
va_list args;
int r;
va_start(args, fmt);
/*
* If a caller overrides the per_cpu printk_func, then it needs
* to disable preemption when calling printk(). Otherwise
* the printk_func should be set to the default. No need to
* disable preemption here.
*/
vprintk_func = this_cpu_read(printk_func);
r = vprintk_func(fmt, args);
va_end(args);
return r;

26
kernel/rcu/update.c
View file

@ -380,29 +380,9 @@ void destroy_rcu_head(struct rcu_head *head)
debug_object_free(head, &rcuhead_debug_descr);
}
/*
* fixup_activate is called when:
* - an active object is activated
* - an unknown object is activated (might be a statically initialized object)
* Activation is performed internally by call_rcu().
*/
static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
static bool rcuhead_is_static_object(void *addr)
{
struct rcu_head *head = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
/*
* This is not really a fixup. We just make sure that it is
* tracked in the object tracker.
*/
debug_object_init(head, &rcuhead_debug_descr);
debug_object_activate(head, &rcuhead_debug_descr);
return 0;
default:
return 1;
}
return true;
}
/**
@ -440,7 +420,7 @@ EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
struct debug_obj_descr rcuhead_debug_descr = {
.name = "rcu_head",
.fixup_activate = rcuhead_fixup_activate,
.is_static_object = rcuhead_is_static_object,
};
EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

18
kernel/sched/core.c
View file

@ -1768,13 +1768,15 @@ void sched_ttwu_pending(void)
cookie = lockdep_pin_lock(&rq->lock);
while (llist) {
int wake_flags = 0;
p = llist_entry(llist, struct task_struct, wake_entry);
llist = llist_next(llist);
/*
* See ttwu_queue(); we only call ttwu_queue_remote() when
* its a x-cpu wakeup.
*/
ttwu_do_activate(rq, p, WF_MIGRATED, cookie);
if (p->sched_remote_wakeup)
wake_flags = WF_MIGRATED;
ttwu_do_activate(rq, p, wake_flags, cookie);
}
lockdep_unpin_lock(&rq->lock, cookie);
@ -1819,10 +1821,12 @@ void scheduler_ipi(void)
irq_exit();
}
static void ttwu_queue_remote(struct task_struct *p, int cpu)
static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
{
struct rq *rq = cpu_rq(cpu);
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
if (!set_nr_if_polling(rq->idle))
smp_send_reschedule(cpu);
@ -1869,7 +1873,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
#if defined(CONFIG_SMP)
if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
sched_clock_cpu(cpu); /* sync clocks x-cpu */
ttwu_queue_remote(p, cpu);
ttwu_queue_remote(p, cpu, wake_flags);
return;
}
#endif

4
kernel/sched/cpufreq_schedutil.c
View file

@ -9,6 +9,8 @@
* published by the Free Software Foundation.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/cpufreq.h>
#include <linux/module.h>
#include <linux/slab.h>
@ -388,7 +390,7 @@ static int sugov_init(struct cpufreq_policy *policy)
mutex_unlock(&global_tunables_lock);
sugov_policy_free(sg_policy);
pr_err("cpufreq: schedutil governor initialization failed (error %d)\n", ret);
pr_err("initialization failed (error %d)\n", ret);
return ret;
}

15
kernel/seccomp.c
View file

@ -513,24 +513,17 @@ static void seccomp_send_sigsys(int syscall, int reason)
* To be fully secure this must be combined with rlimit
* to limit the stack allocations too.
*/
static int mode1_syscalls[] = {
static const int mode1_syscalls[] = {
__NR_seccomp_read, __NR_seccomp_write, __NR_seccomp_exit, __NR_seccomp_sigreturn,
0, /* null terminated */
};
#ifdef CONFIG_COMPAT
static int mode1_syscalls_32[] = {
__NR_seccomp_read_32, __NR_seccomp_write_32, __NR_seccomp_exit_32, __NR_seccomp_sigreturn_32,
0, /* null terminated */
};
#endif
static void __secure_computing_strict(int this_syscall)
{
int *syscall_whitelist = mode1_syscalls;
const int *syscall_whitelist = mode1_syscalls;
#ifdef CONFIG_COMPAT
if (in_compat_syscall())
syscall_whitelist = mode1_syscalls_32;
syscall_whitelist = get_compat_mode1_syscalls();
#endif
do {
if (*syscall_whitelist == this_syscall)
@ -915,7 +908,7 @@ long seccomp_get_filter(struct task_struct *task, unsigned long filter_off,
fprog = filter->prog->orig_prog;
if (!fprog) {
/* This must be a new non-cBPF filter, since we save every
/* This must be a new non-cBPF filter, since we save
* every cBPF filter's orig_prog above when
* CONFIG_CHECKPOINT_RESTORE is enabled.
*/

10
kernel/signal.c
View file

@ -224,7 +224,7 @@ static inline void print_dropped_signal(int sig)
if (!__ratelimit(&ratelimit_state))
return;
printk(KERN_INFO "%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n",
pr_info("%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n",
current->comm, current->pid, sig);
}
@ -1089,10 +1089,10 @@ static int send_signal(int sig, struct siginfo *info, struct task_struct *t,
static void print_fatal_signal(int signr)
{
struct pt_regs *regs = signal_pt_regs();
printk(KERN_INFO "potentially unexpected fatal signal %d.\n", signr);
pr_info("potentially unexpected fatal signal %d.\n", signr);
#if defined(__i386__) && !defined(__arch_um__)
printk(KERN_INFO "code at %08lx: ", regs->ip);
pr_info("code at %08lx: ", regs->ip);
{
int i;
for (i = 0; i < 16; i++) {
@ -1100,10 +1100,10 @@ static void print_fatal_signal(int signr)
if (get_user(insn, (unsigned char *)(regs->ip + i)))
break;
printk(KERN_CONT "%02x ", insn);
pr_cont("%02x ", insn);
}
}
printk(KERN_CONT "\n");
pr_cont("\n");
#endif
preempt_disable();
show_regs(regs);

3
kernel/sys.c
View file

@ -2246,7 +2246,8 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
case PR_SET_THP_DISABLE:
if (arg3 || arg4 || arg5)
return -EINVAL;
down_write(&me->mm->mmap_sem);
if (down_write_killable(&me->mm->mmap_sem))
return -EINTR;
if (arg2)
me->mm->def_flags |= VM_NOHUGEPAGE;
else

18
kernel/sysctl.c
View file

@ -1149,13 +1149,22 @@ static struct ctl_table kern_table[] = {
},
{
.procname = "perf_event_max_stack",
.data = NULL, /* filled in by handler */
.data = &sysctl_perf_event_max_stack,
.maxlen = sizeof(sysctl_perf_event_max_stack),
.mode = 0644,
.proc_handler = perf_event_max_stack_handler,
.extra1 = &zero,
.extra2 = &six_hundred_forty_kb,
},
{
.procname = "perf_event_max_contexts_per_stack",
.data = &sysctl_perf_event_max_contexts_per_stack,
.maxlen = sizeof(sysctl_perf_event_max_contexts_per_stack),
.mode = 0644,
.proc_handler = perf_event_max_stack_handler,
.extra1 = &zero,
.extra2 = &one_thousand,
},
#endif
#ifdef CONFIG_KMEMCHECK
{
@ -1521,6 +1530,13 @@ static struct ctl_table vm_table[] = {
.mode = 0644,
.proc_handler = proc_dointvec_jiffies,
},
{
.procname = "stat_refresh",
.data = NULL,
.maxlen = 0,
.mode = 0600,
.proc_handler = vmstat_refresh,
},
#endif
#ifdef CONFIG_MMU
{

23
kernel/sysctl_binary.c
View file

@ -13,6 +13,7 @@
#include <linux/ctype.h>
#include <linux/netdevice.h>
#include <linux/kernel.h>
#include <linux/uuid.h>
#include <linux/slab.h>
#include <linux/compat.h>
@ -1117,9 +1118,8 @@ static ssize_t bin_uuid(struct file *file,
/* Only supports reads */
if (oldval && oldlen) {
char buf[40], *str = buf;
unsigned char uuid[16];
int i;
char buf[UUID_STRING_LEN + 1];
uuid_be uuid;
result = kernel_read(file, 0, buf, sizeof(buf) - 1);
if (result < 0)
@ -1127,24 +1127,15 @@ static ssize_t bin_uuid(struct file *file,
buf[result] = '\0';
/* Convert the uuid to from a string to binary */
for (i = 0; i < 16; i++) {
result = -EIO;
if (!isxdigit(str[0]) || !isxdigit(str[1]))
goto out;
uuid[i] = (hex_to_bin(str[0]) << 4) |
hex_to_bin(str[1]);
str += 2;
if (*str == '-')
str++;
}
result = -EIO;
if (uuid_be_to_bin(buf, &uuid))
goto out;
if (oldlen > 16)
oldlen = 16;
result = -EFAULT;
if (copy_to_user(oldval, uuid, oldlen))
if (copy_to_user(oldval, &uuid, oldlen))
goto out;
copied = oldlen;

37
kernel/taskstats.c
View file

@ -357,10 +357,6 @@ static int parse(struct nlattr *na, struct cpumask *mask)
return ret;
}
#if defined(CONFIG_64BIT) && !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
#define TASKSTATS_NEEDS_PADDING 1
#endif
static struct taskstats *mk_reply(struct sk_buff *skb, int type, u32 pid)
{
struct nlattr *na, *ret;
@ -370,29 +366,6 @@ static struct taskstats *mk_reply(struct sk_buff *skb, int type, u32 pid)
? TASKSTATS_TYPE_AGGR_PID
: TASKSTATS_TYPE_AGGR_TGID;
/*
* The taskstats structure is internally aligned on 8 byte
* boundaries but the layout of the aggregrate reply, with
* two NLA headers and the pid (each 4 bytes), actually
* force the entire structure to be unaligned. This causes
* the kernel to issue unaligned access warnings on some
* architectures like ia64. Unfortunately, some software out there
* doesn't properly unroll the NLA packet and assumes that the start
* of the taskstats structure will always be 20 bytes from the start
* of the netlink payload. Aligning the start of the taskstats
* structure breaks this software, which we don't want. So, for now
* the alignment only happens on architectures that require it
* and those users will have to update to fixed versions of those
* packages. Space is reserved in the packet only when needed.
* This ifdef should be removed in several years e.g. 2012 once
* we can be confident that fixed versions are installed on most
* systems. We add the padding before the aggregate since the
* aggregate is already a defined type.
*/
#ifdef TASKSTATS_NEEDS_PADDING
if (nla_put(skb, TASKSTATS_TYPE_NULL, 0, NULL) < 0)
goto err;
#endif
na = nla_nest_start(skb, aggr);
if (!na)
goto err;
@ -401,7 +374,8 @@ static struct taskstats *mk_reply(struct sk_buff *skb, int type, u32 pid)
nla_nest_cancel(skb, na);
goto err;
}
ret = nla_reserve(skb, TASKSTATS_TYPE_STATS, sizeof(struct taskstats));
ret = nla_reserve_64bit(skb, TASKSTATS_TYPE_STATS,
sizeof(struct taskstats), TASKSTATS_TYPE_NULL);
if (!ret) {
nla_nest_cancel(skb, na);
goto err;
@ -500,10 +474,9 @@ static size_t taskstats_packet_size(void)
size_t size;
size = nla_total_size(sizeof(u32)) +
nla_total_size(sizeof(struct taskstats)) + nla_total_size(0);
#ifdef TASKSTATS_NEEDS_PADDING
size += nla_total_size(0); /* Padding for alignment */
#endif
nla_total_size_64bit(sizeof(struct taskstats)) +
nla_total_size(0);
return size;
}

23
kernel/time/hrtimer.c
View file

@ -334,7 +334,7 @@ static void *hrtimer_debug_hint(void *addr)
* fixup_init is called when:
* - an active object is initialized
*/
static int hrtimer_fixup_init(void *addr, enum debug_obj_state state)
static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
{
struct hrtimer *timer = addr;
@ -342,30 +342,25 @@ static int hrtimer_fixup_init(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
hrtimer_cancel(timer);
debug_object_init(timer, &hrtimer_debug_descr);
return 1;
return true;
default:
return 0;
return false;
}
}
/*
* fixup_activate is called when:
* - an active object is activated
* - an unknown object is activated (might be a statically initialized object)
* - an unknown non-static object is activated
*/
static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
{
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
WARN_ON_ONCE(1);
return 0;
case ODEBUG_STATE_ACTIVE:
WARN_ON(1);
default:
return 0;
return false;
}
}
@ -373,7 +368,7 @@ static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
* fixup_free is called when:
* - an active object is freed
*/
static int hrtimer_fixup_free(void *addr, enum debug_obj_state state)
static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
{
struct hrtimer *timer = addr;
@ -381,9 +376,9 @@ static int hrtimer_fixup_free(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
hrtimer_cancel(timer);
debug_object_free(timer, &hrtimer_debug_descr);
return 1;
return true;
default:
return 0;
return false;
}
}

21
kernel/time/time.c
View file

@ -769,3 +769,24 @@ struct timespec timespec_add_safe(const struct timespec lhs,
return res;
}
/*
* Add two timespec64 values and do a safety check for overflow.
* It's assumed that both values are valid (>= 0).
* And, each timespec64 is in normalized form.
*/
struct timespec64 timespec64_add_safe(const struct timespec64 lhs,
const struct timespec64 rhs)
{
struct timespec64 res;
set_normalized_timespec64(&res, lhs.tv_sec + rhs.tv_sec,
lhs.tv_nsec + rhs.tv_nsec);
if (unlikely(res.tv_sec < lhs.tv_sec || res.tv_sec < rhs.tv_sec)) {
res.tv_sec = TIME64_MAX;
res.tv_nsec = 0;
}
return res;
}

63
kernel/time/timer.c
View file

@ -489,11 +489,19 @@ static void *timer_debug_hint(void *addr)
return ((struct timer_list *) addr)->function;
}
static bool timer_is_static_object(void *addr)
{
struct timer_list *timer = addr;
return (timer->entry.pprev == NULL &&
timer->entry.next == TIMER_ENTRY_STATIC);
}
/*
* fixup_init is called when:
* - an active object is initialized
*/
static int timer_fixup_init(void *addr, enum debug_obj_state state)
static bool timer_fixup_init(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
@ -501,9 +509,9 @@ static int timer_fixup_init(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
del_timer_sync(timer);
debug_object_init(timer, &timer_debug_descr);
return 1;
return true;
default:
return 0;
return false;
}
}
@ -516,36 +524,22 @@ static void stub_timer(unsigned long data)
/*
* fixup_activate is called when:
* - an active object is activated
* - an unknown object is activated (might be a statically initialized object)
* - an unknown non-static object is activated
*/
static int timer_fixup_activate(void *addr, enum debug_obj_state state)
static bool timer_fixup_activate(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
/*
* This is not really a fixup. The timer was
* statically initialized. We just make sure that it
* is tracked in the object tracker.
*/
if (timer->entry.pprev == NULL &&
timer->entry.next == TIMER_ENTRY_STATIC) {
debug_object_init(timer, &timer_debug_descr);
debug_object_activate(timer, &timer_debug_descr);
return 0;
} else {
setup_timer(timer, stub_timer, 0);
return 1;
}
return 0;
setup_timer(timer, stub_timer, 0);
return true;
case ODEBUG_STATE_ACTIVE:
WARN_ON(1);
default:
return 0;
return false;
}
}
@ -553,7 +547,7 @@ static int timer_fixup_activate(void *addr, enum debug_obj_state state)
* fixup_free is called when:
* - an active object is freed
*/
static int timer_fixup_free(void *addr, enum debug_obj_state state)
static bool timer_fixup_free(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
@ -561,9 +555,9 @@ static int timer_fixup_free(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
del_timer_sync(timer);
debug_object_free(timer, &timer_debug_descr);
return 1;
return true;
default:
return 0;
return false;
}
}
@ -571,32 +565,23 @@ static int timer_fixup_free(void *addr, enum debug_obj_state state)
* fixup_assert_init is called when:
* - an untracked/uninit-ed object is found
*/
static int timer_fixup_assert_init(void *addr, enum debug_obj_state state)
static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
if (timer->entry.next == TIMER_ENTRY_STATIC) {
/*
* This is not really a fixup. The timer was
* statically initialized. We just make sure that it
* is tracked in the object tracker.
*/
debug_object_init(timer, &timer_debug_descr);
return 0;
} else {
setup_timer(timer, stub_timer, 0);
return 1;
}
setup_timer(timer, stub_timer, 0);
return true;
default:
return 0;
return false;
}
}
static struct debug_obj_descr timer_debug_descr = {
.name = "timer_list",
.debug_hint = timer_debug_hint,
.is_static_object = timer_is_static_object,
.fixup_init = timer_fixup_init,
.fixup_activate = timer_fixup_activate,
.fixup_free = timer_fixup_free,

26
kernel/trace/Kconfig
View file

@ -528,6 +528,32 @@ config MMIOTRACE
See Documentation/trace/mmiotrace.txt.
If you are not helping to develop drivers, say N.
config TRACING_MAP
bool
depends on ARCH_HAVE_NMI_SAFE_CMPXCHG
help
tracing_map is a special-purpose lock-free map for tracing,
separated out as a stand-alone facility in order to allow it
to be shared between multiple tracers. It isn't meant to be
generally used outside of that context, and is normally
selected by tracers that use it.
config HIST_TRIGGERS
bool "Histogram triggers"
depends on ARCH_HAVE_NMI_SAFE_CMPXCHG
select TRACING_MAP
default n
help
Hist triggers allow one or more arbitrary trace event fields
to be aggregated into hash tables and dumped to stdout by
reading a debugfs/tracefs file. They're useful for
gathering quick and dirty (though precise) summaries of
event activity as an initial guide for further investigation
using more advanced tools.
See Documentation/trace/events.txt.
If in doubt, say N.
config MMIOTRACE_TEST
tristate "Test module for mmiotrace"
depends on MMIOTRACE && m

2
kernel/trace/Makefile
View file

@ -31,6 +31,7 @@ obj-$(CONFIG_TRACING) += trace_output.o
obj-$(CONFIG_TRACING) += trace_seq.o
obj-$(CONFIG_TRACING) += trace_stat.o
obj-$(CONFIG_TRACING) += trace_printk.o
obj-$(CONFIG_TRACING_MAP) += tracing_map.o
obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
@ -53,6 +54,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_event_perf.o
endif
obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o
obj-$(CONFIG_EVENT_TRACING) += trace_events_trigger.o
obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o
obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o
obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o
obj-$(CONFIG_TRACEPOINTS) += power-traces.o

2
kernel/trace/blktrace.c
View file

@ -1349,6 +1349,7 @@ static enum print_line_t print_one_line(struct trace_iterator *iter,
if (t->action == BLK_TN_MESSAGE) {
log_action(iter, long_act ? "message" : "m");
blk_log_msg(s, iter->ent);
return trace_handle_return(s);
}
if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act)))
@ -1551,6 +1552,7 @@ static const struct {
{ BLK_TC_COMPLETE, "complete" },
{ BLK_TC_FS, "fs" },
{ BLK_TC_PC, "pc" },
{ BLK_TC_NOTIFY, "notify" },
{ BLK_TC_AHEAD, "ahead" },
{ BLK_TC_META, "meta" },
{ BLK_TC_DISCARD, "discard" },

129
kernel/trace/bpf_trace.c
View file

@ -62,17 +62,21 @@ EXPORT_SYMBOL_GPL(trace_call_bpf);
static u64 bpf_probe_read(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
{
void *dst = (void *) (long) r1;
int size = (int) r2;
int ret, size = (int) r2;
void *unsafe_ptr = (void *) (long) r3;
return probe_kernel_read(dst, unsafe_ptr, size);
ret = probe_kernel_read(dst, unsafe_ptr, size);
if (unlikely(ret < 0))
memset(dst, 0, size);
return ret;
}
static const struct bpf_func_proto bpf_probe_read_proto = {
.func = bpf_probe_read,
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_STACK,
.arg1_type = ARG_PTR_TO_RAW_STACK,
.arg2_type = ARG_CONST_STACK_SIZE,
.arg3_type = ARG_ANYTHING,
};
@ -221,11 +225,12 @@ static const struct bpf_func_proto bpf_perf_event_read_proto = {
.arg2_type = ARG_ANYTHING,
};
static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 index, u64 r4, u64 size)
static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)
{
struct pt_regs *regs = (struct pt_regs *) (long) r1;
struct bpf_map *map = (struct bpf_map *) (long) r2;
struct bpf_array *array = container_of(map, struct bpf_array, map);
u64 index = flags & BPF_F_INDEX_MASK;
void *data = (void *) (long) r4;
struct perf_sample_data sample_data;
struct perf_event *event;
@ -235,6 +240,10 @@ static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 index, u64 r4, u64 size)
.data = data,
};
if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
return -EINVAL;
if (index == BPF_F_CURRENT_CPU)
index = raw_smp_processor_id();
if (unlikely(index >= array->map.max_entries))
return -E2BIG;
@ -268,7 +277,34 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {
.arg5_type = ARG_CONST_STACK_SIZE,
};
static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id)
static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);
static u64 bpf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)
{
struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);
perf_fetch_caller_regs(regs);
return bpf_perf_event_output((long)regs, r2, flags, r4, size);
}
static const struct bpf_func_proto bpf_event_output_proto = {
.func = bpf_event_output,
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_STACK,
.arg5_type = ARG_CONST_STACK_SIZE,
};
const struct bpf_func_proto *bpf_get_event_output_proto(void)
{
return &bpf_event_output_proto;
}
static const struct bpf_func_proto *tracing_func_proto(enum bpf_func_id func_id)
{
switch (func_id) {
case BPF_FUNC_map_lookup_elem:
@ -295,12 +331,20 @@ static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func
return &bpf_get_smp_processor_id_proto;
case BPF_FUNC_perf_event_read:
return &bpf_perf_event_read_proto;
default:
return NULL;
}
}
static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id)
{
switch (func_id) {
case BPF_FUNC_perf_event_output:
return &bpf_perf_event_output_proto;
case BPF_FUNC_get_stackid:
return &bpf_get_stackid_proto;
default:
return NULL;
return tracing_func_proto(func_id);
}
}
@ -332,9 +376,82 @@ static struct bpf_prog_type_list kprobe_tl = {
.type = BPF_PROG_TYPE_KPROBE,
};
static u64 bpf_perf_event_output_tp(u64 r1, u64 r2, u64 index, u64 r4, u64 size)
{
/*
* r1 points to perf tracepoint buffer where first 8 bytes are hidden
* from bpf program and contain a pointer to 'struct pt_regs'. Fetch it
* from there and call the same bpf_perf_event_output() helper
*/
u64 ctx = *(long *)(uintptr_t)r1;
return bpf_perf_event_output(ctx, r2, index, r4, size);
}
static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
.func = bpf_perf_event_output_tp,
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_STACK,
.arg5_type = ARG_CONST_STACK_SIZE,
};
static u64 bpf_get_stackid_tp(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
{
u64 ctx = *(long *)(uintptr_t)r1;
return bpf_get_stackid(ctx, r2, r3, r4, r5);
}
static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
.func = bpf_get_stackid_tp,
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
};
static const struct bpf_func_proto *tp_prog_func_proto(enum bpf_func_id func_id)
{
switch (func_id) {
case BPF_FUNC_perf_event_output:
return &bpf_perf_event_output_proto_tp;
case BPF_FUNC_get_stackid:
return &bpf_get_stackid_proto_tp;
default:
return tracing_func_proto(func_id);
}
}
static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type)
{
if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE)
return false;
if (type != BPF_READ)
return false;
if (off % size != 0)
return false;
return true;
}
static const struct bpf_verifier_ops tracepoint_prog_ops = {
.get_func_proto = tp_prog_func_proto,
.is_valid_access = tp_prog_is_valid_access,
};
static struct bpf_prog_type_list tracepoint_tl = {
.ops = &tracepoint_prog_ops,
.type = BPF_PROG_TYPE_TRACEPOINT,
};
static int __init register_kprobe_prog_ops(void)
{
bpf_register_prog_type(&kprobe_tl);
bpf_register_prog_type(&tracepoint_tl);
return 0;
}
late_initcall(register_kprobe_prog_ops);

31
kernel/trace/ftrace.c
View file

@ -1530,7 +1530,19 @@ static int ftrace_cmp_recs(const void *a, const void *b)
return 0;
}
static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
/**
* ftrace_location_range - return the first address of a traced location
* if it touches the given ip range
* @start: start of range to search.
* @end: end of range to search (inclusive). @end points to the last byte
* to check.
*
* Returns rec->ip if the related ftrace location is a least partly within
* the given address range. That is, the first address of the instruction
* that is either a NOP or call to the function tracer. It checks the ftrace
* internal tables to determine if the address belongs or not.
*/
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec;
@ -3444,11 +3456,23 @@ struct ftrace_glob {
int type;
};
/*
* If symbols in an architecture don't correspond exactly to the user-visible
* name of what they represent, it is possible to define this function to
* perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
return str;
}
static int ftrace_match(char *str, struct ftrace_glob *g)
{
int matched = 0;
int slen;
str = arch_ftrace_match_adjust(str, g->search);
switch (g->type) {
case MATCH_FULL:
if (strcmp(str, g->search) == 0)
@ -5713,7 +5737,6 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
int i;
int ret = 0;
unsigned long flags;
int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
struct task_struct *g, *t;
@ -5729,7 +5752,7 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
}
}
read_lock_irqsave(&tasklist_lock, flags);
read_lock(&tasklist_lock);
do_each_thread(g, t) {
if (start == end) {
ret = -EAGAIN;
@ -5747,7 +5770,7 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
} while_each_thread(g, t);
unlock:
read_unlock_irqrestore(&tasklist_lock, flags);
read_unlock(&tasklist_lock);
free:
for (i = start; i < end; i++)
kfree(ret_stack_list[i]);

277
kernel/trace/trace.c
View file

@ -253,6 +253,9 @@ unsigned long long ns2usecs(cycle_t nsec)
#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
/* trace_flags that are default zero for instances */
#define ZEROED_TRACE_FLAGS \
TRACE_ITER_EVENT_FORK
/*
* The global_trace is the descriptor that holds the tracing
@ -303,33 +306,18 @@ void trace_array_put(struct trace_array *this_tr)
mutex_unlock(&trace_types_lock);
}
int filter_check_discard(struct trace_event_file *file, void *rec,
struct ring_buffer *buffer,
struct ring_buffer_event *event)
{
if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
!filter_match_preds(file->filter, rec)) {
ring_buffer_discard_commit(buffer, event);
return 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(filter_check_discard);
int call_filter_check_discard(struct trace_event_call *call, void *rec,
struct ring_buffer *buffer,
struct ring_buffer_event *event)
{
if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
!filter_match_preds(call->filter, rec)) {
ring_buffer_discard_commit(buffer, event);
__trace_event_discard_commit(buffer, event);
return 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(call_filter_check_discard);
static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
{
@ -1672,6 +1660,16 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
}
EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
static __always_inline void
trace_event_setup(struct ring_buffer_event *event,
int type, unsigned long flags, int pc)
{
struct trace_entry *ent = ring_buffer_event_data(event);
tracing_generic_entry_update(ent, flags, pc);
ent->type = type;
}
struct ring_buffer_event *
trace_buffer_lock_reserve(struct ring_buffer *buffer,
int type,
@ -1681,34 +1679,137 @@ trace_buffer_lock_reserve(struct ring_buffer *buffer,
struct ring_buffer_event *event;
event = ring_buffer_lock_reserve(buffer, len);
if (event != NULL) {
struct trace_entry *ent = ring_buffer_event_data(event);
tracing_generic_entry_update(ent, flags, pc);
ent->type = type;
}
if (event != NULL)
trace_event_setup(event, type, flags, pc);
return event;
}
DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
DEFINE_PER_CPU(int, trace_buffered_event_cnt);
static int trace_buffered_event_ref;
/**
* trace_buffered_event_enable - enable buffering events
*
* When events are being filtered, it is quicker to use a temporary
* buffer to write the event data into if there's a likely chance
* that it will not be committed. The discard of the ring buffer
* is not as fast as committing, and is much slower than copying
* a commit.
*
* When an event is to be filtered, allocate per cpu buffers to
* write the event data into, and if the event is filtered and discarded
* it is simply dropped, otherwise, the entire data is to be committed
* in one shot.
*/
void trace_buffered_event_enable(void)
{
struct ring_buffer_event *event;
struct page *page;
int cpu;
WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
if (trace_buffered_event_ref++)
return;
for_each_tracing_cpu(cpu) {
page = alloc_pages_node(cpu_to_node(cpu),
GFP_KERNEL | __GFP_NORETRY, 0);
if (!page)
goto failed;
event = page_address(page);
memset(event, 0, sizeof(*event));
per_cpu(trace_buffered_event, cpu) = event;
preempt_disable();
if (cpu == smp_processor_id() &&
this_cpu_read(trace_buffered_event) !=
per_cpu(trace_buffered_event, cpu))
WARN_ON_ONCE(1);
preempt_enable();
}
return;
failed:
trace_buffered_event_disable();
}
static void enable_trace_buffered_event(void *data)
{
/* Probably not needed, but do it anyway */
smp_rmb();
this_cpu_dec(trace_buffered_event_cnt);
}
static void disable_trace_buffered_event(void *data)
{
this_cpu_inc(trace_buffered_event_cnt);
}
/**
* trace_buffered_event_disable - disable buffering events
*
* When a filter is removed, it is faster to not use the buffered
* events, and to commit directly into the ring buffer. Free up
* the temp buffers when there are no more users. This requires
* special synchronization with current events.
*/
void trace_buffered_event_disable(void)
{
int cpu;
WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
if (WARN_ON_ONCE(!trace_buffered_event_ref))
return;
if (--trace_buffered_event_ref)
return;
preempt_disable();
/* For each CPU, set the buffer as used. */
smp_call_function_many(tracing_buffer_mask,
disable_trace_buffered_event, NULL, 1);
preempt_enable();
/* Wait for all current users to finish */
synchronize_sched();
for_each_tracing_cpu(cpu) {
free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
per_cpu(trace_buffered_event, cpu) = NULL;
}
/*
* Make sure trace_buffered_event is NULL before clearing
* trace_buffered_event_cnt.
*/
smp_wmb();
preempt_disable();
/* Do the work on each cpu */
smp_call_function_many(tracing_buffer_mask,
enable_trace_buffered_event, NULL, 1);
preempt_enable();
}
void
__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
{
__this_cpu_write(trace_cmdline_save, true);
ring_buffer_unlock_commit(buffer, event);
}
void trace_buffer_unlock_commit(struct trace_array *tr,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
unsigned long flags, int pc)
{
__buffer_unlock_commit(buffer, event);
ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
ftrace_trace_userstack(buffer, flags, pc);
/* If this is the temp buffer, we need to commit fully */
if (this_cpu_read(trace_buffered_event) == event) {
/* Length is in event->array[0] */
ring_buffer_write(buffer, event->array[0], &event->array[1]);
/* Release the temp buffer */
this_cpu_dec(trace_buffered_event_cnt);
} else
ring_buffer_unlock_commit(buffer, event);
}
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
static struct ring_buffer *temp_buffer;
@ -1719,8 +1820,23 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
unsigned long flags, int pc)
{
struct ring_buffer_event *entry;
int val;
*current_rb = trace_file->tr->trace_buffer.buffer;
if ((trace_file->flags &
(EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
(entry = this_cpu_read(trace_buffered_event))) {
/* Try to use the per cpu buffer first */
val = this_cpu_inc_return(trace_buffered_event_cnt);
if (val == 1) {
trace_event_setup(entry, type, flags, pc);
entry->array[0] = len;
return entry;
}
this_cpu_dec(trace_buffered_event_cnt);
}
entry = trace_buffer_lock_reserve(*current_rb,
type, len, flags, pc);
/*
@ -1738,17 +1854,6 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
}
EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
struct ring_buffer_event *
trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
int type, unsigned long len,
unsigned long flags, int pc)
{
*current_rb = global_trace.trace_buffer.buffer;
return trace_buffer_lock_reserve(*current_rb,
type, len, flags, pc);
}
EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
void trace_buffer_unlock_commit_regs(struct trace_array *tr,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
@ -1760,14 +1865,6 @@ void trace_buffer_unlock_commit_regs(struct trace_array *tr,
ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
ftrace_trace_userstack(buffer, flags, pc);
}
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
struct ring_buffer_event *event)
{
ring_buffer_discard_commit(buffer, event);
}
EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
void
trace_function(struct trace_array *tr,
@ -3571,6 +3668,9 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
if (mask == TRACE_ITER_RECORD_CMD)
trace_event_enable_cmd_record(enabled);
if (mask == TRACE_ITER_EVENT_FORK)
trace_event_follow_fork(tr, enabled);
if (mask == TRACE_ITER_OVERWRITE) {
ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
#ifdef CONFIG_TRACER_MAX_TRACE
@ -3658,7 +3758,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
if (cnt >= sizeof(buf))
return -EINVAL;
if (copy_from_user(&buf, ubuf, cnt))
if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@ -3804,11 +3904,18 @@ static const char readme_msg[] =
"\t trigger: traceon, traceoff\n"
"\t enable_event:<system>:<event>\n"
"\t disable_event:<system>:<event>\n"
#ifdef CONFIG_HIST_TRIGGERS
"\t enable_hist:<system>:<event>\n"
"\t disable_hist:<system>:<event>\n"
#endif
#ifdef CONFIG_STACKTRACE
"\t\t stacktrace\n"
#endif
#ifdef CONFIG_TRACER_SNAPSHOT
"\t\t snapshot\n"
#endif
#ifdef CONFIG_HIST_TRIGGERS
"\t\t hist (see below)\n"
#endif
"\t example: echo traceoff > events/block/block_unplug/trigger\n"
"\t echo traceoff:3 > events/block/block_unplug/trigger\n"
@ -3825,6 +3932,56 @@ static const char readme_msg[] =
"\t To remove a trigger with a count:\n"
"\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
"\t Filters can be ignored when removing a trigger.\n"
#ifdef CONFIG_HIST_TRIGGERS
" hist trigger\t- If set, event hits are aggregated into a hash table\n"
"\t Format: hist:keys=<field1[,field2,...]>\n"
"\t [:values=<field1[,field2,...]>]\n"
"\t [:sort=<field1[,field2,...]>]\n"
"\t [:size=#entries]\n"
"\t [:pause][:continue][:clear]\n"
"\t [:name=histname1]\n"
"\t [if <filter>]\n\n"
"\t When a matching event is hit, an entry is added to a hash\n"
"\t table using the key(s) and value(s) named, and the value of a\n"
"\t sum called 'hitcount' is incremented. Keys and values\n"
"\t correspond to fields in the event's format description. Keys\n"
"\t can be any field, or the special string 'stacktrace'.\n"
"\t Compound keys consisting of up to two fields can be specified\n"
"\t by the 'keys' keyword. Values must correspond to numeric\n"
"\t fields. Sort keys consisting of up to two fields can be\n"
"\t specified using the 'sort' keyword. The sort direction can\n"
"\t be modified by appending '.descending' or '.ascending' to a\n"
"\t sort field. The 'size' parameter can be used to specify more\n"
"\t or fewer than the default 2048 entries for the hashtable size.\n"
"\t If a hist trigger is given a name using the 'name' parameter,\n"
"\t its histogram data will be shared with other triggers of the\n"
"\t same name, and trigger hits will update this common data.\n\n"
"\t Reading the 'hist' file for the event will dump the hash\n"
"\t table in its entirety to stdout. If there are multiple hist\n"
"\t triggers attached to an event, there will be a table for each\n"
"\t trigger in the output. The table displayed for a named\n"
"\t trigger will be the same as any other instance having the\n"
"\t same name. The default format used to display a given field\n"
"\t can be modified by appending any of the following modifiers\n"
"\t to the field name, as applicable:\n\n"
"\t .hex display a number as a hex value\n"
"\t .sym display an address as a symbol\n"
"\t .sym-offset display an address as a symbol and offset\n"
"\t .execname display a common_pid as a program name\n"
"\t .syscall display a syscall id as a syscall name\n\n"
"\t .log2 display log2 value rather than raw number\n\n"
"\t The 'pause' parameter can be used to pause an existing hist\n"
"\t trigger or to start a hist trigger but not log any events\n"
"\t until told to do so. 'continue' can be used to start or\n"
"\t restart a paused hist trigger.\n\n"
"\t The 'clear' parameter will clear the contents of a running\n"
"\t hist trigger and leave its current paused/active state\n"
"\t unchanged.\n\n"
"\t The enable_hist and disable_hist triggers can be used to\n"
"\t have one event conditionally start and stop another event's\n"
"\t already-attached hist trigger. The syntax is analagous to\n"
"\t the enable_event and disable_event triggers.\n"
#endif
;
static ssize_t
@ -4474,7 +4631,7 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf,
if (cnt > MAX_TRACER_SIZE)
cnt = MAX_TRACER_SIZE;
if (copy_from_user(&buf, ubuf, cnt))
if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@ -5264,7 +5421,7 @@ static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
if (cnt >= sizeof(buf))
return -EINVAL;
if (copy_from_user(&buf, ubuf, cnt))
if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@ -6650,7 +6807,7 @@ static int instance_mkdir(const char *name)
if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
goto out_free_tr;
tr->trace_flags = global_trace.trace_flags;
tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
@ -6724,6 +6881,12 @@ static int instance_rmdir(const char *name)
list_del(&tr->list);
/* Disable all the flags that were enabled coming in */
for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
if ((1 << i) & ZEROED_TRACE_FLAGS)
set_tracer_flag(tr, 1 << i, 0);
}
tracing_set_nop(tr);
event_trace_del_tracer(tr);
ftrace_destroy_function_files(tr);

190
kernel/trace/trace.h
View file

@ -177,9 +177,8 @@ struct trace_options {
};
struct trace_pid_list {
unsigned int nr_pids;
int order;
pid_t *pids;
int pid_max;
unsigned long *pids;
};
/*
@ -656,6 +655,7 @@ static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
extern cycle_t ftrace_now(int cpu);
extern void trace_find_cmdline(int pid, char comm[]);
extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
#ifdef CONFIG_DYNAMIC_FTRACE
extern unsigned long ftrace_update_tot_cnt;
@ -967,6 +967,7 @@ extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
C(STOP_ON_FREE, "disable_on_free"), \
C(IRQ_INFO, "irq-info"), \
C(MARKERS, "markers"), \
C(EVENT_FORK, "event-fork"), \
FUNCTION_FLAGS \
FGRAPH_FLAGS \
STACK_FLAGS \
@ -1064,6 +1065,137 @@ struct trace_subsystem_dir {
int nr_events;
};
extern int call_filter_check_discard(struct trace_event_call *call, void *rec,
struct ring_buffer *buffer,
struct ring_buffer_event *event);
void trace_buffer_unlock_commit_regs(struct trace_array *tr,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
unsigned long flags, int pc,
struct pt_regs *regs);
static inline void trace_buffer_unlock_commit(struct trace_array *tr,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
unsigned long flags, int pc)
{
trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL);
}
DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
DECLARE_PER_CPU(int, trace_buffered_event_cnt);
void trace_buffered_event_disable(void);
void trace_buffered_event_enable(void);
static inline void
__trace_event_discard_commit(struct ring_buffer *buffer,
struct ring_buffer_event *event)
{
if (this_cpu_read(trace_buffered_event) == event) {
/* Simply release the temp buffer */
this_cpu_dec(trace_buffered_event_cnt);
return;
}
ring_buffer_discard_commit(buffer, event);
}
/*
* Helper function for event_trigger_unlock_commit{_regs}().
* If there are event triggers attached to this event that requires
* filtering against its fields, then they wil be called as the
* entry already holds the field information of the current event.
*
* It also checks if the event should be discarded or not.
* It is to be discarded if the event is soft disabled and the
* event was only recorded to process triggers, or if the event
* filter is active and this event did not match the filters.
*
* Returns true if the event is discarded, false otherwise.
*/
static inline bool
__event_trigger_test_discard(struct trace_event_file *file,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
void *entry,
enum event_trigger_type *tt)
{
unsigned long eflags = file->flags;
if (eflags & EVENT_FILE_FL_TRIGGER_COND)
*tt = event_triggers_call(file, entry);
if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
(unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
!filter_match_preds(file->filter, entry))) {
__trace_event_discard_commit(buffer, event);
return true;
}
return false;
}
/**
* event_trigger_unlock_commit - handle triggers and finish event commit
* @file: The file pointer assoctiated to the event
* @buffer: The ring buffer that the event is being written to
* @event: The event meta data in the ring buffer
* @entry: The event itself
* @irq_flags: The state of the interrupts at the start of the event
* @pc: The state of the preempt count at the start of the event.
*
* This is a helper function to handle triggers that require data
* from the event itself. It also tests the event against filters and
* if the event is soft disabled and should be discarded.
*/
static inline void
event_trigger_unlock_commit(struct trace_event_file *file,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
void *entry, unsigned long irq_flags, int pc)
{
enum event_trigger_type tt = ETT_NONE;
if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc);
if (tt)
event_triggers_post_call(file, tt, entry);
}
/**
* event_trigger_unlock_commit_regs - handle triggers and finish event commit
* @file: The file pointer assoctiated to the event
* @buffer: The ring buffer that the event is being written to
* @event: The event meta data in the ring buffer
* @entry: The event itself
* @irq_flags: The state of the interrupts at the start of the event
* @pc: The state of the preempt count at the start of the event.
*
* This is a helper function to handle triggers that require data
* from the event itself. It also tests the event against filters and
* if the event is soft disabled and should be discarded.
*
* Same as event_trigger_unlock_commit() but calls
* trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit().
*/
static inline void
event_trigger_unlock_commit_regs(struct trace_event_file *file,
struct ring_buffer *buffer,
struct ring_buffer_event *event,
void *entry, unsigned long irq_flags, int pc,
struct pt_regs *regs)
{
enum event_trigger_type tt = ETT_NONE;
if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
trace_buffer_unlock_commit_regs(file->tr, buffer, event,
irq_flags, pc, regs);
if (tt)
event_triggers_post_call(file, tt, entry);
}
#define FILTER_PRED_INVALID ((unsigned short)-1)
#define FILTER_PRED_IS_RIGHT (1 << 15)
#define FILTER_PRED_FOLD (1 << 15)
@ -1161,6 +1293,15 @@ extern struct mutex event_mutex;
extern struct list_head ftrace_events;
extern const struct file_operations event_trigger_fops;
extern const struct file_operations event_hist_fops;
#ifdef CONFIG_HIST_TRIGGERS
extern int register_trigger_hist_cmd(void);
extern int register_trigger_hist_enable_disable_cmds(void);
#else
static inline int register_trigger_hist_cmd(void) { return 0; }
static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
#endif
extern int register_trigger_cmds(void);
extern void clear_event_triggers(struct trace_array *tr);
@ -1174,9 +1315,41 @@ struct event_trigger_data {
char *filter_str;
void *private_data;
bool paused;
bool paused_tmp;
struct list_head list;
char *name;
struct list_head named_list;
struct event_trigger_data *named_data;
};
/* Avoid typos */
#define ENABLE_EVENT_STR "enable_event"
#define DISABLE_EVENT_STR "disable_event"
#define ENABLE_HIST_STR "enable_hist"
#define DISABLE_HIST_STR "disable_hist"
struct enable_trigger_data {
struct trace_event_file *file;
bool enable;
bool hist;
};
extern int event_enable_trigger_print(struct seq_file *m,
struct event_trigger_ops *ops,
struct event_trigger_data *data);
extern void event_enable_trigger_free(struct event_trigger_ops *ops,
struct event_trigger_data *data);
extern int event_enable_trigger_func(struct event_command *cmd_ops,
struct trace_event_file *file,
char *glob, char *cmd, char *param);
extern int event_enable_register_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *data,
struct trace_event_file *file);
extern void event_enable_unregister_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *test,
struct trace_event_file *file);
extern void trigger_data_free(struct event_trigger_data *data);
extern int event_trigger_init(struct event_trigger_ops *ops,
struct event_trigger_data *data);
@ -1189,7 +1362,18 @@ extern void unregister_trigger(char *glob, struct event_trigger_ops *ops,
extern int set_trigger_filter(char *filter_str,
struct event_trigger_data *trigger_data,
struct trace_event_file *file);
extern struct event_trigger_data *find_named_trigger(const char *name);
extern bool is_named_trigger(struct event_trigger_data *test);
extern int save_named_trigger(const char *name,
struct event_trigger_data *data);
extern void del_named_trigger(struct event_trigger_data *data);
extern void pause_named_trigger(struct event_trigger_data *data);
extern void unpause_named_trigger(struct event_trigger_data *data);
extern void set_named_trigger_data(struct event_trigger_data *data,
struct event_trigger_data *named_data);
extern int register_event_command(struct event_command *cmd);
extern int unregister_event_command(struct event_command *cmd);
extern int register_trigger_hist_enable_disable_cmds(void);
/**
* struct event_trigger_ops - callbacks for trace event triggers

40
kernel/trace/trace_event_perf.c
View file

@ -263,42 +263,43 @@ void perf_trace_del(struct perf_event *p_event, int flags)
tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);
}
void *perf_trace_buf_prepare(int size, unsigned short type,
struct pt_regs **regs, int *rctxp)
void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
{
struct trace_entry *entry;
unsigned long flags;
char *raw_data;
int pc;
int rctx;
BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
"perf buffer not large enough"))
"perf buffer not large enough"))
return NULL;
pc = preempt_count();
*rctxp = perf_swevent_get_recursion_context();
if (*rctxp < 0)
*rctxp = rctx = perf_swevent_get_recursion_context();
if (rctx < 0)
return NULL;
if (regs)
*regs = this_cpu_ptr(&__perf_regs[*rctxp]);
raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
*regs = this_cpu_ptr(&__perf_regs[rctx]);
raw_data = this_cpu_ptr(perf_trace_buf[rctx]);
/* zero the dead bytes from align to not leak stack to user */
memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
return raw_data;
}
EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
NOKPROBE_SYMBOL(perf_trace_buf_alloc);
void perf_trace_buf_update(void *record, u16 type)
{
struct trace_entry *entry = record;
int pc = preempt_count();
unsigned long flags;
entry = (struct trace_entry *)raw_data;
local_save_flags(flags);
tracing_generic_entry_update(entry, flags, pc);
entry->type = type;
return raw_data;
}
EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
NOKPROBE_SYMBOL(perf_trace_buf_prepare);
NOKPROBE_SYMBOL(perf_trace_buf_update);
#ifdef CONFIG_FUNCTION_TRACER
static void
@ -319,15 +320,16 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);
memset(&regs, 0, sizeof(regs));
perf_fetch_caller_regs(&regs);
entry = perf_trace_buf_prepare(ENTRY_SIZE, TRACE_FN, NULL, &rctx);
entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
if (!entry)
return;
entry->ip = ip;
entry->parent_ip = parent_ip;
perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0,
perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
1, &regs, head, NULL);
#undef ENTRY_SIZE

349
kernel/trace/trace_events.c
View file

@ -15,7 +15,7 @@
#include <linux/kthread.h>
#include <linux/tracefs.h>
#include <linux/uaccess.h>
#include <linux/bsearch.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/ctype.h>
#include <linux/sort.h>
@ -204,6 +204,24 @@ static void trace_destroy_fields(struct trace_event_call *call)
}
}
/*
* run-time version of trace_event_get_offsets_<call>() that returns the last
* accessible offset of trace fields excluding __dynamic_array bytes
*/
int trace_event_get_offsets(struct trace_event_call *call)
{
struct ftrace_event_field *tail;
struct list_head *head;
head = trace_get_fields(call);
/*
* head->next points to the last field with the largest offset,
* since it was added last by trace_define_field()
*/
tail = list_first_entry(head, struct ftrace_event_field, link);
return tail->offset + tail->size;
}
int trace_event_raw_init(struct trace_event_call *call)
{
int id;
@ -363,6 +381,7 @@ static int __ftrace_event_enable_disable(struct trace_event_file *file,
{
struct trace_event_call *call = file->event_call;
struct trace_array *tr = file->tr;
unsigned long file_flags = file->flags;
int ret = 0;
int disable;
@ -445,6 +464,15 @@ static int __ftrace_event_enable_disable(struct trace_event_file *file,
break;
}
/* Enable or disable use of trace_buffered_event */
if ((file_flags & EVENT_FILE_FL_SOFT_DISABLED) !=
(file->flags & EVENT_FILE_FL_SOFT_DISABLED)) {
if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
trace_buffered_event_enable();
else
trace_buffered_event_disable();
}
return ret;
}
@ -471,24 +499,26 @@ static void ftrace_clear_events(struct trace_array *tr)
mutex_unlock(&event_mutex);
}
static int cmp_pid(const void *key, const void *elt)
{
const pid_t *search_pid = key;
const pid_t *pid = elt;
/* Shouldn't this be in a header? */
extern int pid_max;
if (*search_pid == *pid)
return 0;
if (*search_pid < *pid)
return -1;
return 1;
/* Returns true if found in filter */
static bool
find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
{
/*
* If pid_max changed after filtered_pids was created, we
* by default ignore all pids greater than the previous pid_max.
*/
if (search_pid >= filtered_pids->pid_max)
return false;
return test_bit(search_pid, filtered_pids->pids);
}
static bool
check_ignore_pid(struct trace_pid_list *filtered_pids, struct task_struct *task)
ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
{
pid_t search_pid;
pid_t *pid;
/*
* Return false, because if filtered_pids does not exist,
* all pids are good to trace.
@ -496,15 +526,68 @@ check_ignore_pid(struct trace_pid_list *filtered_pids, struct task_struct *task)
if (!filtered_pids)
return false;
search_pid = task->pid;
return !find_filtered_pid(filtered_pids, task->pid);
}
pid = bsearch(&search_pid, filtered_pids->pids,
filtered_pids->nr_pids, sizeof(pid_t),
cmp_pid);
if (!pid)
return true;
static void filter_add_remove_task(struct trace_pid_list *pid_list,
struct task_struct *self,
struct task_struct *task)
{
if (!pid_list)
return;
return false;
/* For forks, we only add if the forking task is listed */
if (self) {
if (!find_filtered_pid(pid_list, self->pid))
return;
}
/* Sorry, but we don't support pid_max changing after setting */
if (task->pid >= pid_list->pid_max)
return;
/* "self" is set for forks, and NULL for exits */
if (self)
set_bit(task->pid, pid_list->pids);
else
clear_bit(task->pid, pid_list->pids);
}
static void
event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = data;
pid_list = rcu_dereference_sched(tr->filtered_pids);
filter_add_remove_task(pid_list, NULL, task);
}
static void
event_filter_pid_sched_process_fork(void *data,
struct task_struct *self,
struct task_struct *task)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = data;
pid_list = rcu_dereference_sched(tr->filtered_pids);
filter_add_remove_task(pid_list, self, task);
}
void trace_event_follow_fork(struct trace_array *tr, bool enable)
{
if (enable) {
register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
tr, INT_MIN);
register_trace_prio_sched_process_exit(event_filter_pid_sched_process_exit,
tr, INT_MAX);
} else {
unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
tr);
unregister_trace_sched_process_exit(event_filter_pid_sched_process_exit,
tr);
}
}
static void
@ -517,8 +600,8 @@ event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
pid_list = rcu_dereference_sched(tr->filtered_pids);
this_cpu_write(tr->trace_buffer.data->ignore_pid,
check_ignore_pid(pid_list, prev) &&
check_ignore_pid(pid_list, next));
ignore_this_task(pid_list, prev) &&
ignore_this_task(pid_list, next));
}
static void
@ -531,7 +614,7 @@ event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
pid_list = rcu_dereference_sched(tr->filtered_pids);
this_cpu_write(tr->trace_buffer.data->ignore_pid,
check_ignore_pid(pid_list, next));
ignore_this_task(pid_list, next));
}
static void
@ -547,7 +630,7 @@ event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
pid_list = rcu_dereference_sched(tr->filtered_pids);
this_cpu_write(tr->trace_buffer.data->ignore_pid,
check_ignore_pid(pid_list, task));
ignore_this_task(pid_list, task));
}
static void
@ -564,7 +647,7 @@ event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
/* Set tracing if current is enabled */
this_cpu_write(tr->trace_buffer.data->ignore_pid,
check_ignore_pid(pid_list, current));
ignore_this_task(pid_list, current));
}
static void __ftrace_clear_event_pids(struct trace_array *tr)
@ -602,7 +685,7 @@ static void __ftrace_clear_event_pids(struct trace_array *tr)
/* Wait till all users are no longer using pid filtering */
synchronize_sched();
free_pages((unsigned long)pid_list->pids, pid_list->order);
vfree(pid_list->pids);
kfree(pid_list);
}
@ -946,11 +1029,32 @@ static void t_stop(struct seq_file *m, void *p)
mutex_unlock(&event_mutex);
}
static void *
p_next(struct seq_file *m, void *v, loff_t *pos)
{
struct trace_array *tr = m->private;
struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);
unsigned long pid = (unsigned long)v;
(*pos)++;
/* pid already is +1 of the actual prevous bit */
pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
/* Return pid + 1 to allow zero to be represented */
if (pid < pid_list->pid_max)
return (void *)(pid + 1);
return NULL;
}
static void *p_start(struct seq_file *m, loff_t *pos)
__acquires(RCU)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = m->private;
unsigned long pid;
loff_t l = 0;
/*
* Grab the mutex, to keep calls to p_next() having the same
@ -963,10 +1067,18 @@ static void *p_start(struct seq_file *m, loff_t *pos)
pid_list = rcu_dereference_sched(tr->filtered_pids);
if (!pid_list || *pos >= pid_list->nr_pids)
if (!pid_list)
return NULL;
return (void *)&pid_list->pids[*pos];
pid = find_first_bit(pid_list->pids, pid_list->pid_max);
if (pid >= pid_list->pid_max)
return NULL;
/* Return pid + 1 so that zero can be the exit value */
for (pid++; pid && l < *pos;
pid = (unsigned long)p_next(m, (void *)pid, &l))
;
return (void *)pid;
}
static void p_stop(struct seq_file *m, void *p)
@ -976,25 +1088,11 @@ static void p_stop(struct seq_file *m, void *p)
mutex_unlock(&event_mutex);
}
static void *
p_next(struct seq_file *m, void *v, loff_t *pos)
{
struct trace_array *tr = m->private;
struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);
(*pos)++;
if (*pos >= pid_list->nr_pids)
return NULL;
return (void *)&pid_list->pids[*pos];
}
static int p_show(struct seq_file *m, void *v)
{
pid_t *pid = v;
unsigned long pid = (unsigned long)v - 1;
seq_printf(m, "%d\n", *pid);
seq_printf(m, "%lu\n", pid);
return 0;
}
@ -1543,11 +1641,6 @@ show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
return r;
}
static int max_pids(struct trace_pid_list *pid_list)
{
return (PAGE_SIZE << pid_list->order) / sizeof(pid_t);
}
static void ignore_task_cpu(void *data)
{
struct trace_array *tr = data;
@ -1561,7 +1654,7 @@ static void ignore_task_cpu(void *data)
mutex_is_locked(&event_mutex));
this_cpu_write(tr->trace_buffer.data->ignore_pid,
check_ignore_pid(pid_list, current));
ignore_this_task(pid_list, current));
}
static ssize_t
@ -1571,7 +1664,7 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
struct seq_file *m = filp->private_data;
struct trace_array *tr = m->private;
struct trace_pid_list *filtered_pids = NULL;
struct trace_pid_list *pid_list = NULL;
struct trace_pid_list *pid_list;
struct trace_event_file *file;
struct trace_parser parser;
unsigned long val;
@ -1579,7 +1672,7 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
ssize_t read = 0;
ssize_t ret = 0;
pid_t pid;
int i;
int nr_pids = 0;
if (!cnt)
return 0;
@ -1592,10 +1685,43 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
return -ENOMEM;
mutex_lock(&event_mutex);
filtered_pids = rcu_dereference_protected(tr->filtered_pids,
lockdep_is_held(&event_mutex));
/*
* Load as many pids into the array before doing a
* swap from the tr->filtered_pids to the new list.
* Always recreate a new array. The write is an all or nothing
* operation. Always create a new array when adding new pids by
* the user. If the operation fails, then the current list is
* not modified.
*/
pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
if (!pid_list) {
read = -ENOMEM;
goto out;
}
pid_list->pid_max = READ_ONCE(pid_max);
/* Only truncating will shrink pid_max */
if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
pid_list->pid_max = filtered_pids->pid_max;
pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
if (!pid_list->pids) {
kfree(pid_list);
read = -ENOMEM;
goto out;
}
if (filtered_pids) {
/* copy the current bits to the new max */
pid = find_first_bit(filtered_pids->pids,
filtered_pids->pid_max);
while (pid < filtered_pids->pid_max) {
set_bit(pid, pid_list->pids);
pid = find_next_bit(filtered_pids->pids,
filtered_pids->pid_max,
pid + 1);
nr_pids++;
}
}
while (cnt > 0) {
this_pos = 0;
@ -1613,92 +1739,35 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
ret = -EINVAL;
if (kstrtoul(parser.buffer, 0, &val))
break;
if (val > INT_MAX)
if (val >= pid_list->pid_max)
break;
pid = (pid_t)val;
ret = -ENOMEM;
if (!pid_list) {
pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
if (!pid_list)
break;
set_bit(pid, pid_list->pids);
nr_pids++;
filtered_pids = rcu_dereference_protected(tr->filtered_pids,
lockdep_is_held(&event_mutex));
if (filtered_pids)
pid_list->order = filtered_pids->order;
else
pid_list->order = 0;
pid_list->pids = (void *)__get_free_pages(GFP_KERNEL,
pid_list->order);
if (!pid_list->pids)
break;
if (filtered_pids) {
pid_list->nr_pids = filtered_pids->nr_pids;
memcpy(pid_list->pids, filtered_pids->pids,
pid_list->nr_pids * sizeof(pid_t));
} else
pid_list->nr_pids = 0;
}
if (pid_list->nr_pids >= max_pids(pid_list)) {
pid_t *pid_page;
pid_page = (void *)__get_free_pages(GFP_KERNEL,
pid_list->order + 1);
if (!pid_page)
break;
memcpy(pid_page, pid_list->pids,
pid_list->nr_pids * sizeof(pid_t));
free_pages((unsigned long)pid_list->pids, pid_list->order);
pid_list->order++;
pid_list->pids = pid_page;
}
pid_list->pids[pid_list->nr_pids++] = pid;
trace_parser_clear(&parser);
ret = 0;
}
trace_parser_put(&parser);
if (ret < 0) {
if (pid_list)
free_pages((unsigned long)pid_list->pids, pid_list->order);
vfree(pid_list->pids);
kfree(pid_list);
mutex_unlock(&event_mutex);
return ret;
read = ret;
goto out;
}
if (!pid_list) {
mutex_unlock(&event_mutex);
return ret;
if (!nr_pids) {
/* Cleared the list of pids */
vfree(pid_list->pids);
kfree(pid_list);
read = ret;
if (!filtered_pids)
goto out;
pid_list = NULL;
}
sort(pid_list->pids, pid_list->nr_pids, sizeof(pid_t), cmp_pid, NULL);
/* Remove duplicates */
for (i = 1; i < pid_list->nr_pids; i++) {
int start = i;
while (i < pid_list->nr_pids &&
pid_list->pids[i - 1] == pid_list->pids[i])
i++;
if (start != i) {
if (i < pid_list->nr_pids) {
memmove(&pid_list->pids[start], &pid_list->pids[i],
(pid_list->nr_pids - i) * sizeof(pid_t));
pid_list->nr_pids -= i - start;
i = start;
} else
pid_list->nr_pids = start;
}
}
rcu_assign_pointer(tr->filtered_pids, pid_list);
list_for_each_entry(file, &tr->events, list) {
@ -1708,7 +1777,7 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
if (filtered_pids) {
synchronize_sched();
free_pages((unsigned long)filtered_pids->pids, filtered_pids->order);
vfree(filtered_pids->pids);
kfree(filtered_pids);
} else {
/*
@ -1745,10 +1814,12 @@ ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
*/
on_each_cpu(ignore_task_cpu, tr, 1);
out:
mutex_unlock(&event_mutex);
ret = read;
*ppos += read;
if (read > 0)
*ppos += read;
return ret;
}
@ -2103,6 +2174,10 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
trace_create_file("trigger", 0644, file->dir, file,
&event_trigger_fops);
#ifdef CONFIG_HIST_TRIGGERS
trace_create_file("hist", 0444, file->dir, file,
&event_hist_fops);
#endif
trace_create_file("format", 0444, file->dir, call,
&ftrace_event_format_fops);
@ -3350,7 +3425,7 @@ static __init void event_trace_self_tests(void)
static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
static struct trace_array *event_tr;
static struct trace_event_file event_trace_file __initdata;
static void __init
function_test_events_call(unsigned long ip, unsigned long parent_ip,
@ -3374,17 +3449,17 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip,
local_save_flags(flags);
event = trace_current_buffer_lock_reserve(&buffer,
TRACE_FN, sizeof(*entry),
flags, pc);
event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
TRACE_FN, sizeof(*entry),
flags, pc);
if (!event)
goto out;
entry = ring_buffer_event_data(event);
entry->ip = ip;
entry->parent_ip = parent_ip;
trace_buffer_unlock_commit(event_tr, buffer, event, flags, pc);
event_trigger_unlock_commit(&event_trace_file, buffer, event,
entry, flags, pc);
out:
atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
preempt_enable_notrace();
@ -3399,9 +3474,11 @@ static struct ftrace_ops trace_ops __initdata =
static __init void event_trace_self_test_with_function(void)
{
int ret;
event_tr = top_trace_array();
if (WARN_ON(!event_tr))
event_trace_file.tr = top_trace_array();
if (WARN_ON(!event_trace_file.tr))
return;
ret = register_ftrace_function(&trace_ops);
if (WARN_ON(ret < 0)) {
pr_info("Failed to enable function tracer for event tests\n");

77
kernel/trace/trace_events_filter.c
View file

@ -689,10 +689,7 @@ static void append_filter_err(struct filter_parse_state *ps,
static inline struct event_filter *event_filter(struct trace_event_file *file)
{
if (file->event_call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
return file->event_call->filter;
else
return file->filter;
return file->filter;
}
/* caller must hold event_mutex */
@ -826,12 +823,12 @@ static void __free_preds(struct event_filter *filter)
static void filter_disable(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
unsigned long old_flags = file->flags;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
call->flags &= ~TRACE_EVENT_FL_FILTERED;
else
file->flags &= ~EVENT_FILE_FL_FILTERED;
file->flags &= ~EVENT_FILE_FL_FILTERED;
if (old_flags != file->flags)
trace_buffered_event_disable();
}
static void __free_filter(struct event_filter *filter)
@ -883,13 +880,8 @@ static int __alloc_preds(struct event_filter *filter, int n_preds)
static inline void __remove_filter(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
filter_disable(file);
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
remove_filter_string(call->filter);
else
remove_filter_string(file->filter);
remove_filter_string(file->filter);
}
static void filter_free_subsystem_preds(struct trace_subsystem_dir *dir,
@ -906,15 +898,8 @@ static void filter_free_subsystem_preds(struct trace_subsystem_dir *dir,
static inline void __free_subsystem_filter(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER) {
__free_filter(call->filter);
call->filter = NULL;
} else {
__free_filter(file->filter);
file->filter = NULL;
}
__free_filter(file->filter);
file->filter = NULL;
}
static void filter_free_subsystem_filters(struct trace_subsystem_dir *dir,
@ -1718,69 +1703,43 @@ fail:
static inline void event_set_filtered_flag(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
unsigned long old_flags = file->flags;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
call->flags |= TRACE_EVENT_FL_FILTERED;
else
file->flags |= EVENT_FILE_FL_FILTERED;
file->flags |= EVENT_FILE_FL_FILTERED;
if (old_flags != file->flags)
trace_buffered_event_enable();
}
static inline void event_set_filter(struct trace_event_file *file,
struct event_filter *filter)
{
struct trace_event_call *call = file->event_call;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
rcu_assign_pointer(call->filter, filter);
else
rcu_assign_pointer(file->filter, filter);
rcu_assign_pointer(file->filter, filter);
}
static inline void event_clear_filter(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
RCU_INIT_POINTER(call->filter, NULL);
else
RCU_INIT_POINTER(file->filter, NULL);
RCU_INIT_POINTER(file->filter, NULL);
}
static inline void
event_set_no_set_filter_flag(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
call->flags |= TRACE_EVENT_FL_NO_SET_FILTER;
else
file->flags |= EVENT_FILE_FL_NO_SET_FILTER;
file->flags |= EVENT_FILE_FL_NO_SET_FILTER;
}
static inline void
event_clear_no_set_filter_flag(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
if (call->flags & TRACE_EVENT_FL_USE_CALL_FILTER)
call->flags &= ~TRACE_EVENT_FL_NO_SET_FILTER;
else
file->flags &= ~EVENT_FILE_FL_NO_SET_FILTER;
file->flags &= ~EVENT_FILE_FL_NO_SET_FILTER;
}
static inline bool
event_no_set_filter_flag(struct trace_event_file *file)
{
struct trace_event_call *call = file->event_call;
if (file->flags & EVENT_FILE_FL_NO_SET_FILTER)
return true;
if ((call->flags & TRACE_EVENT_FL_USE_CALL_FILTER) &&
(call->flags & TRACE_EVENT_FL_NO_SET_FILTER))
return true;
return false;
}

1755
kernel/trace/trace_events_hist.c Normal file
View file

File diff suppressed because it is too large Load diff

215
kernel/trace/trace_events_trigger.c
View file

@ -347,7 +347,7 @@ __init int register_event_command(struct event_command *cmd)
* Currently we only unregister event commands from __init, so mark
* this __init too.
*/
static __init int unregister_event_command(struct event_command *cmd)
__init int unregister_event_command(struct event_command *cmd)
{
struct event_command *p, *n;
int ret = -ENODEV;
@ -641,6 +641,7 @@ event_trigger_callback(struct event_command *cmd_ops,
trigger_data->ops = trigger_ops;
trigger_data->cmd_ops = cmd_ops;
INIT_LIST_HEAD(&trigger_data->list);
INIT_LIST_HEAD(&trigger_data->named_list);
if (glob[0] == '!') {
cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
@ -764,6 +765,148 @@ int set_trigger_filter(char *filter_str,
return ret;
}
static LIST_HEAD(named_triggers);
/**
* find_named_trigger - Find the common named trigger associated with @name
* @name: The name of the set of named triggers to find the common data for
*
* Named triggers are sets of triggers that share a common set of
* trigger data. The first named trigger registered with a given name
* owns the common trigger data that the others subsequently
* registered with the same name will reference. This function
* returns the common trigger data associated with that first
* registered instance.
*
* Return: the common trigger data for the given named trigger on
* success, NULL otherwise.
*/
struct event_trigger_data *find_named_trigger(const char *name)
{
struct event_trigger_data *data;
if (!name)
return NULL;
list_for_each_entry(data, &named_triggers, named_list) {
if (data->named_data)
continue;
if (strcmp(data->name, name) == 0)
return data;
}
return NULL;
}
/**
* is_named_trigger - determine if a given trigger is a named trigger
* @test: The trigger data to test
*
* Return: true if 'test' is a named trigger, false otherwise.
*/
bool is_named_trigger(struct event_trigger_data *test)
{
struct event_trigger_data *data;
list_for_each_entry(data, &named_triggers, named_list) {
if (test == data)
return true;
}
return false;
}
/**
* save_named_trigger - save the trigger in the named trigger list
* @name: The name of the named trigger set
* @data: The trigger data to save
*
* Return: 0 if successful, negative error otherwise.
*/
int save_named_trigger(const char *name, struct event_trigger_data *data)
{
data->name = kstrdup(name, GFP_KERNEL);
if (!data->name)
return -ENOMEM;
list_add(&data->named_list, &named_triggers);
return 0;
}
/**
* del_named_trigger - delete a trigger from the named trigger list
* @data: The trigger data to delete
*/
void del_named_trigger(struct event_trigger_data *data)
{
kfree(data->name);
data->name = NULL;
list_del(&data->named_list);
}
static void __pause_named_trigger(struct event_trigger_data *data, bool pause)
{
struct event_trigger_data *test;
list_for_each_entry(test, &named_triggers, named_list) {
if (strcmp(test->name, data->name) == 0) {
if (pause) {
test->paused_tmp = test->paused;
test->paused = true;
} else {
test->paused = test->paused_tmp;
}
}
}
}
/**
* pause_named_trigger - Pause all named triggers with the same name
* @data: The trigger data of a named trigger to pause
*
* Pauses a named trigger along with all other triggers having the
* same name. Because named triggers share a common set of data,
* pausing only one is meaningless, so pausing one named trigger needs
* to pause all triggers with the same name.
*/
void pause_named_trigger(struct event_trigger_data *data)
{
__pause_named_trigger(data, true);
}
/**
* unpause_named_trigger - Un-pause all named triggers with the same name
* @data: The trigger data of a named trigger to unpause
*
* Un-pauses a named trigger along with all other triggers having the
* same name. Because named triggers share a common set of data,
* unpausing only one is meaningless, so unpausing one named trigger
* needs to unpause all triggers with the same name.
*/
void unpause_named_trigger(struct event_trigger_data *data)
{
__pause_named_trigger(data, false);
}
/**
* set_named_trigger_data - Associate common named trigger data
* @data: The trigger data of a named trigger to unpause
*
* Named triggers are sets of triggers that share a common set of
* trigger data. The first named trigger registered with a given name
* owns the common trigger data that the others subsequently
* registered with the same name will reference. This function
* associates the common trigger data from the first trigger with the
* given trigger.
*/
void set_named_trigger_data(struct event_trigger_data *data,
struct event_trigger_data *named_data)
{
data->named_data = named_data;
}
static void
traceon_trigger(struct event_trigger_data *data, void *rec)
{
@ -1062,15 +1205,6 @@ static __init void unregister_trigger_traceon_traceoff_cmds(void)
unregister_event_command(&trigger_traceoff_cmd);
}
/* Avoid typos */
#define ENABLE_EVENT_STR "enable_event"
#define DISABLE_EVENT_STR "disable_event"
struct enable_trigger_data {
struct trace_event_file *file;
bool enable;
};
static void
event_enable_trigger(struct event_trigger_data *data, void *rec)
{
@ -1100,14 +1234,16 @@ event_enable_count_trigger(struct event_trigger_data *data, void *rec)
event_enable_trigger(data, rec);
}
static int
event_enable_trigger_print(struct seq_file *m, struct event_trigger_ops *ops,
struct event_trigger_data *data)
int event_enable_trigger_print(struct seq_file *m,
struct event_trigger_ops *ops,
struct event_trigger_data *data)
{
struct enable_trigger_data *enable_data = data->private_data;
seq_printf(m, "%s:%s:%s",
enable_data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
enable_data->hist ?
(enable_data->enable ? ENABLE_HIST_STR : DISABLE_HIST_STR) :
(enable_data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR),
enable_data->file->event_call->class->system,
trace_event_name(enable_data->file->event_call));
@ -1124,9 +1260,8 @@ event_enable_trigger_print(struct seq_file *m, struct event_trigger_ops *ops,
return 0;
}
static void
event_enable_trigger_free(struct event_trigger_ops *ops,
struct event_trigger_data *data)
void event_enable_trigger_free(struct event_trigger_ops *ops,
struct event_trigger_data *data)
{
struct enable_trigger_data *enable_data = data->private_data;
@ -1171,10 +1306,9 @@ static struct event_trigger_ops event_disable_count_trigger_ops = {
.free = event_enable_trigger_free,
};
static int
event_enable_trigger_func(struct event_command *cmd_ops,
struct trace_event_file *file,
char *glob, char *cmd, char *param)
int event_enable_trigger_func(struct event_command *cmd_ops,
struct trace_event_file *file,
char *glob, char *cmd, char *param)
{
struct trace_event_file *event_enable_file;
struct enable_trigger_data *enable_data;
@ -1183,6 +1317,7 @@ event_enable_trigger_func(struct event_command *cmd_ops,
struct trace_array *tr = file->tr;
const char *system;
const char *event;
bool hist = false;
char *trigger;
char *number;
bool enable;
@ -1207,8 +1342,15 @@ event_enable_trigger_func(struct event_command *cmd_ops,
if (!event_enable_file)
goto out;
enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
#ifdef CONFIG_HIST_TRIGGERS
hist = ((strcmp(cmd, ENABLE_HIST_STR) == 0) ||
(strcmp(cmd, DISABLE_HIST_STR) == 0));
enable = ((strcmp(cmd, ENABLE_EVENT_STR) == 0) ||
(strcmp(cmd, ENABLE_HIST_STR) == 0));
#else
enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
#endif
trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
ret = -ENOMEM;
@ -1228,6 +1370,7 @@ event_enable_trigger_func(struct event_command *cmd_ops,
INIT_LIST_HEAD(&trigger_data->list);
RCU_INIT_POINTER(trigger_data->filter, NULL);
enable_data->hist = hist;
enable_data->enable = enable;
enable_data->file = event_enable_file;
trigger_data->private_data = enable_data;
@ -1305,10 +1448,10 @@ event_enable_trigger_func(struct event_command *cmd_ops,
goto out;
}
static int event_enable_register_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *data,
struct trace_event_file *file)
int event_enable_register_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *data,
struct trace_event_file *file)
{
struct enable_trigger_data *enable_data = data->private_data;
struct enable_trigger_data *test_enable_data;
@ -1318,6 +1461,8 @@ static int event_enable_register_trigger(char *glob,
list_for_each_entry_rcu(test, &file->triggers, list) {
test_enable_data = test->private_data;
if (test_enable_data &&
(test->cmd_ops->trigger_type ==
data->cmd_ops->trigger_type) &&
(test_enable_data->file == enable_data->file)) {
ret = -EEXIST;
goto out;
@ -1343,10 +1488,10 @@ out:
return ret;
}
static void event_enable_unregister_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *test,
struct trace_event_file *file)
void event_enable_unregister_trigger(char *glob,
struct event_trigger_ops *ops,
struct event_trigger_data *test,
struct trace_event_file *file)
{
struct enable_trigger_data *test_enable_data = test->private_data;
struct enable_trigger_data *enable_data;
@ -1356,6 +1501,8 @@ static void event_enable_unregister_trigger(char *glob,
list_for_each_entry_rcu(data, &file->triggers, list) {
enable_data = data->private_data;
if (enable_data &&
(data->cmd_ops->trigger_type ==
test->cmd_ops->trigger_type) &&
(enable_data->file == test_enable_data->file)) {
unregistered = true;
list_del_rcu(&data->list);
@ -1375,8 +1522,12 @@ event_enable_get_trigger_ops(char *cmd, char *param)
struct event_trigger_ops *ops;
bool enable;
#ifdef CONFIG_HIST_TRIGGERS
enable = ((strcmp(cmd, ENABLE_EVENT_STR) == 0) ||
(strcmp(cmd, ENABLE_HIST_STR) == 0));
#else
enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
#endif
if (enable)
ops = param ? &event_enable_count_trigger_ops :
&event_enable_trigger_ops;
@ -1447,6 +1598,8 @@ __init int register_trigger_cmds(void)
register_trigger_snapshot_cmd();
register_trigger_stacktrace_cmd();
register_trigger_enable_disable_cmds();
register_trigger_hist_enable_disable_cmds();
register_trigger_hist_cmd();
return 0;
}

10
kernel/trace/trace_kprobe.c
View file

@ -1149,14 +1149,15 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
size = ALIGN(__size + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
entry = perf_trace_buf_alloc(size, NULL, &rctx);
if (!entry)
return;
entry->ip = (unsigned long)tk->rp.kp.addr;
memset(&entry[1], 0, dsize);
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
head, NULL);
}
NOKPROBE_SYMBOL(kprobe_perf_func);
@ -1184,14 +1185,15 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
size = ALIGN(__size + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
entry = perf_trace_buf_alloc(size, NULL, &rctx);
if (!entry)
return;
entry->func = (unsigned long)tk->rp.kp.addr;
entry->ret_ip = (unsigned long)ri->ret_addr;
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
head, NULL);
}
NOKPROBE_SYMBOL(kretprobe_perf_func);
#endif /* CONFIG_PERF_EVENTS */

13
kernel/trace/trace_syscalls.c
View file

@ -587,15 +587,16 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
size = ALIGN(size + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
sys_data->enter_event->event.type, NULL, &rctx);
rec = perf_trace_buf_alloc(size, NULL, &rctx);
if (!rec)
return;
rec->nr = syscall_nr;
syscall_get_arguments(current, regs, 0, sys_data->nb_args,
(unsigned long *)&rec->args);
perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
perf_trace_buf_submit(rec, size, rctx,
sys_data->enter_event->event.type, 1, regs,
head, NULL);
}
static int perf_sysenter_enable(struct trace_event_call *call)
@ -660,14 +661,14 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
sys_data->exit_event->event.type, NULL, &rctx);
rec = perf_trace_buf_alloc(size, NULL, &rctx);
if (!rec)
return;
rec->nr = syscall_nr;
rec->ret = syscall_get_return_value(current, regs);
perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
1, regs, head, NULL);
}
static int perf_sysexit_enable(struct trace_event_call *call)

5
kernel/trace/trace_uprobe.c
View file

@ -1131,7 +1131,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
if (hlist_empty(head))
goto out;
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
entry = perf_trace_buf_alloc(size, NULL, &rctx);
if (!entry)
goto out;
@ -1152,7 +1152,8 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
memset(data + len, 0, size - esize - len);
}
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
head, NULL);
out:
preempt_enable();
}

1062
kernel/trace/tracing_map.c Normal file
View file

File diff suppressed because it is too large Load diff

283
kernel/trace/tracing_map.h Normal file
View file

@ -0,0 +1,283 @@
#ifndef __TRACING_MAP_H
#define __TRACING_MAP_H
#define TRACING_MAP_BITS_DEFAULT 11
#define TRACING_MAP_BITS_MAX 17
#define TRACING_MAP_BITS_MIN 7
#define TRACING_MAP_KEYS_MAX 2
#define TRACING_MAP_VALS_MAX 3
#define TRACING_MAP_FIELDS_MAX (TRACING_MAP_KEYS_MAX + \
TRACING_MAP_VALS_MAX)
#define TRACING_MAP_SORT_KEYS_MAX 2
typedef int (*tracing_map_cmp_fn_t) (void *val_a, void *val_b);
/*
* This is an overview of the tracing_map data structures and how they
* relate to the tracing_map API. The details of the algorithms
* aren't discussed here - this is just a general overview of the data
* structures and how they interact with the API.
*
* The central data structure of the tracing_map is an initially
* zeroed array of struct tracing_map_entry (stored in the map field
* of struct tracing_map). tracing_map_entry is a very simple data
* structure containing only two fields: a 32-bit unsigned 'key'
* variable and a pointer named 'val'. This array of struct
* tracing_map_entry is essentially a hash table which will be
* modified by a single function, tracing_map_insert(), but which can
* be traversed and read by a user at any time (though the user does
* this indirectly via an array of tracing_map_sort_entry - see the
* explanation of that data structure in the discussion of the
* sorting-related data structures below).
*
* The central function of the tracing_map API is
* tracing_map_insert(). tracing_map_insert() hashes the
* arbitrarily-sized key passed into it into a 32-bit unsigned key.
* It then uses this key, truncated to the array size, as an index
* into the array of tracing_map_entries. If the value of the 'key'
* field of the tracing_map_entry found at that location is 0, then
* that entry is considered to be free and can be claimed, by
* replacing the 0 in the 'key' field of the tracing_map_entry with
* the new 32-bit hashed key. Once claimed, that tracing_map_entry's
* 'val' field is then used to store a unique element which will be
* forever associated with that 32-bit hashed key in the
* tracing_map_entry.
*
* That unique element now in the tracing_map_entry's 'val' field is
* an instance of tracing_map_elt, where 'elt' in the latter part of
* that variable name is short for 'element'. The purpose of a
* tracing_map_elt is to hold values specific to the particular
* 32-bit hashed key it's assocated with. Things such as the unique
* set of aggregated sums associated with the 32-bit hashed key, along
* with a copy of the full key associated with the entry, and which
* was used to produce the 32-bit hashed key.
*
* When tracing_map_create() is called to create the tracing map, the
* user specifies (indirectly via the map_bits param, the details are
* unimportant for this discussion) the maximum number of elements
* that the map can hold (stored in the max_elts field of struct
* tracing_map). This is the maximum possible number of
* tracing_map_entries in the tracing_map_entry array which can be
* 'claimed' as described in the above discussion, and therefore is
* also the maximum number of tracing_map_elts that can be associated
* with the tracing_map_entry array in the tracing_map. Because of
* the way the insertion algorithm works, the size of the allocated
* tracing_map_entry array is always twice the maximum number of
* elements (2 * max_elts). This value is stored in the map_size
* field of struct tracing_map.
*
* Because tracing_map_insert() needs to work from any context,
* including from within the memory allocation functions themselves,
* both the tracing_map_entry array and a pool of max_elts
* tracing_map_elts are pre-allocated before any call is made to
* tracing_map_insert().
*
* The tracing_map_entry array is allocated as a single block by
* tracing_map_create().
*
* Because the tracing_map_elts are much larger objects and can't
* generally be allocated together as a single large array without
* failure, they're allocated individually, by tracing_map_init().
*
* The pool of tracing_map_elts are allocated by tracing_map_init()
* rather than by tracing_map_create() because at the time
* tracing_map_create() is called, there isn't enough information to
* create the tracing_map_elts. Specifically,the user first needs to
* tell the tracing_map implementation how many fields the
* tracing_map_elts contain, and which types of fields they are (key
* or sum). The user does this via the tracing_map_add_sum_field()
* and tracing_map_add_key_field() functions, following which the user
* calls tracing_map_init() to finish up the tracing map setup. The
* array holding the pointers which make up the pre-allocated pool of
* tracing_map_elts is allocated as a single block and is stored in
* the elts field of struct tracing_map.
*
* There is also a set of structures used for sorting that might
* benefit from some minimal explanation.
*
* struct tracing_map_sort_key is used to drive the sort at any given
* time. By 'any given time' we mean that a different
* tracing_map_sort_key will be used at different times depending on
* whether the sort currently being performed is a primary or a
* secondary sort.
*
* The sort key is very simple, consisting of the field index of the
* tracing_map_elt field to sort on (which the user saved when adding
* the field), and whether the sort should be done in an ascending or
* descending order.
*
* For the convenience of the sorting code, a tracing_map_sort_entry
* is created for each tracing_map_elt, again individually allocated
* to avoid failures that might be expected if allocated as a single
* large array of struct tracing_map_sort_entry.
* tracing_map_sort_entry instances are the objects expected by the
* various internal sorting functions, and are also what the user
* ultimately receives after calling tracing_map_sort_entries().
* Because it doesn't make sense for users to access an unordered and
* sparsely populated tracing_map directly, the
* tracing_map_sort_entries() function is provided so that users can
* retrieve a sorted list of all existing elements. In addition to
* the associated tracing_map_elt 'elt' field contained within the
* tracing_map_sort_entry, which is the object of interest to the
* user, tracing_map_sort_entry objects contain a number of additional
* fields which are used for caching and internal purposes and can
* safely be ignored.
*/
struct tracing_map_field {
tracing_map_cmp_fn_t cmp_fn;
union {
atomic64_t sum;
unsigned int offset;
};
};
struct tracing_map_elt {
struct tracing_map *map;
struct tracing_map_field *fields;
void *key;
void *private_data;
};
struct tracing_map_entry {
u32 key;
struct tracing_map_elt *val;
};
struct tracing_map_sort_key {
unsigned int field_idx;
bool descending;
};
struct tracing_map_sort_entry {
void *key;
struct tracing_map_elt *elt;
bool elt_copied;
bool dup;
};
struct tracing_map_array {
unsigned int entries_per_page;
unsigned int entry_size_shift;
unsigned int entry_shift;
unsigned int entry_mask;
unsigned int n_pages;
void **pages;
};
#define TRACING_MAP_ARRAY_ELT(array, idx) \
(array->pages[idx >> array->entry_shift] + \
((idx & array->entry_mask) << array->entry_size_shift))
#define TRACING_MAP_ENTRY(array, idx) \
((struct tracing_map_entry *)TRACING_MAP_ARRAY_ELT(array, idx))
#define TRACING_MAP_ELT(array, idx) \
((struct tracing_map_elt **)TRACING_MAP_ARRAY_ELT(array, idx))
struct tracing_map {
unsigned int key_size;
unsigned int map_bits;
unsigned int map_size;
unsigned int max_elts;
atomic_t next_elt;
struct tracing_map_array *elts;
struct tracing_map_array *map;
const struct tracing_map_ops *ops;
void *private_data;
struct tracing_map_field fields[TRACING_MAP_FIELDS_MAX];
unsigned int n_fields;
int key_idx[TRACING_MAP_KEYS_MAX];
unsigned int n_keys;
struct tracing_map_sort_key sort_key;
atomic64_t hits;
atomic64_t drops;
};
/**
* struct tracing_map_ops - callbacks for tracing_map
*
* The methods in this structure define callback functions for various
* operations on a tracing_map or objects related to a tracing_map.
*
* For a detailed description of tracing_map_elt objects please see
* the overview of tracing_map data structures at the beginning of
* this file.
*
* All the methods below are optional.
*
* @elt_alloc: When a tracing_map_elt is allocated, this function, if
* defined, will be called and gives clients the opportunity to
* allocate additional data and attach it to the element
* (tracing_map_elt->private_data is meant for that purpose).
* Element allocation occurs before tracing begins, when the
* tracing_map_init() call is made by client code.
*
* @elt_copy: At certain points in the lifetime of an element, it may
* need to be copied. The copy should include a copy of the
* client-allocated data, which can be copied into the 'to'
* element from the 'from' element.
*
* @elt_free: When a tracing_map_elt is freed, this function is called
* and allows client-allocated per-element data to be freed.
*
* @elt_clear: This callback allows per-element client-defined data to
* be cleared, if applicable.
*
* @elt_init: This callback allows per-element client-defined data to
* be initialized when used i.e. when the element is actually
* claimed by tracing_map_insert() in the context of the map
* insertion.
*/
struct tracing_map_ops {
int (*elt_alloc)(struct tracing_map_elt *elt);
void (*elt_copy)(struct tracing_map_elt *to,
struct tracing_map_elt *from);
void (*elt_free)(struct tracing_map_elt *elt);
void (*elt_clear)(struct tracing_map_elt *elt);
void (*elt_init)(struct tracing_map_elt *elt);
};
extern struct tracing_map *
tracing_map_create(unsigned int map_bits,
unsigned int key_size,
const struct tracing_map_ops *ops,
void *private_data);
extern int tracing_map_init(struct tracing_map *map);
extern int tracing_map_add_sum_field(struct tracing_map *map);
extern int tracing_map_add_key_field(struct tracing_map *map,
unsigned int offset,
tracing_map_cmp_fn_t cmp_fn);
extern void tracing_map_destroy(struct tracing_map *map);
extern void tracing_map_clear(struct tracing_map *map);
extern struct tracing_map_elt *
tracing_map_insert(struct tracing_map *map, void *key);
extern struct tracing_map_elt *
tracing_map_lookup(struct tracing_map *map, void *key);
extern tracing_map_cmp_fn_t tracing_map_cmp_num(int field_size,
int field_is_signed);
extern int tracing_map_cmp_string(void *val_a, void *val_b);
extern int tracing_map_cmp_none(void *val_a, void *val_b);
extern void tracing_map_update_sum(struct tracing_map_elt *elt,
unsigned int i, u64 n);
extern u64 tracing_map_read_sum(struct tracing_map_elt *elt, unsigned int i);
extern void tracing_map_set_field_descr(struct tracing_map *map,
unsigned int i,
unsigned int key_offset,
tracing_map_cmp_fn_t cmp_fn);
extern int
tracing_map_sort_entries(struct tracing_map *map,
struct tracing_map_sort_key *sort_keys,
unsigned int n_sort_keys,
struct tracing_map_sort_entry ***sort_entries);
extern void
tracing_map_destroy_sort_entries(struct tracing_map_sort_entry **entries,
unsigned int n_entries);
#endif /* __TRACING_MAP_H */

54
kernel/workqueue.c
View file

@ -433,11 +433,18 @@ static void *work_debug_hint(void *addr)
return ((struct work_struct *) addr)->func;
}
static bool work_is_static_object(void *addr)
{
struct work_struct *work = addr;
return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
}
/*
* fixup_init is called when:
* - an active object is initialized
*/
static int work_fixup_init(void *addr, enum debug_obj_state state)
static bool work_fixup_init(void *addr, enum debug_obj_state state)
{
struct work_struct *work = addr;
@ -445,42 +452,9 @@ static int work_fixup_init(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
cancel_work_sync(work);
debug_object_init(work, &work_debug_descr);
return 1;
return true;
default:
return 0;
}
}
/*
* fixup_activate is called when:
* - an active object is activated
* - an unknown object is activated (might be a statically initialized object)
*/
static int work_fixup_activate(void *addr, enum debug_obj_state state)
{
struct work_struct *work = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
/*
* This is not really a fixup. The work struct was
* statically initialized. We just make sure that it
* is tracked in the object tracker.
*/
if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
debug_object_init(work, &work_debug_descr);
debug_object_activate(work, &work_debug_descr);
return 0;
}
WARN_ON_ONCE(1);
return 0;
case ODEBUG_STATE_ACTIVE:
WARN_ON(1);
default:
return 0;
return false;
}
}
@ -488,7 +462,7 @@ static int work_fixup_activate(void *addr, enum debug_obj_state state)
* fixup_free is called when:
* - an active object is freed
*/
static int work_fixup_free(void *addr, enum debug_obj_state state)
static bool work_fixup_free(void *addr, enum debug_obj_state state)
{
struct work_struct *work = addr;
@ -496,17 +470,17 @@ static int work_fixup_free(void *addr, enum debug_obj_state state)
case ODEBUG_STATE_ACTIVE:
cancel_work_sync(work);
debug_object_free(work, &work_debug_descr);
return 1;
return true;
default:
return 0;
return false;
}
}
static struct debug_obj_descr work_debug_descr = {
.name = "work_struct",
.debug_hint = work_debug_hint,
.is_static_object = work_is_static_object,
.fixup_init = work_fixup_init,
.fixup_activate = work_fixup_activate,
.fixup_free = work_fixup_free,
};