A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-25 22:23:14 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'sched/urgent' into sched/core, to avoid conflicts

Browse source 
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Ingo Molnar 2014-05-07 13:15:46 +02:00
commit 2fe5de9ce7
7124 changed files with 438977 additions and 144779 deletions
Download patch file Download diff file Expand all files Collapse all files

27
kernel/audit.c
View file

@ -182,7 +182,7 @@ struct audit_buffer {
struct audit_reply {
__u32 portid;
struct net *net;
struct net *net;
struct sk_buff *skb;
};
@ -396,7 +396,7 @@ static void audit_printk_skb(struct sk_buff *skb)
if (printk_ratelimit())
pr_notice("type=%d %s\n", nlh->nlmsg_type, data);
else
audit_log_lost("printk limit exceeded\n");
audit_log_lost("printk limit exceeded");
}
audit_hold_skb(skb);
@ -412,7 +412,7 @@ static void kauditd_send_skb(struct sk_buff *skb)
BUG_ON(err != -ECONNREFUSED); /* Shouldn't happen */
if (audit_pid) {
pr_err("*NO* daemon at audit_pid=%d\n", audit_pid);
audit_log_lost("auditd disappeared\n");
audit_log_lost("auditd disappeared");
audit_pid = 0;
audit_sock = NULL;
}
@ -607,7 +607,7 @@ static int audit_netlink_ok(struct sk_buff *skb, u16 msg_type)
{
int err = 0;
/* Only support the initial namespaces for now. */
/* Only support initial user namespace for now. */
/*
* We return ECONNREFUSED because it tricks userspace into thinking
* that audit was not configured into the kernel. Lots of users
@ -618,8 +618,7 @@ static int audit_netlink_ok(struct sk_buff *skb, u16 msg_type)
* userspace will reject all logins. This should be removed when we
* support non init namespaces!!
*/
if ((current_user_ns() != &init_user_ns) ||
(task_active_pid_ns(current) != &init_pid_ns))
if (current_user_ns() != &init_user_ns)
return -ECONNREFUSED;
switch (msg_type) {
@ -639,6 +638,11 @@ static int audit_netlink_ok(struct sk_buff *skb, u16 msg_type)
case AUDIT_TTY_SET:
case AUDIT_TRIM:
case AUDIT_MAKE_EQUIV:
/* Only support auditd and auditctl in initial pid namespace
* for now. */
if ((task_active_pid_ns(current) != &init_pid_ns))
return -EPERM;
if (!capable(CAP_AUDIT_CONTROL))
err = -EPERM;
break;
@ -659,6 +663,7 @@ static int audit_log_common_recv_msg(struct audit_buffer **ab, u16 msg_type)
{
int rc = 0;
uid_t uid = from_kuid(&init_user_ns, current_uid());
pid_t pid = task_tgid_nr(current);
if (!audit_enabled && msg_type != AUDIT_USER_AVC) {
*ab = NULL;
@ -668,7 +673,7 @@ static int audit_log_common_recv_msg(struct audit_buffer **ab, u16 msg_type)
*ab = audit_log_start(NULL, GFP_KERNEL, msg_type);
if (unlikely(!*ab))
return rc;
audit_log_format(*ab, "pid=%d uid=%u", task_tgid_vnr(current), uid);
audit_log_format(*ab, "pid=%d uid=%u", pid, uid);
audit_log_session_info(*ab);
audit_log_task_context(*ab);
@ -1097,7 +1102,7 @@ static void __net_exit audit_net_exit(struct net *net)
audit_sock = NULL;
}
rcu_assign_pointer(aunet->nlsk, NULL);
RCU_INIT_POINTER(aunet->nlsk, NULL);
synchronize_net();
netlink_kernel_release(sock);
}
@ -1829,11 +1834,11 @@ void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
spin_unlock_irq(&tsk->sighand->siglock);
audit_log_format(ab,
" ppid=%ld pid=%d auid=%u uid=%u gid=%u"
" ppid=%d pid=%d auid=%u uid=%u gid=%u"
" euid=%u suid=%u fsuid=%u"
" egid=%u sgid=%u fsgid=%u tty=%s ses=%u",
sys_getppid(),
tsk->pid,
task_ppid_nr(tsk),
task_pid_nr(tsk),
from_kuid(&init_user_ns, audit_get_loginuid(tsk)),
from_kuid(&init_user_ns, cred->uid),
from_kgid(&init_user_ns, cred->gid),

6
kernel/audit.h
View file

@ -106,6 +106,11 @@ struct audit_names {
bool should_free;
};
struct audit_proctitle {
int len; /* length of the cmdline field. */
char *value; /* the cmdline field */
};
/* The per-task audit context. */
struct audit_context {
int dummy; /* must be the first element */
@ -202,6 +207,7 @@ struct audit_context {
} execve;
};
int fds[2];
struct audit_proctitle proctitle;
#if AUDIT_DEBUG
int put_count;

33
kernel/auditfilter.c
View file

@ -19,6 +19,8 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/audit.h>
#include <linux/kthread.h>
@ -226,7 +228,7 @@ static int audit_match_signal(struct audit_entry *entry)
#endif
/* Common user-space to kernel rule translation. */
static inline struct audit_entry *audit_to_entry_common(struct audit_rule *rule)
static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data *rule)
{
unsigned listnr;
struct audit_entry *entry;
@ -249,7 +251,7 @@ static inline struct audit_entry *audit_to_entry_common(struct audit_rule *rule)
;
}
if (unlikely(rule->action == AUDIT_POSSIBLE)) {
printk(KERN_ERR "AUDIT_POSSIBLE is deprecated\n");
pr_err("AUDIT_POSSIBLE is deprecated\n");
goto exit_err;
}
if (rule->action != AUDIT_NEVER && rule->action != AUDIT_ALWAYS)
@ -403,7 +405,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
int i;
char *str;
entry = audit_to_entry_common((struct audit_rule *)data);
entry = audit_to_entry_common(data);
if (IS_ERR(entry))
goto exit_nofree;
@ -431,6 +433,19 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
f->val = 0;
}
if ((f->type == AUDIT_PID) || (f->type == AUDIT_PPID)) {
struct pid *pid;
rcu_read_lock();
pid = find_vpid(f->val);
if (!pid) {
rcu_read_unlock();
err = -ESRCH;
goto exit_free;
}
f->val = pid_nr(pid);
rcu_read_unlock();
}
err = audit_field_valid(entry, f);
if (err)
goto exit_free;
@ -479,8 +494,8 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
/* Keep currently invalid fields around in case they
* become valid after a policy reload. */
if (err == -EINVAL) {
printk(KERN_WARNING "audit rule for LSM "
"\'%s\' is invalid\n", str);
pr_warn("audit rule for LSM \'%s\' is invalid\n",
str);
err = 0;
}
if (err) {
@ -709,8 +724,8 @@ static inline int audit_dupe_lsm_field(struct audit_field *df,
/* Keep currently invalid fields around in case they
* become valid after a policy reload. */
if (ret == -EINVAL) {
printk(KERN_WARNING "audit rule for LSM \'%s\' is "
"invalid\n", df->lsm_str);
pr_warn("audit rule for LSM \'%s\' is invalid\n",
df->lsm_str);
ret = 0;
}
@ -1240,12 +1255,14 @@ static int audit_filter_user_rules(struct audit_krule *rule, int type,
for (i = 0; i < rule->field_count; i++) {
struct audit_field *f = &rule->fields[i];
pid_t pid;
int result = 0;
u32 sid;
switch (f->type) {
case AUDIT_PID:
result = audit_comparator(task_pid_vnr(current), f->op, f->val);
pid = task_pid_nr(current);
result = audit_comparator(pid, f->op, f->val);
break;
case AUDIT_UID:
result = audit_uid_comparator(current_uid(), f->op, f->uid);

133
kernel/auditsc.c
View file

@ -42,6 +42,8 @@
* and <dustin.kirkland@us.ibm.com> for LSPP certification compliance.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/init.h>
#include <asm/types.h>
#include <linux/atomic.h>
@ -68,6 +70,7 @@
#include <linux/capability.h>
#include <linux/fs_struct.h>
#include <linux/compat.h>
#include <linux/ctype.h>
#include "audit.h"
@ -79,6 +82,9 @@
/* no execve audit message should be longer than this (userspace limits) */
#define MAX_EXECVE_AUDIT_LEN 7500
/* max length to print of cmdline/proctitle value during audit */
#define MAX_PROCTITLE_AUDIT_LEN 128
/* number of audit rules */
int audit_n_rules;
@ -451,15 +457,17 @@ static int audit_filter_rules(struct task_struct *tsk,
struct audit_field *f = &rule->fields[i];
struct audit_names *n;
int result = 0;
pid_t pid;
switch (f->type) {
case AUDIT_PID:
result = audit_comparator(tsk->pid, f->op, f->val);
pid = task_pid_nr(tsk);
result = audit_comparator(pid, f->op, f->val);
break;
case AUDIT_PPID:
if (ctx) {
if (!ctx->ppid)
ctx->ppid = sys_getppid();
ctx->ppid = task_ppid_nr(tsk);
result = audit_comparator(ctx->ppid, f->op, f->val);
}
break;
@ -805,7 +813,8 @@ void audit_filter_inodes(struct task_struct *tsk, struct audit_context *ctx)
rcu_read_unlock();
}
static inline struct audit_context *audit_get_context(struct task_struct *tsk,
/* Transfer the audit context pointer to the caller, clearing it in the tsk's struct */
static inline struct audit_context *audit_take_context(struct task_struct *tsk,
int return_valid,
long return_code)
{
@ -842,6 +851,13 @@ static inline struct audit_context *audit_get_context(struct task_struct *tsk,
return context;
}
static inline void audit_proctitle_free(struct audit_context *context)
{
kfree(context->proctitle.value);
context->proctitle.value = NULL;
context->proctitle.len = 0;
}
static inline void audit_free_names(struct audit_context *context)
{
struct audit_names *n, *next;
@ -850,16 +866,15 @@ static inline void audit_free_names(struct audit_context *context)
if (context->put_count + context->ino_count != context->name_count) {
int i = 0;
printk(KERN_ERR "%s:%d(:%d): major=%d in_syscall=%d"
" name_count=%d put_count=%d"
" ino_count=%d [NOT freeing]\n",
__FILE__, __LINE__,
pr_err("%s:%d(:%d): major=%d in_syscall=%d"
" name_count=%d put_count=%d ino_count=%d"
" [NOT freeing]\n", __FILE__, __LINE__,
context->serial, context->major, context->in_syscall,
context->name_count, context->put_count,
context->ino_count);
list_for_each_entry(n, &context->names_list, list) {
printk(KERN_ERR "names[%d] = %p = %s\n", i++,
n->name, n->name->name ?: "(null)");
pr_err("names[%d] = %p = %s\n", i++, n->name,
n->name->name ?: "(null)");
}
dump_stack();
return;
@ -955,6 +970,7 @@ static inline void audit_free_context(struct audit_context *context)
audit_free_aux(context);
kfree(context->filterkey);
kfree(context->sockaddr);
audit_proctitle_free(context);
kfree(context);
}
@ -1157,7 +1173,7 @@ static void audit_log_execve_info(struct audit_context *context,
*/
buf = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL);
if (!buf) {
audit_panic("out of memory for argv string\n");
audit_panic("out of memory for argv string");
return;
}
@ -1271,6 +1287,59 @@ static void show_special(struct audit_context *context, int *call_panic)
audit_log_end(ab);
}
static inline int audit_proctitle_rtrim(char *proctitle, int len)
{
char *end = proctitle + len - 1;
while (end > proctitle && !isprint(*end))
end--;
/* catch the case where proctitle is only 1 non-print character */
len = end - proctitle + 1;
len -= isprint(proctitle[len-1]) == 0;
return len;
}
static void audit_log_proctitle(struct task_struct *tsk,
struct audit_context *context)
{
int res;
char *buf;
char *msg = "(null)";
int len = strlen(msg);
struct audit_buffer *ab;
ab = audit_log_start(context, GFP_KERNEL, AUDIT_PROCTITLE);
if (!ab)
return; /* audit_panic or being filtered */
audit_log_format(ab, "proctitle=");
/* Not cached */
if (!context->proctitle.value) {
buf = kmalloc(MAX_PROCTITLE_AUDIT_LEN, GFP_KERNEL);
if (!buf)
goto out;
/* Historically called this from procfs naming */
res = get_cmdline(tsk, buf, MAX_PROCTITLE_AUDIT_LEN);
if (res == 0) {
kfree(buf);
goto out;
}
res = audit_proctitle_rtrim(buf, res);
if (res == 0) {
kfree(buf);
goto out;
}
context->proctitle.value = buf;
context->proctitle.len = res;
}
msg = context->proctitle.value;
len = context->proctitle.len;
out:
audit_log_n_untrustedstring(ab, msg, len);
audit_log_end(ab);
}
static void audit_log_exit(struct audit_context *context, struct task_struct *tsk)
{
int i, call_panic = 0;
@ -1388,6 +1457,8 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
audit_log_name(context, n, NULL, i++, &call_panic);
}
audit_log_proctitle(tsk, context);
/* Send end of event record to help user space know we are finished */
ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE);
if (ab)
@ -1406,7 +1477,7 @@ void __audit_free(struct task_struct *tsk)
{
struct audit_context *context;
context = audit_get_context(tsk, 0, 0);
context = audit_take_context(tsk, 0, 0);
if (!context)
return;
@ -1500,7 +1571,7 @@ void __audit_syscall_exit(int success, long return_code)
else
success = AUDITSC_FAILURE;
context = audit_get_context(tsk, success, return_code);
context = audit_take_context(tsk, success, return_code);
if (!context)
return;
@ -1550,7 +1621,7 @@ static inline void handle_one(const struct inode *inode)
if (likely(put_tree_ref(context, chunk)))
return;
if (unlikely(!grow_tree_refs(context))) {
printk(KERN_WARNING "out of memory, audit has lost a tree reference\n");
pr_warn("out of memory, audit has lost a tree reference\n");
audit_set_auditable(context);
audit_put_chunk(chunk);
unroll_tree_refs(context, p, count);
@ -1609,8 +1680,7 @@ retry:
goto retry;
}
/* too bad */
printk(KERN_WARNING
"out of memory, audit has lost a tree reference\n");
pr_warn("out of memory, audit has lost a tree reference\n");
unroll_tree_refs(context, p, count);
audit_set_auditable(context);
return;
@ -1682,7 +1752,7 @@ void __audit_getname(struct filename *name)
if (!context->in_syscall) {
#if AUDIT_DEBUG == 2
printk(KERN_ERR "%s:%d(:%d): ignoring getname(%p)\n",
pr_err("%s:%d(:%d): ignoring getname(%p)\n",
__FILE__, __LINE__, context->serial, name);
dump_stack();
#endif
@ -1721,15 +1791,15 @@ void audit_putname(struct filename *name)
BUG_ON(!context);
if (!name->aname || !context->in_syscall) {
#if AUDIT_DEBUG == 2
printk(KERN_ERR "%s:%d(:%d): final_putname(%p)\n",
pr_err("%s:%d(:%d): final_putname(%p)\n",
__FILE__, __LINE__, context->serial, name);
if (context->name_count) {
struct audit_names *n;
int i = 0;
list_for_each_entry(n, &context->names_list, list)
printk(KERN_ERR "name[%d] = %p = %s\n", i++,
n->name, n->name->name ?: "(null)");
pr_err("name[%d] = %p = %s\n", i++, n->name,
n->name->name ?: "(null)");
}
#endif
final_putname(name);
@ -1738,9 +1808,8 @@ void audit_putname(struct filename *name)
else {
++context->put_count;
if (context->put_count > context->name_count) {
printk(KERN_ERR "%s:%d(:%d): major=%d"
" in_syscall=%d putname(%p) name_count=%d"
" put_count=%d\n",
pr_err("%s:%d(:%d): major=%d in_syscall=%d putname(%p)"
" name_count=%d put_count=%d\n",
__FILE__, __LINE__,
context->serial, context->major,
context->in_syscall, name->name,
@ -1981,12 +2050,10 @@ static void audit_log_set_loginuid(kuid_t koldloginuid, kuid_t kloginuid,
ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN);
if (!ab)
return;
audit_log_format(ab, "pid=%d uid=%u"
" old-auid=%u new-auid=%u old-ses=%u new-ses=%u"
" res=%d",
current->pid, uid,
oldloginuid, loginuid, oldsessionid, sessionid,
!rc);
audit_log_format(ab, "pid=%d uid=%u", task_pid_nr(current), uid);
audit_log_task_context(ab);
audit_log_format(ab, " old-auid=%u auid=%u old-ses=%u ses=%u res=%d",
oldloginuid, loginuid, oldsessionid, sessionid, !rc);
audit_log_end(ab);
}
@ -2208,7 +2275,7 @@ void __audit_ptrace(struct task_struct *t)
{
struct audit_context *context = current->audit_context;
context->target_pid = t->pid;
context->target_pid = task_pid_nr(t);
context->target_auid = audit_get_loginuid(t);
context->target_uid = task_uid(t);
context->target_sessionid = audit_get_sessionid(t);
@ -2233,7 +2300,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
if (audit_pid && t->tgid == audit_pid) {
if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1 || sig == SIGUSR2) {
audit_sig_pid = tsk->pid;
audit_sig_pid = task_pid_nr(tsk);
if (uid_valid(tsk->loginuid))
audit_sig_uid = tsk->loginuid;
else
@ -2247,7 +2314,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
/* optimize the common case by putting first signal recipient directly
* in audit_context */
if (!ctx->target_pid) {
ctx->target_pid = t->tgid;
ctx->target_pid = task_tgid_nr(t);
ctx->target_auid = audit_get_loginuid(t);
ctx->target_uid = t_uid;
ctx->target_sessionid = audit_get_sessionid(t);
@ -2268,7 +2335,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
}
BUG_ON(axp->pid_count >= AUDIT_AUX_PIDS);
axp->target_pid[axp->pid_count] = t->tgid;
axp->target_pid[axp->pid_count] = task_tgid_nr(t);
axp->target_auid[axp->pid_count] = audit_get_loginuid(t);
axp->target_uid[axp->pid_count] = t_uid;
axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t);
@ -2368,7 +2435,7 @@ static void audit_log_task(struct audit_buffer *ab)
from_kgid(&init_user_ns, gid),
sessionid);
audit_log_task_context(ab);
audit_log_format(ab, " pid=%d comm=", current->pid);
audit_log_format(ab, " pid=%d comm=", task_pid_nr(current));
audit_log_untrustedstring(ab, current->comm);
if (mm) {
down_read(&mm->mmap_sem);

27
kernel/capability.c
View file

@ -7,6 +7,8 @@
* 30 May 2002: Cleanup, Robert M. Love <rml@tech9.net>
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/mm.h>
@ -42,15 +44,10 @@ __setup("no_file_caps", file_caps_disable);
static void warn_legacy_capability_use(void)
{
static int warned;
if (!warned) {
char name[sizeof(current->comm)];
char name[sizeof(current->comm)];
printk(KERN_INFO "warning: `%s' uses 32-bit capabilities"
" (legacy support in use)\n",
get_task_comm(name, current));
warned = 1;
}
pr_info_once("warning: `%s' uses 32-bit capabilities (legacy support in use)\n",
get_task_comm(name, current));
}
/*
@ -71,16 +68,10 @@ static void warn_legacy_capability_use(void)
static void warn_deprecated_v2(void)
{
static int warned;
char name[sizeof(current->comm)];
if (!warned) {
char name[sizeof(current->comm)];
printk(KERN_INFO "warning: `%s' uses deprecated v2"
" capabilities in a way that may be insecure.\n",
get_task_comm(name, current));
warned = 1;
}
pr_info_once("warning: `%s' uses deprecated v2 capabilities in a way that may be insecure\n",
get_task_comm(name, current));
}
/*
@ -380,7 +371,7 @@ bool has_capability_noaudit(struct task_struct *t, int cap)
bool ns_capable(struct user_namespace *ns, int cap)
{
if (unlikely(!cap_valid(cap))) {
printk(KERN_CRIT "capable() called with invalid cap=%u\n", cap);
pr_crit("capable() called with invalid cap=%u\n", cap);
BUG();
}

3765
kernel/cgroup.c
View file

File diff suppressed because it is too large Load diff

40
kernel/cgroup_freezer.c
View file

@ -52,7 +52,7 @@ static inline struct freezer *css_freezer(struct cgroup_subsys_state *css)
static inline struct freezer *task_freezer(struct task_struct *task)
{
return css_freezer(task_css(task, freezer_subsys_id));
return css_freezer(task_css(task, freezer_cgrp_id));
}
static struct freezer *parent_freezer(struct freezer *freezer)
@ -84,8 +84,6 @@ static const char *freezer_state_strs(unsigned int state)
return "THAWED";
};
struct cgroup_subsys freezer_subsys;
static struct cgroup_subsys_state *
freezer_css_alloc(struct cgroup_subsys_state *parent_css)
{
@ -189,7 +187,7 @@ static void freezer_attach(struct cgroup_subsys_state *new_css,
* current state before executing the following - !frozen tasks may
* be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
*/
cgroup_taskset_for_each(task, new_css, tset) {
cgroup_taskset_for_each(task, tset) {
if (!(freezer->state & CGROUP_FREEZING)) {
__thaw_task(task);
} else {
@ -216,6 +214,16 @@ static void freezer_attach(struct cgroup_subsys_state *new_css,
}
}
/**
* freezer_fork - cgroup post fork callback
* @task: a task which has just been forked
*
* @task has just been created and should conform to the current state of
* the cgroup_freezer it belongs to. This function may race against
* freezer_attach(). Losing to freezer_attach() means that we don't have
* to do anything as freezer_attach() will put @task into the appropriate
* state.
*/
static void freezer_fork(struct task_struct *task)
{
struct freezer *freezer;
@ -224,14 +232,26 @@ static void freezer_fork(struct task_struct *task)
freezer = task_freezer(task);
/*
* The root cgroup is non-freezable, so we can skip the
* following check.
* The root cgroup is non-freezable, so we can skip locking the
* freezer. This is safe regardless of race with task migration.
* If we didn't race or won, skipping is obviously the right thing
* to do. If we lost and root is the new cgroup, noop is still the
* right thing to do.
*/
if (!parent_freezer(freezer))
goto out;
/*
* Grab @freezer->lock and freeze @task after verifying @task still
* belongs to @freezer and it's freezing. The former is for the
* case where we have raced against task migration and lost and
* @task is already in a different cgroup which may not be frozen.
* This isn't strictly necessary as freeze_task() is allowed to be
* called spuriously but let's do it anyway for, if nothing else,
* documentation.
*/
spin_lock_irq(&freezer->lock);
if (freezer->state & CGROUP_FREEZING)
if (freezer == task_freezer(task) && (freezer->state & CGROUP_FREEZING))
freeze_task(task);
spin_unlock_irq(&freezer->lock);
out:
@ -422,7 +442,7 @@ static void freezer_change_state(struct freezer *freezer, bool freeze)
}
static int freezer_write(struct cgroup_subsys_state *css, struct cftype *cft,
const char *buffer)
char *buffer)
{
bool freeze;
@ -473,13 +493,11 @@ static struct cftype files[] = {
{ } /* terminate */
};
struct cgroup_subsys freezer_subsys = {
.name = "freezer",
struct cgroup_subsys freezer_cgrp_subsys = {
.css_alloc = freezer_css_alloc,
.css_online = freezer_css_online,
.css_offline = freezer_css_offline,
.css_free = freezer_css_free,
.subsys_id = freezer_subsys_id,
.attach = freezer_attach,
.fork = freezer_fork,
.base_cftypes = files,

38
kernel/cpu.c
View file

@ -19,6 +19,7 @@
#include <linux/mutex.h>
#include <linux/gfp.h>
#include <linux/suspend.h>
#include <linux/lockdep.h>
#include "smpboot.h"
@ -27,18 +28,23 @@
static DEFINE_MUTEX(cpu_add_remove_lock);
/*
* The following two API's must be used when attempting
* to serialize the updates to cpu_online_mask, cpu_present_mask.
* The following two APIs (cpu_maps_update_begin/done) must be used when
* attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
* The APIs cpu_notifier_register_begin/done() must be used to protect CPU
* hotplug callback (un)registration performed using __register_cpu_notifier()
* or __unregister_cpu_notifier().
*/
void cpu_maps_update_begin(void)
{
mutex_lock(&cpu_add_remove_lock);
}
EXPORT_SYMBOL(cpu_notifier_register_begin);
void cpu_maps_update_done(void)
{
mutex_unlock(&cpu_add_remove_lock);
}
EXPORT_SYMBOL(cpu_notifier_register_done);
static RAW_NOTIFIER_HEAD(cpu_chain);
@ -57,17 +63,30 @@ static struct {
* an ongoing cpu hotplug operation.
*/
int refcount;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
} cpu_hotplug = {
.active_writer = NULL,
.lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
.refcount = 0,
#ifdef CONFIG_DEBUG_LOCK_ALLOC
.dep_map = {.name = "cpu_hotplug.lock" },
#endif
};
/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
void get_online_cpus(void)
{
might_sleep();
if (cpu_hotplug.active_writer == current)
return;
cpuhp_lock_acquire_read();
mutex_lock(&cpu_hotplug.lock);
cpu_hotplug.refcount++;
mutex_unlock(&cpu_hotplug.lock);
@ -87,6 +106,7 @@ void put_online_cpus(void)
if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
wake_up_process(cpu_hotplug.active_writer);
mutex_unlock(&cpu_hotplug.lock);
cpuhp_lock_release();
}
EXPORT_SYMBOL_GPL(put_online_cpus);
@ -117,6 +137,7 @@ void cpu_hotplug_begin(void)
{
cpu_hotplug.active_writer = current;
cpuhp_lock_acquire();
for (;;) {
mutex_lock(&cpu_hotplug.lock);
if (likely(!cpu_hotplug.refcount))
@ -131,6 +152,7 @@ void cpu_hotplug_done(void)
{
cpu_hotplug.active_writer = NULL;
mutex_unlock(&cpu_hotplug.lock);
cpuhp_lock_release();
}
/*
@ -166,6 +188,11 @@ int __ref register_cpu_notifier(struct notifier_block *nb)
return ret;
}
int __ref __register_cpu_notifier(struct notifier_block *nb)
{
return raw_notifier_chain_register(&cpu_chain, nb);
}
static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
int *nr_calls)
{
@ -189,6 +216,7 @@ static void cpu_notify_nofail(unsigned long val, void *v)
BUG_ON(cpu_notify(val, v));
}
EXPORT_SYMBOL(register_cpu_notifier);
EXPORT_SYMBOL(__register_cpu_notifier);
void __ref unregister_cpu_notifier(struct notifier_block *nb)
{
@ -198,6 +226,12 @@ void __ref unregister_cpu_notifier(struct notifier_block *nb)
}
EXPORT_SYMBOL(unregister_cpu_notifier);
void __ref __unregister_cpu_notifier(struct notifier_block *nb)
{
raw_notifier_chain_unregister(&cpu_chain, nb);
}
EXPORT_SYMBOL(__unregister_cpu_notifier);
/**
* clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
* @cpu: a CPU id

264
kernel/cpuset.c
View file

@ -119,7 +119,7 @@ static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
/* Retrieve the cpuset for a task */
static inline struct cpuset *task_cs(struct task_struct *task)
{
return css_cs(task_css(task, cpuset_subsys_id));
return css_cs(task_css(task, cpuset_cgrp_id));
}
static inline struct cpuset *parent_cs(struct cpuset *cs)
@ -467,7 +467,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial)
* be changed to have empty cpus_allowed or mems_allowed.
*/
ret = -ENOSPC;
if ((cgroup_task_count(cur->css.cgroup) || cur->attach_in_progress)) {
if ((cgroup_has_tasks(cur->css.cgroup) || cur->attach_in_progress)) {
if (!cpumask_empty(cur->cpus_allowed) &&
cpumask_empty(trial->cpus_allowed))
goto out;
@ -828,56 +828,37 @@ static struct cpuset *effective_nodemask_cpuset(struct cpuset *cs)
return cs;
}
/**
* cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
* @tsk: task to test
* @data: cpuset to @tsk belongs to
*
* Called by css_scan_tasks() for each task in a cgroup whose cpus_allowed
* mask needs to be changed.
*
* We don't need to re-check for the cgroup/cpuset membership, since we're
* holding cpuset_mutex at this point.
*/
static void cpuset_change_cpumask(struct task_struct *tsk, void *data)
{
struct cpuset *cs = data;
struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
set_cpus_allowed_ptr(tsk, cpus_cs->cpus_allowed);
}
/**
* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
* @heap: if NULL, defer allocating heap memory to css_scan_tasks()
*
* Called with cpuset_mutex held
*
* The css_scan_tasks() function will scan all the tasks in a cgroup,
* calling callback functions for each.
*
* No return value. It's guaranteed that css_scan_tasks() always returns 0
* if @heap != NULL.
* Iterate through each task of @cs updating its cpus_allowed to the
* effective cpuset's. As this function is called with cpuset_mutex held,
* cpuset membership stays stable.
*/
static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap)
static void update_tasks_cpumask(struct cpuset *cs)
{
css_scan_tasks(&cs->css, NULL, cpuset_change_cpumask, cs, heap);
struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
struct css_task_iter it;
struct task_struct *task;
css_task_iter_start(&cs->css, &it);
while ((task = css_task_iter_next(&it)))
set_cpus_allowed_ptr(task, cpus_cs->cpus_allowed);
css_task_iter_end(&it);
}
/*
* update_tasks_cpumask_hier - Update the cpumasks of tasks in the hierarchy.
* @root_cs: the root cpuset of the hierarchy
* @update_root: update root cpuset or not?
* @heap: the heap used by css_scan_tasks()
*
* This will update cpumasks of tasks in @root_cs and all other empty cpusets
* which take on cpumask of @root_cs.
*
* Called with cpuset_mutex held
*/
static void update_tasks_cpumask_hier(struct cpuset *root_cs,
bool update_root, struct ptr_heap *heap)
static void update_tasks_cpumask_hier(struct cpuset *root_cs, bool update_root)
{
struct cpuset *cp;
struct cgroup_subsys_state *pos_css;
@ -898,7 +879,7 @@ static void update_tasks_cpumask_hier(struct cpuset *root_cs,
continue;
rcu_read_unlock();
update_tasks_cpumask(cp, heap);
update_tasks_cpumask(cp);
rcu_read_lock();
css_put(&cp->css);
@ -914,7 +895,6 @@ static void update_tasks_cpumask_hier(struct cpuset *root_cs,
static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
const char *buf)
{
struct ptr_heap heap;
int retval;
int is_load_balanced;
@ -947,19 +927,13 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
if (retval < 0)
return retval;
retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
if (retval)
return retval;
is_load_balanced = is_sched_load_balance(trialcs);
mutex_lock(&callback_mutex);
cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
mutex_unlock(&callback_mutex);
update_tasks_cpumask_hier(cs, true, &heap);
heap_free(&heap);
update_tasks_cpumask_hier(cs, true);
if (is_load_balanced)
rebuild_sched_domains_locked();
@ -1022,7 +996,7 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk,
task_lock(tsk);
/*
* Determine if a loop is necessary if another thread is doing
* get_mems_allowed(). If at least one node remains unchanged and
* read_mems_allowed_begin(). If at least one node remains unchanged and
* tsk does not have a mempolicy, then an empty nodemask will not be
* possible when mems_allowed is larger than a word.
*/
@ -1048,53 +1022,22 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk,
task_unlock(tsk);
}
struct cpuset_change_nodemask_arg {
struct cpuset *cs;
nodemask_t *newmems;
};
/*
* Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
* of it to cpuset's new mems_allowed, and migrate pages to new nodes if
* memory_migrate flag is set. Called with cpuset_mutex held.
*/
static void cpuset_change_nodemask(struct task_struct *p, void *data)
{
struct cpuset_change_nodemask_arg *arg = data;
struct cpuset *cs = arg->cs;
struct mm_struct *mm;
int migrate;
cpuset_change_task_nodemask(p, arg->newmems);
mm = get_task_mm(p);
if (!mm)
return;
migrate = is_memory_migrate(cs);
mpol_rebind_mm(mm, &cs->mems_allowed);
if (migrate)
cpuset_migrate_mm(mm, &cs->old_mems_allowed, arg->newmems);
mmput(mm);
}
static void *cpuset_being_rebound;
/**
* update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
* @cs: the cpuset in which each task's mems_allowed mask needs to be changed
* @heap: if NULL, defer allocating heap memory to css_scan_tasks()
*
* Called with cpuset_mutex held. No return value. It's guaranteed that
* css_scan_tasks() always returns 0 if @heap != NULL.
* Iterate through each task of @cs updating its mems_allowed to the
* effective cpuset's. As this function is called with cpuset_mutex held,
* cpuset membership stays stable.
*/
static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
static void update_tasks_nodemask(struct cpuset *cs)
{
static nodemask_t newmems; /* protected by cpuset_mutex */
struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
struct cpuset_change_nodemask_arg arg = { .cs = cs,
.newmems = &newmems };
struct css_task_iter it;
struct task_struct *task;
cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
@ -1110,7 +1053,25 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
* It's ok if we rebind the same mm twice; mpol_rebind_mm()
* is idempotent. Also migrate pages in each mm to new nodes.
*/
css_scan_tasks(&cs->css, NULL, cpuset_change_nodemask, &arg, heap);
css_task_iter_start(&cs->css, &it);
while ((task = css_task_iter_next(&it))) {
struct mm_struct *mm;
bool migrate;
cpuset_change_task_nodemask(task, &newmems);
mm = get_task_mm(task);
if (!mm)
continue;
migrate = is_memory_migrate(cs);
mpol_rebind_mm(mm, &cs->mems_allowed);
if (migrate)
cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
mmput(mm);
}
css_task_iter_end(&it);
/*
* All the tasks' nodemasks have been updated, update
@ -1126,15 +1087,13 @@ static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
* update_tasks_nodemask_hier - Update the nodemasks of tasks in the hierarchy.
* @cs: the root cpuset of the hierarchy
* @update_root: update the root cpuset or not?
* @heap: the heap used by css_scan_tasks()
*
* This will update nodemasks of tasks in @root_cs and all other empty cpusets
* which take on nodemask of @root_cs.
*
* Called with cpuset_mutex held
*/
static void update_tasks_nodemask_hier(struct cpuset *root_cs,
bool update_root, struct ptr_heap *heap)
static void update_tasks_nodemask_hier(struct cpuset *root_cs, bool update_root)
{
struct cpuset *cp;
struct cgroup_subsys_state *pos_css;
@ -1155,7 +1114,7 @@ static void update_tasks_nodemask_hier(struct cpuset *root_cs,
continue;
rcu_read_unlock();
update_tasks_nodemask(cp, heap);
update_tasks_nodemask(cp);
rcu_read_lock();
css_put(&cp->css);
@ -1180,7 +1139,6 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
const char *buf)
{
int retval;
struct ptr_heap heap;
/*
* top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
@ -1219,17 +1177,11 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
if (retval < 0)
goto done;
retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
if (retval < 0)
goto done;
mutex_lock(&callback_mutex);
cs->mems_allowed = trialcs->mems_allowed;
mutex_unlock(&callback_mutex);
update_tasks_nodemask_hier(cs, true, &heap);
heap_free(&heap);
update_tasks_nodemask_hier(cs, true);
done:
return retval;
}
@ -1256,39 +1208,23 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
return 0;
}
/**
* cpuset_change_flag - make a task's spread flags the same as its cpuset's
* @tsk: task to be updated
* @data: cpuset to @tsk belongs to
*
* Called by css_scan_tasks() for each task in a cgroup.
*
* We don't need to re-check for the cgroup/cpuset membership, since we're
* holding cpuset_mutex at this point.
*/
static void cpuset_change_flag(struct task_struct *tsk, void *data)
{
struct cpuset *cs = data;
cpuset_update_task_spread_flag(cs, tsk);
}
/**
* update_tasks_flags - update the spread flags of tasks in the cpuset.
* @cs: the cpuset in which each task's spread flags needs to be changed
* @heap: if NULL, defer allocating heap memory to css_scan_tasks()
*
* Called with cpuset_mutex held
*
* The css_scan_tasks() function will scan all the tasks in a cgroup,
* calling callback functions for each.
*
* No return value. It's guaranteed that css_scan_tasks() always returns 0
* if @heap != NULL.
* Iterate through each task of @cs updating its spread flags. As this
* function is called with cpuset_mutex held, cpuset membership stays
* stable.
*/
static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
static void update_tasks_flags(struct cpuset *cs)
{
css_scan_tasks(&cs->css, NULL, cpuset_change_flag, cs, heap);
struct css_task_iter it;
struct task_struct *task;
css_task_iter_start(&cs->css, &it);
while ((task = css_task_iter_next(&it)))
cpuset_update_task_spread_flag(cs, task);
css_task_iter_end(&it);
}
/*
@ -1306,7 +1242,6 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
struct cpuset *trialcs;
int balance_flag_changed;
int spread_flag_changed;
struct ptr_heap heap;
int err;
trialcs = alloc_trial_cpuset(cs);
@ -1322,10 +1257,6 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
if (err < 0)
goto out;
err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
if (err < 0)
goto out;
balance_flag_changed = (is_sched_load_balance(cs) !=
is_sched_load_balance(trialcs));
@ -1340,8 +1271,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
rebuild_sched_domains_locked();
if (spread_flag_changed)
update_tasks_flags(cs, &heap);
heap_free(&heap);
update_tasks_flags(cs);
out:
free_trial_cpuset(trialcs);
return err;
@ -1445,6 +1375,8 @@ static int fmeter_getrate(struct fmeter *fmp)
return val;
}
static struct cpuset *cpuset_attach_old_cs;
/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
static int cpuset_can_attach(struct cgroup_subsys_state *css,
struct cgroup_taskset *tset)
@ -1453,6 +1385,9 @@ static int cpuset_can_attach(struct cgroup_subsys_state *css,
struct task_struct *task;
int ret;
/* used later by cpuset_attach() */
cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset));
mutex_lock(&cpuset_mutex);
/*
@ -1464,7 +1399,7 @@ static int cpuset_can_attach(struct cgroup_subsys_state *css,
(cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
goto out_unlock;
cgroup_taskset_for_each(task, css, tset) {
cgroup_taskset_for_each(task, tset) {
/*
* Kthreads which disallow setaffinity shouldn't be moved
* to a new cpuset; we don't want to change their cpu
@ -1516,10 +1451,8 @@ static void cpuset_attach(struct cgroup_subsys_state *css,
struct mm_struct *mm;
struct task_struct *task;
struct task_struct *leader = cgroup_taskset_first(tset);
struct cgroup_subsys_state *oldcss = cgroup_taskset_cur_css(tset,
cpuset_subsys_id);
struct cpuset *cs = css_cs(css);
struct cpuset *oldcs = css_cs(oldcss);
struct cpuset *oldcs = cpuset_attach_old_cs;
struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
@ -1533,7 +1466,7 @@ static void cpuset_attach(struct cgroup_subsys_state *css,
guarantee_online_mems(mems_cs, &cpuset_attach_nodemask_to);
cgroup_taskset_for_each(task, css, tset) {
cgroup_taskset_for_each(task, tset) {
/*
* can_attach beforehand should guarantee that this doesn't
* fail. TODO: have a better way to handle failure here
@ -1673,7 +1606,7 @@ out_unlock:
* Common handling for a write to a "cpus" or "mems" file.
*/
static int cpuset_write_resmask(struct cgroup_subsys_state *css,
struct cftype *cft, const char *buf)
struct cftype *cft, char *buf)
{
struct cpuset *cs = css_cs(css);
struct cpuset *trialcs;
@ -2020,8 +1953,7 @@ static void cpuset_css_free(struct cgroup_subsys_state *css)
kfree(cs);
}
struct cgroup_subsys cpuset_subsys = {
.name = "cpuset",
struct cgroup_subsys cpuset_cgrp_subsys = {
.css_alloc = cpuset_css_alloc,
.css_online = cpuset_css_online,
.css_offline = cpuset_css_offline,
@ -2029,7 +1961,6 @@ struct cgroup_subsys cpuset_subsys = {
.can_attach = cpuset_can_attach,
.cancel_attach = cpuset_cancel_attach,
.attach = cpuset_attach,
.subsys_id = cpuset_subsys_id,
.base_cftypes = files,
.early_init = 1,
};
@ -2086,10 +2017,9 @@ static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
parent = parent_cs(parent);
if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
rcu_read_lock();
printk(KERN_ERR "cpuset: failed to transfer tasks out of empty cpuset %s\n",
cgroup_name(cs->css.cgroup));
rcu_read_unlock();
printk(KERN_ERR "cpuset: failed to transfer tasks out of empty cpuset ");
pr_cont_cgroup_name(cs->css.cgroup);
pr_cont("\n");
}
}
@ -2137,7 +2067,7 @@ retry:
*/
if ((sane && cpumask_empty(cs->cpus_allowed)) ||
(!cpumask_empty(&off_cpus) && !cpumask_empty(cs->cpus_allowed)))
update_tasks_cpumask(cs, NULL);
update_tasks_cpumask(cs);
mutex_lock(&callback_mutex);
nodes_andnot(cs->mems_allowed, cs->mems_allowed, off_mems);
@ -2151,7 +2081,7 @@ retry:
*/
if ((sane && nodes_empty(cs->mems_allowed)) ||
(!nodes_empty(off_mems) && !nodes_empty(cs->mems_allowed)))
update_tasks_nodemask(cs, NULL);
update_tasks_nodemask(cs);
is_empty = cpumask_empty(cs->cpus_allowed) ||
nodes_empty(cs->mems_allowed);
@ -2213,7 +2143,7 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
mutex_lock(&callback_mutex);
top_cpuset.mems_allowed = new_mems;
mutex_unlock(&callback_mutex);
update_tasks_nodemask(&top_cpuset, NULL);
update_tasks_nodemask(&top_cpuset);
}
mutex_unlock(&cpuset_mutex);
@ -2305,10 +2235,10 @@ void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
struct cpuset *cpus_cs;
mutex_lock(&callback_mutex);
task_lock(tsk);
rcu_read_lock();
cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
guarantee_online_cpus(cpus_cs, pmask);
task_unlock(tsk);
rcu_read_unlock();
mutex_unlock(&callback_mutex);
}
@ -2361,10 +2291,10 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
nodemask_t mask;
mutex_lock(&callback_mutex);
task_lock(tsk);
rcu_read_lock();
mems_cs = effective_nodemask_cpuset(task_cs(tsk));
guarantee_online_mems(mems_cs, &mask);
task_unlock(tsk);
rcu_read_unlock();
mutex_unlock(&callback_mutex);
return mask;
@ -2480,10 +2410,10 @@ int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
/* Not hardwall and node outside mems_allowed: scan up cpusets */
mutex_lock(&callback_mutex);
task_lock(current);
rcu_read_lock();
cs = nearest_hardwall_ancestor(task_cs(current));
allowed = node_isset(node, cs->mems_allowed);
task_unlock(current);
rcu_read_unlock();
mutex_unlock(&callback_mutex);
return allowed;
@ -2609,27 +2539,27 @@ int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
* @task: pointer to task_struct of some task.
*
* Description: Prints @task's name, cpuset name, and cached copy of its
* mems_allowed to the kernel log. Must hold task_lock(task) to allow
* dereferencing task_cs(task).
* mems_allowed to the kernel log.
*/
void cpuset_print_task_mems_allowed(struct task_struct *tsk)
{
/* Statically allocated to prevent using excess stack. */
static char cpuset_nodelist[CPUSET_NODELIST_LEN];
static DEFINE_SPINLOCK(cpuset_buffer_lock);
struct cgroup *cgrp;
struct cgroup *cgrp = task_cs(tsk)->css.cgroup;
rcu_read_lock();
spin_lock(&cpuset_buffer_lock);
rcu_read_lock();
cgrp = task_cs(tsk)->css.cgroup;
nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
tsk->mems_allowed);
printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",
tsk->comm, cgroup_name(cgrp), cpuset_nodelist);
printk(KERN_INFO "%s cpuset=", tsk->comm);
pr_cont_cgroup_name(cgrp);
pr_cont(" mems_allowed=%s\n", cpuset_nodelist);
spin_unlock(&cpuset_buffer_lock);
rcu_read_unlock();
spin_unlock(&cpuset_buffer_lock);
}
/*
@ -2660,9 +2590,9 @@ int cpuset_memory_pressure_enabled __read_mostly;
void __cpuset_memory_pressure_bump(void)
{
task_lock(current);
rcu_read_lock();
fmeter_markevent(&task_cs(current)->fmeter);
task_unlock(current);
rcu_read_unlock();
}
#ifdef CONFIG_PROC_PID_CPUSET
@ -2679,12 +2609,12 @@ int proc_cpuset_show(struct seq_file *m, void *unused_v)
{
struct pid *pid;
struct task_struct *tsk;
char *buf;
char *buf, *p;
struct cgroup_subsys_state *css;
int retval;
retval = -ENOMEM;
buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
buf = kmalloc(PATH_MAX, GFP_KERNEL);
if (!buf)
goto out;
@ -2694,14 +2624,16 @@ int proc_cpuset_show(struct seq_file *m, void *unused_v)
if (!tsk)
goto out_free;
retval = -ENAMETOOLONG;
rcu_read_lock();
css = task_css(tsk, cpuset_subsys_id);
retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
css = task_css(tsk, cpuset_cgrp_id);
p = cgroup_path(css->cgroup, buf, PATH_MAX);
rcu_read_unlock();
if (retval < 0)
if (!p)
goto out_put_task;
seq_puts(m, buf);
seq_puts(m, p);
seq_putc(m, '\n');
retval = 0;
out_put_task:
put_task_struct(tsk);
out_free:

14
kernel/debug/debug_core.c
View file

@ -49,6 +49,7 @@
#include <linux/pid.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/vmacache.h>
#include <linux/rcupdate.h>
#include <asm/cacheflush.h>
@ -224,10 +225,17 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)
if (!CACHE_FLUSH_IS_SAFE)
return;
if (current->mm && current->mm->mmap_cache) {
flush_cache_range(current->mm->mmap_cache,
addr, addr + BREAK_INSTR_SIZE);
if (current->mm) {
int i;
for (i = 0; i < VMACACHE_SIZE; i++) {
if (!current->vmacache[i])
continue;
flush_cache_range(current->vmacache[i],
addr, addr + BREAK_INSTR_SIZE);
}
}
/* Force flush instruction cache if it was outside the mm */
flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
}

25
kernel/events/core.c
View file

@ -361,7 +361,7 @@ struct perf_cgroup {
static inline struct perf_cgroup *
perf_cgroup_from_task(struct task_struct *task)
{
return container_of(task_css(task, perf_subsys_id),
return container_of(task_css(task, perf_event_cgrp_id),
struct perf_cgroup, css);
}
@ -389,11 +389,6 @@ perf_cgroup_match(struct perf_event *event)
event->cgrp->css.cgroup);
}
static inline bool perf_tryget_cgroup(struct perf_event *event)
{
return css_tryget(&event->cgrp->css);
}
static inline void perf_put_cgroup(struct perf_event *event)
{
css_put(&event->cgrp->css);
@ -612,9 +607,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
if (!f.file)
return -EBADF;
rcu_read_lock();
css = css_from_dir(f.file->f_dentry, &perf_subsys);
css = css_tryget_from_dir(f.file->f_dentry, &perf_event_cgrp_subsys);
if (IS_ERR(css)) {
ret = PTR_ERR(css);
goto out;
@ -623,13 +616,6 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
cgrp = container_of(css, struct perf_cgroup, css);
event->cgrp = cgrp;
/* must be done before we fput() the file */
if (!perf_tryget_cgroup(event)) {
event->cgrp = NULL;
ret = -ENOENT;
goto out;
}
/*
* all events in a group must monitor
* the same cgroup because a task belongs
@ -640,7 +626,6 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
ret = -EINVAL;
}
out:
rcu_read_unlock();
fdput(f);
return ret;
}
@ -8053,7 +8038,7 @@ static void perf_cgroup_attach(struct cgroup_subsys_state *css,
{
struct task_struct *task;
cgroup_taskset_for_each(task, css, tset)
cgroup_taskset_for_each(task, tset)
task_function_call(task, __perf_cgroup_move, task);
}
@ -8072,9 +8057,7 @@ static void perf_cgroup_exit(struct cgroup_subsys_state *css,
task_function_call(task, __perf_cgroup_move, task);
}
struct cgroup_subsys perf_subsys = {
.name = "perf_event",
.subsys_id = perf_subsys_id,
struct cgroup_subsys perf_event_cgrp_subsys = {
.css_alloc = perf_cgroup_css_alloc,
.css_free = perf_cgroup_css_free,
.exit = perf_cgroup_exit,

9
kernel/events/uprobes.c
View file

@ -1804,6 +1804,11 @@ static bool handle_trampoline(struct pt_regs *regs)
return true;
}
bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
{
return false;
}
/*
* Run handler and ask thread to singlestep.
* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
@ -1858,7 +1863,11 @@ static void handle_swbp(struct pt_regs *regs)
if (!get_utask())
goto out;
if (arch_uprobe_ignore(&uprobe->arch, regs))
goto out;
handler_chain(uprobe, regs);
if (can_skip_sstep(uprobe, regs))
goto out;

118
kernel/exit.c
View file

@ -570,7 +570,7 @@ static void reparent_leader(struct task_struct *father, struct task_struct *p,
if (same_thread_group(p->real_parent, father))
return;
/* We don't want people slaying init. */
/* We don't want people slaying init. */
p->exit_signal = SIGCHLD;
/* If it has exited notify the new parent about this child's death. */
@ -784,9 +784,10 @@ void do_exit(long code)
exit_shm(tsk);
exit_files(tsk);
exit_fs(tsk);
if (group_dead)
disassociate_ctty(1);
exit_task_namespaces(tsk);
exit_task_work(tsk);
check_stack_usage();
exit_thread();
/*
@ -797,21 +798,17 @@ void do_exit(long code)
*/
perf_event_exit_task(tsk);
cgroup_exit(tsk, 1);
if (group_dead)
disassociate_ctty(1);
cgroup_exit(tsk);
module_put(task_thread_info(tsk)->exec_domain->module);
proc_exit_connector(tsk);
/*
* FIXME: do that only when needed, using sched_exit tracepoint
*/
flush_ptrace_hw_breakpoint(tsk);
exit_notify(tsk, group_dead);
proc_exit_connector(tsk);
#ifdef CONFIG_NUMA
task_lock(tsk);
mpol_put(tsk->mempolicy);
@ -844,6 +841,7 @@ void do_exit(long code)
validate_creds_for_do_exit(tsk);
check_stack_usage();
preempt_disable();
if (tsk->nr_dirtied)
__this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
@ -1038,17 +1036,13 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
return wait_noreap_copyout(wo, p, pid, uid, why, status);
}
/*
* Try to move the task's state to DEAD
* only one thread is allowed to do this:
*/
state = xchg(&p->exit_state, EXIT_DEAD);
if (state != EXIT_ZOMBIE) {
BUG_ON(state != EXIT_DEAD);
return 0;
}
traced = ptrace_reparented(p);
/*
* Move the task's state to DEAD/TRACE, only one thread can do this.
*/
state = traced && thread_group_leader(p) ? EXIT_TRACE : EXIT_DEAD;
if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
return 0;
/*
* It can be ptraced but not reparented, check
* thread_group_leader() to filter out sub-threads.
@ -1109,7 +1103,7 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
/*
* Now we are sure this task is interesting, and no other
* thread can reap it because we set its state to EXIT_DEAD.
* thread can reap it because we its state == DEAD/TRACE.
*/
read_unlock(&tasklist_lock);
@ -1146,22 +1140,19 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
if (!retval)
retval = pid;
if (traced) {
if (state == EXIT_TRACE) {
write_lock_irq(&tasklist_lock);
/* We dropped tasklist, ptracer could die and untrace */
ptrace_unlink(p);
/*
* If this is not a sub-thread, notify the parent.
* If parent wants a zombie, don't release it now.
*/
if (thread_group_leader(p) &&
!do_notify_parent(p, p->exit_signal)) {
p->exit_state = EXIT_ZOMBIE;
p = NULL;
}
/* If parent wants a zombie, don't release it now */
state = EXIT_ZOMBIE;
if (do_notify_parent(p, p->exit_signal))
state = EXIT_DEAD;
p->exit_state = state;
write_unlock_irq(&tasklist_lock);
}
if (p != NULL)
if (state == EXIT_DEAD)
release_task(p);
return retval;
@ -1338,7 +1329,12 @@ static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
static int wait_consider_task(struct wait_opts *wo, int ptrace,
struct task_struct *p)
{
int ret = eligible_child(wo, p);
int ret;
if (unlikely(p->exit_state == EXIT_DEAD))
return 0;
ret = eligible_child(wo, p);
if (!ret)
return ret;
@ -1356,33 +1352,44 @@ static int wait_consider_task(struct wait_opts *wo, int ptrace,
return 0;
}
/* dead body doesn't have much to contribute */
if (unlikely(p->exit_state == EXIT_DEAD)) {
if (unlikely(p->exit_state == EXIT_TRACE)) {
/*
* But do not ignore this task until the tracer does
* wait_task_zombie()->do_notify_parent().
* ptrace == 0 means we are the natural parent. In this case
* we should clear notask_error, debugger will notify us.
*/
if (likely(!ptrace) && unlikely(ptrace_reparented(p)))
if (likely(!ptrace))
wo->notask_error = 0;
return 0;
}
if (likely(!ptrace) && unlikely(p->ptrace)) {
/*
* If it is traced by its real parent's group, just pretend
* the caller is ptrace_do_wait() and reap this child if it
* is zombie.
*
* This also hides group stop state from real parent; otherwise
* a single stop can be reported twice as group and ptrace stop.
* If a ptracer wants to distinguish these two events for its
* own children it should create a separate process which takes
* the role of real parent.
*/
if (!ptrace_reparented(p))
ptrace = 1;
}
/* slay zombie? */
if (p->exit_state == EXIT_ZOMBIE) {
/*
* A zombie ptracee is only visible to its ptracer.
* Notification and reaping will be cascaded to the real
* parent when the ptracer detaches.
*/
if (likely(!ptrace) && unlikely(p->ptrace)) {
/* it will become visible, clear notask_error */
wo->notask_error = 0;
return 0;
}
/* we don't reap group leaders with subthreads */
if (!delay_group_leader(p))
return wait_task_zombie(wo, p);
if (!delay_group_leader(p)) {
/*
* A zombie ptracee is only visible to its ptracer.
* Notification and reaping will be cascaded to the
* real parent when the ptracer detaches.
*/
if (unlikely(ptrace) || likely(!p->ptrace))
return wait_task_zombie(wo, p);
}
/*
* Allow access to stopped/continued state via zombie by
@ -1407,19 +1414,6 @@ static int wait_consider_task(struct wait_opts *wo, int ptrace,
if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
wo->notask_error = 0;
} else {
/*
* If @p is ptraced by a task in its real parent's group,
* hide group stop/continued state when looking at @p as
* the real parent; otherwise, a single stop can be
* reported twice as group and ptrace stops.
*
* If a ptracer wants to distinguish the two events for its
* own children, it should create a separate process which
* takes the role of real parent.
*/
if (likely(!ptrace) && p->ptrace && !ptrace_reparented(p))
return 0;
/*
* @p is alive and it's gonna stop, continue or exit, so
* there always is something to wait for.

39
kernel/fork.c
View file

@ -28,6 +28,8 @@
#include <linux/mman.h>
#include <linux/mmu_notifier.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/vmacache.h>
#include <linux/nsproxy.h>
#include <linux/capability.h>
#include <linux/cpu.h>
@ -71,6 +73,7 @@
#include <linux/signalfd.h>
#include <linux/uprobes.h>
#include <linux/aio.h>
#include <linux/compiler.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
@ -284,7 +287,7 @@ void __init fork_init(unsigned long mempages)
init_task.signal->rlim[RLIMIT_NPROC];
}
int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
int __weak arch_dup_task_struct(struct task_struct *dst,
struct task_struct *src)
{
*dst = *src;
@ -364,7 +367,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
mm->locked_vm = 0;
mm->mmap = NULL;
mm->mmap_cache = NULL;
mm->vmacache_seqnum = 0;
mm->map_count = 0;
cpumask_clear(mm_cpumask(mm));
mm->mm_rb = RB_ROOT;
@ -530,8 +533,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
atomic_set(&mm->mm_count, 1);
init_rwsem(&mm->mmap_sem);
INIT_LIST_HEAD(&mm->mmlist);
mm->flags = (current->mm) ?
(current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
mm->core_state = NULL;
atomic_long_set(&mm->nr_ptes, 0);
memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
@ -540,8 +541,15 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
mm_init_owner(mm, p);
clear_tlb_flush_pending(mm);
if (likely(!mm_alloc_pgd(mm))) {
if (current->mm) {
mm->flags = current->mm->flags & MMF_INIT_MASK;
mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
} else {
mm->flags = default_dump_filter;
mm->def_flags = 0;
}
if (likely(!mm_alloc_pgd(mm))) {
mmu_notifier_mm_init(mm);
return mm;
}
@ -877,6 +885,9 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
if (!oldmm)
return 0;
/* initialize the new vmacache entries */
vmacache_flush(tsk);
if (clone_flags & CLONE_VM) {
atomic_inc(&oldmm->mm_users);
mm = oldmm;
@ -1070,15 +1081,6 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
return 0;
}
static void copy_flags(unsigned long clone_flags, struct task_struct *p)
{
unsigned long new_flags = p->flags;
new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
new_flags |= PF_FORKNOEXEC;
p->flags = new_flags;
}
SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
{
current->clear_child_tid = tidptr;
@ -1228,7 +1230,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
goto bad_fork_cleanup_count;
delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
copy_flags(clone_flags, p);
p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
p->flags |= PF_FORKNOEXEC;
INIT_LIST_HEAD(&p->children);
INIT_LIST_HEAD(&p->sibling);
rcu_copy_process(p);
@ -1272,9 +1275,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
if (IS_ERR(p->mempolicy)) {
retval = PTR_ERR(p->mempolicy);
p->mempolicy = NULL;
goto bad_fork_cleanup_cgroup;
goto bad_fork_cleanup_threadgroup_lock;
}
mpol_fix_fork_child_flag(p);
#endif
#ifdef CONFIG_CPUSETS
p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
@ -1525,11 +1527,10 @@ bad_fork_cleanup_policy:
perf_event_free_task(p);
#ifdef CONFIG_NUMA
mpol_put(p->mempolicy);
bad_fork_cleanup_cgroup:
bad_fork_cleanup_threadgroup_lock:
#endif
if (clone_flags & CLONE_THREAD)
threadgroup_change_end(current);
cgroup_exit(p, 0);
delayacct_tsk_free(p);
module_put(task_thread_info(p)->exec_domain->module);
bad_fork_cleanup_count:

37
kernel/futex.c
View file

@ -70,7 +70,10 @@
#include "locking/rtmutex_common.h"
/*
* Basic futex operation and ordering guarantees:
* READ this before attempting to hack on futexes!
*
* Basic futex operation and ordering guarantees
* =============================================
*
* The waiter reads the futex value in user space and calls
* futex_wait(). This function computes the hash bucket and acquires
@ -119,7 +122,7 @@
* sys_futex(WAIT, futex, val);
* futex_wait(futex, val);
*
* waiters++;
* waiters++; (a)
* mb(); (A) <-- paired with -.
* |
* lock(hash_bucket(futex)); |
@ -135,14 +138,14 @@
* unlock(hash_bucket(futex));
* schedule(); if (waiters)
* lock(hash_bucket(futex));
* wake_waiters(futex);
* unlock(hash_bucket(futex));
* else wake_waiters(futex);
* waiters--; (b) unlock(hash_bucket(futex));
*
* Where (A) orders the waiters increment and the futex value read -- this
* is guaranteed by the head counter in the hb spinlock; and where (B)
* orders the write to futex and the waiters read -- this is done by the
* barriers in get_futex_key_refs(), through either ihold or atomic_inc,
* depending on the futex type.
* Where (A) orders the waiters increment and the futex value read through
* atomic operations (see hb_waiters_inc) and where (B) orders the write
* to futex and the waiters read -- this is done by the barriers in
* get_futex_key_refs(), through either ihold or atomic_inc, depending on the
* futex type.
*
* This yields the following case (where X:=waiters, Y:=futex):
*
@ -155,6 +158,17 @@
* Which guarantees that x==0 && y==0 is impossible; which translates back into
* the guarantee that we cannot both miss the futex variable change and the
* enqueue.
*
* Note that a new waiter is accounted for in (a) even when it is possible that
* the wait call can return error, in which case we backtrack from it in (b).
* Refer to the comment in queue_lock().
*
* Similarly, in order to account for waiters being requeued on another
* address we always increment the waiters for the destination bucket before
* acquiring the lock. It then decrements them again after releasing it -
* the code that actually moves the futex(es) between hash buckets (requeue_futex)
* will do the additional required waiter count housekeeping. This is done for
* double_lock_hb() and double_unlock_hb(), respectively.
*/
#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
@ -1452,6 +1466,7 @@ retry:
hb2 = hash_futex(&key2);
retry_private:
hb_waiters_inc(hb2);
double_lock_hb(hb1, hb2);
if (likely(cmpval != NULL)) {
@ -1461,6 +1476,7 @@ retry_private:
if (unlikely(ret)) {
double_unlock_hb(hb1, hb2);
hb_waiters_dec(hb2);
ret = get_user(curval, uaddr1);
if (ret)
@ -1510,6 +1526,7 @@ retry_private:
break;
case -EFAULT:
double_unlock_hb(hb1, hb2);
hb_waiters_dec(hb2);
put_futex_key(&key2);
put_futex_key(&key1);
ret = fault_in_user_writeable(uaddr2);
@ -1519,6 +1536,7 @@ retry_private:
case -EAGAIN:
/* The owner was exiting, try again. */
double_unlock_hb(hb1, hb2);
hb_waiters_dec(hb2);
put_futex_key(&key2);
put_futex_key(&key1);
cond_resched();
@ -1594,6 +1612,7 @@ retry_private:
out_unlock:
double_unlock_hb(hb1, hb2);
hb_waiters_dec(hb2);
/*
* drop_futex_key_refs() must be called outside the spinlocks. During

14
kernel/groups.c
View file

@ -157,17 +157,13 @@ int groups_search(const struct group_info *group_info, kgid_t grp)
* set_groups - Change a group subscription in a set of credentials
* @new: The newly prepared set of credentials to alter
* @group_info: The group list to install
*
* Validate a group subscription and, if valid, insert it into a set
* of credentials.
*/
int set_groups(struct cred *new, struct group_info *group_info)
void set_groups(struct cred *new, struct group_info *group_info)
{
put_group_info(new->group_info);
groups_sort(group_info);
get_group_info(group_info);
new->group_info = group_info;
return 0;
}
EXPORT_SYMBOL(set_groups);
@ -182,18 +178,12 @@ EXPORT_SYMBOL(set_groups);
int set_current_groups(struct group_info *group_info)
{
struct cred *new;
int ret;
new = prepare_creds();
if (!new)
return -ENOMEM;
ret = set_groups(new, group_info);
if (ret < 0) {
abort_creds(new);
return ret;
}
set_groups(new, group_info);
return commit_creds(new);
}

3
kernel/hung_task.c
View file

@ -246,5 +246,4 @@ static int __init hung_task_init(void)
return 0;
}
module_init(hung_task_init);
subsys_initcall(hung_task_init);

11
kernel/kallsyms.c
View file

@ -23,6 +23,7 @@
#include <linux/mm.h>
#include <linux/ctype.h>
#include <linux/slab.h>
#include <linux/compiler.h>
#include <asm/sections.h>
@ -36,8 +37,8 @@
* These will be re-linked against their real values
* during the second link stage.
*/
extern const unsigned long kallsyms_addresses[] __attribute__((weak));
extern const u8 kallsyms_names[] __attribute__((weak));
extern const unsigned long kallsyms_addresses[] __weak;
extern const u8 kallsyms_names[] __weak;
/*
* Tell the compiler that the count isn't in the small data section if the arch
@ -46,10 +47,10 @@ extern const u8 kallsyms_names[] __attribute__((weak));
extern const unsigned long kallsyms_num_syms
__attribute__((weak, section(".rodata")));
extern const u8 kallsyms_token_table[] __attribute__((weak));
extern const u16 kallsyms_token_index[] __attribute__((weak));
extern const u8 kallsyms_token_table[] __weak;
extern const u16 kallsyms_token_index[] __weak;
extern const unsigned long kallsyms_markers[] __attribute__((weak));
extern const unsigned long kallsyms_markers[] __weak;
static inline int is_kernel_inittext(unsigned long addr)
{

9
kernel/kexec.c
View file

@ -32,6 +32,7 @@
#include <linux/vmalloc.h>
#include <linux/swap.h>
#include <linux/syscore_ops.h>
#include <linux/compiler.h>
#include <asm/page.h>
#include <asm/uaccess.h>
@ -1235,7 +1236,7 @@ static int __init crash_notes_memory_init(void)
}
return 0;
}
module_init(crash_notes_memory_init)
subsys_initcall(crash_notes_memory_init);
/*
@ -1551,10 +1552,10 @@ void vmcoreinfo_append_str(const char *fmt, ...)
* provide an empty default implementation here -- architecture
* code may override this
*/
void __attribute__ ((weak)) arch_crash_save_vmcoreinfo(void)
void __weak arch_crash_save_vmcoreinfo(void)
{}
unsigned long __attribute__ ((weak)) paddr_vmcoreinfo_note(void)
unsigned long __weak paddr_vmcoreinfo_note(void)
{
return __pa((unsigned long)(char *)&vmcoreinfo_note);
}
@ -1629,7 +1630,7 @@ static int __init crash_save_vmcoreinfo_init(void)
return 0;
}
module_init(crash_save_vmcoreinfo_init)
subsys_initcall(crash_save_vmcoreinfo_init);
/*
* Move into place and start executing a preloaded standalone

5
kernel/ksysfs.c
View file

@ -18,6 +18,7 @@
#include <linux/stat.h>
#include <linux/sched.h>
#include <linux/capability.h>
#include <linux/compiler.h>
#include <linux/rcupdate.h> /* rcu_expedited */
@ -162,8 +163,8 @@ KERNEL_ATTR_RW(rcu_expedited);
/*
* Make /sys/kernel/notes give the raw contents of our kernel .notes section.
*/
extern const void __start_notes __attribute__((weak));
extern const void __stop_notes __attribute__((weak));
extern const void __start_notes __weak;
extern const void __stop_notes __weak;
#define notes_size (&__stop_notes - &__start_notes)
static ssize_t notes_read(struct file *filp, struct kobject *kobj,

4
kernel/kthread.c
View file

@ -217,7 +217,7 @@ int tsk_fork_get_node(struct task_struct *tsk)
if (tsk == kthreadd_task)
return tsk->pref_node_fork;
#endif
return numa_node_id();
return NUMA_NO_NODE;
}
static void create_kthread(struct kthread_create_info *create)
@ -369,7 +369,7 @@ struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
{
struct task_struct *p;
p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
p = kthread_create_on_node(threadfn, data, cpu_to_mem(cpu), namefmt,
cpu);
if (IS_ERR(p))
return p;

3
kernel/locking/Makefile
View file

@ -1,5 +1,5 @@
obj-y += mutex.o semaphore.o rwsem.o lglock.o mcs_spinlock.o
obj-y += mutex.o semaphore.o rwsem.o mcs_spinlock.o
ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_lockdep.o = -pg
@ -14,6 +14,7 @@ ifeq ($(CONFIG_PROC_FS),y)
obj-$(CONFIG_LOCKDEP) += lockdep_proc.o
endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o

19
kernel/locking/mutex-debug.c
View file

@ -71,18 +71,17 @@ void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
void debug_mutex_unlock(struct mutex *lock)
{
if (unlikely(!debug_locks))
return;
if (likely(debug_locks)) {
DEBUG_LOCKS_WARN_ON(lock->magic != lock);
DEBUG_LOCKS_WARN_ON(lock->magic != lock);
if (!lock->owner)
DEBUG_LOCKS_WARN_ON(!lock->owner);
else
DEBUG_LOCKS_WARN_ON(lock->owner != current);
if (!lock->owner)
DEBUG_LOCKS_WARN_ON(!lock->owner);
else
DEBUG_LOCKS_WARN_ON(lock->owner != current);
DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
mutex_clear_owner(lock);
DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
mutex_clear_owner(lock);
}
/*
* __mutex_slowpath_needs_to_unlock() is explicitly 0 for debug

12
kernel/module.c
View file

@ -640,7 +640,7 @@ static int module_unload_init(struct module *mod)
INIT_LIST_HEAD(&mod->target_list);
/* Hold reference count during initialization. */
__this_cpu_write(mod->refptr->incs, 1);
raw_cpu_write(mod->refptr->incs, 1);
return 0;
}
@ -1013,6 +1013,8 @@ static size_t module_flags_taint(struct module *mod, char *buf)
buf[l++] = 'F';
if (mod->taints & (1 << TAINT_CRAP))
buf[l++] = 'C';
if (mod->taints & (1 << TAINT_UNSIGNED_MODULE))
buf[l++] = 'E';
/*
* TAINT_FORCED_RMMOD: could be added.
* TAINT_CPU_OUT_OF_SPEC, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't
@ -3218,7 +3220,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
pr_notice_once("%s: module verification failed: signature "
"and/or required key missing - tainting "
"kernel\n", mod->name);
add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_STILL_OK);
add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
}
#endif
@ -3813,12 +3815,12 @@ void print_modules(void)
list_for_each_entry_rcu(mod, &modules, list) {
if (mod->state == MODULE_STATE_UNFORMED)
continue;
printk(" %s%s", mod->name, module_flags(mod, buf));
pr_cont(" %s%s", mod->name, module_flags(mod, buf));
}
preempt_enable();
if (last_unloaded_module[0])
printk(" [last unloaded: %s]", last_unloaded_module);
printk("\n");
pr_cont(" [last unloaded: %s]", last_unloaded_module);
pr_cont("\n");
}
#ifdef CONFIG_MODVERSIONS

15
kernel/panic.c
View file

@ -100,7 +100,7 @@ void panic(const char *fmt, ...)
va_start(args, fmt);
vsnprintf(buf, sizeof(buf), fmt, args);
va_end(args);
printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf);
pr_emerg("Kernel panic - not syncing: %s\n", buf);
#ifdef CONFIG_DEBUG_BUGVERBOSE
/*
* Avoid nested stack-dumping if a panic occurs during oops processing
@ -141,7 +141,7 @@ void panic(const char *fmt, ...)
* Delay timeout seconds before rebooting the machine.
* We can't use the "normal" timers since we just panicked.
*/
printk(KERN_EMERG "Rebooting in %d seconds..", panic_timeout);
pr_emerg("Rebooting in %d seconds..", panic_timeout);
for (i = 0; i < panic_timeout * 1000; i += PANIC_TIMER_STEP) {
touch_nmi_watchdog();
@ -165,7 +165,7 @@ void panic(const char *fmt, ...)
extern int stop_a_enabled;
/* Make sure the user can actually press Stop-A (L1-A) */
stop_a_enabled = 1;
printk(KERN_EMERG "Press Stop-A (L1-A) to return to the boot prom\n");
pr_emerg("Press Stop-A (L1-A) to return to the boot prom\n");
}
#endif
#if defined(CONFIG_S390)
@ -176,6 +176,7 @@ void panic(const char *fmt, ...)
disabled_wait(caller);
}
#endif
pr_emerg("---[ end Kernel panic - not syncing: %s\n", buf);
local_irq_enable();
for (i = 0; ; i += PANIC_TIMER_STEP) {
touch_softlockup_watchdog();
@ -210,6 +211,7 @@ static const struct tnt tnts[] = {
{ TAINT_CRAP, 'C', ' ' },
{ TAINT_FIRMWARE_WORKAROUND, 'I', ' ' },
{ TAINT_OOT_MODULE, 'O', ' ' },
{ TAINT_UNSIGNED_MODULE, 'E', ' ' },
};
/**
@ -228,6 +230,7 @@ static const struct tnt tnts[] = {
* 'C' - modules from drivers/staging are loaded.
* 'I' - Working around severe firmware bug.
* 'O' - Out-of-tree module has been loaded.
* 'E' - Unsigned module has been loaded.
*
* The string is overwritten by the next call to print_tainted().
*/
@ -274,8 +277,7 @@ unsigned long get_taint(void)
void add_taint(unsigned flag, enum lockdep_ok lockdep_ok)
{
if (lockdep_ok == LOCKDEP_NOW_UNRELIABLE && __debug_locks_off())
printk(KERN_WARNING
"Disabling lock debugging due to kernel taint\n");
pr_warn("Disabling lock debugging due to kernel taint\n");
set_bit(flag, &tainted_mask);
}
@ -380,8 +382,7 @@ late_initcall(init_oops_id);
void print_oops_end_marker(void)
{
init_oops_id();
printk(KERN_WARNING "---[ end trace %016llx ]---\n",
(unsigned long long)oops_id);
pr_warn("---[ end trace %016llx ]---\n", (unsigned long long)oops_id);
}
/*

4
kernel/pid_namespace.c
View file

@ -318,7 +318,9 @@ static void *pidns_get(struct task_struct *task)
struct pid_namespace *ns;
rcu_read_lock();
ns = get_pid_ns(task_active_pid_ns(task));
ns = task_active_pid_ns(task);
if (ns)
get_pid_ns(ns);
rcu_read_unlock();
return ns;

3
kernel/power/power.h
View file

@ -2,6 +2,7 @@
#include <linux/suspend_ioctls.h>
#include <linux/utsname.h>
#include <linux/freezer.h>
#include <linux/compiler.h>
struct swsusp_info {
struct new_utsname uts;
@ -11,7 +12,7 @@ struct swsusp_info {
unsigned long image_pages;
unsigned long pages;
unsigned long size;
} __attribute__((aligned(PAGE_SIZE)));
} __aligned(PAGE_SIZE);
#ifdef CONFIG_HIBERNATION
/* kernel/power/snapshot.c */

3
kernel/power/snapshot.c
View file

@ -27,6 +27,7 @@
#include <linux/highmem.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/compiler.h>
#include <asm/uaccess.h>
#include <asm/mmu_context.h>
@ -155,7 +156,7 @@ static inline void free_image_page(void *addr, int clear_nosave_free)
struct linked_page {
struct linked_page *next;
char data[LINKED_PAGE_DATA_SIZE];
} __attribute__((packed));
} __packed;
static inline void
free_list_of_pages(struct linked_page *list, int clear_page_nosave)

5
kernel/power/suspend.c
View file

@ -26,6 +26,7 @@
#include <linux/syscore_ops.h>
#include <linux/ftrace.h>
#include <trace/events/power.h>
#include <linux/compiler.h>
#include "power.h"
@ -156,13 +157,13 @@ static int suspend_prepare(suspend_state_t state)
}
/* default implementation */
void __attribute__ ((weak)) arch_suspend_disable_irqs(void)
void __weak arch_suspend_disable_irqs(void)
{
local_irq_disable();
}
/* default implementation */
void __attribute__ ((weak)) arch_suspend_enable_irqs(void)
void __weak arch_suspend_enable_irqs(void)
{
local_irq_enable();
}

2
kernel/power/swap.c
View file

@ -101,7 +101,7 @@ struct swsusp_header {
unsigned int flags; /* Flags to pass to the "boot" kernel */
char orig_sig[10];
char sig[10];
} __attribute__((packed));
} __packed;
static struct swsusp_header *swsusp_header;

15
kernel/printk/printk.c
View file

@ -319,7 +319,7 @@ static void log_store(int facility, int level,
else
free = log_first_idx - log_next_idx;
if (free > size + sizeof(struct printk_log))
if (free >= size + sizeof(struct printk_log))
break;
/* drop old messages until we have enough contiuous space */
@ -327,7 +327,7 @@ static void log_store(int facility, int level,
log_first_seq++;
}
if (log_next_idx + size + sizeof(struct printk_log) >= log_buf_len) {
if (log_next_idx + size + sizeof(struct printk_log) > log_buf_len) {
/*
* This message + an additional empty header does not fit
* at the end of the buffer. Add an empty header with len == 0
@ -351,7 +351,7 @@ static void log_store(int facility, int level,
else
msg->ts_nsec = local_clock();
memset(log_dict(msg) + dict_len, 0, pad_len);
msg->len = sizeof(struct printk_log) + text_len + dict_len + pad_len;
msg->len = size;
/* insert message */
log_next_idx += msg->len;
@ -1560,9 +1560,12 @@ asmlinkage int vprintk_emit(int facility, int level,
level = kern_level - '0';
case 'd': /* KERN_DEFAULT */
lflags |= LOG_PREFIX;
case 'c': /* KERN_CONT */
break;
}
/*
* No need to check length here because vscnprintf
* put '\0' at the end of the string. Only valid and
* newly printed level is detected.
*/
text_len -= end_of_header - text;
text = (char *)end_of_header;
}
@ -1880,6 +1883,7 @@ void suspend_console(void)
console_lock();
console_suspended = 1;
up(&console_sem);
mutex_release(&console_lock_dep_map, 1, _RET_IP_);
}
void resume_console(void)
@ -1887,6 +1891,7 @@ void resume_console(void)
if (!console_suspend_enabled)
return;
down(&console_sem);
mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);
console_suspended = 0;
console_unlock();
}

22
kernel/profile.c
View file

@ -591,18 +591,28 @@ out_cleanup:
int __ref create_proc_profile(void) /* false positive from hotcpu_notifier */
{
struct proc_dir_entry *entry;
int err = 0;
if (!prof_on)
return 0;
if (create_hash_tables())
return -ENOMEM;
cpu_notifier_register_begin();
if (create_hash_tables()) {
err = -ENOMEM;
goto out;
}
entry = proc_create("profile", S_IWUSR | S_IRUGO,
NULL, &proc_profile_operations);
if (!entry)
return 0;
goto out;
proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
hotcpu_notifier(profile_cpu_callback, 0);
return 0;
__hotcpu_notifier(profile_cpu_callback, 0);
out:
cpu_notifier_register_done();
return err;
}
module_init(create_proc_profile);
subsys_initcall(create_proc_profile);
#endif /* CONFIG_PROC_FS */

6
kernel/relay.c
View file

@ -227,7 +227,7 @@ static void relay_destroy_buf(struct rchan_buf *buf)
* relay_remove_buf - remove a channel buffer
* @kref: target kernel reference that contains the relay buffer
*
* Removes the file from the fileystem, which also frees the
* Removes the file from the filesystem, which also frees the
* rchan_buf_struct and the channel buffer. Should only be called from
* kref_put().
*/
@ -1195,8 +1195,6 @@ static void relay_pipe_buf_release(struct pipe_inode_info *pipe,
static const struct pipe_buf_operations relay_pipe_buf_ops = {
.can_merge = 0,
.map = generic_pipe_buf_map,
.unmap = generic_pipe_buf_unmap,
.confirm = generic_pipe_buf_confirm,
.release = relay_pipe_buf_release,
.steal = generic_pipe_buf_steal,
@ -1253,7 +1251,7 @@ static ssize_t subbuf_splice_actor(struct file *in,
subbuf_pages = rbuf->chan->alloc_size >> PAGE_SHIFT;
pidx = (read_start / PAGE_SIZE) % subbuf_pages;
poff = read_start & ~PAGE_MASK;
nr_pages = min_t(unsigned int, subbuf_pages, pipe->buffers);
nr_pages = min_t(unsigned int, subbuf_pages, spd.nr_pages_max);
for (total_len = 0; spd.nr_pages < nr_pages; spd.nr_pages++) {
unsigned int this_len, this_end, private;

23
kernel/res_counter.c
View file

@ -22,8 +22,18 @@ void res_counter_init(struct res_counter *counter, struct res_counter *parent)
counter->parent = parent;
}
int res_counter_charge_locked(struct res_counter *counter, unsigned long val,
bool force)
static u64 res_counter_uncharge_locked(struct res_counter *counter,
unsigned long val)
{
if (WARN_ON(counter->usage < val))
val = counter->usage;
counter->usage -= val;
return counter->usage;
}
static int res_counter_charge_locked(struct res_counter *counter,
unsigned long val, bool force)
{
int ret = 0;
@ -86,15 +96,6 @@ int res_counter_charge_nofail(struct res_counter *counter, unsigned long val,
return __res_counter_charge(counter, val, limit_fail_at, true);
}
u64 res_counter_uncharge_locked(struct res_counter *counter, unsigned long val)
{
if (WARN_ON(counter->usage < val))
val = counter->usage;
counter->usage -= val;
return counter->usage;
}
u64 res_counter_uncharge_until(struct res_counter *counter,
struct res_counter *top,
unsigned long val)

2
kernel/resource.c
View file

@ -511,7 +511,7 @@ static int find_resource(struct resource *root, struct resource *new,
* @newsize: new size of the resource descriptor
* @constraint: the size and alignment constraints to be met.
*/
int reallocate_resource(struct resource *root, struct resource *old,
static int reallocate_resource(struct resource *root, struct resource *old,
resource_size_t newsize,
struct resource_constraint *constraint)
{

3
kernel/sched/clock.c
View file

@ -60,13 +60,14 @@
#include <linux/sched.h>
#include <linux/static_key.h>
#include <linux/workqueue.h>
#include <linux/compiler.h>
/*
* Scheduler clock - returns current time in nanosec units.
* This is default implementation.
* Architectures and sub-architectures can override this.
*/
unsigned long long __attribute__((weak)) sched_clock(void)
unsigned long long __weak sched_clock(void)
{
return (unsigned long long)(jiffies - INITIAL_JIFFIES)
* (NSEC_PER_SEC / HZ);

74
kernel/sched/core.c
View file

@ -73,6 +73,7 @@
#include <linux/init_task.h>
#include <linux/binfmts.h>
#include <linux/context_tracking.h>
#include <linux/compiler.h>
#include <asm/switch_to.h>
#include <asm/tlb.h>
@ -2591,8 +2592,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev)
if (likely(prev->sched_class == class &&
rq->nr_running == rq->cfs.h_nr_running)) {
p = fair_sched_class.pick_next_task(rq, prev);
if (likely(p && p != RETRY_TASK))
return p;
if (unlikely(p == RETRY_TASK))
goto again;
/* assumes fair_sched_class->next == idle_sched_class */
if (unlikely(!p))
p = idle_sched_class.pick_next_task(rq, prev);
return p;
}
again:
@ -2845,52 +2852,6 @@ int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
}
EXPORT_SYMBOL(default_wake_function);
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
{
unsigned long flags;
wait_queue_t wait;
init_waitqueue_entry(&wait, current);
__set_current_state(state);
spin_lock_irqsave(&q->lock, flags);
__add_wait_queue(q, &wait);
spin_unlock(&q->lock);
timeout = schedule_timeout(timeout);
spin_lock_irq(&q->lock);
__remove_wait_queue(q, &wait);
spin_unlock_irqrestore(&q->lock, flags);
return timeout;
}
void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
}
EXPORT_SYMBOL(interruptible_sleep_on);
long __sched
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
{
return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);
void __sched sleep_on(wait_queue_head_t *q)
{
sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
}
EXPORT_SYMBOL(sleep_on);
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
{
return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
}
EXPORT_SYMBOL(sleep_on_timeout);
#ifdef CONFIG_RT_MUTEXES
/*
@ -3169,6 +3130,7 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
dl_se->dl_throttled = 0;
dl_se->dl_new = 1;
dl_se->dl_yielded = 0;
}
static void __setscheduler_params(struct task_struct *p,
@ -3684,6 +3646,7 @@ SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
* sys_sched_setattr - same as above, but with extended sched_attr
* @pid: the pid in question.
* @uattr: structure containing the extended parameters.
* @flags: for future extension.
*/
SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
unsigned int, flags)
@ -3828,6 +3791,7 @@ err_size:
* @pid: the pid in question.
* @uattr: structure containing the extended parameters.
* @size: sizeof(attr) for fwd/bwd comp.
* @flags: for future extension.
*/
SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
unsigned int, size, unsigned int, flags)
@ -6062,6 +6026,8 @@ sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
,
.last_balance = jiffies,
.balance_interval = sd_weight,
.max_newidle_lb_cost = 0,
.next_decay_max_lb_cost = jiffies,
};
SD_INIT_NAME(sd, NUMA);
sd->private = &tl->data;
@ -6498,7 +6464,7 @@ static cpumask_var_t fallback_doms;
* cpu core maps. It is supposed to return 1 if the topology changed
* or 0 if it stayed the same.
*/
int __attribute__((weak)) arch_update_cpu_topology(void)
int __weak arch_update_cpu_topology(void)
{
return 0;
}
@ -7230,7 +7196,7 @@ void sched_move_task(struct task_struct *tsk)
if (unlikely(running))
tsk->sched_class->put_prev_task(rq, tsk);
tg = container_of(task_css_check(tsk, cpu_cgroup_subsys_id,
tg = container_of(task_css_check(tsk, cpu_cgrp_id,
lockdep_is_held(&tsk->sighand->siglock)),
struct task_group, css);
tg = autogroup_task_group(tsk, tg);
@ -7657,7 +7623,7 @@ static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
{
struct task_struct *task;
cgroup_taskset_for_each(task, css, tset) {
cgroup_taskset_for_each(task, tset) {
#ifdef CONFIG_RT_GROUP_SCHED
if (!sched_rt_can_attach(css_tg(css), task))
return -EINVAL;
@ -7675,7 +7641,7 @@ static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
{
struct task_struct *task;
cgroup_taskset_for_each(task, css, tset)
cgroup_taskset_for_each(task, tset)
sched_move_task(task);
}
@ -8014,8 +7980,7 @@ static struct cftype cpu_files[] = {
{ } /* terminate */
};
struct cgroup_subsys cpu_cgroup_subsys = {
.name = "cpu",
struct cgroup_subsys cpu_cgrp_subsys = {
.css_alloc = cpu_cgroup_css_alloc,
.css_free = cpu_cgroup_css_free,
.css_online = cpu_cgroup_css_online,
@ -8023,7 +7988,6 @@ struct cgroup_subsys cpu_cgroup_subsys = {
.can_attach = cpu_cgroup_can_attach,
.attach = cpu_cgroup_attach,
.exit = cpu_cgroup_exit,
.subsys_id = cpu_cgroup_subsys_id,
.base_cftypes = cpu_files,
.early_init = 1,
};

6
kernel/sched/cpuacct.c
View file

@ -41,7 +41,7 @@ static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css)
/* return cpu accounting group to which this task belongs */
static inline struct cpuacct *task_ca(struct task_struct *tsk)
{
return css_ca(task_css(tsk, cpuacct_subsys_id));
return css_ca(task_css(tsk, cpuacct_cgrp_id));
}
static inline struct cpuacct *parent_ca(struct cpuacct *ca)
@ -275,11 +275,9 @@ void cpuacct_account_field(struct task_struct *p, int index, u64 val)
rcu_read_unlock();
}
struct cgroup_subsys cpuacct_subsys = {
.name = "cpuacct",
struct cgroup_subsys cpuacct_cgrp_subsys = {
.css_alloc = cpuacct_css_alloc,
.css_free = cpuacct_css_free,
.subsys_id = cpuacct_subsys_id,
.base_cftypes = files,
.early_init = 1,
};

4
kernel/sched/cpudeadline.c
View file

@ -210,7 +210,5 @@ int cpudl_init(struct cpudl *cp)
*/
void cpudl_cleanup(struct cpudl *cp)
{
/*
* nothing to do for the moment
*/
free_cpumask_var(cp->free_cpus);
}

3
kernel/sched/cpupri.c
View file

@ -70,8 +70,7 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
int idx = 0;
int task_pri = convert_prio(p->prio);
if (task_pri >= MAX_RT_PRIO)
return 0;
BUG_ON(task_pri >= CPUPRI_NR_PRIORITIES);
for (idx = 0; idx < task_pri; idx++) {
struct cpupri_vec *vec = &cp->pri_to_cpu[idx];

32
kernel/sched/cputime.c
View file

@ -332,50 +332,50 @@ out:
* softirq as those do not count in task exec_runtime any more.
*/
static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
struct rq *rq)
struct rq *rq, int ticks)
{
cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
cputime_t scaled = cputime_to_scaled(cputime_one_jiffy);
u64 cputime = (__force u64) cputime_one_jiffy;
u64 *cpustat = kcpustat_this_cpu->cpustat;
if (steal_account_process_tick())
return;
cputime *= ticks;
scaled *= ticks;
if (irqtime_account_hi_update()) {
cpustat[CPUTIME_IRQ] += (__force u64) cputime_one_jiffy;
cpustat[CPUTIME_IRQ] += cputime;
} else if (irqtime_account_si_update()) {
cpustat[CPUTIME_SOFTIRQ] += (__force u64) cputime_one_jiffy;
cpustat[CPUTIME_SOFTIRQ] += cputime;
} else if (this_cpu_ksoftirqd() == p) {
/*
* ksoftirqd time do not get accounted in cpu_softirq_time.
* So, we have to handle it separately here.
* Also, p->stime needs to be updated for ksoftirqd.
*/
__account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
CPUTIME_SOFTIRQ);
__account_system_time(p, cputime, scaled, CPUTIME_SOFTIRQ);
} else if (user_tick) {
account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
account_user_time(p, cputime, scaled);
} else if (p == rq->idle) {
account_idle_time(cputime_one_jiffy);
account_idle_time(cputime);
} else if (p->flags & PF_VCPU) { /* System time or guest time */
account_guest_time(p, cputime_one_jiffy, one_jiffy_scaled);
account_guest_time(p, cputime, scaled);
} else {
__account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
CPUTIME_SYSTEM);
__account_system_time(p, cputime, scaled, CPUTIME_SYSTEM);
}
}
static void irqtime_account_idle_ticks(int ticks)
{
int i;
struct rq *rq = this_rq();
for (i = 0; i < ticks; i++)
irqtime_account_process_tick(current, 0, rq);
irqtime_account_process_tick(current, 0, rq, ticks);
}
#else /* CONFIG_IRQ_TIME_ACCOUNTING */
static inline void irqtime_account_idle_ticks(int ticks) {}
static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
struct rq *rq) {}
struct rq *rq, int nr_ticks) {}
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
/*
@ -464,7 +464,7 @@ void account_process_tick(struct task_struct *p, int user_tick)
return;
if (sched_clock_irqtime) {
irqtime_account_process_tick(p, user_tick, rq);
irqtime_account_process_tick(p, user_tick, rq, 1);
return;
}

5
kernel/sched/deadline.c
View file

@ -528,6 +528,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer)
sched_clock_tick();
update_rq_clock(rq);
dl_se->dl_throttled = 0;
dl_se->dl_yielded = 0;
if (p->on_rq) {
enqueue_task_dl(rq, p, ENQUEUE_REPLENISH);
if (task_has_dl_policy(rq->curr))
@ -893,10 +894,10 @@ static void yield_task_dl(struct rq *rq)
* We make the task go to sleep until its current deadline by
* forcing its runtime to zero. This way, update_curr_dl() stops
* it and the bandwidth timer will wake it up and will give it
* new scheduling parameters (thanks to dl_new=1).
* new scheduling parameters (thanks to dl_yielded=1).
*/
if (p->dl.runtime > 0) {
rq->curr->dl.dl_new = 1;
rq->curr->dl.dl_yielded = 1;
p->dl.runtime = 0;
}
update_curr_dl(rq);

3
kernel/sched/debug.c
View file

@ -111,8 +111,7 @@ static char *task_group_path(struct task_group *tg)
if (autogroup_path(tg, group_path, PATH_MAX))
return group_path;
cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
return group_path;
return cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
}
#endif

16
kernel/sched/fair.c
View file

@ -6658,6 +6658,7 @@ static int idle_balance(struct rq *this_rq)
int this_cpu = this_rq->cpu;
idle_enter_fair(this_rq);
/*
* We must set idle_stamp _before_ calling idle_balance(), such that we
* measure the duration of idle_balance() as idle time.
@ -6710,14 +6711,16 @@ static int idle_balance(struct rq *this_rq)
raw_spin_lock(&this_rq->lock);
if (curr_cost > this_rq->max_idle_balance_cost)
this_rq->max_idle_balance_cost = curr_cost;
/*
* While browsing the domains, we released the rq lock.
* A task could have be enqueued in the meantime
* While browsing the domains, we released the rq lock, a task could
* have been enqueued in the meantime. Since we're not going idle,
* pretend we pulled a task.
*/
if (this_rq->cfs.h_nr_running && !pulled_task) {
if (this_rq->cfs.h_nr_running && !pulled_task)
pulled_task = 1;
goto out;
}
if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
/*
@ -6727,9 +6730,6 @@ static int idle_balance(struct rq *this_rq)
this_rq->next_balance = next_balance;
}
if (curr_cost > this_rq->max_idle_balance_cost)
this_rq->max_idle_balance_cost = curr_cost;
out:
/* Is there a task of a high priority class? */
if (this_rq->nr_running != this_rq->cfs.h_nr_running)

150
kernel/sched/idle.c
View file

@ -63,6 +63,136 @@ void __weak arch_cpu_idle(void)
local_irq_enable();
}
/**
* cpuidle_idle_call - the main idle function
*
* NOTE: no locks or semaphores should be used here
* return non-zero on failure
*/
static int cpuidle_idle_call(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
int next_state, entered_state, ret;
bool broadcast;
/*
* Check if the idle task must be rescheduled. If it is the
* case, exit the function after re-enabling the local irq and
* set again the polling flag
*/
if (current_clr_polling_and_test()) {
local_irq_enable();
__current_set_polling();
return 0;
}
/*
* During the idle period, stop measuring the disabled irqs
* critical sections latencies
*/
stop_critical_timings();
/*
* Tell the RCU framework we are entering an idle section,
* so no more rcu read side critical sections and one more
* step to the grace period
*/
rcu_idle_enter();
/*
* Check if the cpuidle framework is ready, otherwise fallback
* to the default arch specific idle method
*/
ret = cpuidle_enabled(drv, dev);
if (!ret) {
/*
* Ask the governor to choose an idle state it thinks
* it is convenient to go to. There is *always* a
* convenient idle state
*/
next_state = cpuidle_select(drv, dev);
/*
* The idle task must be scheduled, it is pointless to
* go to idle, just update no idle residency and get
* out of this function
*/
if (current_clr_polling_and_test()) {
dev->last_residency = 0;
entered_state = next_state;
local_irq_enable();
} else {
broadcast = !!(drv->states[next_state].flags &
CPUIDLE_FLAG_TIMER_STOP);
if (broadcast)
/*
* Tell the time framework to switch
* to a broadcast timer because our
* local timer will be shutdown. If a
* local timer is used from another
* cpu as a broadcast timer, this call
* may fail if it is not available
*/
ret = clockevents_notify(
CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
&dev->cpu);
if (!ret) {
trace_cpu_idle_rcuidle(next_state, dev->cpu);
/*
* Enter the idle state previously
* returned by the governor
* decision. This function will block
* until an interrupt occurs and will
* take care of re-enabling the local
* interrupts
*/
entered_state = cpuidle_enter(drv, dev,
next_state);
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT,
dev->cpu);
if (broadcast)
clockevents_notify(
CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
&dev->cpu);
/*
* Give the governor an opportunity to reflect on the
* outcome
*/
cpuidle_reflect(dev, entered_state);
}
}
}
/*
* We can't use the cpuidle framework, let's use the default
* idle routine
*/
if (ret)
arch_cpu_idle();
__current_set_polling();
/*
* It is up to the idle functions to enable back the local
* interrupt
*/
if (WARN_ON_ONCE(irqs_disabled()))
local_irq_enable();
rcu_idle_exit();
start_critical_timings();
return 0;
}
/*
* Generic idle loop implementation
*/
@ -90,23 +220,11 @@ static void cpu_idle_loop(void)
* know that the IPI is going to arrive right
* away
*/
if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
if (cpu_idle_force_poll || tick_check_broadcast_expired())
cpu_idle_poll();
} else {
if (!current_clr_polling_and_test()) {
stop_critical_timings();
rcu_idle_enter();
if (cpuidle_idle_call())
arch_cpu_idle();
if (WARN_ON_ONCE(irqs_disabled()))
local_irq_enable();
rcu_idle_exit();
start_critical_timings();
} else {
local_irq_enable();
}
__current_set_polling();
}
else
cpuidle_idle_call();
arch_cpu_idle_exit();
}

2
kernel/sched/stats.c
View file

@ -142,4 +142,4 @@ static int __init proc_schedstat_init(void)
proc_create("schedstat", 0, NULL, &proc_schedstat_operations);
return 0;
}
module_init(proc_schedstat_init);
subsys_initcall(proc_schedstat_init);

124
kernel/seccomp.c
View file

@ -55,60 +55,32 @@ struct seccomp_filter {
atomic_t usage;
struct seccomp_filter *prev;
unsigned short len; /* Instruction count */
struct sock_filter insns[];
struct sock_filter_int insnsi[];
};
/* Limit any path through the tree to 256KB worth of instructions. */
#define MAX_INSNS_PER_PATH ((1 << 18) / sizeof(struct sock_filter))
/**
* get_u32 - returns a u32 offset into data
* @data: a unsigned 64 bit value
* @index: 0 or 1 to return the first or second 32-bits
*
* This inline exists to hide the length of unsigned long. If a 32-bit
* unsigned long is passed in, it will be extended and the top 32-bits will be
* 0. If it is a 64-bit unsigned long, then whatever data is resident will be
* properly returned.
*
/*
* Endianness is explicitly ignored and left for BPF program authors to manage
* as per the specific architecture.
*/
static inline u32 get_u32(u64 data, int index)
static void populate_seccomp_data(struct seccomp_data *sd)
{
return ((u32 *)&data)[index];
}
struct task_struct *task = current;
struct pt_regs *regs = task_pt_regs(task);
unsigned long args[6];
/* Helper for bpf_load below. */
#define BPF_DATA(_name) offsetof(struct seccomp_data, _name)
/**
* bpf_load: checks and returns a pointer to the requested offset
* @off: offset into struct seccomp_data to load from
*
* Returns the requested 32-bits of data.
* seccomp_check_filter() should assure that @off is 32-bit aligned
* and not out of bounds. Failure to do so is a BUG.
*/
u32 seccomp_bpf_load(int off)
{
struct pt_regs *regs = task_pt_regs(current);
if (off == BPF_DATA(nr))
return syscall_get_nr(current, regs);
if (off == BPF_DATA(arch))
return syscall_get_arch(current, regs);
if (off >= BPF_DATA(args[0]) && off < BPF_DATA(args[6])) {
unsigned long value;
int arg = (off - BPF_DATA(args[0])) / sizeof(u64);
int index = !!(off % sizeof(u64));
syscall_get_arguments(current, regs, arg, 1, &value);
return get_u32(value, index);
}
if (off == BPF_DATA(instruction_pointer))
return get_u32(KSTK_EIP(current), 0);
if (off == BPF_DATA(instruction_pointer) + sizeof(u32))
return get_u32(KSTK_EIP(current), 1);
/* seccomp_check_filter should make this impossible. */
BUG();
sd->nr = syscall_get_nr(task, regs);
sd->arch = syscall_get_arch();
syscall_get_arguments(task, regs, 0, 6, args);
sd->args[0] = args[0];
sd->args[1] = args[1];
sd->args[2] = args[2];
sd->args[3] = args[3];
sd->args[4] = args[4];
sd->args[5] = args[5];
sd->instruction_pointer = KSTK_EIP(task);
}
/**
@ -133,17 +105,17 @@ static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen)
switch (code) {
case BPF_S_LD_W_ABS:
ftest->code = BPF_S_ANC_SECCOMP_LD_W;
ftest->code = BPF_LDX | BPF_W | BPF_ABS;
/* 32-bit aligned and not out of bounds. */
if (k >= sizeof(struct seccomp_data) || k & 3)
return -EINVAL;
continue;
case BPF_S_LD_W_LEN:
ftest->code = BPF_S_LD_IMM;
ftest->code = BPF_LD | BPF_IMM;
ftest->k = sizeof(struct seccomp_data);
continue;
case BPF_S_LDX_W_LEN:
ftest->code = BPF_S_LDX_IMM;
ftest->code = BPF_LDX | BPF_IMM;
ftest->k = sizeof(struct seccomp_data);
continue;
/* Explicitly include allowed calls. */
@ -185,6 +157,7 @@ static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen)
case BPF_S_JMP_JGT_X:
case BPF_S_JMP_JSET_K:
case BPF_S_JMP_JSET_X:
sk_decode_filter(ftest, ftest);
continue;
default:
return -EINVAL;
@ -202,18 +175,21 @@ static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen)
static u32 seccomp_run_filters(int syscall)
{
struct seccomp_filter *f;
struct seccomp_data sd;
u32 ret = SECCOMP_RET_ALLOW;
/* Ensure unexpected behavior doesn't result in failing open. */
if (WARN_ON(current->seccomp.filter == NULL))
return SECCOMP_RET_KILL;
populate_seccomp_data(&sd);
/*
* All filters in the list are evaluated and the lowest BPF return
* value always takes priority (ignoring the DATA).
*/
for (f = current->seccomp.filter; f; f = f->prev) {
u32 cur_ret = sk_run_filter(NULL, f->insns);
u32 cur_ret = sk_run_filter_int_seccomp(&sd, f->insnsi);
if ((cur_ret & SECCOMP_RET_ACTION) < (ret & SECCOMP_RET_ACTION))
ret = cur_ret;
}
@ -231,6 +207,8 @@ static long seccomp_attach_filter(struct sock_fprog *fprog)
struct seccomp_filter *filter;
unsigned long fp_size = fprog->len * sizeof(struct sock_filter);
unsigned long total_insns = fprog->len;
struct sock_filter *fp;
int new_len;
long ret;
if (fprog->len == 0 || fprog->len > BPF_MAXINSNS)
@ -252,28 +230,45 @@ static long seccomp_attach_filter(struct sock_fprog *fprog)
CAP_SYS_ADMIN) != 0)
return -EACCES;
/* Allocate a new seccomp_filter */
filter = kzalloc(sizeof(struct seccomp_filter) + fp_size,
GFP_KERNEL|__GFP_NOWARN);
if (!filter)
fp = kzalloc(fp_size, GFP_KERNEL|__GFP_NOWARN);
if (!fp)
return -ENOMEM;
atomic_set(&filter->usage, 1);
filter->len = fprog->len;
/* Copy the instructions from fprog. */
ret = -EFAULT;
if (copy_from_user(filter->insns, fprog->filter, fp_size))
goto fail;
if (copy_from_user(fp, fprog->filter, fp_size))
goto free_prog;
/* Check and rewrite the fprog via the skb checker */
ret = sk_chk_filter(filter->insns, filter->len);
ret = sk_chk_filter(fp, fprog->len);
if (ret)
goto fail;
goto free_prog;
/* Check and rewrite the fprog for seccomp use */
ret = seccomp_check_filter(filter->insns, filter->len);
ret = seccomp_check_filter(fp, fprog->len);
if (ret)
goto fail;
goto free_prog;
/* Convert 'sock_filter' insns to 'sock_filter_int' insns */
ret = sk_convert_filter(fp, fprog->len, NULL, &new_len);
if (ret)
goto free_prog;
/* Allocate a new seccomp_filter */
ret = -ENOMEM;
filter = kzalloc(sizeof(struct seccomp_filter) +
sizeof(struct sock_filter_int) * new_len,
GFP_KERNEL|__GFP_NOWARN);
if (!filter)
goto free_prog;
ret = sk_convert_filter(fp, fprog->len, filter->insnsi, &new_len);
if (ret)
goto free_filter;
kfree(fp);
atomic_set(&filter->usage, 1);
filter->len = new_len;
/*
* If there is an existing filter, make it the prev and don't drop its
@ -282,8 +277,11 @@ static long seccomp_attach_filter(struct sock_fprog *fprog)
filter->prev = current->seccomp.filter;
current->seccomp.filter = filter;
return 0;
fail:
free_filter:
kfree(filter);
free_prog:
kfree(fp);
return ret;
}
@ -293,7 +291,7 @@ fail:
*
* Returns 0 on success and non-zero otherwise.
*/
long seccomp_attach_user_filter(char __user *user_filter)
static long seccomp_attach_user_filter(char __user *user_filter)
{
struct sock_fprog fprog;
long ret = -EFAULT;
@ -351,7 +349,7 @@ static void seccomp_send_sigsys(int syscall, int reason)
info.si_code = SYS_SECCOMP;
info.si_call_addr = (void __user *)KSTK_EIP(current);
info.si_errno = reason;
info.si_arch = syscall_get_arch(current, task_pt_regs(current));
info.si_arch = syscall_get_arch();
info.si_syscall = syscall;
force_sig_info(SIGSYS, &info, current);
}

6
kernel/signal.c
View file

@ -33,6 +33,8 @@
#include <linux/uprobes.h>
#include <linux/compat.h>
#include <linux/cn_proc.h>
#include <linux/compiler.h>
#define CREATE_TRACE_POINTS
#include <trace/events/signal.h>
@ -2382,7 +2384,7 @@ relock:
* @regs: user register state
* @stepping: nonzero if debugger single-step or block-step in use
*
* This function should be called when a signal has succesfully been
* This function should be called when a signal has successfully been
* delivered. It updates the blocked signals accordingly (@ka->sa.sa_mask
* is always blocked, and the signal itself is blocked unless %SA_NODEFER
* is set in @ka->sa.sa_flags. Tracing is notified.
@ -3618,7 +3620,7 @@ SYSCALL_DEFINE3(sigsuspend, int, unused1, int, unused2, old_sigset_t, mask)
}
#endif
__attribute__((weak)) const char *arch_vma_name(struct vm_area_struct *vma)
__weak const char *arch_vma_name(struct vm_area_struct *vma)
{
return NULL;
}

15
kernel/sys.c
View file

@ -1996,6 +1996,21 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
if (arg2 || arg3 || arg4 || arg5)
return -EINVAL;
return current->no_new_privs ? 1 : 0;
case PR_GET_THP_DISABLE:
if (arg2 || arg3 || arg4 || arg5)
return -EINVAL;
error = !!(me->mm->def_flags & VM_NOHUGEPAGE);
break;
case PR_SET_THP_DISABLE:
if (arg3 || arg4 || arg5)
return -EINVAL;
down_write(&me->mm->mmap_sem);
if (arg2)
me->mm->def_flags |= VM_NOHUGEPAGE;
else
me->mm->def_flags &= ~VM_NOHUGEPAGE;
up_write(&me->mm->mmap_sem);
break;
default:
error = -EINVAL;
break;

2
kernel/sys_ni.c
View file

@ -146,11 +146,13 @@ cond_syscall(sys_io_destroy);
cond_syscall(sys_io_submit);
cond_syscall(sys_io_cancel);
cond_syscall(sys_io_getevents);
cond_syscall(sys_sysfs);
cond_syscall(sys_syslog);
cond_syscall(sys_process_vm_readv);
cond_syscall(sys_process_vm_writev);
cond_syscall(compat_sys_process_vm_readv);
cond_syscall(compat_sys_process_vm_writev);
cond_syscall(sys_uselib);
/* arch-specific weak syscall entries */
cond_syscall(sys_pciconfig_read);

10
kernel/sysctl.c
View file

@ -123,7 +123,7 @@ static int __maybe_unused neg_one = -1;
static int zero;
static int __maybe_unused one = 1;
static int __maybe_unused two = 2;
static int __maybe_unused three = 3;
static int __maybe_unused four = 4;
static unsigned long one_ul = 1;
static int one_hundred = 100;
#ifdef CONFIG_PRINTK
@ -141,6 +141,11 @@ static int min_percpu_pagelist_fract = 8;
static int ngroups_max = NGROUPS_MAX;
static const int cap_last_cap = CAP_LAST_CAP;
/*this is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs */
#ifdef CONFIG_DETECT_HUNG_TASK
static unsigned long hung_task_timeout_max = (LONG_MAX/HZ);
#endif
#ifdef CONFIG_INOTIFY_USER
#include <linux/inotify.h>
#endif
@ -985,6 +990,7 @@ static struct ctl_table kern_table[] = {
.maxlen = sizeof(unsigned long),
.mode = 0644,
.proc_handler = proc_dohung_task_timeout_secs,
.extra2 = &hung_task_timeout_max,
},
{
.procname = "hung_task_warnings",
@ -1264,7 +1270,7 @@ static struct ctl_table vm_table[] = {
.mode = 0644,
.proc_handler = drop_caches_sysctl_handler,
.extra1 = &one,
.extra2 = &three,
.extra2 = &four,
},
#ifdef CONFIG_COMPACTION
{

2
kernel/time/tick-common.c
View file

@ -276,7 +276,7 @@ static bool tick_check_preferred(struct clock_event_device *curdev,
bool tick_check_replacement(struct clock_event_device *curdev,
struct clock_event_device *newdev)
{
if (tick_check_percpu(curdev, newdev, smp_processor_id()))
if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
return false;
return tick_check_preferred(curdev, newdev);

5
kernel/time/tick-sched.c
View file

@ -84,6 +84,9 @@ static void tick_do_update_jiffies64(ktime_t now)
/* Keep the tick_next_period variable up to date */
tick_next_period = ktime_add(last_jiffies_update, tick_period);
} else {
write_sequnlock(&jiffies_lock);
return;
}
write_sequnlock(&jiffies_lock);
update_wall_time();
@ -967,7 +970,7 @@ static void tick_nohz_switch_to_nohz(void)
struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
ktime_t next;
if (!tick_nohz_active)
if (!tick_nohz_enabled)
return;
local_irq_disable();

5
kernel/time/timekeeping.c
View file

@ -22,6 +22,7 @@
#include <linux/tick.h>
#include <linux/stop_machine.h>
#include <linux/pvclock_gtod.h>
#include <linux/compiler.h>
#include "tick-internal.h"
#include "ntp_internal.h"
@ -760,7 +761,7 @@ u64 timekeeping_max_deferment(void)
*
* XXX - Do be sure to remove it once all arches implement it.
*/
void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
void __weak read_persistent_clock(struct timespec *ts)
{
ts->tv_sec = 0;
ts->tv_nsec = 0;
@ -775,7 +776,7 @@ void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
*
* XXX - Do be sure to remove it once all arches implement it.
*/
void __attribute__((weak)) read_boot_clock(struct timespec *ts)
void __weak read_boot_clock(struct timespec *ts)
{
ts->tv_sec = 0;
ts->tv_nsec = 0;

1
kernel/trace/Kconfig
View file

@ -424,6 +424,7 @@ config UPROBE_EVENT
bool "Enable uprobes-based dynamic events"
depends on ARCH_SUPPORTS_UPROBES
depends on MMU
depends on PERF_EVENTS
select UPROBES
select PROBE_EVENTS
select TRACING

3
kernel/trace/blktrace.c
View file

@ -1429,7 +1429,8 @@ static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
return print_one_line(iter, true);
}
static int blk_tracer_set_flag(u32 old_flags, u32 bit, int set)
static int
blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
/* don't output context-info for blk_classic output */
if (bit == TRACE_BLK_OPT_CLASSIC) {

162
kernel/trace/ftrace.c
View file

@ -237,14 +237,13 @@ static int control_ops_alloc(struct ftrace_ops *ops)
return 0;
}
static void control_ops_free(struct ftrace_ops *ops)
{
free_percpu(ops->disabled);
}
static void update_global_ops(void)
{
ftrace_func_t func;
ftrace_func_t func = ftrace_global_list_func;
void *private = NULL;
/* The list has its own recursion protection. */
global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
/*
* If there's only one function registered, then call that
@ -254,23 +253,17 @@ static void update_global_ops(void)
if (ftrace_global_list == &ftrace_list_end ||
ftrace_global_list->next == &ftrace_list_end) {
func = ftrace_global_list->func;
private = ftrace_global_list->private;
/*
* As we are calling the function directly.
* If it does not have recursion protection,
* the function_trace_op needs to be updated
* accordingly.
*/
if (ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE)
global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
else
if (!(ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE))
global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;
} else {
func = ftrace_global_list_func;
/* The list has its own recursion protection. */
global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
}
/* If we filter on pids, update to use the pid function */
if (!list_empty(&ftrace_pids)) {
set_ftrace_pid_function(func);
@ -278,6 +271,7 @@ static void update_global_ops(void)
}
global_ops.func = func;
global_ops.private = private;
}
static void ftrace_sync(struct work_struct *work)
@ -437,6 +431,9 @@ static int remove_ftrace_list_ops(struct ftrace_ops **list,
static int __register_ftrace_function(struct ftrace_ops *ops)
{
if (ops->flags & FTRACE_OPS_FL_DELETED)
return -EINVAL;
if (FTRACE_WARN_ON(ops == &global_ops))
return -EINVAL;
@ -1172,8 +1169,6 @@ struct ftrace_page {
int size;
};
static struct ftrace_page *ftrace_new_pgs;
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
@ -1560,7 +1555,7 @@ unsigned long ftrace_location(unsigned long ip)
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
int ftrace_text_reserved(void *start, void *end)
int ftrace_text_reserved(const void *start, const void *end)
{
unsigned long ret;
@ -1994,6 +1989,7 @@ int __weak ftrace_arch_code_modify_post_process(void)
void ftrace_modify_all_code(int command)
{
int update = command & FTRACE_UPDATE_TRACE_FUNC;
int err = 0;
/*
* If the ftrace_caller calls a ftrace_ops func directly,
@ -2005,8 +2001,11 @@ void ftrace_modify_all_code(int command)
* to make sure the ops are having the right functions
* traced.
*/
if (update)
ftrace_update_ftrace_func(ftrace_ops_list_func);
if (update) {
err = ftrace_update_ftrace_func(ftrace_ops_list_func);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_UPDATE_CALLS)
ftrace_replace_code(1);
@ -2019,13 +2018,16 @@ void ftrace_modify_all_code(int command)
/* If irqs are disabled, we are in stop machine */
if (!irqs_disabled())
smp_call_function(ftrace_sync_ipi, NULL, 1);
ftrace_update_ftrace_func(ftrace_trace_function);
err = ftrace_update_ftrace_func(ftrace_trace_function);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_START_FUNC_RET)
ftrace_enable_ftrace_graph_caller();
err = ftrace_enable_ftrace_graph_caller();
else if (command & FTRACE_STOP_FUNC_RET)
ftrace_disable_ftrace_graph_caller();
err = ftrace_disable_ftrace_graph_caller();
FTRACE_WARN_ON(err);
}
static int __ftrace_modify_code(void *data)
@ -2093,6 +2095,11 @@ static ftrace_func_t saved_ftrace_func;
static int ftrace_start_up;
static int global_start_up;
static void control_ops_free(struct ftrace_ops *ops)
{
free_percpu(ops->disabled);
}
static void ftrace_startup_enable(int command)
{
if (saved_ftrace_func != ftrace_trace_function) {
@ -2244,7 +2251,6 @@ static void ftrace_shutdown_sysctl(void)
}
static cycle_t ftrace_update_time;
static unsigned long ftrace_update_cnt;
unsigned long ftrace_update_tot_cnt;
static inline int ops_traces_mod(struct ftrace_ops *ops)
@ -2300,11 +2306,12 @@ static int referenced_filters(struct dyn_ftrace *rec)
return cnt;
}
static int ftrace_update_code(struct module *mod)
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
{
struct ftrace_page *pg;
struct dyn_ftrace *p;
cycle_t start, stop;
unsigned long update_cnt = 0;
unsigned long ref = 0;
bool test = false;
int i;
@ -2330,9 +2337,8 @@ static int ftrace_update_code(struct module *mod)
}
start = ftrace_now(raw_smp_processor_id());
ftrace_update_cnt = 0;
for (pg = ftrace_new_pgs; pg; pg = pg->next) {
for (pg = new_pgs; pg; pg = pg->next) {
for (i = 0; i < pg->index; i++) {
int cnt = ref;
@ -2353,7 +2359,7 @@ static int ftrace_update_code(struct module *mod)
if (!ftrace_code_disable(mod, p))
break;
ftrace_update_cnt++;
update_cnt++;
/*
* If the tracing is enabled, go ahead and enable the record.
@ -2372,11 +2378,9 @@ static int ftrace_update_code(struct module *mod)
}
}
ftrace_new_pgs = NULL;
stop = ftrace_now(raw_smp_processor_id());
ftrace_update_time = stop - start;
ftrace_update_tot_cnt += ftrace_update_cnt;
ftrace_update_tot_cnt += update_cnt;
return 0;
}
@ -2468,22 +2472,6 @@ ftrace_allocate_pages(unsigned long num_to_init)
return NULL;
}
static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
{
int cnt;
if (!num_to_init) {
pr_info("ftrace: No functions to be traced?\n");
return -1;
}
cnt = num_to_init / ENTRIES_PER_PAGE;
pr_info("ftrace: allocating %ld entries in %d pages\n",
num_to_init, cnt + 1);
return 0;
}
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
struct ftrace_iterator {
@ -2871,7 +2859,9 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag,
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
return ftrace_regex_open(&global_ops,
struct ftrace_ops *ops = inode->i_private;
return ftrace_regex_open(ops,
FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
inode, file);
}
@ -2879,7 +2869,9 @@ ftrace_filter_open(struct inode *inode, struct file *file)
static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
struct ftrace_ops *ops = inode->i_private;
return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
inode, file);
}
@ -4109,6 +4101,36 @@ static const struct file_operations ftrace_graph_notrace_fops = {
};
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
void ftrace_create_filter_files(struct ftrace_ops *ops,
struct dentry *parent)
{
trace_create_file("set_ftrace_filter", 0644, parent,
ops, &ftrace_filter_fops);
trace_create_file("set_ftrace_notrace", 0644, parent,
ops, &ftrace_notrace_fops);
}
/*
* The name "destroy_filter_files" is really a misnomer. Although
* in the future, it may actualy delete the files, but this is
* really intended to make sure the ops passed in are disabled
* and that when this function returns, the caller is free to
* free the ops.
*
* The "destroy" name is only to match the "create" name that this
* should be paired with.
*/
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
mutex_lock(&ftrace_lock);
if (ops->flags & FTRACE_OPS_FL_ENABLED)
ftrace_shutdown(ops, 0);
ops->flags |= FTRACE_OPS_FL_DELETED;
mutex_unlock(&ftrace_lock);
}
static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
{
@ -4118,11 +4140,7 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
trace_create_file("enabled_functions", 0444,
d_tracer, NULL, &ftrace_enabled_fops);
trace_create_file("set_ftrace_filter", 0644, d_tracer,
NULL, &ftrace_filter_fops);
trace_create_file("set_ftrace_notrace", 0644, d_tracer,
NULL, &ftrace_notrace_fops);
ftrace_create_filter_files(&global_ops, d_tracer);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
trace_create_file("set_graph_function", 0444, d_tracer,
@ -4238,9 +4256,6 @@ static int ftrace_process_locs(struct module *mod,
/* Assign the last page to ftrace_pages */
ftrace_pages = pg;
/* These new locations need to be initialized */
ftrace_new_pgs = start_pg;
/*
* We only need to disable interrupts on start up
* because we are modifying code that an interrupt
@ -4251,7 +4266,7 @@ static int ftrace_process_locs(struct module *mod,
*/
if (!mod)
local_irq_save(flags);
ftrace_update_code(mod);
ftrace_update_code(mod, start_pg);
if (!mod)
local_irq_restore(flags);
ret = 0;
@ -4360,30 +4375,27 @@ struct notifier_block ftrace_module_exit_nb = {
.priority = INT_MIN, /* Run after anything that can remove kprobes */
};
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];
void __init ftrace_init(void)
{
unsigned long count, addr, flags;
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];
unsigned long count, flags;
int ret;
/* Keep the ftrace pointer to the stub */
addr = (unsigned long)ftrace_stub;
local_irq_save(flags);
ftrace_dyn_arch_init(&addr);
ret = ftrace_dyn_arch_init();
local_irq_restore(flags);
/* ftrace_dyn_arch_init places the return code in addr */
if (addr)
if (ret)
goto failed;
count = __stop_mcount_loc - __start_mcount_loc;
ret = ftrace_dyn_table_alloc(count);
if (ret)
if (!count) {
pr_info("ftrace: No functions to be traced?\n");
goto failed;
}
pr_info("ftrace: allocating %ld entries in %ld pages\n",
count, count / ENTRIES_PER_PAGE + 1);
last_ftrace_enabled = ftrace_enabled = 1;
@ -4431,7 +4443,13 @@ static inline void ftrace_startup_enable(int command) { }
(ops)->flags |= FTRACE_OPS_FL_ENABLED; \
___ret; \
})
# define ftrace_shutdown(ops, command) __unregister_ftrace_function(ops)
# define ftrace_shutdown(ops, command) \
({ \
int ___ret = __unregister_ftrace_function(ops); \
if (!___ret) \
(ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
___ret; \
})
# define ftrace_startup_sysctl() do { } while (0)
# define ftrace_shutdown_sysctl() do { } while (0)

19
kernel/trace/ring_buffer.c
View file

@ -1301,7 +1301,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
* In that off case, we need to allocate for all possible cpus.
*/
#ifdef CONFIG_HOTPLUG_CPU
get_online_cpus();
cpu_notifier_register_begin();
cpumask_copy(buffer->cpumask, cpu_online_mask);
#else
cpumask_copy(buffer->cpumask, cpu_possible_mask);
@ -1324,10 +1324,10 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
#ifdef CONFIG_HOTPLUG_CPU
buffer->cpu_notify.notifier_call = rb_cpu_notify;
buffer->cpu_notify.priority = 0;
register_cpu_notifier(&buffer->cpu_notify);
__register_cpu_notifier(&buffer->cpu_notify);
cpu_notifier_register_done();
#endif
put_online_cpus();
mutex_init(&buffer->mutex);
return buffer;
@ -1341,7 +1341,9 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
fail_free_cpumask:
free_cpumask_var(buffer->cpumask);
put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
cpu_notifier_register_done();
#endif
fail_free_buffer:
kfree(buffer);
@ -1358,16 +1360,17 @@ ring_buffer_free(struct ring_buffer *buffer)
{
int cpu;
get_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
unregister_cpu_notifier(&buffer->cpu_notify);
cpu_notifier_register_begin();
__unregister_cpu_notifier(&buffer->cpu_notify);
#endif
for_each_buffer_cpu(buffer, cpu)
rb_free_cpu_buffer(buffer->buffers[cpu]);
put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
cpu_notifier_register_done();
#endif
kfree(buffer->buffers);
free_cpumask_var(buffer->cpumask);

197
kernel/trace/trace.c
View file

@ -73,7 +73,8 @@ static struct tracer_flags dummy_tracer_flags = {
.opts = dummy_tracer_opt
};
static int dummy_set_flag(u32 old_flags, u32 bit, int set)
static int
dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
return 0;
}
@ -118,7 +119,7 @@ enum ftrace_dump_mode ftrace_dump_on_oops;
/* When set, tracing will stop when a WARN*() is hit */
int __disable_trace_on_warning;
static int tracing_set_tracer(const char *buf);
static int tracing_set_tracer(struct trace_array *tr, const char *buf);
#define MAX_TRACER_SIZE 100
static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
@ -180,6 +181,17 @@ static int __init set_trace_boot_options(char *str)
}
__setup("trace_options=", set_trace_boot_options);
static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
static char *trace_boot_clock __initdata;
static int __init set_trace_boot_clock(char *str)
{
strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
trace_boot_clock = trace_boot_clock_buf;
return 0;
}
__setup("trace_clock=", set_trace_boot_clock);
unsigned long long ns2usecs(cycle_t nsec)
{
@ -1230,7 +1242,7 @@ int register_tracer(struct tracer *type)
printk(KERN_INFO "Starting tracer '%s'\n", type->name);
/* Do we want this tracer to start on bootup? */
tracing_set_tracer(type->name);
tracing_set_tracer(&global_trace, type->name);
default_bootup_tracer = NULL;
/* disable other selftests, since this will break it. */
tracing_selftest_disabled = true;
@ -3137,27 +3149,52 @@ static int tracing_open(struct inode *inode, struct file *file)
return ret;
}
/*
* Some tracers are not suitable for instance buffers.
* A tracer is always available for the global array (toplevel)
* or if it explicitly states that it is.
*/
static bool
trace_ok_for_array(struct tracer *t, struct trace_array *tr)
{
return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
}
/* Find the next tracer that this trace array may use */
static struct tracer *
get_tracer_for_array(struct trace_array *tr, struct tracer *t)
{
while (t && !trace_ok_for_array(t, tr))
t = t->next;
return t;
}
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
struct trace_array *tr = m->private;
struct tracer *t = v;
(*pos)++;
if (t)
t = t->next;
t = get_tracer_for_array(tr, t->next);
return t;
}
static void *t_start(struct seq_file *m, loff_t *pos)
{
struct trace_array *tr = m->private;
struct tracer *t;
loff_t l = 0;
mutex_lock(&trace_types_lock);
for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
;
t = get_tracer_for_array(tr, trace_types);
for (; t && l < *pos; t = t_next(m, t, &l))
;
return t;
}
@ -3192,10 +3229,21 @@ static const struct seq_operations show_traces_seq_ops = {
static int show_traces_open(struct inode *inode, struct file *file)
{
struct trace_array *tr = inode->i_private;
struct seq_file *m;
int ret;
if (tracing_disabled)
return -ENODEV;
return seq_open(file, &show_traces_seq_ops);
ret = seq_open(file, &show_traces_seq_ops);
if (ret)
return ret;
m = file->private_data;
m->private = tr;
return 0;
}
static ssize_t
@ -3355,13 +3403,14 @@ static int tracing_trace_options_show(struct seq_file *m, void *v)
return 0;
}
static int __set_tracer_option(struct tracer *trace,
static int __set_tracer_option(struct trace_array *tr,
struct tracer_flags *tracer_flags,
struct tracer_opt *opts, int neg)
{
struct tracer *trace = tr->current_trace;
int ret;
ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
if (ret)
return ret;
@ -3373,8 +3422,9 @@ static int __set_tracer_option(struct tracer *trace,
}
/* Try to assign a tracer specific option */
static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
{
struct tracer *trace = tr->current_trace;
struct tracer_flags *tracer_flags = trace->flags;
struct tracer_opt *opts = NULL;
int i;
@ -3383,8 +3433,7 @@ static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
opts = &tracer_flags->opts[i];
if (strcmp(cmp, opts->name) == 0)
return __set_tracer_option(trace, trace->flags,
opts, neg);
return __set_tracer_option(tr, trace->flags, opts, neg);
}
return -EINVAL;
@ -3407,7 +3456,7 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
/* Give the tracer a chance to approve the change */
if (tr->current_trace->flag_changed)
if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
if (tr->current_trace->flag_changed(tr, mask, !!enabled))
return -EINVAL;
if (enabled)
@ -3456,7 +3505,7 @@ static int trace_set_options(struct trace_array *tr, char *option)
/* If no option could be set, test the specific tracer options */
if (!trace_options[i])
ret = set_tracer_option(tr->current_trace, cmp, neg);
ret = set_tracer_option(tr, cmp, neg);
mutex_unlock(&trace_types_lock);
@ -3562,6 +3611,8 @@ static const char readme_msg[] =
#ifdef CONFIG_TRACER_SNAPSHOT
"\t\t snapshot\n"
#endif
"\t\t dump\n"
"\t\t cpudump\n"
"\t example: echo do_fault:traceoff > set_ftrace_filter\n"
"\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
"\t The first one will disable tracing every time do_fault is hit\n"
@ -3885,10 +3936,26 @@ create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
static void
destroy_trace_option_files(struct trace_option_dentry *topts);
static int tracing_set_tracer(const char *buf)
/*
* Used to clear out the tracer before deletion of an instance.
* Must have trace_types_lock held.
*/
static void tracing_set_nop(struct trace_array *tr)
{
if (tr->current_trace == &nop_trace)
return;
tr->current_trace->enabled--;
if (tr->current_trace->reset)
tr->current_trace->reset(tr);
tr->current_trace = &nop_trace;
}
static int tracing_set_tracer(struct trace_array *tr, const char *buf)
{
static struct trace_option_dentry *topts;
struct trace_array *tr = &global_trace;
struct tracer *t;
#ifdef CONFIG_TRACER_MAX_TRACE
bool had_max_tr;
@ -3916,9 +3983,15 @@ static int tracing_set_tracer(const char *buf)
if (t == tr->current_trace)
goto out;
/* Some tracers are only allowed for the top level buffer */
if (!trace_ok_for_array(t, tr)) {
ret = -EINVAL;
goto out;
}
trace_branch_disable();
tr->current_trace->enabled = false;
tr->current_trace->enabled--;
if (tr->current_trace->reset)
tr->current_trace->reset(tr);
@ -3941,9 +4014,11 @@ static int tracing_set_tracer(const char *buf)
free_snapshot(tr);
}
#endif
destroy_trace_option_files(topts);
topts = create_trace_option_files(tr, t);
/* Currently, only the top instance has options */
if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
destroy_trace_option_files(topts);
topts = create_trace_option_files(tr, t);
}
#ifdef CONFIG_TRACER_MAX_TRACE
if (t->use_max_tr && !had_max_tr) {
@ -3960,7 +4035,7 @@ static int tracing_set_tracer(const char *buf)
}
tr->current_trace = t;
tr->current_trace->enabled = true;
tr->current_trace->enabled++;
trace_branch_enable(tr);
out:
mutex_unlock(&trace_types_lock);
@ -3972,6 +4047,7 @@ static ssize_t
tracing_set_trace_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
struct trace_array *tr = filp->private_data;
char buf[MAX_TRACER_SIZE+1];
int i;
size_t ret;
@ -3991,7 +4067,7 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf,
for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
buf[i] = 0;
err = tracing_set_tracer(buf);
err = tracing_set_tracer(tr, buf);
if (err)
return err;
@ -4316,8 +4392,6 @@ static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
static const struct pipe_buf_operations tracing_pipe_buf_ops = {
.can_merge = 0,
.map = generic_pipe_buf_map,
.unmap = generic_pipe_buf_unmap,
.confirm = generic_pipe_buf_confirm,
.release = generic_pipe_buf_release,
.steal = generic_pipe_buf_steal,
@ -4412,7 +4486,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,
trace_access_lock(iter->cpu_file);
/* Fill as many pages as possible. */
for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
spd.pages[i] = alloc_page(GFP_KERNEL);
if (!spd.pages[i])
break;
@ -4699,25 +4773,10 @@ static int tracing_clock_show(struct seq_file *m, void *v)
return 0;
}
static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *fpos)
static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
{
struct seq_file *m = filp->private_data;
struct trace_array *tr = m->private;
char buf[64];
const char *clockstr;
int i;
if (cnt >= sizeof(buf))
return -EINVAL;
if (copy_from_user(&buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
clockstr = strstrip(buf);
for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
if (strcmp(trace_clocks[i].name, clockstr) == 0)
break;
@ -4745,6 +4804,32 @@ static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
mutex_unlock(&trace_types_lock);
return 0;
}
static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *fpos)
{
struct seq_file *m = filp->private_data;
struct trace_array *tr = m->private;
char buf[64];
const char *clockstr;
int ret;
if (cnt >= sizeof(buf))
return -EINVAL;
if (copy_from_user(&buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
clockstr = strstrip(buf);
ret = tracing_set_clock(tr, clockstr);
if (ret)
return ret;
*fpos += cnt;
return cnt;
@ -5194,8 +5279,6 @@ static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
/* Pipe buffer operations for a buffer. */
static const struct pipe_buf_operations buffer_pipe_buf_ops = {
.can_merge = 0,
.map = generic_pipe_buf_map,
.unmap = generic_pipe_buf_unmap,
.confirm = generic_pipe_buf_confirm,
.release = buffer_pipe_buf_release,
.steal = generic_pipe_buf_steal,
@ -5271,7 +5354,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
trace_access_lock(iter->cpu_file);
entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
struct page *page;
int r;
@ -5705,7 +5788,7 @@ trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
if (!!(topt->flags->val & topt->opt->bit) != val) {
mutex_lock(&trace_types_lock);
ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
ret = __set_tracer_option(topt->tr, topt->flags,
topt->opt, !val);
mutex_unlock(&trace_types_lock);
if (ret)
@ -6112,7 +6195,9 @@ static int instance_delete(const char *name)
list_del(&tr->list);
tracing_set_nop(tr);
event_trace_del_tracer(tr);
ftrace_destroy_function_files(tr);
debugfs_remove_recursive(tr->dir);
free_percpu(tr->trace_buffer.data);
ring_buffer_free(tr->trace_buffer.buffer);
@ -6207,6 +6292,12 @@ init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
{
int cpu;
trace_create_file("available_tracers", 0444, d_tracer,
tr, &show_traces_fops);
trace_create_file("current_tracer", 0644, d_tracer,
tr, &set_tracer_fops);
trace_create_file("tracing_cpumask", 0644, d_tracer,
tr, &tracing_cpumask_fops);
@ -6237,6 +6328,9 @@ init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
trace_create_file("tracing_on", 0644, d_tracer,
tr, &rb_simple_fops);
if (ftrace_create_function_files(tr, d_tracer))
WARN(1, "Could not allocate function filter files");
#ifdef CONFIG_TRACER_SNAPSHOT
trace_create_file("snapshot", 0644, d_tracer,
tr, &snapshot_fops);
@ -6259,12 +6353,6 @@ static __init int tracer_init_debugfs(void)
init_tracer_debugfs(&global_trace, d_tracer);
trace_create_file("available_tracers", 0444, d_tracer,
&global_trace, &show_traces_fops);
trace_create_file("current_tracer", 0644, d_tracer,
&global_trace, &set_tracer_fops);
#ifdef CONFIG_TRACER_MAX_TRACE
trace_create_file("tracing_max_latency", 0644, d_tracer,
&tracing_max_latency, &tracing_max_lat_fops);
@ -6527,6 +6615,13 @@ __init static int tracer_alloc_buffers(void)
trace_init_cmdlines();
if (trace_boot_clock) {
ret = tracing_set_clock(&global_trace, trace_boot_clock);
if (ret < 0)
pr_warning("Trace clock %s not defined, going back to default\n",
trace_boot_clock);
}
/*
* register_tracer() might reference current_trace, so it
* needs to be set before we register anything. This is

41
kernel/trace/trace.h
View file

@ -13,6 +13,7 @@
#include <linux/hw_breakpoint.h>
#include <linux/trace_seq.h>
#include <linux/ftrace_event.h>
#include <linux/compiler.h>
#ifdef CONFIG_FTRACE_SYSCALLS
#include <asm/unistd.h> /* For NR_SYSCALLS */
@ -210,6 +211,11 @@ struct trace_array {
struct list_head events;
cpumask_var_t tracing_cpumask; /* only trace on set CPUs */
int ref;
#ifdef CONFIG_FUNCTION_TRACER
struct ftrace_ops *ops;
/* function tracing enabled */
int function_enabled;
#endif
};
enum {
@ -355,14 +361,16 @@ struct tracer {
void (*print_header)(struct seq_file *m);
enum print_line_t (*print_line)(struct trace_iterator *iter);
/* If you handled the flag setting, return 0 */
int (*set_flag)(u32 old_flags, u32 bit, int set);
int (*set_flag)(struct trace_array *tr,
u32 old_flags, u32 bit, int set);
/* Return 0 if OK with change, else return non-zero */
int (*flag_changed)(struct tracer *tracer,
int (*flag_changed)(struct trace_array *tr,
u32 mask, int set);
struct tracer *next;
struct tracer_flags *flags;
int enabled;
bool print_max;
bool enabled;
bool allow_instances;
#ifdef CONFIG_TRACER_MAX_TRACE
bool use_max_tr;
#endif
@ -812,13 +820,36 @@ static inline int ftrace_trace_task(struct task_struct *task)
return test_tsk_trace_trace(task);
}
extern int ftrace_is_dead(void);
int ftrace_create_function_files(struct trace_array *tr,
struct dentry *parent);
void ftrace_destroy_function_files(struct trace_array *tr);
#else
static inline int ftrace_trace_task(struct task_struct *task)
{
return 1;
}
static inline int ftrace_is_dead(void) { return 0; }
#endif
static inline int
ftrace_create_function_files(struct trace_array *tr,
struct dentry *parent)
{
return 0;
}
static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
#endif /* CONFIG_FUNCTION_TRACER */
#if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
void ftrace_create_filter_files(struct ftrace_ops *ops,
struct dentry *parent);
void ftrace_destroy_filter_files(struct ftrace_ops *ops);
#else
/*
* The ops parameter passed in is usually undefined.
* This must be a macro.
*/
#define ftrace_create_filter_files(ops, parent) do { } while (0)
#define ftrace_destroy_filter_files(ops) do { } while (0)
#endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */
int ftrace_event_is_function(struct ftrace_event_call *call);
@ -1249,7 +1280,7 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
#undef FTRACE_ENTRY
#define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
extern struct ftrace_event_call \
__attribute__((__aligned__(4))) event_##call;
__aligned(4) event_##call;
#undef FTRACE_ENTRY_DUP
#define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print, filter) \
FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print), \

85
kernel/trace/trace_events.c
View file

@ -188,29 +188,60 @@ int trace_event_raw_init(struct ftrace_event_call *call)
}
EXPORT_SYMBOL_GPL(trace_event_raw_init);
void *ftrace_event_buffer_reserve(struct ftrace_event_buffer *fbuffer,
struct ftrace_event_file *ftrace_file,
unsigned long len)
{
struct ftrace_event_call *event_call = ftrace_file->event_call;
local_save_flags(fbuffer->flags);
fbuffer->pc = preempt_count();
fbuffer->ftrace_file = ftrace_file;
fbuffer->event =
trace_event_buffer_lock_reserve(&fbuffer->buffer, ftrace_file,
event_call->event.type, len,
fbuffer->flags, fbuffer->pc);
if (!fbuffer->event)
return NULL;
fbuffer->entry = ring_buffer_event_data(fbuffer->event);
return fbuffer->entry;
}
EXPORT_SYMBOL_GPL(ftrace_event_buffer_reserve);
void ftrace_event_buffer_commit(struct ftrace_event_buffer *fbuffer)
{
event_trigger_unlock_commit(fbuffer->ftrace_file, fbuffer->buffer,
fbuffer->event, fbuffer->entry,
fbuffer->flags, fbuffer->pc);
}
EXPORT_SYMBOL_GPL(ftrace_event_buffer_commit);
int ftrace_event_reg(struct ftrace_event_call *call,
enum trace_reg type, void *data)
{
struct ftrace_event_file *file = data;
WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
switch (type) {
case TRACE_REG_REGISTER:
return tracepoint_probe_register(call->name,
return tracepoint_probe_register(call->tp,
call->class->probe,
file);
case TRACE_REG_UNREGISTER:
tracepoint_probe_unregister(call->name,
tracepoint_probe_unregister(call->tp,
call->class->probe,
file);
return 0;
#ifdef CONFIG_PERF_EVENTS
case TRACE_REG_PERF_REGISTER:
return tracepoint_probe_register(call->name,
return tracepoint_probe_register(call->tp,
call->class->perf_probe,
call);
case TRACE_REG_PERF_UNREGISTER:
tracepoint_probe_unregister(call->name,
tracepoint_probe_unregister(call->tp,
call->class->perf_probe,
call);
return 0;
@ -322,7 +353,7 @@ static int __ftrace_event_enable_disable(struct ftrace_event_file *file,
if (ret) {
tracing_stop_cmdline_record();
pr_info("event trace: Could not enable event "
"%s\n", call->name);
"%s\n", ftrace_event_name(call));
break;
}
set_bit(FTRACE_EVENT_FL_ENABLED_BIT, &file->flags);
@ -451,27 +482,29 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
{
struct ftrace_event_file *file;
struct ftrace_event_call *call;
const char *name;
int ret = -EINVAL;
list_for_each_entry(file, &tr->events, list) {
call = file->event_call;
name = ftrace_event_name(call);
if (!call->name || !call->class || !call->class->reg)
if (!name || !call->class || !call->class->reg)
continue;
if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
continue;
if (match &&
strcmp(match, call->name) != 0 &&
strcmp(match, name) != 0 &&
strcmp(match, call->class->system) != 0)
continue;
if (sub && strcmp(sub, call->class->system) != 0)
continue;
if (event && strcmp(event, call->name) != 0)
if (event && strcmp(event, name) != 0)
continue;
ftrace_event_enable_disable(file, set);
@ -669,7 +702,7 @@ static int t_show(struct seq_file *m, void *v)
if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
seq_printf(m, "%s:", call->class->system);
seq_printf(m, "%s\n", call->name);
seq_printf(m, "%s\n", ftrace_event_name(call));
return 0;
}
@ -762,7 +795,7 @@ system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
mutex_lock(&event_mutex);
list_for_each_entry(file, &tr->events, list) {
call = file->event_call;
if (!call->name || !call->class || !call->class->reg)
if (!ftrace_event_name(call) || !call->class || !call->class->reg)
continue;
if (system && strcmp(call->class->system, system->name) != 0)
@ -877,7 +910,7 @@ static int f_show(struct seq_file *m, void *v)
switch ((unsigned long)v) {
case FORMAT_HEADER:
seq_printf(m, "name: %s\n", call->name);
seq_printf(m, "name: %s\n", ftrace_event_name(call));
seq_printf(m, "ID: %d\n", call->event.type);
seq_printf(m, "format:\n");
return 0;
@ -1497,6 +1530,7 @@ event_create_dir(struct dentry *parent, struct ftrace_event_file *file)
struct trace_array *tr = file->tr;
struct list_head *head;
struct dentry *d_events;
const char *name;
int ret;
/*
@ -1510,10 +1544,11 @@ event_create_dir(struct dentry *parent, struct ftrace_event_file *file)
} else
d_events = parent;
file->dir = debugfs_create_dir(call->name, d_events);
name = ftrace_event_name(call);
file->dir = debugfs_create_dir(name, d_events);
if (!file->dir) {
pr_warning("Could not create debugfs '%s' directory\n",
call->name);
name);
return -1;
}
@ -1537,7 +1572,7 @@ event_create_dir(struct dentry *parent, struct ftrace_event_file *file)
ret = call->class->define_fields(call);
if (ret < 0) {
pr_warning("Could not initialize trace point"
" events/%s\n", call->name);
" events/%s\n", name);
return -1;
}
}
@ -1601,15 +1636,17 @@ static void event_remove(struct ftrace_event_call *call)
static int event_init(struct ftrace_event_call *call)
{
int ret = 0;
const char *name;
if (WARN_ON(!call->name))
name = ftrace_event_name(call);
if (WARN_ON(!name))
return -EINVAL;
if (call->class->raw_init) {
ret = call->class->raw_init(call);
if (ret < 0 && ret != -ENOSYS)
pr_warn("Could not initialize trace events/%s\n",
call->name);
name);
}
return ret;
@ -1855,7 +1892,7 @@ __trace_add_event_dirs(struct trace_array *tr)
ret = __trace_add_new_event(call, tr);
if (ret < 0)
pr_warning("Could not create directory for event %s\n",
call->name);
ftrace_event_name(call));
}
}
@ -1864,18 +1901,20 @@ find_event_file(struct trace_array *tr, const char *system, const char *event)
{
struct ftrace_event_file *file;
struct ftrace_event_call *call;
const char *name;
list_for_each_entry(file, &tr->events, list) {
call = file->event_call;
name = ftrace_event_name(call);
if (!call->name || !call->class || !call->class->reg)
if (!name || !call->class || !call->class->reg)
continue;
if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
continue;
if (strcmp(event, call->name) == 0 &&
if (strcmp(event, name) == 0 &&
strcmp(system, call->class->system) == 0)
return file;
}
@ -1943,7 +1982,7 @@ event_enable_print(struct seq_file *m, unsigned long ip,
seq_printf(m, "%s:%s:%s",
data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
data->file->event_call->class->system,
data->file->event_call->name);
ftrace_event_name(data->file->event_call));
if (data->count == -1)
seq_printf(m, ":unlimited\n");
@ -2163,7 +2202,7 @@ __trace_early_add_event_dirs(struct trace_array *tr)
ret = event_create_dir(tr->event_dir, file);
if (ret < 0)
pr_warning("Could not create directory for event %s\n",
file->event_call->name);
ftrace_event_name(file->event_call));
}
}
@ -2187,7 +2226,7 @@ __trace_early_add_events(struct trace_array *tr)
ret = __trace_early_add_new_event(call, tr);
if (ret < 0)
pr_warning("Could not create early event %s\n",
call->name);
ftrace_event_name(call));
}
}
@ -2519,7 +2558,7 @@ static __init void event_trace_self_tests(void)
continue;
#endif
pr_info("Testing event %s: ", call->name);
pr_info("Testing event %s: ", ftrace_event_name(call));
/*
* If an event is already enabled, someone is using

2
kernel/trace/trace_events_trigger.c
View file

@ -1095,7 +1095,7 @@ event_enable_trigger_print(struct seq_file *m, struct event_trigger_ops *ops,
seq_printf(m, "%s:%s:%s",
enable_data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
enable_data->file->event_call->class->system,
enable_data->file->event_call->name);
ftrace_event_name(enable_data->file->event_call));
if (data->count == -1)
seq_puts(m, ":unlimited");

6
kernel/trace/trace_export.c
View file

@ -173,9 +173,11 @@ struct ftrace_event_class __refdata event_class_ftrace_##call = { \
}; \
\
struct ftrace_event_call __used event_##call = { \
.name = #call, \
.event.type = etype, \
.class = &event_class_ftrace_##call, \
{ \
.name = #call, \
}, \
.event.type = etype, \
.print_fmt = print, \
.flags = TRACE_EVENT_FL_IGNORE_ENABLE | TRACE_EVENT_FL_USE_CALL_FILTER, \
}; \

147
kernel/trace/trace_functions.c
View file

@ -13,32 +13,110 @@
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include "trace.h"
/* function tracing enabled */
static int ftrace_function_enabled;
static void tracing_start_function_trace(struct trace_array *tr);
static void tracing_stop_function_trace(struct trace_array *tr);
static void
function_trace_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *pt_regs);
static void
function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *pt_regs);
static struct ftrace_ops trace_ops;
static struct ftrace_ops trace_stack_ops;
static struct tracer_flags func_flags;
static struct trace_array *func_trace;
/* Our option */
enum {
TRACE_FUNC_OPT_STACK = 0x1,
};
static void tracing_start_function_trace(void);
static void tracing_stop_function_trace(void);
static int allocate_ftrace_ops(struct trace_array *tr)
{
struct ftrace_ops *ops;
ops = kzalloc(sizeof(*ops), GFP_KERNEL);
if (!ops)
return -ENOMEM;
/* Currently only the non stack verision is supported */
ops->func = function_trace_call;
ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;
tr->ops = ops;
ops->private = tr;
return 0;
}
int ftrace_create_function_files(struct trace_array *tr,
struct dentry *parent)
{
int ret;
/*
* The top level array uses the "global_ops", and the files are
* created on boot up.
*/
if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
return 0;
ret = allocate_ftrace_ops(tr);
if (ret)
return ret;
ftrace_create_filter_files(tr->ops, parent);
return 0;
}
void ftrace_destroy_function_files(struct trace_array *tr)
{
ftrace_destroy_filter_files(tr->ops);
kfree(tr->ops);
tr->ops = NULL;
}
static int function_trace_init(struct trace_array *tr)
{
func_trace = tr;
struct ftrace_ops *ops;
if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
/* There's only one global tr */
if (!trace_ops.private) {
trace_ops.private = tr;
trace_stack_ops.private = tr;
}
if (func_flags.val & TRACE_FUNC_OPT_STACK)
ops = &trace_stack_ops;
else
ops = &trace_ops;
tr->ops = ops;
} else if (!tr->ops) {
/*
* Instance trace_arrays get their ops allocated
* at instance creation. Unless it failed
* the allocation.
*/
return -ENOMEM;
}
tr->trace_buffer.cpu = get_cpu();
put_cpu();
tracing_start_cmdline_record();
tracing_start_function_trace();
tracing_start_function_trace(tr);
return 0;
}
static void function_trace_reset(struct trace_array *tr)
{
tracing_stop_function_trace();
tracing_stop_function_trace(tr);
tracing_stop_cmdline_record();
}
@ -47,25 +125,18 @@ static void function_trace_start(struct trace_array *tr)
tracing_reset_online_cpus(&tr->trace_buffer);
}
/* Our option */
enum {
TRACE_FUNC_OPT_STACK = 0x1,
};
static struct tracer_flags func_flags;
static void
function_trace_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *pt_regs)
{
struct trace_array *tr = func_trace;
struct trace_array *tr = op->private;
struct trace_array_cpu *data;
unsigned long flags;
int bit;
int cpu;
int pc;
if (unlikely(!ftrace_function_enabled))
if (unlikely(!tr->function_enabled))
return;
pc = preempt_count();
@ -91,14 +162,14 @@ static void
function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *pt_regs)
{
struct trace_array *tr = func_trace;
struct trace_array *tr = op->private;
struct trace_array_cpu *data;
unsigned long flags;
long disabled;
int cpu;
int pc;
if (unlikely(!ftrace_function_enabled))
if (unlikely(!tr->function_enabled))
return;
/*
@ -128,7 +199,6 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
local_irq_restore(flags);
}
static struct ftrace_ops trace_ops __read_mostly =
{
.func = function_trace_call,
@ -153,29 +223,21 @@ static struct tracer_flags func_flags = {
.opts = func_opts
};
static void tracing_start_function_trace(void)
static void tracing_start_function_trace(struct trace_array *tr)
{
ftrace_function_enabled = 0;
if (func_flags.val & TRACE_FUNC_OPT_STACK)
register_ftrace_function(&trace_stack_ops);
else
register_ftrace_function(&trace_ops);
ftrace_function_enabled = 1;
tr->function_enabled = 0;
register_ftrace_function(tr->ops);
tr->function_enabled = 1;
}
static void tracing_stop_function_trace(void)
static void tracing_stop_function_trace(struct trace_array *tr)
{
ftrace_function_enabled = 0;
if (func_flags.val & TRACE_FUNC_OPT_STACK)
unregister_ftrace_function(&trace_stack_ops);
else
unregister_ftrace_function(&trace_ops);
tr->function_enabled = 0;
unregister_ftrace_function(tr->ops);
}
static int func_set_flag(u32 old_flags, u32 bit, int set)
static int
func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
switch (bit) {
case TRACE_FUNC_OPT_STACK:
@ -183,12 +245,14 @@ static int func_set_flag(u32 old_flags, u32 bit, int set)
if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
break;
unregister_ftrace_function(tr->ops);
if (set) {
unregister_ftrace_function(&trace_ops);
register_ftrace_function(&trace_stack_ops);
tr->ops = &trace_stack_ops;
register_ftrace_function(tr->ops);
} else {
unregister_ftrace_function(&trace_stack_ops);
register_ftrace_function(&trace_ops);
tr->ops = &trace_ops;
register_ftrace_function(tr->ops);
}
break;
@ -208,6 +272,7 @@ static struct tracer function_trace __tracer_data =
.wait_pipe = poll_wait_pipe,
.flags = &func_flags,
.set_flag = func_set_flag,
.allow_instances = true,
#ifdef CONFIG_FTRACE_SELFTEST
.selftest = trace_selftest_startup_function,
#endif

3
kernel/trace/trace_functions_graph.c
View file

@ -1476,7 +1476,8 @@ void graph_trace_close(struct trace_iterator *iter)
}
}
static int func_graph_set_flag(u32 old_flags, u32 bit, int set)
static int
func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
if (bit == TRACE_GRAPH_PRINT_IRQS)
ftrace_graph_skip_irqs = !set;

10
kernel/trace/trace_irqsoff.c
View file

@ -160,7 +160,8 @@ static struct ftrace_ops trace_ops __read_mostly =
#endif /* CONFIG_FUNCTION_TRACER */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int irqsoff_set_flag(u32 old_flags, u32 bit, int set)
static int
irqsoff_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
int cpu;
@ -266,7 +267,8 @@ __trace_function(struct trace_array *tr,
#else
#define __trace_function trace_function
static int irqsoff_set_flag(u32 old_flags, u32 bit, int set)
static int
irqsoff_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
return -EINVAL;
}
@ -570,8 +572,10 @@ static void irqsoff_function_set(int set)
unregister_irqsoff_function(is_graph());
}
static int irqsoff_flag_changed(struct tracer *tracer, u32 mask, int set)
static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set)
{
struct tracer *tracer = tr->current_trace;
if (mask & TRACE_ITER_FUNCTION)
irqsoff_function_set(set);

38
kernel/trace/trace_kprobe.c
View file

@ -35,11 +35,6 @@ struct trace_kprobe {
struct trace_probe tp;
};
struct event_file_link {
struct ftrace_event_file *file;
struct list_head list;
};
#define SIZEOF_TRACE_KPROBE(n) \
(offsetof(struct trace_kprobe, tp.args) + \
(sizeof(struct probe_arg) * (n)))
@ -346,7 +341,7 @@ static struct trace_kprobe *find_trace_kprobe(const char *event,
struct trace_kprobe *tk;
list_for_each_entry(tk, &probe_list, list)
if (strcmp(tk->tp.call.name, event) == 0 &&
if (strcmp(ftrace_event_name(&tk->tp.call), event) == 0 &&
strcmp(tk->tp.call.class->system, group) == 0)
return tk;
return NULL;
@ -387,18 +382,6 @@ enable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
return ret;
}
static struct event_file_link *
find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
{
struct event_file_link *link;
list_for_each_entry(link, &tp->files, list)
if (link->file == file)
return link;
return NULL;
}
/*
* Disable trace_probe
* if the file is NULL, disable "perf" handler, or disable "trace" handler.
@ -533,7 +516,8 @@ static int register_trace_kprobe(struct trace_kprobe *tk)
mutex_lock(&probe_lock);
/* Delete old (same name) event if exist */
old_tk = find_trace_kprobe(tk->tp.call.name, tk->tp.call.class->system);
old_tk = find_trace_kprobe(ftrace_event_name(&tk->tp.call),
tk->tp.call.class->system);
if (old_tk) {
ret = unregister_trace_kprobe(old_tk);
if (ret < 0)
@ -581,7 +565,8 @@ static int trace_kprobe_module_callback(struct notifier_block *nb,
if (ret)
pr_warning("Failed to re-register probe %s on"
"%s: %d\n",
tk->tp.call.name, mod->name, ret);
ftrace_event_name(&tk->tp.call),
mod->name, ret);
}
}
mutex_unlock(&probe_lock);
@ -835,7 +820,8 @@ static int probes_seq_show(struct seq_file *m, void *v)
int i;
seq_printf(m, "%c", trace_kprobe_is_return(tk) ? 'r' : 'p');
seq_printf(m, ":%s/%s", tk->tp.call.class->system, tk->tp.call.name);
seq_printf(m, ":%s/%s", tk->tp.call.class->system,
ftrace_event_name(&tk->tp.call));
if (!tk->symbol)
seq_printf(m, " 0x%p", tk->rp.kp.addr);
@ -893,7 +879,8 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
{
struct trace_kprobe *tk = v;
seq_printf(m, " %-44s %15lu %15lu\n", tk->tp.call.name, tk->nhit,
seq_printf(m, " %-44s %15lu %15lu\n",
ftrace_event_name(&tk->tp.call), tk->nhit,
tk->rp.kp.nmissed);
return 0;
@ -1028,7 +1015,7 @@ print_kprobe_event(struct trace_iterator *iter, int flags,
field = (struct kprobe_trace_entry_head *)iter->ent;
tp = container_of(event, struct trace_probe, call.event);
if (!trace_seq_printf(s, "%s: (", tp->call.name))
if (!trace_seq_printf(s, "%s: (", ftrace_event_name(&tp->call)))
goto partial;
if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
@ -1064,7 +1051,7 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
field = (struct kretprobe_trace_entry_head *)iter->ent;
tp = container_of(event, struct trace_probe, call.event);
if (!trace_seq_printf(s, "%s: (", tp->call.name))
if (!trace_seq_printf(s, "%s: (", ftrace_event_name(&tp->call)))
goto partial;
if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
@ -1303,7 +1290,8 @@ static int register_kprobe_event(struct trace_kprobe *tk)
call->data = tk;
ret = trace_add_event_call(call);
if (ret) {
pr_info("Failed to register kprobe event: %s\n", call->name);
pr_info("Failed to register kprobe event: %s\n",
ftrace_event_name(call));
kfree(call->print_fmt);
unregister_ftrace_event(&call->event);
}

5
kernel/trace/trace_nop.c
View file

@ -62,7 +62,7 @@ static void nop_trace_reset(struct trace_array *tr)
* If you don't implement it, then the flag setting will be
* automatically accepted.
*/
static int nop_set_flag(u32 old_flags, u32 bit, int set)
static int nop_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
/*
* Note that you don't need to update nop_flags.val yourself.
@ -96,6 +96,7 @@ struct tracer nop_trace __read_mostly =
.selftest = trace_selftest_startup_nop,
#endif
.flags = &nop_flags,
.set_flag = nop_set_flag
.set_flag = nop_set_flag,
.allow_instances = true,
};

33
kernel/trace/trace_output.c
View file

@ -431,7 +431,7 @@ int ftrace_raw_output_prep(struct trace_iterator *iter,
}
trace_seq_init(p);
ret = trace_seq_printf(s, "%s: ", event->name);
ret = trace_seq_printf(s, "%s: ", ftrace_event_name(event));
if (!ret)
return TRACE_TYPE_PARTIAL_LINE;
@ -439,6 +439,37 @@ int ftrace_raw_output_prep(struct trace_iterator *iter,
}
EXPORT_SYMBOL(ftrace_raw_output_prep);
static int ftrace_output_raw(struct trace_iterator *iter, char *name,
char *fmt, va_list ap)
{
struct trace_seq *s = &iter->seq;
int ret;
ret = trace_seq_printf(s, "%s: ", name);
if (!ret)
return TRACE_TYPE_PARTIAL_LINE;
ret = trace_seq_vprintf(s, fmt, ap);
if (!ret)
return TRACE_TYPE_PARTIAL_LINE;
return TRACE_TYPE_HANDLED;
}
int ftrace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
{
va_list ap;
int ret;
va_start(ap, fmt);
ret = ftrace_output_raw(iter, name, fmt, ap);
va_end(ap);
return ret;
}
EXPORT_SYMBOL_GPL(ftrace_output_call);
#ifdef CONFIG_KRETPROBES
static inline const char *kretprobed(const char *name)
{

17
kernel/trace/trace_probe.h
View file

@ -288,6 +288,11 @@ struct trace_probe {
struct probe_arg args[];
};
struct event_file_link {
struct ftrace_event_file *file;
struct list_head list;
};
static inline bool trace_probe_is_enabled(struct trace_probe *tp)
{
return !!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE));
@ -316,6 +321,18 @@ static inline int is_good_name(const char *name)
return 1;
}
static inline struct event_file_link *
find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
{
struct event_file_link *link;
list_for_each_entry(link, &tp->files, list)
if (link->file == file)
return link;
return NULL;
}
extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
struct probe_arg *parg, bool is_return, bool is_kprobe);

10
kernel/trace/trace_sched_wakeup.c
View file

@ -179,8 +179,10 @@ static void wakeup_function_set(int set)
unregister_wakeup_function(is_graph());
}
static int wakeup_flag_changed(struct tracer *tracer, u32 mask, int set)
static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
{
struct tracer *tracer = tr->current_trace;
if (mask & TRACE_ITER_FUNCTION)
wakeup_function_set(set);
@ -209,7 +211,8 @@ static void stop_func_tracer(int graph)
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
static int
wakeup_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
if (!(bit & TRACE_DISPLAY_GRAPH))
@ -311,7 +314,8 @@ __trace_function(struct trace_array *tr,
#else
#define __trace_function trace_function
static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
static int
wakeup_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
return -EINVAL;
}

3
kernel/trace/trace_stack.c
View file

@ -13,6 +13,7 @@
#include <linux/sysctl.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/magic.h>
#include <asm/setup.h>
@ -144,6 +145,8 @@ check_stack(unsigned long ip, unsigned long *stack)
i++;
}
BUG_ON(current != &init_task &&
*(end_of_stack(current)) != STACK_END_MAGIC);
out:
arch_spin_unlock(&max_stack_lock);
local_irq_restore(flags);

217
kernel/trace/trace_uprobe.c
View file

@ -260,6 +260,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
goto error;
INIT_LIST_HEAD(&tu->list);
INIT_LIST_HEAD(&tu->tp.files);
tu->consumer.handler = uprobe_dispatcher;
if (is_ret)
tu->consumer.ret_handler = uretprobe_dispatcher;
@ -293,7 +294,7 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou
struct trace_uprobe *tu;
list_for_each_entry(tu, &uprobe_list, list)
if (strcmp(tu->tp.call.name, event) == 0 &&
if (strcmp(ftrace_event_name(&tu->tp.call), event) == 0 &&
strcmp(tu->tp.call.class->system, group) == 0)
return tu;
@ -323,7 +324,8 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
mutex_lock(&uprobe_lock);
/* register as an event */
old_tu = find_probe_event(tu->tp.call.name, tu->tp.call.class->system);
old_tu = find_probe_event(ftrace_event_name(&tu->tp.call),
tu->tp.call.class->system);
if (old_tu) {
/* delete old event */
ret = unregister_trace_uprobe(old_tu);
@ -598,7 +600,8 @@ static int probes_seq_show(struct seq_file *m, void *v)
char c = is_ret_probe(tu) ? 'r' : 'p';
int i;
seq_printf(m, "%c:%s/%s", c, tu->tp.call.class->system, tu->tp.call.name);
seq_printf(m, "%c:%s/%s", c, tu->tp.call.class->system,
ftrace_event_name(&tu->tp.call));
seq_printf(m, " %s:0x%p", tu->filename, (void *)tu->offset);
for (i = 0; i < tu->tp.nr_args; i++)
@ -648,7 +651,8 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
{
struct trace_uprobe *tu = v;
seq_printf(m, " %s %-44s %15lu\n", tu->filename, tu->tp.call.name, tu->nhit);
seq_printf(m, " %s %-44s %15lu\n", tu->filename,
ftrace_event_name(&tu->tp.call), tu->nhit);
return 0;
}
@ -728,9 +732,15 @@ static int uprobe_buffer_enable(void)
static void uprobe_buffer_disable(void)
{
int cpu;
BUG_ON(!mutex_is_locked(&event_mutex));
if (--uprobe_buffer_refcnt == 0) {
for_each_possible_cpu(cpu)
free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
cpu)->buf);
free_percpu(uprobe_cpu_buffer);
uprobe_cpu_buffer = NULL;
}
@ -758,31 +768,32 @@ static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
mutex_unlock(&ucb->mutex);
}
static void uprobe_trace_print(struct trace_uprobe *tu,
unsigned long func, struct pt_regs *regs)
static void __uprobe_trace_func(struct trace_uprobe *tu,
unsigned long func, struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize,
struct ftrace_event_file *ftrace_file)
{
struct uprobe_trace_entry_head *entry;
struct ring_buffer_event *event;
struct ring_buffer *buffer;
struct uprobe_cpu_buffer *ucb;
void *data;
int size, dsize, esize;
int size, esize;
struct ftrace_event_call *call = &tu->tp.call;
dsize = __get_data_size(&tu->tp, regs);
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
WARN_ON(call != ftrace_file->event_call);
if (WARN_ON_ONCE(!uprobe_cpu_buffer || tu->tp.size + dsize > PAGE_SIZE))
if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
return;
ucb = uprobe_buffer_get();
store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
if (ftrace_trigger_soft_disabled(ftrace_file))
return;
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
size = esize + tu->tp.size + dsize;
event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
size, 0, 0);
event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
call->event.type, size, 0, 0);
if (!event)
goto out;
return;
entry = ring_buffer_event_data(event);
if (is_ret_probe(tu)) {
@ -796,25 +807,36 @@ static void uprobe_trace_print(struct trace_uprobe *tu,
memcpy(data, ucb->buf, tu->tp.size + dsize);
if (!call_filter_check_discard(call, entry, buffer, event))
trace_buffer_unlock_commit(buffer, event, 0, 0);
out:
uprobe_buffer_put(ucb);
event_trigger_unlock_commit(ftrace_file, buffer, event, entry, 0, 0);
}
/* uprobe handler */
static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs)
static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize)
{
if (!is_ret_probe(tu))
uprobe_trace_print(tu, 0, regs);
struct event_file_link *link;
if (is_ret_probe(tu))
return 0;
rcu_read_lock();
list_for_each_entry_rcu(link, &tu->tp.files, list)
__uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
rcu_read_unlock();
return 0;
}
static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
struct pt_regs *regs)
struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize)
{
uprobe_trace_print(tu, func, regs);
struct event_file_link *link;
rcu_read_lock();
list_for_each_entry_rcu(link, &tu->tp.files, list)
__uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
rcu_read_unlock();
}
/* Event entry printers */
@ -831,12 +853,14 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
tu = container_of(event, struct trace_uprobe, tp.call.event);
if (is_ret_probe(tu)) {
if (!trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", tu->tp.call.name,
if (!trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
ftrace_event_name(&tu->tp.call),
entry->vaddr[1], entry->vaddr[0]))
goto partial;
data = DATAOF_TRACE_ENTRY(entry, true);
} else {
if (!trace_seq_printf(s, "%s: (0x%lx)", tu->tp.call.name,
if (!trace_seq_printf(s, "%s: (0x%lx)",
ftrace_event_name(&tu->tp.call),
entry->vaddr[0]))
goto partial;
data = DATAOF_TRACE_ENTRY(entry, false);
@ -861,12 +885,24 @@ typedef bool (*filter_func_t)(struct uprobe_consumer *self,
struct mm_struct *mm);
static int
probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
probe_event_enable(struct trace_uprobe *tu, struct ftrace_event_file *file,
filter_func_t filter)
{
int ret = 0;
bool enabled = trace_probe_is_enabled(&tu->tp);
struct event_file_link *link = NULL;
int ret;
if (trace_probe_is_enabled(&tu->tp))
return -EINTR;
if (file) {
link = kmalloc(sizeof(*link), GFP_KERNEL);
if (!link)
return -ENOMEM;
link->file = file;
list_add_tail_rcu(&link->list, &tu->tp.files);
tu->tp.flags |= TP_FLAG_TRACE;
} else
tu->tp.flags |= TP_FLAG_PROFILE;
ret = uprobe_buffer_enable();
if (ret < 0)
@ -874,24 +910,49 @@ probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
WARN_ON(!uprobe_filter_is_empty(&tu->filter));
tu->tp.flags |= flag;
if (enabled)
return 0;
tu->consumer.filter = filter;
ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
if (ret)
tu->tp.flags &= ~flag;
if (ret) {
if (file) {
list_del(&link->list);
kfree(link);
tu->tp.flags &= ~TP_FLAG_TRACE;
} else
tu->tp.flags &= ~TP_FLAG_PROFILE;
}
return ret;
}
static void probe_event_disable(struct trace_uprobe *tu, int flag)
static void
probe_event_disable(struct trace_uprobe *tu, struct ftrace_event_file *file)
{
if (!trace_probe_is_enabled(&tu->tp))
return;
if (file) {
struct event_file_link *link;
link = find_event_file_link(&tu->tp, file);
if (!link)
return;
list_del_rcu(&link->list);
/* synchronize with u{,ret}probe_trace_func */
synchronize_sched();
kfree(link);
if (!list_empty(&tu->tp.files))
return;
}
WARN_ON(!uprobe_filter_is_empty(&tu->filter));
uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
tu->tp.flags &= ~flag;
tu->tp.flags &= file ? ~TP_FLAG_TRACE : ~TP_FLAG_PROFILE;
uprobe_buffer_disable();
}
@ -1014,31 +1075,24 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc,
return ret;
}
static void uprobe_perf_print(struct trace_uprobe *tu,
unsigned long func, struct pt_regs *regs)
static void __uprobe_perf_func(struct trace_uprobe *tu,
unsigned long func, struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize)
{
struct ftrace_event_call *call = &tu->tp.call;
struct uprobe_trace_entry_head *entry;
struct hlist_head *head;
struct uprobe_cpu_buffer *ucb;
void *data;
int size, dsize, esize;
int size, esize;
int rctx;
dsize = __get_data_size(&tu->tp, regs);
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
if (WARN_ON_ONCE(!uprobe_cpu_buffer))
return;
size = esize + tu->tp.size + dsize;
size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
return;
ucb = uprobe_buffer_get();
store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
preempt_disable();
head = this_cpu_ptr(call->perf_events);
if (hlist_empty(head))
@ -1068,46 +1122,49 @@ static void uprobe_perf_print(struct trace_uprobe *tu,
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
out:
preempt_enable();
uprobe_buffer_put(ucb);
}
/* uprobe profile handler */
static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs)
static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize)
{
if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
return UPROBE_HANDLER_REMOVE;
if (!is_ret_probe(tu))
uprobe_perf_print(tu, 0, regs);
__uprobe_perf_func(tu, 0, regs, ucb, dsize);
return 0;
}
static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
struct pt_regs *regs)
struct pt_regs *regs,
struct uprobe_cpu_buffer *ucb, int dsize)
{
uprobe_perf_print(tu, func, regs);
__uprobe_perf_func(tu, func, regs, ucb, dsize);
}
#endif /* CONFIG_PERF_EVENTS */
static
int trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type, void *data)
static int
trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type,
void *data)
{
struct trace_uprobe *tu = event->data;
struct ftrace_event_file *file = data;
switch (type) {
case TRACE_REG_REGISTER:
return probe_event_enable(tu, TP_FLAG_TRACE, NULL);
return probe_event_enable(tu, file, NULL);
case TRACE_REG_UNREGISTER:
probe_event_disable(tu, TP_FLAG_TRACE);
probe_event_disable(tu, file);
return 0;
#ifdef CONFIG_PERF_EVENTS
case TRACE_REG_PERF_REGISTER:
return probe_event_enable(tu, TP_FLAG_PROFILE, uprobe_perf_filter);
return probe_event_enable(tu, NULL, uprobe_perf_filter);
case TRACE_REG_PERF_UNREGISTER:
probe_event_disable(tu, TP_FLAG_PROFILE);
probe_event_disable(tu, NULL);
return 0;
case TRACE_REG_PERF_OPEN:
@ -1127,8 +1184,11 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
{
struct trace_uprobe *tu;
struct uprobe_dispatch_data udd;
struct uprobe_cpu_buffer *ucb;
int dsize, esize;
int ret = 0;
tu = container_of(con, struct trace_uprobe, consumer);
tu->nhit++;
@ -1137,13 +1197,29 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
current->utask->vaddr = (unsigned long) &udd;
#ifdef CONFIG_PERF_EVENTS
if ((tu->tp.flags & TP_FLAG_TRACE) == 0 &&
!uprobe_perf_filter(&tu->consumer, 0, current->mm))
return UPROBE_HANDLER_REMOVE;
#endif
if (WARN_ON_ONCE(!uprobe_cpu_buffer))
return 0;
dsize = __get_data_size(&tu->tp, regs);
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
ucb = uprobe_buffer_get();
store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
if (tu->tp.flags & TP_FLAG_TRACE)
ret |= uprobe_trace_func(tu, regs);
ret |= uprobe_trace_func(tu, regs, ucb, dsize);
#ifdef CONFIG_PERF_EVENTS
if (tu->tp.flags & TP_FLAG_PROFILE)
ret |= uprobe_perf_func(tu, regs);
ret |= uprobe_perf_func(tu, regs, ucb, dsize);
#endif
uprobe_buffer_put(ucb);
return ret;
}
@ -1152,6 +1228,8 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
{
struct trace_uprobe *tu;
struct uprobe_dispatch_data udd;
struct uprobe_cpu_buffer *ucb;
int dsize, esize;
tu = container_of(con, struct trace_uprobe, consumer);
@ -1160,13 +1238,23 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
current->utask->vaddr = (unsigned long) &udd;
if (WARN_ON_ONCE(!uprobe_cpu_buffer))
return 0;
dsize = __get_data_size(&tu->tp, regs);
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
ucb = uprobe_buffer_get();
store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);
if (tu->tp.flags & TP_FLAG_TRACE)
uretprobe_trace_func(tu, func, regs);
uretprobe_trace_func(tu, func, regs, ucb, dsize);
#ifdef CONFIG_PERF_EVENTS
if (tu->tp.flags & TP_FLAG_PROFILE)
uretprobe_perf_func(tu, func, regs);
uretprobe_perf_func(tu, func, regs, ucb, dsize);
#endif
uprobe_buffer_put(ucb);
return 0;
}
@ -1198,7 +1286,8 @@ static int register_uprobe_event(struct trace_uprobe *tu)
ret = trace_add_event_call(call);
if (ret) {
pr_info("Failed to register uprobe event: %s\n", call->name);
pr_info("Failed to register uprobe event: %s\n",
ftrace_event_name(call));
kfree(call->print_fmt);
unregister_ftrace_event(&call->event);
}

728
kernel/tracepoint.c
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2008 Mathieu Desnoyers
* Copyright (C) 2008-2014 Mathieu Desnoyers
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -33,43 +33,29 @@ extern struct tracepoint * const __stop___tracepoints_ptrs[];
/* Set to 1 to enable tracepoint debug output */
static const int tracepoint_debug;
/*
* Tracepoints mutex protects the builtin and module tracepoints and the hash
* table, as well as the local module list.
*/
static DEFINE_MUTEX(tracepoints_mutex);
#ifdef CONFIG_MODULES
/* Local list of struct module */
/*
* Tracepoint module list mutex protects the local module list.
*/
static DEFINE_MUTEX(tracepoint_module_list_mutex);
/* Local list of struct tp_module */
static LIST_HEAD(tracepoint_module_list);
#endif /* CONFIG_MODULES */
/*
* Tracepoint hash table, containing the active tracepoints.
* Protected by tracepoints_mutex.
* tracepoints_mutex protects the builtin and module tracepoints.
* tracepoints_mutex nests inside tracepoint_module_list_mutex.
*/
#define TRACEPOINT_HASH_BITS 6
#define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
static DEFINE_MUTEX(tracepoints_mutex);
/*
* Note about RCU :
* It is used to delay the free of multiple probes array until a quiescent
* state is reached.
* Tracepoint entries modifications are protected by the tracepoints_mutex.
*/
struct tracepoint_entry {
struct hlist_node hlist;
struct tracepoint_func *funcs;
int refcount; /* Number of times armed. 0 if disarmed. */
char name[0];
};
struct tp_probes {
union {
struct rcu_head rcu;
struct list_head list;
} u;
struct rcu_head rcu;
struct tracepoint_func probes[0];
};
@ -82,7 +68,7 @@ static inline void *allocate_probes(int count)
static void rcu_free_old_probes(struct rcu_head *head)
{
kfree(container_of(head, struct tp_probes, u.rcu));
kfree(container_of(head, struct tp_probes, rcu));
}
static inline void release_probes(struct tracepoint_func *old)
@ -90,38 +76,37 @@ static inline void release_probes(struct tracepoint_func *old)
if (old) {
struct tp_probes *tp_probes = container_of(old,
struct tp_probes, probes[0]);
call_rcu_sched(&tp_probes->u.rcu, rcu_free_old_probes);
call_rcu_sched(&tp_probes->rcu, rcu_free_old_probes);
}
}
static void debug_print_probes(struct tracepoint_entry *entry)
static void debug_print_probes(struct tracepoint_func *funcs)
{
int i;
if (!tracepoint_debug || !entry->funcs)
if (!tracepoint_debug || !funcs)
return;
for (i = 0; entry->funcs[i].func; i++)
printk(KERN_DEBUG "Probe %d : %p\n", i, entry->funcs[i].func);
for (i = 0; funcs[i].func; i++)
printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func);
}
static struct tracepoint_func *
tracepoint_entry_add_probe(struct tracepoint_entry *entry,
void *probe, void *data)
static struct tracepoint_func *func_add(struct tracepoint_func **funcs,
struct tracepoint_func *tp_func)
{
int nr_probes = 0;
struct tracepoint_func *old, *new;
if (WARN_ON(!probe))
if (WARN_ON(!tp_func->func))
return ERR_PTR(-EINVAL);
debug_print_probes(entry);
old = entry->funcs;
debug_print_probes(*funcs);
old = *funcs;
if (old) {
/* (N -> N+1), (N != 0, 1) probes */
for (nr_probes = 0; old[nr_probes].func; nr_probes++)
if (old[nr_probes].func == probe &&
old[nr_probes].data == data)
if (old[nr_probes].func == tp_func->func &&
old[nr_probes].data == tp_func->data)
return ERR_PTR(-EEXIST);
}
/* + 2 : one for new probe, one for NULL func */
@ -130,33 +115,30 @@ tracepoint_entry_add_probe(struct tracepoint_entry *entry,
return ERR_PTR(-ENOMEM);
if (old)
memcpy(new, old, nr_probes * sizeof(struct tracepoint_func));
new[nr_probes].func = probe;
new[nr_probes].data = data;
new[nr_probes] = *tp_func;
new[nr_probes + 1].func = NULL;
entry->refcount = nr_probes + 1;
entry->funcs = new;
debug_print_probes(entry);
*funcs = new;
debug_print_probes(*funcs);
return old;
}
static void *
tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
void *probe, void *data)
static void *func_remove(struct tracepoint_func **funcs,
struct tracepoint_func *tp_func)
{
int nr_probes = 0, nr_del = 0, i;
struct tracepoint_func *old, *new;
old = entry->funcs;
old = *funcs;
if (!old)
return ERR_PTR(-ENOENT);
debug_print_probes(entry);
debug_print_probes(*funcs);
/* (N -> M), (N > 1, M >= 0) probes */
if (probe) {
if (tp_func->func) {
for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
if (old[nr_probes].func == probe &&
old[nr_probes].data == data)
if (old[nr_probes].func == tp_func->func &&
old[nr_probes].data == tp_func->data)
nr_del++;
}
}
@ -167,9 +149,8 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
*/
if (nr_probes - nr_del == 0) {
/* N -> 0, (N > 1) */
entry->funcs = NULL;
entry->refcount = 0;
debug_print_probes(entry);
*funcs = NULL;
debug_print_probes(*funcs);
return old;
} else {
int j = 0;
@ -179,90 +160,35 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
if (new == NULL)
return ERR_PTR(-ENOMEM);
for (i = 0; old[i].func; i++)
if (old[i].func != probe || old[i].data != data)
if (old[i].func != tp_func->func
|| old[i].data != tp_func->data)
new[j++] = old[i];
new[nr_probes - nr_del].func = NULL;
entry->refcount = nr_probes - nr_del;
entry->funcs = new;
*funcs = new;
}
debug_print_probes(entry);
debug_print_probes(*funcs);
return old;
}
/*
* Get tracepoint if the tracepoint is present in the tracepoint hash table.
* Must be called with tracepoints_mutex held.
* Returns NULL if not present.
* Add the probe function to a tracepoint.
*/
static struct tracepoint_entry *get_tracepoint(const char *name)
static int tracepoint_add_func(struct tracepoint *tp,
struct tracepoint_func *func)
{
struct hlist_head *head;
struct tracepoint_entry *e;
u32 hash = jhash(name, strlen(name), 0);
struct tracepoint_func *old, *tp_funcs;
head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
hlist_for_each_entry(e, head, hlist) {
if (!strcmp(name, e->name))
return e;
if (tp->regfunc && !static_key_enabled(&tp->key))
tp->regfunc();
tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
old = func_add(&tp_funcs, func);
if (IS_ERR(old)) {
WARN_ON_ONCE(1);
return PTR_ERR(old);
}
return NULL;
}
/*
* Add the tracepoint to the tracepoint hash table. Must be called with
* tracepoints_mutex held.
*/
static struct tracepoint_entry *add_tracepoint(const char *name)
{
struct hlist_head *head;
struct tracepoint_entry *e;
size_t name_len = strlen(name) + 1;
u32 hash = jhash(name, name_len-1, 0);
head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
hlist_for_each_entry(e, head, hlist) {
if (!strcmp(name, e->name)) {
printk(KERN_NOTICE
"tracepoint %s busy\n", name);
return ERR_PTR(-EEXIST); /* Already there */
}
}
/*
* Using kmalloc here to allocate a variable length element. Could
* cause some memory fragmentation if overused.
*/
e = kmalloc(sizeof(struct tracepoint_entry) + name_len, GFP_KERNEL);
if (!e)
return ERR_PTR(-ENOMEM);
memcpy(&e->name[0], name, name_len);
e->funcs = NULL;
e->refcount = 0;
hlist_add_head(&e->hlist, head);
return e;
}
/*
* Remove the tracepoint from the tracepoint hash table. Must be called with
* mutex_lock held.
*/
static inline void remove_tracepoint(struct tracepoint_entry *e)
{
hlist_del(&e->hlist);
kfree(e);
}
/*
* Sets the probe callback corresponding to one tracepoint.
*/
static void set_tracepoint(struct tracepoint_entry **entry,
struct tracepoint *elem, int active)
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
if (elem->regfunc && !static_key_enabled(&elem->key) && active)
elem->regfunc();
else if (elem->unregfunc && static_key_enabled(&elem->key) && !active)
elem->unregfunc();
release_probes(old);
/*
* rcu_assign_pointer has a smp_wmb() which makes sure that the new
@ -271,426 +197,215 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* include/linux/tracepoints.h. A matching smp_read_barrier_depends()
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
if (active && !static_key_enabled(&elem->key))
static_key_slow_inc(&elem->key);
else if (!active && static_key_enabled(&elem->key))
static_key_slow_dec(&elem->key);
rcu_assign_pointer(tp->funcs, tp_funcs);
if (!static_key_enabled(&tp->key))
static_key_slow_inc(&tp->key);
return 0;
}
/*
* Disable a tracepoint and its probe callback.
* Remove a probe function from a tracepoint.
* Note: only waiting an RCU period after setting elem->call to the empty
* function insures that the original callback is not used anymore. This insured
* by preempt_disable around the call site.
*/
static void disable_tracepoint(struct tracepoint *elem)
static int tracepoint_remove_func(struct tracepoint *tp,
struct tracepoint_func *func)
{
if (elem->unregfunc && static_key_enabled(&elem->key))
elem->unregfunc();
struct tracepoint_func *old, *tp_funcs;
if (static_key_enabled(&elem->key))
static_key_slow_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}
/**
* tracepoint_update_probe_range - Update a probe range
* @begin: beginning of the range
* @end: end of the range
*
* Updates the probe callback corresponding to a range of tracepoints.
* Called with tracepoints_mutex held.
*/
static void tracepoint_update_probe_range(struct tracepoint * const *begin,
struct tracepoint * const *end)
{
struct tracepoint * const *iter;
struct tracepoint_entry *mark_entry;
if (!begin)
return;
for (iter = begin; iter < end; iter++) {
mark_entry = get_tracepoint((*iter)->name);
if (mark_entry) {
set_tracepoint(&mark_entry, *iter,
!!mark_entry->refcount);
} else {
disable_tracepoint(*iter);
}
tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
old = func_remove(&tp_funcs, func);
if (IS_ERR(old)) {
WARN_ON_ONCE(1);
return PTR_ERR(old);
}
}
release_probes(old);
#ifdef CONFIG_MODULES
void module_update_tracepoints(void)
{
struct tp_module *tp_mod;
if (!tp_funcs) {
/* Removed last function */
if (tp->unregfunc && static_key_enabled(&tp->key))
tp->unregfunc();
list_for_each_entry(tp_mod, &tracepoint_module_list, list)
tracepoint_update_probe_range(tp_mod->tracepoints_ptrs,
tp_mod->tracepoints_ptrs + tp_mod->num_tracepoints);
}
#else /* CONFIG_MODULES */
void module_update_tracepoints(void)
{
}
#endif /* CONFIG_MODULES */
/*
* Update probes, removing the faulty probes.
* Called with tracepoints_mutex held.
*/
static void tracepoint_update_probes(void)
{
/* Core kernel tracepoints */
tracepoint_update_probe_range(__start___tracepoints_ptrs,
__stop___tracepoints_ptrs);
/* tracepoints in modules. */
module_update_tracepoints();
}
static struct tracepoint_func *
tracepoint_add_probe(const char *name, void *probe, void *data)
{
struct tracepoint_entry *entry;
struct tracepoint_func *old;
entry = get_tracepoint(name);
if (!entry) {
entry = add_tracepoint(name);
if (IS_ERR(entry))
return (struct tracepoint_func *)entry;
if (static_key_enabled(&tp->key))
static_key_slow_dec(&tp->key);
}
old = tracepoint_entry_add_probe(entry, probe, data);
if (IS_ERR(old) && !entry->refcount)
remove_tracepoint(entry);
return old;
rcu_assign_pointer(tp->funcs, tp_funcs);
return 0;
}
/**
* tracepoint_probe_register - Connect a probe to a tracepoint
* @name: tracepoint name
* @tp: tracepoint
* @probe: probe handler
*
* Returns 0 if ok, error value on error.
* The probe address must at least be aligned on the architecture pointer size.
* Note: if @tp is within a module, the caller is responsible for
* unregistering the probe before the module is gone. This can be
* performed either with a tracepoint module going notifier, or from
* within module exit functions.
*/
int tracepoint_probe_register(const char *name, void *probe, void *data)
int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
{
struct tracepoint_func *old;
struct tracepoint_func tp_func;
int ret;
mutex_lock(&tracepoints_mutex);
old = tracepoint_add_probe(name, probe, data);
if (IS_ERR(old)) {
mutex_unlock(&tracepoints_mutex);
return PTR_ERR(old);
}
tracepoint_update_probes(); /* may update entry */
tp_func.func = probe;
tp_func.data = data;
ret = tracepoint_add_func(tp, &tp_func);
mutex_unlock(&tracepoints_mutex);
release_probes(old);
return 0;
return ret;
}
EXPORT_SYMBOL_GPL(tracepoint_probe_register);
static struct tracepoint_func *
tracepoint_remove_probe(const char *name, void *probe, void *data)
{
struct tracepoint_entry *entry;
struct tracepoint_func *old;
entry = get_tracepoint(name);
if (!entry)
return ERR_PTR(-ENOENT);
old = tracepoint_entry_remove_probe(entry, probe, data);
if (IS_ERR(old))
return old;
if (!entry->refcount)
remove_tracepoint(entry);
return old;
}
/**
* tracepoint_probe_unregister - Disconnect a probe from a tracepoint
* @name: tracepoint name
* @tp: tracepoint
* @probe: probe function pointer
*
* We do not need to call a synchronize_sched to make sure the probes have
* finished running before doing a module unload, because the module unload
* itself uses stop_machine(), which insures that every preempt disabled section
* have finished.
* Returns 0 if ok, error value on error.
*/
int tracepoint_probe_unregister(const char *name, void *probe, void *data)
int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
{
struct tracepoint_func *old;
struct tracepoint_func tp_func;
int ret;
mutex_lock(&tracepoints_mutex);
old = tracepoint_remove_probe(name, probe, data);
if (IS_ERR(old)) {
mutex_unlock(&tracepoints_mutex);
return PTR_ERR(old);
}
tracepoint_update_probes(); /* may update entry */
tp_func.func = probe;
tp_func.data = data;
ret = tracepoint_remove_func(tp, &tp_func);
mutex_unlock(&tracepoints_mutex);
release_probes(old);
return 0;
return ret;
}
EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
static LIST_HEAD(old_probes);
static int need_update;
static void tracepoint_add_old_probes(void *old)
{
need_update = 1;
if (old) {
struct tp_probes *tp_probes = container_of(old,
struct tp_probes, probes[0]);
list_add(&tp_probes->u.list, &old_probes);
}
}
/**
* tracepoint_probe_register_noupdate - register a probe but not connect
* @name: tracepoint name
* @probe: probe handler
*
* caller must call tracepoint_probe_update_all()
*/
int tracepoint_probe_register_noupdate(const char *name, void *probe,
void *data)
{
struct tracepoint_func *old;
mutex_lock(&tracepoints_mutex);
old = tracepoint_add_probe(name, probe, data);
if (IS_ERR(old)) {
mutex_unlock(&tracepoints_mutex);
return PTR_ERR(old);
}
tracepoint_add_old_probes(old);
mutex_unlock(&tracepoints_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(tracepoint_probe_register_noupdate);
/**
* tracepoint_probe_unregister_noupdate - remove a probe but not disconnect
* @name: tracepoint name
* @probe: probe function pointer
*
* caller must call tracepoint_probe_update_all()
*/
int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
void *data)
{
struct tracepoint_func *old;
mutex_lock(&tracepoints_mutex);
old = tracepoint_remove_probe(name, probe, data);
if (IS_ERR(old)) {
mutex_unlock(&tracepoints_mutex);
return PTR_ERR(old);
}
tracepoint_add_old_probes(old);
mutex_unlock(&tracepoints_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(tracepoint_probe_unregister_noupdate);
/**
* tracepoint_probe_update_all - update tracepoints
*/
void tracepoint_probe_update_all(void)
{
LIST_HEAD(release_probes);
struct tp_probes *pos, *next;
mutex_lock(&tracepoints_mutex);
if (!need_update) {
mutex_unlock(&tracepoints_mutex);
return;
}
if (!list_empty(&old_probes))
list_replace_init(&old_probes, &release_probes);
need_update = 0;
tracepoint_update_probes();
mutex_unlock(&tracepoints_mutex);
list_for_each_entry_safe(pos, next, &release_probes, u.list) {
list_del(&pos->u.list);
call_rcu_sched(&pos->u.rcu, rcu_free_old_probes);
}
}
EXPORT_SYMBOL_GPL(tracepoint_probe_update_all);
/**
* tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
* @tracepoint: current tracepoints (in), next tracepoint (out)
* @begin: beginning of the range
* @end: end of the range
*
* Returns whether a next tracepoint has been found (1) or not (0).
* Will return the first tracepoint in the range if the input tracepoint is
* NULL.
*/
static int tracepoint_get_iter_range(struct tracepoint * const **tracepoint,
struct tracepoint * const *begin, struct tracepoint * const *end)
{
if (!*tracepoint && begin != end) {
*tracepoint = begin;
return 1;
}
if (*tracepoint >= begin && *tracepoint < end)
return 1;
return 0;
}
#ifdef CONFIG_MODULES
static void tracepoint_get_iter(struct tracepoint_iter *iter)
{
int found = 0;
struct tp_module *iter_mod;
/* Core kernel tracepoints */
if (!iter->module) {
found = tracepoint_get_iter_range(&iter->tracepoint,
__start___tracepoints_ptrs,
__stop___tracepoints_ptrs);
if (found)
goto end;
}
/* Tracepoints in modules */
mutex_lock(&tracepoints_mutex);
list_for_each_entry(iter_mod, &tracepoint_module_list, list) {
/*
* Sorted module list
*/
if (iter_mod < iter->module)
continue;
else if (iter_mod > iter->module)
iter->tracepoint = NULL;
found = tracepoint_get_iter_range(&iter->tracepoint,
iter_mod->tracepoints_ptrs,
iter_mod->tracepoints_ptrs
+ iter_mod->num_tracepoints);
if (found) {
iter->module = iter_mod;
break;
}
}
mutex_unlock(&tracepoints_mutex);
end:
if (!found)
tracepoint_iter_reset(iter);
}
#else /* CONFIG_MODULES */
static void tracepoint_get_iter(struct tracepoint_iter *iter)
{
int found = 0;
/* Core kernel tracepoints */
found = tracepoint_get_iter_range(&iter->tracepoint,
__start___tracepoints_ptrs,
__stop___tracepoints_ptrs);
if (!found)
tracepoint_iter_reset(iter);
}
#endif /* CONFIG_MODULES */
void tracepoint_iter_start(struct tracepoint_iter *iter)
{
tracepoint_get_iter(iter);
}
EXPORT_SYMBOL_GPL(tracepoint_iter_start);
void tracepoint_iter_next(struct tracepoint_iter *iter)
{
iter->tracepoint++;
/*
* iter->tracepoint may be invalid because we blindly incremented it.
* Make sure it is valid by marshalling on the tracepoints, getting the
* tracepoints from following modules if necessary.
*/
tracepoint_get_iter(iter);
}
EXPORT_SYMBOL_GPL(tracepoint_iter_next);
void tracepoint_iter_stop(struct tracepoint_iter *iter)
{
}
EXPORT_SYMBOL_GPL(tracepoint_iter_stop);
void tracepoint_iter_reset(struct tracepoint_iter *iter)
{
#ifdef CONFIG_MODULES
iter->module = NULL;
#endif /* CONFIG_MODULES */
iter->tracepoint = NULL;
}
EXPORT_SYMBOL_GPL(tracepoint_iter_reset);
#ifdef CONFIG_MODULES
bool trace_module_has_bad_taint(struct module *mod)
{
return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP));
return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
(1 << TAINT_UNSIGNED_MODULE));
}
static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);
/**
* register_tracepoint_notifier - register tracepoint coming/going notifier
* @nb: notifier block
*
* Notifiers registered with this function are called on module
* coming/going with the tracepoint_module_list_mutex held.
* The notifier block callback should expect a "struct tp_module" data
* pointer.
*/
int register_tracepoint_module_notifier(struct notifier_block *nb)
{
struct tp_module *tp_mod;
int ret;
mutex_lock(&tracepoint_module_list_mutex);
ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb);
if (ret)
goto end;
list_for_each_entry(tp_mod, &tracepoint_module_list, list)
(void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod);
end:
mutex_unlock(&tracepoint_module_list_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier);
/**
* unregister_tracepoint_notifier - unregister tracepoint coming/going notifier
* @nb: notifier block
*
* The notifier block callback should expect a "struct tp_module" data
* pointer.
*/
int unregister_tracepoint_module_notifier(struct notifier_block *nb)
{
struct tp_module *tp_mod;
int ret;
mutex_lock(&tracepoint_module_list_mutex);
ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb);
if (ret)
goto end;
list_for_each_entry(tp_mod, &tracepoint_module_list, list)
(void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod);
end:
mutex_unlock(&tracepoint_module_list_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier);
/*
* Ensure the tracer unregistered the module's probes before the module
* teardown is performed. Prevents leaks of probe and data pointers.
*/
static void tp_module_going_check_quiescent(struct tracepoint * const *begin,
struct tracepoint * const *end)
{
struct tracepoint * const *iter;
if (!begin)
return;
for (iter = begin; iter < end; iter++)
WARN_ON_ONCE((*iter)->funcs);
}
static int tracepoint_module_coming(struct module *mod)
{
struct tp_module *tp_mod, *iter;
struct tp_module *tp_mod;
int ret = 0;
if (!mod->num_tracepoints)
return 0;
/*
* We skip modules that taint the kernel, especially those with different
* module headers (for forced load), to make sure we don't cause a crash.
* Staging and out-of-tree GPL modules are fine.
* Staging, out-of-tree, and unsigned GPL modules are fine.
*/
if (trace_module_has_bad_taint(mod))
return 0;
mutex_lock(&tracepoints_mutex);
mutex_lock(&tracepoint_module_list_mutex);
tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
if (!tp_mod) {
ret = -ENOMEM;
goto end;
}
tp_mod->num_tracepoints = mod->num_tracepoints;
tp_mod->tracepoints_ptrs = mod->tracepoints_ptrs;
/*
* tracepoint_module_list is kept sorted by struct module pointer
* address for iteration on tracepoints from a seq_file that can release
* the mutex between calls.
*/
list_for_each_entry_reverse(iter, &tracepoint_module_list, list) {
BUG_ON(iter == tp_mod); /* Should never be in the list twice */
if (iter < tp_mod) {
/* We belong to the location right after iter. */
list_add(&tp_mod->list, &iter->list);
goto module_added;
}
}
/* We belong to the beginning of the list */
list_add(&tp_mod->list, &tracepoint_module_list);
module_added:
tracepoint_update_probe_range(mod->tracepoints_ptrs,
mod->tracepoints_ptrs + mod->num_tracepoints);
tp_mod->mod = mod;
list_add_tail(&tp_mod->list, &tracepoint_module_list);
blocking_notifier_call_chain(&tracepoint_notify_list,
MODULE_STATE_COMING, tp_mod);
end:
mutex_unlock(&tracepoints_mutex);
mutex_unlock(&tracepoint_module_list_mutex);
return ret;
}
static int tracepoint_module_going(struct module *mod)
static void tracepoint_module_going(struct module *mod)
{
struct tp_module *pos;
struct tp_module *tp_mod;
mutex_lock(&tracepoints_mutex);
tracepoint_update_probe_range(mod->tracepoints_ptrs,
mod->tracepoints_ptrs + mod->num_tracepoints);
list_for_each_entry(pos, &tracepoint_module_list, list) {
if (pos->tracepoints_ptrs == mod->tracepoints_ptrs) {
list_del(&pos->list);
kfree(pos);
if (!mod->num_tracepoints)
return;
mutex_lock(&tracepoint_module_list_mutex);
list_for_each_entry(tp_mod, &tracepoint_module_list, list) {
if (tp_mod->mod == mod) {
blocking_notifier_call_chain(&tracepoint_notify_list,
MODULE_STATE_GOING, tp_mod);
list_del(&tp_mod->list);
kfree(tp_mod);
/*
* Called the going notifier before checking for
* quiescence.
*/
tp_module_going_check_quiescent(mod->tracepoints_ptrs,
mod->tracepoints_ptrs + mod->num_tracepoints);
break;
}
}
@ -700,12 +415,11 @@ static int tracepoint_module_going(struct module *mod)
* flag on "going", in case a module taints the kernel only after being
* loaded.
*/
mutex_unlock(&tracepoints_mutex);
return 0;
mutex_unlock(&tracepoint_module_list_mutex);
}
int tracepoint_module_notify(struct notifier_block *self,
unsigned long val, void *data)
static int tracepoint_module_notify(struct notifier_block *self,
unsigned long val, void *data)
{
struct module *mod = data;
int ret = 0;
@ -717,24 +431,58 @@ int tracepoint_module_notify(struct notifier_block *self,
case MODULE_STATE_LIVE:
break;
case MODULE_STATE_GOING:
ret = tracepoint_module_going(mod);
tracepoint_module_going(mod);
break;
case MODULE_STATE_UNFORMED:
break;
}
return ret;
}
struct notifier_block tracepoint_module_nb = {
static struct notifier_block tracepoint_module_nb = {
.notifier_call = tracepoint_module_notify,
.priority = 0,
};
static int init_tracepoints(void)
static __init int init_tracepoints(void)
{
return register_module_notifier(&tracepoint_module_nb);
int ret;
ret = register_module_notifier(&tracepoint_module_nb);
if (ret)
pr_warning("Failed to register tracepoint module enter notifier\n");
return ret;
}
__initcall(init_tracepoints);
#endif /* CONFIG_MODULES */
static void for_each_tracepoint_range(struct tracepoint * const *begin,
struct tracepoint * const *end,
void (*fct)(struct tracepoint *tp, void *priv),
void *priv)
{
struct tracepoint * const *iter;
if (!begin)
return;
for (iter = begin; iter < end; iter++)
fct(*iter, priv);
}
/**
* for_each_kernel_tracepoint - iteration on all kernel tracepoints
* @fct: callback
* @priv: private data
*/
void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
void *priv)
{
for_each_tracepoint_range(__start___tracepoints_ptrs,
__stop___tracepoints_ptrs, fct, priv);
}
EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
/* NB: reg/unreg are called while guarded with the tracepoints_mutex */

3
kernel/user.c
View file

@ -222,5 +222,4 @@ static int __init uid_cache_init(void)
return 0;
}
module_init(uid_cache_init);
subsys_initcall(uid_cache_init);

13
kernel/user_namespace.c
View file

@ -152,7 +152,7 @@ static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count)
/* Find the matching extent */
extents = map->nr_extents;
smp_read_barrier_depends();
smp_rmb();
for (idx = 0; idx < extents; idx++) {
first = map->extent[idx].first;
last = first + map->extent[idx].count - 1;
@ -176,7 +176,7 @@ static u32 map_id_down(struct uid_gid_map *map, u32 id)
/* Find the matching extent */
extents = map->nr_extents;
smp_read_barrier_depends();
smp_rmb();
for (idx = 0; idx < extents; idx++) {
first = map->extent[idx].first;
last = first + map->extent[idx].count - 1;
@ -199,7 +199,7 @@ static u32 map_id_up(struct uid_gid_map *map, u32 id)
/* Find the matching extent */
extents = map->nr_extents;
smp_read_barrier_depends();
smp_rmb();
for (idx = 0; idx < extents; idx++) {
first = map->extent[idx].lower_first;
last = first + map->extent[idx].count - 1;
@ -615,9 +615,8 @@ static ssize_t map_write(struct file *file, const char __user *buf,
* were written before the count of the extents.
*
* To achieve this smp_wmb() is used on guarantee the write
* order and smp_read_barrier_depends() is guaranteed that we
* don't have crazy architectures returning stale data.
*
* order and smp_rmb() is guaranteed that we don't have crazy
* architectures returning stale data.
*/
mutex_lock(&id_map_mutex);
@ -902,4 +901,4 @@ static __init int user_namespaces_init(void)
user_ns_cachep = KMEM_CACHE(user_namespace, SLAB_PANIC);
return 0;
}
module_init(user_namespaces_init);
subsys_initcall(user_namespaces_init);

22
kernel/watchdog.c
View file

@ -138,7 +138,11 @@ static void __touch_watchdog(void)
void touch_softlockup_watchdog(void)
{
__this_cpu_write(watchdog_touch_ts, 0);
/*
* Preemption can be enabled. It doesn't matter which CPU's timestamp
* gets zeroed here, so use the raw_ operation.
*/
raw_cpu_write(watchdog_touch_ts, 0);
}
EXPORT_SYMBOL(touch_softlockup_watchdog);
@ -158,14 +162,14 @@ void touch_all_softlockup_watchdogs(void)
#ifdef CONFIG_HARDLOCKUP_DETECTOR
void touch_nmi_watchdog(void)
{
if (watchdog_user_enabled) {
unsigned cpu;
for_each_present_cpu(cpu) {
if (per_cpu(watchdog_nmi_touch, cpu) != true)
per_cpu(watchdog_nmi_touch, cpu) = true;
}
}
/*
* Using __raw here because some code paths have
* preemption enabled. If preemption is enabled
* then interrupts should be enabled too, in which
* case we shouldn't have to worry about the watchdog
* going off.
*/
__raw_get_cpu_var(watchdog_nmi_touch) = true;
touch_softlockup_watchdog();
}
EXPORT_SYMBOL(touch_nmi_watchdog);