mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-24 13:44:26 -04:00
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller:
"Highlights:
1) Maintain the TCP retransmit queue using an rbtree, with 1GB
windows at 100Gb this really has become necessary. From Eric
Dumazet.
2) Multi-program support for cgroup+bpf, from Alexei Starovoitov.
3) Perform broadcast flooding in hardware in mv88e6xxx, from Andrew
Lunn.
4) Add meter action support to openvswitch, from Andy Zhou.
5) Add a data meta pointer for BPF accessible packets, from Daniel
Borkmann.
6) Namespace-ify almost all TCP sysctl knobs, from Eric Dumazet.
7) Turn on Broadcom Tags in b53 driver, from Florian Fainelli.
8) More work to move the RTNL mutex down, from Florian Westphal.
9) Add 'bpftool' utility, to help with bpf program introspection.
From Jakub Kicinski.
10) Add new 'cpumap' type for XDP_REDIRECT action, from Jesper
Dangaard Brouer.
11) Support 'blocks' of transformations in the packet scheduler which
can span multiple network devices, from Jiri Pirko.
12) TC flower offload support in cxgb4, from Kumar Sanghvi.
13) Priority based stream scheduler for SCTP, from Marcelo Ricardo
Leitner.
14) Thunderbolt networking driver, from Amir Levy and Mika Westerberg.
15) Add RED qdisc offloadability, and use it in mlxsw driver. From
Nogah Frankel.
16) eBPF based device controller for cgroup v2, from Roman Gushchin.
17) Add some fundamental tracepoints for TCP, from Song Liu.
18) Remove garbage collection from ipv6 route layer, this is a
significant accomplishment. From Wei Wang.
19) Add multicast route offload support to mlxsw, from Yotam Gigi"
* git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next: (2177 commits)
tcp: highest_sack fix
geneve: fix fill_info when link down
bpf: fix lockdep splat
net: cdc_ncm: GetNtbFormat endian fix
openvswitch: meter: fix NULL pointer dereference in ovs_meter_cmd_reply_start
netem: remove unnecessary 64 bit modulus
netem: use 64 bit divide by rate
tcp: Namespace-ify sysctl_tcp_default_congestion_control
net: Protect iterations over net::fib_notifier_ops in fib_seq_sum()
ipv6: set all.accept_dad to 0 by default
uapi: fix linux/tls.h userspace compilation error
usbnet: ipheth: prevent TX queue timeouts when device not ready
vhost_net: conditionally enable tx polling
uapi: fix linux/rxrpc.h userspace compilation errors
net: stmmac: fix LPI transitioning for dwmac4
atm: horizon: Fix irq release error
net-sysfs: trigger netlink notification on ifalias change via sysfs
openvswitch: Using kfree_rcu() to simplify the code
openvswitch: Make local function ovs_nsh_key_attr_size() static
openvswitch: Fix return value check in ovs_meter_cmd_features()
...
This commit is contained in:
Linus Torvalds
commit
5bbcc0f595
1617 changed files with 91536 additions and 27256 deletions
|
|
@ -3,8 +3,11 @@ obj-y := core.o
|
|||
|
||||
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += disasm.o
|
||||
ifeq ($(CONFIG_NET),y)
|
||||
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += offload.o
|
||||
ifeq ($(CONFIG_STREAM_PARSER),y)
|
||||
obj-$(CONFIG_BPF_SYSCALL) += sockmap.o
|
||||
endif
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@
|
|||
|
||||
#include "map_in_map.h"
|
||||
|
||||
#define ARRAY_CREATE_FLAG_MASK \
|
||||
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
static void bpf_array_free_percpu(struct bpf_array *array)
|
||||
{
|
||||
int i;
|
||||
|
|
@ -56,7 +59,8 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
|
|||
|
||||
/* check sanity of attributes */
|
||||
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
||||
attr->value_size == 0 || attr->map_flags & ~BPF_F_NUMA_NODE ||
|
||||
attr->value_size == 0 ||
|
||||
attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
|
||||
(percpu && numa_node != NUMA_NO_NODE))
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
|
|
|
|||
|
|
@ -27,129 +27,405 @@ void cgroup_bpf_put(struct cgroup *cgrp)
|
|||
{
|
||||
unsigned int type;
|
||||
|
||||
for (type = 0; type < ARRAY_SIZE(cgrp->bpf.prog); type++) {
|
||||
struct bpf_prog *prog = cgrp->bpf.prog[type];
|
||||
for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
|
||||
struct list_head *progs = &cgrp->bpf.progs[type];
|
||||
struct bpf_prog_list *pl, *tmp;
|
||||
|
||||
if (prog) {
|
||||
bpf_prog_put(prog);
|
||||
list_for_each_entry_safe(pl, tmp, progs, node) {
|
||||
list_del(&pl->node);
|
||||
bpf_prog_put(pl->prog);
|
||||
kfree(pl);
|
||||
static_branch_dec(&cgroup_bpf_enabled_key);
|
||||
}
|
||||
bpf_prog_array_free(cgrp->bpf.effective[type]);
|
||||
}
|
||||
}
|
||||
|
||||
/* count number of elements in the list.
|
||||
* it's slow but the list cannot be long
|
||||
*/
|
||||
static u32 prog_list_length(struct list_head *head)
|
||||
{
|
||||
struct bpf_prog_list *pl;
|
||||
u32 cnt = 0;
|
||||
|
||||
list_for_each_entry(pl, head, node) {
|
||||
if (!pl->prog)
|
||||
continue;
|
||||
cnt++;
|
||||
}
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/* if parent has non-overridable prog attached,
|
||||
* disallow attaching new programs to the descendent cgroup.
|
||||
* if parent has overridable or multi-prog, allow attaching
|
||||
*/
|
||||
static bool hierarchy_allows_attach(struct cgroup *cgrp,
|
||||
enum bpf_attach_type type,
|
||||
u32 new_flags)
|
||||
{
|
||||
struct cgroup *p;
|
||||
|
||||
p = cgroup_parent(cgrp);
|
||||
if (!p)
|
||||
return true;
|
||||
do {
|
||||
u32 flags = p->bpf.flags[type];
|
||||
u32 cnt;
|
||||
|
||||
if (flags & BPF_F_ALLOW_MULTI)
|
||||
return true;
|
||||
cnt = prog_list_length(&p->bpf.progs[type]);
|
||||
WARN_ON_ONCE(cnt > 1);
|
||||
if (cnt == 1)
|
||||
return !!(flags & BPF_F_ALLOW_OVERRIDE);
|
||||
p = cgroup_parent(p);
|
||||
} while (p);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* compute a chain of effective programs for a given cgroup:
|
||||
* start from the list of programs in this cgroup and add
|
||||
* all parent programs.
|
||||
* Note that parent's F_ALLOW_OVERRIDE-type program is yielding
|
||||
* to programs in this cgroup
|
||||
*/
|
||||
static int compute_effective_progs(struct cgroup *cgrp,
|
||||
enum bpf_attach_type type,
|
||||
struct bpf_prog_array __rcu **array)
|
||||
{
|
||||
struct bpf_prog_array __rcu *progs;
|
||||
struct bpf_prog_list *pl;
|
||||
struct cgroup *p = cgrp;
|
||||
int cnt = 0;
|
||||
|
||||
/* count number of effective programs by walking parents */
|
||||
do {
|
||||
if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
|
||||
cnt += prog_list_length(&p->bpf.progs[type]);
|
||||
p = cgroup_parent(p);
|
||||
} while (p);
|
||||
|
||||
progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
|
||||
if (!progs)
|
||||
return -ENOMEM;
|
||||
|
||||
/* populate the array with effective progs */
|
||||
cnt = 0;
|
||||
p = cgrp;
|
||||
do {
|
||||
if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
|
||||
list_for_each_entry(pl,
|
||||
&p->bpf.progs[type], node) {
|
||||
if (!pl->prog)
|
||||
continue;
|
||||
rcu_dereference_protected(progs, 1)->
|
||||
progs[cnt++] = pl->prog;
|
||||
}
|
||||
p = cgroup_parent(p);
|
||||
} while (p);
|
||||
|
||||
*array = progs;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void activate_effective_progs(struct cgroup *cgrp,
|
||||
enum bpf_attach_type type,
|
||||
struct bpf_prog_array __rcu *array)
|
||||
{
|
||||
struct bpf_prog_array __rcu *old_array;
|
||||
|
||||
old_array = xchg(&cgrp->bpf.effective[type], array);
|
||||
/* free prog array after grace period, since __cgroup_bpf_run_*()
|
||||
* might be still walking the array
|
||||
*/
|
||||
bpf_prog_array_free(old_array);
|
||||
}
|
||||
|
||||
/**
|
||||
* cgroup_bpf_inherit() - inherit effective programs from parent
|
||||
* @cgrp: the cgroup to modify
|
||||
* @parent: the parent to inherit from
|
||||
*/
|
||||
void cgroup_bpf_inherit(struct cgroup *cgrp, struct cgroup *parent)
|
||||
int cgroup_bpf_inherit(struct cgroup *cgrp)
|
||||
{
|
||||
unsigned int type;
|
||||
/* has to use marco instead of const int, since compiler thinks
|
||||
* that array below is variable length
|
||||
*/
|
||||
#define NR ARRAY_SIZE(cgrp->bpf.effective)
|
||||
struct bpf_prog_array __rcu *arrays[NR] = {};
|
||||
int i;
|
||||
|
||||
for (type = 0; type < ARRAY_SIZE(cgrp->bpf.effective); type++) {
|
||||
struct bpf_prog *e;
|
||||
for (i = 0; i < NR; i++)
|
||||
INIT_LIST_HEAD(&cgrp->bpf.progs[i]);
|
||||
|
||||
e = rcu_dereference_protected(parent->bpf.effective[type],
|
||||
lockdep_is_held(&cgroup_mutex));
|
||||
rcu_assign_pointer(cgrp->bpf.effective[type], e);
|
||||
cgrp->bpf.disallow_override[type] = parent->bpf.disallow_override[type];
|
||||
}
|
||||
for (i = 0; i < NR; i++)
|
||||
if (compute_effective_progs(cgrp, i, &arrays[i]))
|
||||
goto cleanup;
|
||||
|
||||
for (i = 0; i < NR; i++)
|
||||
activate_effective_progs(cgrp, i, arrays[i]);
|
||||
|
||||
return 0;
|
||||
cleanup:
|
||||
for (i = 0; i < NR; i++)
|
||||
bpf_prog_array_free(arrays[i]);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
#define BPF_CGROUP_MAX_PROGS 64
|
||||
|
||||
/**
|
||||
* __cgroup_bpf_update() - Update the pinned program of a cgroup, and
|
||||
* __cgroup_bpf_attach() - Attach the program to a cgroup, and
|
||||
* propagate the change to descendants
|
||||
* @cgrp: The cgroup which descendants to traverse
|
||||
* @parent: The parent of @cgrp, or %NULL if @cgrp is the root
|
||||
* @prog: A new program to pin
|
||||
* @type: Type of pinning operation (ingress/egress)
|
||||
*
|
||||
* Each cgroup has a set of two pointers for bpf programs; one for eBPF
|
||||
* programs it owns, and which is effective for execution.
|
||||
*
|
||||
* If @prog is not %NULL, this function attaches a new program to the cgroup
|
||||
* and releases the one that is currently attached, if any. @prog is then made
|
||||
* the effective program of type @type in that cgroup.
|
||||
*
|
||||
* If @prog is %NULL, the currently attached program of type @type is released,
|
||||
* and the effective program of the parent cgroup (if any) is inherited to
|
||||
* @cgrp.
|
||||
*
|
||||
* Then, the descendants of @cgrp are walked and the effective program for
|
||||
* each of them is set to the effective program of @cgrp unless the
|
||||
* descendant has its own program attached, in which case the subbranch is
|
||||
* skipped. This ensures that delegated subcgroups with own programs are left
|
||||
* untouched.
|
||||
* @prog: A program to attach
|
||||
* @type: Type of attach operation
|
||||
*
|
||||
* Must be called with cgroup_mutex held.
|
||||
*/
|
||||
int __cgroup_bpf_update(struct cgroup *cgrp, struct cgroup *parent,
|
||||
struct bpf_prog *prog, enum bpf_attach_type type,
|
||||
bool new_overridable)
|
||||
int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
|
||||
enum bpf_attach_type type, u32 flags)
|
||||
{
|
||||
struct bpf_prog *old_prog, *effective = NULL;
|
||||
struct cgroup_subsys_state *pos;
|
||||
bool overridable = true;
|
||||
struct list_head *progs = &cgrp->bpf.progs[type];
|
||||
struct bpf_prog *old_prog = NULL;
|
||||
struct cgroup_subsys_state *css;
|
||||
struct bpf_prog_list *pl;
|
||||
bool pl_was_allocated;
|
||||
int err;
|
||||
|
||||
if (parent) {
|
||||
overridable = !parent->bpf.disallow_override[type];
|
||||
effective = rcu_dereference_protected(parent->bpf.effective[type],
|
||||
lockdep_is_held(&cgroup_mutex));
|
||||
}
|
||||
if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
|
||||
/* invalid combination */
|
||||
return -EINVAL;
|
||||
|
||||
if (prog && effective && !overridable)
|
||||
/* if parent has non-overridable prog attached, disallow
|
||||
* attaching new programs to descendent cgroup
|
||||
if (!hierarchy_allows_attach(cgrp, type, flags))
|
||||
return -EPERM;
|
||||
|
||||
if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)
|
||||
/* Disallow attaching non-overridable on top
|
||||
* of existing overridable in this cgroup.
|
||||
* Disallow attaching multi-prog if overridable or none
|
||||
*/
|
||||
return -EPERM;
|
||||
|
||||
if (prog && effective && overridable != new_overridable)
|
||||
/* if parent has overridable prog attached, only
|
||||
* allow overridable programs in descendent cgroup
|
||||
*/
|
||||
return -EPERM;
|
||||
if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
|
||||
return -E2BIG;
|
||||
|
||||
old_prog = cgrp->bpf.prog[type];
|
||||
if (flags & BPF_F_ALLOW_MULTI) {
|
||||
list_for_each_entry(pl, progs, node)
|
||||
if (pl->prog == prog)
|
||||
/* disallow attaching the same prog twice */
|
||||
return -EINVAL;
|
||||
|
||||
if (prog) {
|
||||
overridable = new_overridable;
|
||||
effective = prog;
|
||||
if (old_prog &&
|
||||
cgrp->bpf.disallow_override[type] == new_overridable)
|
||||
/* disallow attaching non-overridable on top
|
||||
* of existing overridable in this cgroup
|
||||
* and vice versa
|
||||
*/
|
||||
return -EPERM;
|
||||
}
|
||||
|
||||
if (!prog && !old_prog)
|
||||
/* report error when trying to detach and nothing is attached */
|
||||
return -ENOENT;
|
||||
|
||||
cgrp->bpf.prog[type] = prog;
|
||||
|
||||
css_for_each_descendant_pre(pos, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(pos, struct cgroup, self);
|
||||
|
||||
/* skip the subtree if the descendant has its own program */
|
||||
if (desc->bpf.prog[type] && desc != cgrp) {
|
||||
pos = css_rightmost_descendant(pos);
|
||||
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
|
||||
if (!pl)
|
||||
return -ENOMEM;
|
||||
pl_was_allocated = true;
|
||||
pl->prog = prog;
|
||||
list_add_tail(&pl->node, progs);
|
||||
} else {
|
||||
if (list_empty(progs)) {
|
||||
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
|
||||
if (!pl)
|
||||
return -ENOMEM;
|
||||
pl_was_allocated = true;
|
||||
list_add_tail(&pl->node, progs);
|
||||
} else {
|
||||
rcu_assign_pointer(desc->bpf.effective[type],
|
||||
effective);
|
||||
desc->bpf.disallow_override[type] = !overridable;
|
||||
pl = list_first_entry(progs, typeof(*pl), node);
|
||||
old_prog = pl->prog;
|
||||
pl_was_allocated = false;
|
||||
}
|
||||
pl->prog = prog;
|
||||
}
|
||||
|
||||
if (prog)
|
||||
static_branch_inc(&cgroup_bpf_enabled_key);
|
||||
cgrp->bpf.flags[type] = flags;
|
||||
|
||||
/* allocate and recompute effective prog arrays */
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
err = compute_effective_progs(desc, type, &desc->bpf.inactive);
|
||||
if (err)
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* all allocations were successful. Activate all prog arrays */
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
activate_effective_progs(desc, type, desc->bpf.inactive);
|
||||
desc->bpf.inactive = NULL;
|
||||
}
|
||||
|
||||
static_branch_inc(&cgroup_bpf_enabled_key);
|
||||
if (old_prog) {
|
||||
bpf_prog_put(old_prog);
|
||||
static_branch_dec(&cgroup_bpf_enabled_key);
|
||||
}
|
||||
return 0;
|
||||
|
||||
cleanup:
|
||||
/* oom while computing effective. Free all computed effective arrays
|
||||
* since they were not activated
|
||||
*/
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
bpf_prog_array_free(desc->bpf.inactive);
|
||||
desc->bpf.inactive = NULL;
|
||||
}
|
||||
|
||||
/* and cleanup the prog list */
|
||||
pl->prog = old_prog;
|
||||
if (pl_was_allocated) {
|
||||
list_del(&pl->node);
|
||||
kfree(pl);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
/**
|
||||
* __cgroup_bpf_detach() - Detach the program from a cgroup, and
|
||||
* propagate the change to descendants
|
||||
* @cgrp: The cgroup which descendants to traverse
|
||||
* @prog: A program to detach or NULL
|
||||
* @type: Type of detach operation
|
||||
*
|
||||
* Must be called with cgroup_mutex held.
|
||||
*/
|
||||
int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
|
||||
enum bpf_attach_type type, u32 unused_flags)
|
||||
{
|
||||
struct list_head *progs = &cgrp->bpf.progs[type];
|
||||
u32 flags = cgrp->bpf.flags[type];
|
||||
struct bpf_prog *old_prog = NULL;
|
||||
struct cgroup_subsys_state *css;
|
||||
struct bpf_prog_list *pl;
|
||||
int err;
|
||||
|
||||
if (flags & BPF_F_ALLOW_MULTI) {
|
||||
if (!prog)
|
||||
/* to detach MULTI prog the user has to specify valid FD
|
||||
* of the program to be detached
|
||||
*/
|
||||
return -EINVAL;
|
||||
} else {
|
||||
if (list_empty(progs))
|
||||
/* report error when trying to detach and nothing is attached */
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
if (flags & BPF_F_ALLOW_MULTI) {
|
||||
/* find the prog and detach it */
|
||||
list_for_each_entry(pl, progs, node) {
|
||||
if (pl->prog != prog)
|
||||
continue;
|
||||
old_prog = prog;
|
||||
/* mark it deleted, so it's ignored while
|
||||
* recomputing effective
|
||||
*/
|
||||
pl->prog = NULL;
|
||||
break;
|
||||
}
|
||||
if (!old_prog)
|
||||
return -ENOENT;
|
||||
} else {
|
||||
/* to maintain backward compatibility NONE and OVERRIDE cgroups
|
||||
* allow detaching with invalid FD (prog==NULL)
|
||||
*/
|
||||
pl = list_first_entry(progs, typeof(*pl), node);
|
||||
old_prog = pl->prog;
|
||||
pl->prog = NULL;
|
||||
}
|
||||
|
||||
/* allocate and recompute effective prog arrays */
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
err = compute_effective_progs(desc, type, &desc->bpf.inactive);
|
||||
if (err)
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* all allocations were successful. Activate all prog arrays */
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
activate_effective_progs(desc, type, desc->bpf.inactive);
|
||||
desc->bpf.inactive = NULL;
|
||||
}
|
||||
|
||||
/* now can actually delete it from this cgroup list */
|
||||
list_del(&pl->node);
|
||||
kfree(pl);
|
||||
if (list_empty(progs))
|
||||
/* last program was detached, reset flags to zero */
|
||||
cgrp->bpf.flags[type] = 0;
|
||||
|
||||
bpf_prog_put(old_prog);
|
||||
static_branch_dec(&cgroup_bpf_enabled_key);
|
||||
return 0;
|
||||
|
||||
cleanup:
|
||||
/* oom while computing effective. Free all computed effective arrays
|
||||
* since they were not activated
|
||||
*/
|
||||
css_for_each_descendant_pre(css, &cgrp->self) {
|
||||
struct cgroup *desc = container_of(css, struct cgroup, self);
|
||||
|
||||
bpf_prog_array_free(desc->bpf.inactive);
|
||||
desc->bpf.inactive = NULL;
|
||||
}
|
||||
|
||||
/* and restore back old_prog */
|
||||
pl->prog = old_prog;
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Must be called with cgroup_mutex held to avoid races. */
|
||||
int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
|
||||
union bpf_attr __user *uattr)
|
||||
{
|
||||
__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
|
||||
enum bpf_attach_type type = attr->query.attach_type;
|
||||
struct list_head *progs = &cgrp->bpf.progs[type];
|
||||
u32 flags = cgrp->bpf.flags[type];
|
||||
int cnt, ret = 0, i;
|
||||
|
||||
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
|
||||
cnt = bpf_prog_array_length(cgrp->bpf.effective[type]);
|
||||
else
|
||||
cnt = prog_list_length(progs);
|
||||
|
||||
if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
|
||||
return -EFAULT;
|
||||
if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt)))
|
||||
return -EFAULT;
|
||||
if (attr->query.prog_cnt == 0 || !prog_ids || !cnt)
|
||||
/* return early if user requested only program count + flags */
|
||||
return 0;
|
||||
if (attr->query.prog_cnt < cnt) {
|
||||
cnt = attr->query.prog_cnt;
|
||||
ret = -ENOSPC;
|
||||
}
|
||||
|
||||
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
|
||||
return bpf_prog_array_copy_to_user(cgrp->bpf.effective[type],
|
||||
prog_ids, cnt);
|
||||
} else {
|
||||
struct bpf_prog_list *pl;
|
||||
u32 id;
|
||||
|
||||
i = 0;
|
||||
list_for_each_entry(pl, progs, node) {
|
||||
id = pl->prog->aux->id;
|
||||
if (copy_to_user(prog_ids + i, &id, sizeof(id)))
|
||||
return -EFAULT;
|
||||
if (++i == cnt)
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -171,36 +447,26 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
|
|||
struct sk_buff *skb,
|
||||
enum bpf_attach_type type)
|
||||
{
|
||||
struct bpf_prog *prog;
|
||||
unsigned int offset = skb->data - skb_network_header(skb);
|
||||
struct sock *save_sk;
|
||||
struct cgroup *cgrp;
|
||||
int ret = 0;
|
||||
int ret;
|
||||
|
||||
if (!sk || !sk_fullsock(sk))
|
||||
return 0;
|
||||
|
||||
if (sk->sk_family != AF_INET &&
|
||||
sk->sk_family != AF_INET6)
|
||||
if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
|
||||
return 0;
|
||||
|
||||
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
prog = rcu_dereference(cgrp->bpf.effective[type]);
|
||||
if (prog) {
|
||||
unsigned int offset = skb->data - skb_network_header(skb);
|
||||
struct sock *save_sk = skb->sk;
|
||||
|
||||
skb->sk = sk;
|
||||
__skb_push(skb, offset);
|
||||
ret = bpf_prog_run_save_cb(prog, skb) == 1 ? 0 : -EPERM;
|
||||
__skb_pull(skb, offset);
|
||||
skb->sk = save_sk;
|
||||
}
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
return ret;
|
||||
save_sk = skb->sk;
|
||||
skb->sk = sk;
|
||||
__skb_push(skb, offset);
|
||||
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
|
||||
bpf_prog_run_save_cb);
|
||||
__skb_pull(skb, offset);
|
||||
skb->sk = save_sk;
|
||||
return ret == 1 ? 0 : -EPERM;
|
||||
}
|
||||
EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);
|
||||
|
||||
|
|
@ -221,19 +487,10 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk,
|
|||
enum bpf_attach_type type)
|
||||
{
|
||||
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
||||
struct bpf_prog *prog;
|
||||
int ret = 0;
|
||||
int ret;
|
||||
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
prog = rcu_dereference(cgrp->bpf.effective[type]);
|
||||
if (prog)
|
||||
ret = BPF_PROG_RUN(prog, sk) == 1 ? 0 : -EPERM;
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
return ret;
|
||||
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);
|
||||
return ret == 1 ? 0 : -EPERM;
|
||||
}
|
||||
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
|
||||
|
||||
|
|
@ -258,18 +515,77 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
|
|||
enum bpf_attach_type type)
|
||||
{
|
||||
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
|
||||
struct bpf_prog *prog;
|
||||
int ret = 0;
|
||||
int ret;
|
||||
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
prog = rcu_dereference(cgrp->bpf.effective[type]);
|
||||
if (prog)
|
||||
ret = BPF_PROG_RUN(prog, sock_ops) == 1 ? 0 : -EPERM;
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
return ret;
|
||||
ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,
|
||||
BPF_PROG_RUN);
|
||||
return ret == 1 ? 0 : -EPERM;
|
||||
}
|
||||
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
|
||||
|
||||
int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
|
||||
short access, enum bpf_attach_type type)
|
||||
{
|
||||
struct cgroup *cgrp;
|
||||
struct bpf_cgroup_dev_ctx ctx = {
|
||||
.access_type = (access << 16) | dev_type,
|
||||
.major = major,
|
||||
.minor = minor,
|
||||
};
|
||||
int allow = 1;
|
||||
|
||||
rcu_read_lock();
|
||||
cgrp = task_dfl_cgroup(current);
|
||||
allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx,
|
||||
BPF_PROG_RUN);
|
||||
rcu_read_unlock();
|
||||
|
||||
return !allow;
|
||||
}
|
||||
EXPORT_SYMBOL(__cgroup_bpf_check_dev_permission);
|
||||
|
||||
static const struct bpf_func_proto *
|
||||
cgroup_dev_func_proto(enum bpf_func_id func_id)
|
||||
{
|
||||
switch (func_id) {
|
||||
case BPF_FUNC_map_lookup_elem:
|
||||
return &bpf_map_lookup_elem_proto;
|
||||
case BPF_FUNC_map_update_elem:
|
||||
return &bpf_map_update_elem_proto;
|
||||
case BPF_FUNC_map_delete_elem:
|
||||
return &bpf_map_delete_elem_proto;
|
||||
case BPF_FUNC_get_current_uid_gid:
|
||||
return &bpf_get_current_uid_gid_proto;
|
||||
case BPF_FUNC_trace_printk:
|
||||
if (capable(CAP_SYS_ADMIN))
|
||||
return bpf_get_trace_printk_proto();
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool cgroup_dev_is_valid_access(int off, int size,
|
||||
enum bpf_access_type type,
|
||||
struct bpf_insn_access_aux *info)
|
||||
{
|
||||
if (type == BPF_WRITE)
|
||||
return false;
|
||||
|
||||
if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx))
|
||||
return false;
|
||||
/* The verifier guarantees that size > 0. */
|
||||
if (off % size != 0)
|
||||
return false;
|
||||
if (size != sizeof(__u32))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const struct bpf_prog_ops cg_dev_prog_ops = {
|
||||
};
|
||||
|
||||
const struct bpf_verifier_ops cg_dev_verifier_ops = {
|
||||
.get_func_proto = cgroup_dev_func_proto,
|
||||
.is_valid_access = cgroup_dev_is_valid_access,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -309,12 +309,25 @@ bpf_get_prog_addr_region(const struct bpf_prog *prog,
|
|||
|
||||
static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
|
||||
{
|
||||
const char *end = sym + KSYM_NAME_LEN;
|
||||
|
||||
BUILD_BUG_ON(sizeof("bpf_prog_") +
|
||||
sizeof(prog->tag) * 2 + 1 > KSYM_NAME_LEN);
|
||||
sizeof(prog->tag) * 2 +
|
||||
/* name has been null terminated.
|
||||
* We should need +1 for the '_' preceding
|
||||
* the name. However, the null character
|
||||
* is double counted between the name and the
|
||||
* sizeof("bpf_prog_") above, so we omit
|
||||
* the +1 here.
|
||||
*/
|
||||
sizeof(prog->aux->name) > KSYM_NAME_LEN);
|
||||
|
||||
sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_");
|
||||
sym = bin2hex(sym, prog->tag, sizeof(prog->tag));
|
||||
*sym = 0;
|
||||
if (prog->aux->name[0])
|
||||
snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
|
||||
else
|
||||
*sym = 0;
|
||||
}
|
||||
|
||||
static __always_inline unsigned long
|
||||
|
|
@ -1367,7 +1380,13 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
|
|||
* valid program, which in this case would simply not
|
||||
* be JITed, but falls back to the interpreter.
|
||||
*/
|
||||
fp = bpf_int_jit_compile(fp);
|
||||
if (!bpf_prog_is_dev_bound(fp->aux)) {
|
||||
fp = bpf_int_jit_compile(fp);
|
||||
} else {
|
||||
*err = bpf_prog_offload_compile(fp);
|
||||
if (*err)
|
||||
return fp;
|
||||
}
|
||||
bpf_prog_lock_ro(fp);
|
||||
|
||||
/* The tail call compatibility check can only be done at
|
||||
|
|
@ -1381,11 +1400,163 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
|
||||
|
||||
static unsigned int __bpf_prog_ret1(const void *ctx,
|
||||
const struct bpf_insn *insn)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
static struct bpf_prog_dummy {
|
||||
struct bpf_prog prog;
|
||||
} dummy_bpf_prog = {
|
||||
.prog = {
|
||||
.bpf_func = __bpf_prog_ret1,
|
||||
},
|
||||
};
|
||||
|
||||
/* to avoid allocating empty bpf_prog_array for cgroups that
|
||||
* don't have bpf program attached use one global 'empty_prog_array'
|
||||
* It will not be modified the caller of bpf_prog_array_alloc()
|
||||
* (since caller requested prog_cnt == 0)
|
||||
* that pointer should be 'freed' by bpf_prog_array_free()
|
||||
*/
|
||||
static struct {
|
||||
struct bpf_prog_array hdr;
|
||||
struct bpf_prog *null_prog;
|
||||
} empty_prog_array = {
|
||||
.null_prog = NULL,
|
||||
};
|
||||
|
||||
struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
|
||||
{
|
||||
if (prog_cnt)
|
||||
return kzalloc(sizeof(struct bpf_prog_array) +
|
||||
sizeof(struct bpf_prog *) * (prog_cnt + 1),
|
||||
flags);
|
||||
|
||||
return &empty_prog_array.hdr;
|
||||
}
|
||||
|
||||
void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
|
||||
{
|
||||
if (!progs ||
|
||||
progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
|
||||
return;
|
||||
kfree_rcu(progs, rcu);
|
||||
}
|
||||
|
||||
int bpf_prog_array_length(struct bpf_prog_array __rcu *progs)
|
||||
{
|
||||
struct bpf_prog **prog;
|
||||
u32 cnt = 0;
|
||||
|
||||
rcu_read_lock();
|
||||
prog = rcu_dereference(progs)->progs;
|
||||
for (; *prog; prog++)
|
||||
cnt++;
|
||||
rcu_read_unlock();
|
||||
return cnt;
|
||||
}
|
||||
|
||||
int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
|
||||
__u32 __user *prog_ids, u32 cnt)
|
||||
{
|
||||
struct bpf_prog **prog;
|
||||
u32 i = 0, id;
|
||||
|
||||
rcu_read_lock();
|
||||
prog = rcu_dereference(progs)->progs;
|
||||
for (; *prog; prog++) {
|
||||
id = (*prog)->aux->id;
|
||||
if (copy_to_user(prog_ids + i, &id, sizeof(id))) {
|
||||
rcu_read_unlock();
|
||||
return -EFAULT;
|
||||
}
|
||||
if (++i == cnt) {
|
||||
prog++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
rcu_read_unlock();
|
||||
if (*prog)
|
||||
return -ENOSPC;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
|
||||
struct bpf_prog *old_prog)
|
||||
{
|
||||
struct bpf_prog **prog = progs->progs;
|
||||
|
||||
for (; *prog; prog++)
|
||||
if (*prog == old_prog) {
|
||||
WRITE_ONCE(*prog, &dummy_bpf_prog.prog);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
|
||||
struct bpf_prog *exclude_prog,
|
||||
struct bpf_prog *include_prog,
|
||||
struct bpf_prog_array **new_array)
|
||||
{
|
||||
int new_prog_cnt, carry_prog_cnt = 0;
|
||||
struct bpf_prog **existing_prog;
|
||||
struct bpf_prog_array *array;
|
||||
int new_prog_idx = 0;
|
||||
|
||||
/* Figure out how many existing progs we need to carry over to
|
||||
* the new array.
|
||||
*/
|
||||
if (old_array) {
|
||||
existing_prog = old_array->progs;
|
||||
for (; *existing_prog; existing_prog++) {
|
||||
if (*existing_prog != exclude_prog &&
|
||||
*existing_prog != &dummy_bpf_prog.prog)
|
||||
carry_prog_cnt++;
|
||||
if (*existing_prog == include_prog)
|
||||
return -EEXIST;
|
||||
}
|
||||
}
|
||||
|
||||
/* How many progs (not NULL) will be in the new array? */
|
||||
new_prog_cnt = carry_prog_cnt;
|
||||
if (include_prog)
|
||||
new_prog_cnt += 1;
|
||||
|
||||
/* Do we have any prog (not NULL) in the new array? */
|
||||
if (!new_prog_cnt) {
|
||||
*new_array = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* +1 as the end of prog_array is marked with NULL */
|
||||
array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL);
|
||||
if (!array)
|
||||
return -ENOMEM;
|
||||
|
||||
/* Fill in the new prog array */
|
||||
if (carry_prog_cnt) {
|
||||
existing_prog = old_array->progs;
|
||||
for (; *existing_prog; existing_prog++)
|
||||
if (*existing_prog != exclude_prog &&
|
||||
*existing_prog != &dummy_bpf_prog.prog)
|
||||
array->progs[new_prog_idx++] = *existing_prog;
|
||||
}
|
||||
if (include_prog)
|
||||
array->progs[new_prog_idx++] = include_prog;
|
||||
array->progs[new_prog_idx] = NULL;
|
||||
*new_array = array;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void bpf_prog_free_deferred(struct work_struct *work)
|
||||
{
|
||||
struct bpf_prog_aux *aux;
|
||||
|
||||
aux = container_of(work, struct bpf_prog_aux, work);
|
||||
if (bpf_prog_is_dev_bound(aux))
|
||||
bpf_prog_offload_destroy(aux->prog);
|
||||
bpf_jit_free(aux->prog);
|
||||
}
|
||||
|
||||
|
|
@ -1498,5 +1669,8 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to,
|
|||
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);
|
||||
|
||||
/* These are only used within the BPF_SYSCALL code */
|
||||
#ifdef CONFIG_BPF_SYSCALL
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(bpf_prog_get_type);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(bpf_prog_put_rcu);
|
||||
#endif
|
||||
|
|
|
|||
706
kernel/bpf/cpumap.c
Normal file
706
kernel/bpf/cpumap.c
Normal file
|
|
@ -0,0 +1,706 @@
|
|||
/* bpf/cpumap.c
|
||||
*
|
||||
* Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
|
||||
* Released under terms in GPL version 2. See COPYING.
|
||||
*/
|
||||
|
||||
/* The 'cpumap' is primarily used as a backend map for XDP BPF helper
|
||||
* call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
|
||||
*
|
||||
* Unlike devmap which redirects XDP frames out another NIC device,
|
||||
* this map type redirects raw XDP frames to another CPU. The remote
|
||||
* CPU will do SKB-allocation and call the normal network stack.
|
||||
*
|
||||
* This is a scalability and isolation mechanism, that allow
|
||||
* separating the early driver network XDP layer, from the rest of the
|
||||
* netstack, and assigning dedicated CPUs for this stage. This
|
||||
* basically allows for 10G wirespeed pre-filtering via bpf.
|
||||
*/
|
||||
#include <linux/bpf.h>
|
||||
#include <linux/filter.h>
|
||||
#include <linux/ptr_ring.h>
|
||||
|
||||
#include <linux/sched.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <linux/capability.h>
|
||||
#include <trace/events/xdp.h>
|
||||
|
||||
#include <linux/netdevice.h> /* netif_receive_skb_core */
|
||||
#include <linux/etherdevice.h> /* eth_type_trans */
|
||||
|
||||
/* General idea: XDP packets getting XDP redirected to another CPU,
|
||||
* will maximum be stored/queued for one driver ->poll() call. It is
|
||||
* guaranteed that setting flush bit and flush operation happen on
|
||||
* same CPU. Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
|
||||
* which queue in bpf_cpu_map_entry contains packets.
|
||||
*/
|
||||
|
||||
#define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */
|
||||
struct xdp_bulk_queue {
|
||||
void *q[CPU_MAP_BULK_SIZE];
|
||||
unsigned int count;
|
||||
};
|
||||
|
||||
/* Struct for every remote "destination" CPU in map */
|
||||
struct bpf_cpu_map_entry {
|
||||
u32 cpu; /* kthread CPU and map index */
|
||||
int map_id; /* Back reference to map */
|
||||
u32 qsize; /* Queue size placeholder for map lookup */
|
||||
|
||||
/* XDP can run multiple RX-ring queues, need __percpu enqueue store */
|
||||
struct xdp_bulk_queue __percpu *bulkq;
|
||||
|
||||
/* Queue with potential multi-producers, and single-consumer kthread */
|
||||
struct ptr_ring *queue;
|
||||
struct task_struct *kthread;
|
||||
struct work_struct kthread_stop_wq;
|
||||
|
||||
atomic_t refcnt; /* Control when this struct can be free'ed */
|
||||
struct rcu_head rcu;
|
||||
};
|
||||
|
||||
struct bpf_cpu_map {
|
||||
struct bpf_map map;
|
||||
/* Below members specific for map type */
|
||||
struct bpf_cpu_map_entry **cpu_map;
|
||||
unsigned long __percpu *flush_needed;
|
||||
};
|
||||
|
||||
static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
|
||||
struct xdp_bulk_queue *bq);
|
||||
|
||||
static u64 cpu_map_bitmap_size(const union bpf_attr *attr)
|
||||
{
|
||||
return BITS_TO_LONGS(attr->max_entries) * sizeof(unsigned long);
|
||||
}
|
||||
|
||||
static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
|
||||
{
|
||||
struct bpf_cpu_map *cmap;
|
||||
int err = -ENOMEM;
|
||||
u64 cost;
|
||||
int ret;
|
||||
|
||||
if (!capable(CAP_SYS_ADMIN))
|
||||
return ERR_PTR(-EPERM);
|
||||
|
||||
/* check sanity of attributes */
|
||||
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
||||
attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
cmap = kzalloc(sizeof(*cmap), GFP_USER);
|
||||
if (!cmap)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
/* mandatory map attributes */
|
||||
cmap->map.map_type = attr->map_type;
|
||||
cmap->map.key_size = attr->key_size;
|
||||
cmap->map.value_size = attr->value_size;
|
||||
cmap->map.max_entries = attr->max_entries;
|
||||
cmap->map.map_flags = attr->map_flags;
|
||||
cmap->map.numa_node = bpf_map_attr_numa_node(attr);
|
||||
|
||||
/* Pre-limit array size based on NR_CPUS, not final CPU check */
|
||||
if (cmap->map.max_entries > NR_CPUS) {
|
||||
err = -E2BIG;
|
||||
goto free_cmap;
|
||||
}
|
||||
|
||||
/* make sure page count doesn't overflow */
|
||||
cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
|
||||
cost += cpu_map_bitmap_size(attr) * num_possible_cpus();
|
||||
if (cost >= U32_MAX - PAGE_SIZE)
|
||||
goto free_cmap;
|
||||
cmap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
|
||||
|
||||
/* Notice returns -EPERM on if map size is larger than memlock limit */
|
||||
ret = bpf_map_precharge_memlock(cmap->map.pages);
|
||||
if (ret) {
|
||||
err = ret;
|
||||
goto free_cmap;
|
||||
}
|
||||
|
||||
/* A per cpu bitfield with a bit per possible CPU in map */
|
||||
cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),
|
||||
__alignof__(unsigned long));
|
||||
if (!cmap->flush_needed)
|
||||
goto free_cmap;
|
||||
|
||||
/* Alloc array for possible remote "destination" CPUs */
|
||||
cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
|
||||
sizeof(struct bpf_cpu_map_entry *),
|
||||
cmap->map.numa_node);
|
||||
if (!cmap->cpu_map)
|
||||
goto free_percpu;
|
||||
|
||||
return &cmap->map;
|
||||
free_percpu:
|
||||
free_percpu(cmap->flush_needed);
|
||||
free_cmap:
|
||||
kfree(cmap);
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
|
||||
void __cpu_map_queue_destructor(void *ptr)
|
||||
{
|
||||
/* The tear-down procedure should have made sure that queue is
|
||||
* empty. See __cpu_map_entry_replace() and work-queue
|
||||
* invoked cpu_map_kthread_stop(). Catch any broken behaviour
|
||||
* gracefully and warn once.
|
||||
*/
|
||||
if (WARN_ON_ONCE(ptr))
|
||||
page_frag_free(ptr);
|
||||
}
|
||||
|
||||
static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
|
||||
{
|
||||
if (atomic_dec_and_test(&rcpu->refcnt)) {
|
||||
/* The queue should be empty at this point */
|
||||
ptr_ring_cleanup(rcpu->queue, __cpu_map_queue_destructor);
|
||||
kfree(rcpu->queue);
|
||||
kfree(rcpu);
|
||||
}
|
||||
}
|
||||
|
||||
static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
|
||||
{
|
||||
atomic_inc(&rcpu->refcnt);
|
||||
}
|
||||
|
||||
/* called from workqueue, to workaround syscall using preempt_disable */
|
||||
static void cpu_map_kthread_stop(struct work_struct *work)
|
||||
{
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
|
||||
rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);
|
||||
|
||||
/* Wait for flush in __cpu_map_entry_free(), via full RCU barrier,
|
||||
* as it waits until all in-flight call_rcu() callbacks complete.
|
||||
*/
|
||||
rcu_barrier();
|
||||
|
||||
/* kthread_stop will wake_up_process and wait for it to complete */
|
||||
kthread_stop(rcpu->kthread);
|
||||
}
|
||||
|
||||
/* For now, xdp_pkt is a cpumap internal data structure, with info
|
||||
* carried between enqueue to dequeue. It is mapped into the top
|
||||
* headroom of the packet, to avoid allocating separate mem.
|
||||
*/
|
||||
struct xdp_pkt {
|
||||
void *data;
|
||||
u16 len;
|
||||
u16 headroom;
|
||||
u16 metasize;
|
||||
struct net_device *dev_rx;
|
||||
};
|
||||
|
||||
/* Convert xdp_buff to xdp_pkt */
|
||||
static struct xdp_pkt *convert_to_xdp_pkt(struct xdp_buff *xdp)
|
||||
{
|
||||
struct xdp_pkt *xdp_pkt;
|
||||
int metasize;
|
||||
int headroom;
|
||||
|
||||
/* Assure headroom is available for storing info */
|
||||
headroom = xdp->data - xdp->data_hard_start;
|
||||
metasize = xdp->data - xdp->data_meta;
|
||||
metasize = metasize > 0 ? metasize : 0;
|
||||
if (unlikely((headroom - metasize) < sizeof(*xdp_pkt)))
|
||||
return NULL;
|
||||
|
||||
/* Store info in top of packet */
|
||||
xdp_pkt = xdp->data_hard_start;
|
||||
|
||||
xdp_pkt->data = xdp->data;
|
||||
xdp_pkt->len = xdp->data_end - xdp->data;
|
||||
xdp_pkt->headroom = headroom - sizeof(*xdp_pkt);
|
||||
xdp_pkt->metasize = metasize;
|
||||
|
||||
return xdp_pkt;
|
||||
}
|
||||
|
||||
struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu,
|
||||
struct xdp_pkt *xdp_pkt)
|
||||
{
|
||||
unsigned int frame_size;
|
||||
void *pkt_data_start;
|
||||
struct sk_buff *skb;
|
||||
|
||||
/* build_skb need to place skb_shared_info after SKB end, and
|
||||
* also want to know the memory "truesize". Thus, need to
|
||||
* know the memory frame size backing xdp_buff.
|
||||
*
|
||||
* XDP was designed to have PAGE_SIZE frames, but this
|
||||
* assumption is not longer true with ixgbe and i40e. It
|
||||
* would be preferred to set frame_size to 2048 or 4096
|
||||
* depending on the driver.
|
||||
* frame_size = 2048;
|
||||
* frame_len = frame_size - sizeof(*xdp_pkt);
|
||||
*
|
||||
* Instead, with info avail, skb_shared_info in placed after
|
||||
* packet len. This, unfortunately fakes the truesize.
|
||||
* Another disadvantage of this approach, the skb_shared_info
|
||||
* is not at a fixed memory location, with mixed length
|
||||
* packets, which is bad for cache-line hotness.
|
||||
*/
|
||||
frame_size = SKB_DATA_ALIGN(xdp_pkt->len) + xdp_pkt->headroom +
|
||||
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
|
||||
|
||||
pkt_data_start = xdp_pkt->data - xdp_pkt->headroom;
|
||||
skb = build_skb(pkt_data_start, frame_size);
|
||||
if (!skb)
|
||||
return NULL;
|
||||
|
||||
skb_reserve(skb, xdp_pkt->headroom);
|
||||
__skb_put(skb, xdp_pkt->len);
|
||||
if (xdp_pkt->metasize)
|
||||
skb_metadata_set(skb, xdp_pkt->metasize);
|
||||
|
||||
/* Essential SKB info: protocol and skb->dev */
|
||||
skb->protocol = eth_type_trans(skb, xdp_pkt->dev_rx);
|
||||
|
||||
/* Optional SKB info, currently missing:
|
||||
* - HW checksum info (skb->ip_summed)
|
||||
* - HW RX hash (skb_set_hash)
|
||||
* - RX ring dev queue index (skb_record_rx_queue)
|
||||
*/
|
||||
|
||||
return skb;
|
||||
}
|
||||
|
||||
static int cpu_map_kthread_run(void *data)
|
||||
{
|
||||
struct bpf_cpu_map_entry *rcpu = data;
|
||||
|
||||
set_current_state(TASK_INTERRUPTIBLE);
|
||||
|
||||
/* When kthread gives stop order, then rcpu have been disconnected
|
||||
* from map, thus no new packets can enter. Remaining in-flight
|
||||
* per CPU stored packets are flushed to this queue. Wait honoring
|
||||
* kthread_stop signal until queue is empty.
|
||||
*/
|
||||
while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {
|
||||
unsigned int processed = 0, drops = 0, sched = 0;
|
||||
struct xdp_pkt *xdp_pkt;
|
||||
|
||||
/* Release CPU reschedule checks */
|
||||
if (__ptr_ring_empty(rcpu->queue)) {
|
||||
set_current_state(TASK_INTERRUPTIBLE);
|
||||
/* Recheck to avoid lost wake-up */
|
||||
if (__ptr_ring_empty(rcpu->queue)) {
|
||||
schedule();
|
||||
sched = 1;
|
||||
} else {
|
||||
__set_current_state(TASK_RUNNING);
|
||||
}
|
||||
} else {
|
||||
sched = cond_resched();
|
||||
}
|
||||
|
||||
/* Process packets in rcpu->queue */
|
||||
local_bh_disable();
|
||||
/*
|
||||
* The bpf_cpu_map_entry is single consumer, with this
|
||||
* kthread CPU pinned. Lockless access to ptr_ring
|
||||
* consume side valid as no-resize allowed of queue.
|
||||
*/
|
||||
while ((xdp_pkt = __ptr_ring_consume(rcpu->queue))) {
|
||||
struct sk_buff *skb;
|
||||
int ret;
|
||||
|
||||
skb = cpu_map_build_skb(rcpu, xdp_pkt);
|
||||
if (!skb) {
|
||||
page_frag_free(xdp_pkt);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Inject into network stack */
|
||||
ret = netif_receive_skb_core(skb);
|
||||
if (ret == NET_RX_DROP)
|
||||
drops++;
|
||||
|
||||
/* Limit BH-disable period */
|
||||
if (++processed == 8)
|
||||
break;
|
||||
}
|
||||
/* Feedback loop via tracepoint */
|
||||
trace_xdp_cpumap_kthread(rcpu->map_id, processed, drops, sched);
|
||||
|
||||
local_bh_enable(); /* resched point, may call do_softirq() */
|
||||
}
|
||||
__set_current_state(TASK_RUNNING);
|
||||
|
||||
put_cpu_map_entry(rcpu);
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, int map_id)
|
||||
{
|
||||
gfp_t gfp = GFP_ATOMIC|__GFP_NOWARN;
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
int numa, err;
|
||||
|
||||
/* Have map->numa_node, but choose node of redirect target CPU */
|
||||
numa = cpu_to_node(cpu);
|
||||
|
||||
rcpu = kzalloc_node(sizeof(*rcpu), gfp, numa);
|
||||
if (!rcpu)
|
||||
return NULL;
|
||||
|
||||
/* Alloc percpu bulkq */
|
||||
rcpu->bulkq = __alloc_percpu_gfp(sizeof(*rcpu->bulkq),
|
||||
sizeof(void *), gfp);
|
||||
if (!rcpu->bulkq)
|
||||
goto free_rcu;
|
||||
|
||||
/* Alloc queue */
|
||||
rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
|
||||
if (!rcpu->queue)
|
||||
goto free_bulkq;
|
||||
|
||||
err = ptr_ring_init(rcpu->queue, qsize, gfp);
|
||||
if (err)
|
||||
goto free_queue;
|
||||
|
||||
rcpu->cpu = cpu;
|
||||
rcpu->map_id = map_id;
|
||||
rcpu->qsize = qsize;
|
||||
|
||||
/* Setup kthread */
|
||||
rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
|
||||
"cpumap/%d/map:%d", cpu, map_id);
|
||||
if (IS_ERR(rcpu->kthread))
|
||||
goto free_ptr_ring;
|
||||
|
||||
get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
|
||||
get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */
|
||||
|
||||
/* Make sure kthread runs on a single CPU */
|
||||
kthread_bind(rcpu->kthread, cpu);
|
||||
wake_up_process(rcpu->kthread);
|
||||
|
||||
return rcpu;
|
||||
|
||||
free_ptr_ring:
|
||||
ptr_ring_cleanup(rcpu->queue, NULL);
|
||||
free_queue:
|
||||
kfree(rcpu->queue);
|
||||
free_bulkq:
|
||||
free_percpu(rcpu->bulkq);
|
||||
free_rcu:
|
||||
kfree(rcpu);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void __cpu_map_entry_free(struct rcu_head *rcu)
|
||||
{
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
int cpu;
|
||||
|
||||
/* This cpu_map_entry have been disconnected from map and one
|
||||
* RCU graze-period have elapsed. Thus, XDP cannot queue any
|
||||
* new packets and cannot change/set flush_needed that can
|
||||
* find this entry.
|
||||
*/
|
||||
rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);
|
||||
|
||||
/* Flush remaining packets in percpu bulkq */
|
||||
for_each_online_cpu(cpu) {
|
||||
struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu);
|
||||
|
||||
/* No concurrent bq_enqueue can run at this point */
|
||||
bq_flush_to_queue(rcpu, bq);
|
||||
}
|
||||
free_percpu(rcpu->bulkq);
|
||||
/* Cannot kthread_stop() here, last put free rcpu resources */
|
||||
put_cpu_map_entry(rcpu);
|
||||
}
|
||||
|
||||
/* After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
|
||||
* ensure any driver rcu critical sections have completed, but this
|
||||
* does not guarantee a flush has happened yet. Because driver side
|
||||
* rcu_read_lock/unlock only protects the running XDP program. The
|
||||
* atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
|
||||
* pending flush op doesn't fail.
|
||||
*
|
||||
* The bpf_cpu_map_entry is still used by the kthread, and there can
|
||||
* still be pending packets (in queue and percpu bulkq). A refcnt
|
||||
* makes sure to last user (kthread_stop vs. call_rcu) free memory
|
||||
* resources.
|
||||
*
|
||||
* The rcu callback __cpu_map_entry_free flush remaining packets in
|
||||
* percpu bulkq to queue. Due to caller map_delete_elem() disable
|
||||
* preemption, cannot call kthread_stop() to make sure queue is empty.
|
||||
* Instead a work_queue is started for stopping kthread,
|
||||
* cpu_map_kthread_stop, which waits for an RCU graze period before
|
||||
* stopping kthread, emptying the queue.
|
||||
*/
|
||||
void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
|
||||
u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
|
||||
{
|
||||
struct bpf_cpu_map_entry *old_rcpu;
|
||||
|
||||
old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
|
||||
if (old_rcpu) {
|
||||
call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
|
||||
INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
|
||||
schedule_work(&old_rcpu->kthread_stop_wq);
|
||||
}
|
||||
}
|
||||
|
||||
int cpu_map_delete_elem(struct bpf_map *map, void *key)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
u32 key_cpu = *(u32 *)key;
|
||||
|
||||
if (key_cpu >= map->max_entries)
|
||||
return -EINVAL;
|
||||
|
||||
/* notice caller map_delete_elem() use preempt_disable() */
|
||||
__cpu_map_entry_replace(cmap, key_cpu, NULL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
|
||||
u64 map_flags)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
|
||||
/* Array index key correspond to CPU number */
|
||||
u32 key_cpu = *(u32 *)key;
|
||||
/* Value is the queue size */
|
||||
u32 qsize = *(u32 *)value;
|
||||
|
||||
if (unlikely(map_flags > BPF_EXIST))
|
||||
return -EINVAL;
|
||||
if (unlikely(key_cpu >= cmap->map.max_entries))
|
||||
return -E2BIG;
|
||||
if (unlikely(map_flags == BPF_NOEXIST))
|
||||
return -EEXIST;
|
||||
if (unlikely(qsize > 16384)) /* sanity limit on qsize */
|
||||
return -EOVERFLOW;
|
||||
|
||||
/* Make sure CPU is a valid possible cpu */
|
||||
if (!cpu_possible(key_cpu))
|
||||
return -ENODEV;
|
||||
|
||||
if (qsize == 0) {
|
||||
rcpu = NULL; /* Same as deleting */
|
||||
} else {
|
||||
/* Updating qsize cause re-allocation of bpf_cpu_map_entry */
|
||||
rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
|
||||
if (!rcpu)
|
||||
return -ENOMEM;
|
||||
}
|
||||
rcu_read_lock();
|
||||
__cpu_map_entry_replace(cmap, key_cpu, rcpu);
|
||||
rcu_read_unlock();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cpu_map_free(struct bpf_map *map)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
int cpu;
|
||||
u32 i;
|
||||
|
||||
/* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
|
||||
* so the bpf programs (can be more than one that used this map) were
|
||||
* disconnected from events. Wait for outstanding critical sections in
|
||||
* these programs to complete. The rcu critical section only guarantees
|
||||
* no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
|
||||
* It does __not__ ensure pending flush operations (if any) are
|
||||
* complete.
|
||||
*/
|
||||
synchronize_rcu();
|
||||
|
||||
/* To ensure all pending flush operations have completed wait for flush
|
||||
* bitmap to indicate all flush_needed bits to be zero on _all_ cpus.
|
||||
* Because the above synchronize_rcu() ensures the map is disconnected
|
||||
* from the program we can assume no new bits will be set.
|
||||
*/
|
||||
for_each_online_cpu(cpu) {
|
||||
unsigned long *bitmap = per_cpu_ptr(cmap->flush_needed, cpu);
|
||||
|
||||
while (!bitmap_empty(bitmap, cmap->map.max_entries))
|
||||
cond_resched();
|
||||
}
|
||||
|
||||
/* For cpu_map the remote CPUs can still be using the entries
|
||||
* (struct bpf_cpu_map_entry).
|
||||
*/
|
||||
for (i = 0; i < cmap->map.max_entries; i++) {
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
|
||||
rcpu = READ_ONCE(cmap->cpu_map[i]);
|
||||
if (!rcpu)
|
||||
continue;
|
||||
|
||||
/* bq flush and cleanup happens after RCU graze-period */
|
||||
__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
|
||||
}
|
||||
free_percpu(cmap->flush_needed);
|
||||
bpf_map_area_free(cmap->cpu_map);
|
||||
kfree(cmap);
|
||||
}
|
||||
|
||||
struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
struct bpf_cpu_map_entry *rcpu;
|
||||
|
||||
if (key >= map->max_entries)
|
||||
return NULL;
|
||||
|
||||
rcpu = READ_ONCE(cmap->cpu_map[key]);
|
||||
return rcpu;
|
||||
}
|
||||
|
||||
static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
|
||||
{
|
||||
struct bpf_cpu_map_entry *rcpu =
|
||||
__cpu_map_lookup_elem(map, *(u32 *)key);
|
||||
|
||||
return rcpu ? &rcpu->qsize : NULL;
|
||||
}
|
||||
|
||||
static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
u32 index = key ? *(u32 *)key : U32_MAX;
|
||||
u32 *next = next_key;
|
||||
|
||||
if (index >= cmap->map.max_entries) {
|
||||
*next = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (index == cmap->map.max_entries - 1)
|
||||
return -ENOENT;
|
||||
*next = index + 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
const struct bpf_map_ops cpu_map_ops = {
|
||||
.map_alloc = cpu_map_alloc,
|
||||
.map_free = cpu_map_free,
|
||||
.map_delete_elem = cpu_map_delete_elem,
|
||||
.map_update_elem = cpu_map_update_elem,
|
||||
.map_lookup_elem = cpu_map_lookup_elem,
|
||||
.map_get_next_key = cpu_map_get_next_key,
|
||||
};
|
||||
|
||||
static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
|
||||
struct xdp_bulk_queue *bq)
|
||||
{
|
||||
unsigned int processed = 0, drops = 0;
|
||||
const int to_cpu = rcpu->cpu;
|
||||
struct ptr_ring *q;
|
||||
int i;
|
||||
|
||||
if (unlikely(!bq->count))
|
||||
return 0;
|
||||
|
||||
q = rcpu->queue;
|
||||
spin_lock(&q->producer_lock);
|
||||
|
||||
for (i = 0; i < bq->count; i++) {
|
||||
void *xdp_pkt = bq->q[i];
|
||||
int err;
|
||||
|
||||
err = __ptr_ring_produce(q, xdp_pkt);
|
||||
if (err) {
|
||||
drops++;
|
||||
page_frag_free(xdp_pkt); /* Free xdp_pkt */
|
||||
}
|
||||
processed++;
|
||||
}
|
||||
bq->count = 0;
|
||||
spin_unlock(&q->producer_lock);
|
||||
|
||||
/* Feedback loop via tracepoints */
|
||||
trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Runs under RCU-read-side, plus in softirq under NAPI protection.
|
||||
* Thus, safe percpu variable access.
|
||||
*/
|
||||
static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
|
||||
{
|
||||
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
|
||||
|
||||
if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
|
||||
bq_flush_to_queue(rcpu, bq);
|
||||
|
||||
/* Notice, xdp_buff/page MUST be queued here, long enough for
|
||||
* driver to code invoking us to finished, due to driver
|
||||
* (e.g. ixgbe) recycle tricks based on page-refcnt.
|
||||
*
|
||||
* Thus, incoming xdp_pkt is always queued here (else we race
|
||||
* with another CPU on page-refcnt and remaining driver code).
|
||||
* Queue time is very short, as driver will invoke flush
|
||||
* operation, when completing napi->poll call.
|
||||
*/
|
||||
bq->q[bq->count++] = xdp_pkt;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
|
||||
struct net_device *dev_rx)
|
||||
{
|
||||
struct xdp_pkt *xdp_pkt;
|
||||
|
||||
xdp_pkt = convert_to_xdp_pkt(xdp);
|
||||
if (unlikely(!xdp_pkt))
|
||||
return -EOVERFLOW;
|
||||
|
||||
/* Info needed when constructing SKB on remote CPU */
|
||||
xdp_pkt->dev_rx = dev_rx;
|
||||
|
||||
bq_enqueue(rcpu, xdp_pkt);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void __cpu_map_insert_ctx(struct bpf_map *map, u32 bit)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
|
||||
|
||||
__set_bit(bit, bitmap);
|
||||
}
|
||||
|
||||
void __cpu_map_flush(struct bpf_map *map)
|
||||
{
|
||||
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
||||
unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
|
||||
u32 bit;
|
||||
|
||||
/* The napi->poll softirq makes sure __cpu_map_insert_ctx()
|
||||
* and __cpu_map_flush() happen on same CPU. Thus, the percpu
|
||||
* bitmap indicate which percpu bulkq have packets.
|
||||
*/
|
||||
for_each_set_bit(bit, bitmap, map->max_entries) {
|
||||
struct bpf_cpu_map_entry *rcpu = READ_ONCE(cmap->cpu_map[bit]);
|
||||
struct xdp_bulk_queue *bq;
|
||||
|
||||
/* This is possible if entry is removed by user space
|
||||
* between xdp redirect and flush op.
|
||||
*/
|
||||
if (unlikely(!rcpu))
|
||||
continue;
|
||||
|
||||
__clear_bit(bit, bitmap);
|
||||
|
||||
/* Flush all frames in bulkq to real queue */
|
||||
bq = this_cpu_ptr(rcpu->bulkq);
|
||||
bq_flush_to_queue(rcpu, bq);
|
||||
|
||||
/* If already running, costs spin_lock_irqsave + smb_mb */
|
||||
wake_up_process(rcpu->kthread);
|
||||
}
|
||||
}
|
||||
|
|
@ -50,6 +50,9 @@
|
|||
#include <linux/bpf.h>
|
||||
#include <linux/filter.h>
|
||||
|
||||
#define DEV_CREATE_FLAG_MASK \
|
||||
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
struct bpf_dtab_netdev {
|
||||
struct net_device *dev;
|
||||
struct bpf_dtab *dtab;
|
||||
|
|
@ -83,7 +86,7 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
|
|||
|
||||
/* check sanity of attributes */
|
||||
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
||||
attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
|
||||
attr->value_size != 4 || attr->map_flags & ~DEV_CREATE_FLAG_MASK)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
dtab = kzalloc(sizeof(*dtab), GFP_USER);
|
||||
|
|
|
|||
214
kernel/bpf/disasm.c
Normal file
214
kernel/bpf/disasm.c
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
|
||||
* Copyright (c) 2016 Facebook
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of version 2 of the GNU General Public
|
||||
* License as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*/
|
||||
|
||||
#include <linux/bpf.h>
|
||||
|
||||
#include "disasm.h"
|
||||
|
||||
#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x)
|
||||
static const char * const func_id_str[] = {
|
||||
__BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN)
|
||||
};
|
||||
#undef __BPF_FUNC_STR_FN
|
||||
|
||||
const char *func_id_name(int id)
|
||||
{
|
||||
BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
|
||||
|
||||
if (id >= 0 && id < __BPF_FUNC_MAX_ID && func_id_str[id])
|
||||
return func_id_str[id];
|
||||
else
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
const char *const bpf_class_string[8] = {
|
||||
[BPF_LD] = "ld",
|
||||
[BPF_LDX] = "ldx",
|
||||
[BPF_ST] = "st",
|
||||
[BPF_STX] = "stx",
|
||||
[BPF_ALU] = "alu",
|
||||
[BPF_JMP] = "jmp",
|
||||
[BPF_RET] = "BUG",
|
||||
[BPF_ALU64] = "alu64",
|
||||
};
|
||||
|
||||
const char *const bpf_alu_string[16] = {
|
||||
[BPF_ADD >> 4] = "+=",
|
||||
[BPF_SUB >> 4] = "-=",
|
||||
[BPF_MUL >> 4] = "*=",
|
||||
[BPF_DIV >> 4] = "/=",
|
||||
[BPF_OR >> 4] = "|=",
|
||||
[BPF_AND >> 4] = "&=",
|
||||
[BPF_LSH >> 4] = "<<=",
|
||||
[BPF_RSH >> 4] = ">>=",
|
||||
[BPF_NEG >> 4] = "neg",
|
||||
[BPF_MOD >> 4] = "%=",
|
||||
[BPF_XOR >> 4] = "^=",
|
||||
[BPF_MOV >> 4] = "=",
|
||||
[BPF_ARSH >> 4] = "s>>=",
|
||||
[BPF_END >> 4] = "endian",
|
||||
};
|
||||
|
||||
static const char *const bpf_ldst_string[] = {
|
||||
[BPF_W >> 3] = "u32",
|
||||
[BPF_H >> 3] = "u16",
|
||||
[BPF_B >> 3] = "u8",
|
||||
[BPF_DW >> 3] = "u64",
|
||||
};
|
||||
|
||||
static const char *const bpf_jmp_string[16] = {
|
||||
[BPF_JA >> 4] = "jmp",
|
||||
[BPF_JEQ >> 4] = "==",
|
||||
[BPF_JGT >> 4] = ">",
|
||||
[BPF_JLT >> 4] = "<",
|
||||
[BPF_JGE >> 4] = ">=",
|
||||
[BPF_JLE >> 4] = "<=",
|
||||
[BPF_JSET >> 4] = "&",
|
||||
[BPF_JNE >> 4] = "!=",
|
||||
[BPF_JSGT >> 4] = "s>",
|
||||
[BPF_JSLT >> 4] = "s<",
|
||||
[BPF_JSGE >> 4] = "s>=",
|
||||
[BPF_JSLE >> 4] = "s<=",
|
||||
[BPF_CALL >> 4] = "call",
|
||||
[BPF_EXIT >> 4] = "exit",
|
||||
};
|
||||
|
||||
static void print_bpf_end_insn(bpf_insn_print_cb verbose,
|
||||
struct bpf_verifier_env *env,
|
||||
const struct bpf_insn *insn)
|
||||
{
|
||||
verbose(env, "(%02x) r%d = %s%d r%d\n", insn->code, insn->dst_reg,
|
||||
BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le",
|
||||
insn->imm, insn->dst_reg);
|
||||
}
|
||||
|
||||
void print_bpf_insn(bpf_insn_print_cb verbose, struct bpf_verifier_env *env,
|
||||
const struct bpf_insn *insn, bool allow_ptr_leaks)
|
||||
{
|
||||
u8 class = BPF_CLASS(insn->code);
|
||||
|
||||
if (class == BPF_ALU || class == BPF_ALU64) {
|
||||
if (BPF_OP(insn->code) == BPF_END) {
|
||||
if (class == BPF_ALU64)
|
||||
verbose(env, "BUG_alu64_%02x\n", insn->code);
|
||||
else
|
||||
print_bpf_end_insn(verbose, env, insn);
|
||||
} else if (BPF_OP(insn->code) == BPF_NEG) {
|
||||
verbose(env, "(%02x) r%d = %s-r%d\n",
|
||||
insn->code, insn->dst_reg,
|
||||
class == BPF_ALU ? "(u32) " : "",
|
||||
insn->dst_reg);
|
||||
} else if (BPF_SRC(insn->code) == BPF_X) {
|
||||
verbose(env, "(%02x) %sr%d %s %sr%d\n",
|
||||
insn->code, class == BPF_ALU ? "(u32) " : "",
|
||||
insn->dst_reg,
|
||||
bpf_alu_string[BPF_OP(insn->code) >> 4],
|
||||
class == BPF_ALU ? "(u32) " : "",
|
||||
insn->src_reg);
|
||||
} else {
|
||||
verbose(env, "(%02x) %sr%d %s %s%d\n",
|
||||
insn->code, class == BPF_ALU ? "(u32) " : "",
|
||||
insn->dst_reg,
|
||||
bpf_alu_string[BPF_OP(insn->code) >> 4],
|
||||
class == BPF_ALU ? "(u32) " : "",
|
||||
insn->imm);
|
||||
}
|
||||
} else if (class == BPF_STX) {
|
||||
if (BPF_MODE(insn->code) == BPF_MEM)
|
||||
verbose(env, "(%02x) *(%s *)(r%d %+d) = r%d\n",
|
||||
insn->code,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->dst_reg,
|
||||
insn->off, insn->src_reg);
|
||||
else if (BPF_MODE(insn->code) == BPF_XADD)
|
||||
verbose(env, "(%02x) lock *(%s *)(r%d %+d) += r%d\n",
|
||||
insn->code,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->dst_reg, insn->off,
|
||||
insn->src_reg);
|
||||
else
|
||||
verbose(env, "BUG_%02x\n", insn->code);
|
||||
} else if (class == BPF_ST) {
|
||||
if (BPF_MODE(insn->code) != BPF_MEM) {
|
||||
verbose(env, "BUG_st_%02x\n", insn->code);
|
||||
return;
|
||||
}
|
||||
verbose(env, "(%02x) *(%s *)(r%d %+d) = %d\n",
|
||||
insn->code,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->dst_reg,
|
||||
insn->off, insn->imm);
|
||||
} else if (class == BPF_LDX) {
|
||||
if (BPF_MODE(insn->code) != BPF_MEM) {
|
||||
verbose(env, "BUG_ldx_%02x\n", insn->code);
|
||||
return;
|
||||
}
|
||||
verbose(env, "(%02x) r%d = *(%s *)(r%d %+d)\n",
|
||||
insn->code, insn->dst_reg,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->src_reg, insn->off);
|
||||
} else if (class == BPF_LD) {
|
||||
if (BPF_MODE(insn->code) == BPF_ABS) {
|
||||
verbose(env, "(%02x) r0 = *(%s *)skb[%d]\n",
|
||||
insn->code,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->imm);
|
||||
} else if (BPF_MODE(insn->code) == BPF_IND) {
|
||||
verbose(env, "(%02x) r0 = *(%s *)skb[r%d + %d]\n",
|
||||
insn->code,
|
||||
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
|
||||
insn->src_reg, insn->imm);
|
||||
} else if (BPF_MODE(insn->code) == BPF_IMM &&
|
||||
BPF_SIZE(insn->code) == BPF_DW) {
|
||||
/* At this point, we already made sure that the second
|
||||
* part of the ldimm64 insn is accessible.
|
||||
*/
|
||||
u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
|
||||
bool map_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD;
|
||||
|
||||
if (map_ptr && !allow_ptr_leaks)
|
||||
imm = 0;
|
||||
|
||||
verbose(env, "(%02x) r%d = 0x%llx\n", insn->code,
|
||||
insn->dst_reg, (unsigned long long)imm);
|
||||
} else {
|
||||
verbose(env, "BUG_ld_%02x\n", insn->code);
|
||||
return;
|
||||
}
|
||||
} else if (class == BPF_JMP) {
|
||||
u8 opcode = BPF_OP(insn->code);
|
||||
|
||||
if (opcode == BPF_CALL) {
|
||||
verbose(env, "(%02x) call %s#%d\n", insn->code,
|
||||
func_id_name(insn->imm), insn->imm);
|
||||
} else if (insn->code == (BPF_JMP | BPF_JA)) {
|
||||
verbose(env, "(%02x) goto pc%+d\n",
|
||||
insn->code, insn->off);
|
||||
} else if (insn->code == (BPF_JMP | BPF_EXIT)) {
|
||||
verbose(env, "(%02x) exit\n", insn->code);
|
||||
} else if (BPF_SRC(insn->code) == BPF_X) {
|
||||
verbose(env, "(%02x) if r%d %s r%d goto pc%+d\n",
|
||||
insn->code, insn->dst_reg,
|
||||
bpf_jmp_string[BPF_OP(insn->code) >> 4],
|
||||
insn->src_reg, insn->off);
|
||||
} else {
|
||||
verbose(env, "(%02x) if r%d %s 0x%x goto pc%+d\n",
|
||||
insn->code, insn->dst_reg,
|
||||
bpf_jmp_string[BPF_OP(insn->code) >> 4],
|
||||
insn->imm, insn->off);
|
||||
}
|
||||
} else {
|
||||
verbose(env, "(%02x) %s\n",
|
||||
insn->code, bpf_class_string[class]);
|
||||
}
|
||||
}
|
||||
32
kernel/bpf/disasm.h
Normal file
32
kernel/bpf/disasm.h
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
|
||||
* Copyright (c) 2016 Facebook
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of version 2 of the GNU General Public
|
||||
* License as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*/
|
||||
|
||||
#ifndef __BPF_DISASM_H__
|
||||
#define __BPF_DISASM_H__
|
||||
|
||||
#include <linux/bpf.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/stringify.h>
|
||||
|
||||
extern const char *const bpf_alu_string[16];
|
||||
extern const char *const bpf_class_string[8];
|
||||
|
||||
const char *func_id_name(int id);
|
||||
|
||||
struct bpf_verifier_env;
|
||||
typedef void (*bpf_insn_print_cb)(struct bpf_verifier_env *env,
|
||||
const char *, ...);
|
||||
void print_bpf_insn(bpf_insn_print_cb verbose, struct bpf_verifier_env *env,
|
||||
const struct bpf_insn *insn, bool allow_ptr_leaks);
|
||||
|
||||
#endif
|
||||
|
|
@ -18,8 +18,9 @@
|
|||
#include "bpf_lru_list.h"
|
||||
#include "map_in_map.h"
|
||||
|
||||
#define HTAB_CREATE_FLAG_MASK \
|
||||
(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE)
|
||||
#define HTAB_CREATE_FLAG_MASK \
|
||||
(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \
|
||||
BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
struct bucket {
|
||||
struct hlist_nulls_head head;
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ out:
|
|||
}
|
||||
|
||||
static void *bpf_obj_do_get(const struct filename *pathname,
|
||||
enum bpf_type *type)
|
||||
enum bpf_type *type, int flags)
|
||||
{
|
||||
struct inode *inode;
|
||||
struct path path;
|
||||
|
|
@ -307,7 +307,7 @@ static void *bpf_obj_do_get(const struct filename *pathname,
|
|||
return ERR_PTR(ret);
|
||||
|
||||
inode = d_backing_inode(path.dentry);
|
||||
ret = inode_permission(inode, MAY_WRITE);
|
||||
ret = inode_permission(inode, ACC_MODE(flags));
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
|
|
@ -326,18 +326,23 @@ out:
|
|||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
int bpf_obj_get_user(const char __user *pathname)
|
||||
int bpf_obj_get_user(const char __user *pathname, int flags)
|
||||
{
|
||||
enum bpf_type type = BPF_TYPE_UNSPEC;
|
||||
struct filename *pname;
|
||||
int ret = -ENOENT;
|
||||
int f_flags;
|
||||
void *raw;
|
||||
|
||||
f_flags = bpf_get_file_flag(flags);
|
||||
if (f_flags < 0)
|
||||
return f_flags;
|
||||
|
||||
pname = getname(pathname);
|
||||
if (IS_ERR(pname))
|
||||
return PTR_ERR(pname);
|
||||
|
||||
raw = bpf_obj_do_get(pname, &type);
|
||||
raw = bpf_obj_do_get(pname, &type, f_flags);
|
||||
if (IS_ERR(raw)) {
|
||||
ret = PTR_ERR(raw);
|
||||
goto out;
|
||||
|
|
@ -346,7 +351,7 @@ int bpf_obj_get_user(const char __user *pathname)
|
|||
if (type == BPF_TYPE_PROG)
|
||||
ret = bpf_prog_new_fd(raw);
|
||||
else if (type == BPF_TYPE_MAP)
|
||||
ret = bpf_map_new_fd(raw);
|
||||
ret = bpf_map_new_fd(raw, f_flags);
|
||||
else
|
||||
goto out;
|
||||
|
||||
|
|
|
|||
|
|
@ -389,10 +389,99 @@ out:
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int trie_delete_elem(struct bpf_map *map, void *key)
|
||||
/* Called from syscall or from eBPF program */
|
||||
static int trie_delete_elem(struct bpf_map *map, void *_key)
|
||||
{
|
||||
/* TODO */
|
||||
return -ENOSYS;
|
||||
struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
|
||||
struct bpf_lpm_trie_key *key = _key;
|
||||
struct lpm_trie_node __rcu **trim, **trim2;
|
||||
struct lpm_trie_node *node, *parent;
|
||||
unsigned long irq_flags;
|
||||
unsigned int next_bit;
|
||||
size_t matchlen = 0;
|
||||
int ret = 0;
|
||||
|
||||
if (key->prefixlen > trie->max_prefixlen)
|
||||
return -EINVAL;
|
||||
|
||||
raw_spin_lock_irqsave(&trie->lock, irq_flags);
|
||||
|
||||
/* Walk the tree looking for an exact key/length match and keeping
|
||||
* track of the path we traverse. We will need to know the node
|
||||
* we wish to delete, and the slot that points to the node we want
|
||||
* to delete. We may also need to know the nodes parent and the
|
||||
* slot that contains it.
|
||||
*/
|
||||
trim = &trie->root;
|
||||
trim2 = trim;
|
||||
parent = NULL;
|
||||
while ((node = rcu_dereference_protected(
|
||||
*trim, lockdep_is_held(&trie->lock)))) {
|
||||
matchlen = longest_prefix_match(trie, node, key);
|
||||
|
||||
if (node->prefixlen != matchlen ||
|
||||
node->prefixlen == key->prefixlen)
|
||||
break;
|
||||
|
||||
parent = node;
|
||||
trim2 = trim;
|
||||
next_bit = extract_bit(key->data, node->prefixlen);
|
||||
trim = &node->child[next_bit];
|
||||
}
|
||||
|
||||
if (!node || node->prefixlen != key->prefixlen ||
|
||||
(node->flags & LPM_TREE_NODE_FLAG_IM)) {
|
||||
ret = -ENOENT;
|
||||
goto out;
|
||||
}
|
||||
|
||||
trie->n_entries--;
|
||||
|
||||
/* If the node we are removing has two children, simply mark it
|
||||
* as intermediate and we are done.
|
||||
*/
|
||||
if (rcu_access_pointer(node->child[0]) &&
|
||||
rcu_access_pointer(node->child[1])) {
|
||||
node->flags |= LPM_TREE_NODE_FLAG_IM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* If the parent of the node we are about to delete is an intermediate
|
||||
* node, and the deleted node doesn't have any children, we can delete
|
||||
* the intermediate parent as well and promote its other child
|
||||
* up the tree. Doing this maintains the invariant that all
|
||||
* intermediate nodes have exactly 2 children and that there are no
|
||||
* unnecessary intermediate nodes in the tree.
|
||||
*/
|
||||
if (parent && (parent->flags & LPM_TREE_NODE_FLAG_IM) &&
|
||||
!node->child[0] && !node->child[1]) {
|
||||
if (node == rcu_access_pointer(parent->child[0]))
|
||||
rcu_assign_pointer(
|
||||
*trim2, rcu_access_pointer(parent->child[1]));
|
||||
else
|
||||
rcu_assign_pointer(
|
||||
*trim2, rcu_access_pointer(parent->child[0]));
|
||||
kfree_rcu(parent, rcu);
|
||||
kfree_rcu(node, rcu);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* The node we are removing has either zero or one child. If there
|
||||
* is a child, move it into the removed node's slot then delete
|
||||
* the node. Otherwise just clear the slot and delete the node.
|
||||
*/
|
||||
if (node->child[0])
|
||||
rcu_assign_pointer(*trim, rcu_access_pointer(node->child[0]));
|
||||
else if (node->child[1])
|
||||
rcu_assign_pointer(*trim, rcu_access_pointer(node->child[1]));
|
||||
else
|
||||
RCU_INIT_POINTER(*trim, NULL);
|
||||
kfree_rcu(node, rcu);
|
||||
|
||||
out:
|
||||
raw_spin_unlock_irqrestore(&trie->lock, irq_flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define LPM_DATA_SIZE_MAX 256
|
||||
|
|
@ -406,7 +495,8 @@ static int trie_delete_elem(struct bpf_map *map, void *key)
|
|||
#define LPM_KEY_SIZE_MAX LPM_KEY_SIZE(LPM_DATA_SIZE_MAX)
|
||||
#define LPM_KEY_SIZE_MIN LPM_KEY_SIZE(LPM_DATA_SIZE_MIN)
|
||||
|
||||
#define LPM_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_NUMA_NODE)
|
||||
#define LPM_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_NUMA_NODE | \
|
||||
BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
static struct bpf_map *trie_alloc(union bpf_attr *attr)
|
||||
{
|
||||
|
|
|
|||
194
kernel/bpf/offload.c
Normal file
194
kernel/bpf/offload.c
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
#include <linux/bpf.h>
|
||||
#include <linux/bpf_verifier.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/netdevice.h>
|
||||
#include <linux/printk.h>
|
||||
#include <linux/rtnetlink.h>
|
||||
|
||||
/* protected by RTNL */
|
||||
static LIST_HEAD(bpf_prog_offload_devs);
|
||||
|
||||
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
|
||||
{
|
||||
struct net *net = current->nsproxy->net_ns;
|
||||
struct bpf_dev_offload *offload;
|
||||
|
||||
if (!capable(CAP_SYS_ADMIN))
|
||||
return -EPERM;
|
||||
|
||||
if (attr->prog_flags)
|
||||
return -EINVAL;
|
||||
|
||||
offload = kzalloc(sizeof(*offload), GFP_USER);
|
||||
if (!offload)
|
||||
return -ENOMEM;
|
||||
|
||||
offload->prog = prog;
|
||||
init_waitqueue_head(&offload->verifier_done);
|
||||
|
||||
rtnl_lock();
|
||||
offload->netdev = __dev_get_by_index(net, attr->prog_target_ifindex);
|
||||
if (!offload->netdev) {
|
||||
rtnl_unlock();
|
||||
kfree(offload);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
prog->aux->offload = offload;
|
||||
list_add_tail(&offload->offloads, &bpf_prog_offload_devs);
|
||||
rtnl_unlock();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __bpf_offload_ndo(struct bpf_prog *prog, enum bpf_netdev_command cmd,
|
||||
struct netdev_bpf *data)
|
||||
{
|
||||
struct net_device *netdev = prog->aux->offload->netdev;
|
||||
|
||||
ASSERT_RTNL();
|
||||
|
||||
if (!netdev)
|
||||
return -ENODEV;
|
||||
if (!netdev->netdev_ops->ndo_bpf)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
data->command = cmd;
|
||||
|
||||
return netdev->netdev_ops->ndo_bpf(netdev, data);
|
||||
}
|
||||
|
||||
int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env)
|
||||
{
|
||||
struct netdev_bpf data = {};
|
||||
int err;
|
||||
|
||||
data.verifier.prog = env->prog;
|
||||
|
||||
rtnl_lock();
|
||||
err = __bpf_offload_ndo(env->prog, BPF_OFFLOAD_VERIFIER_PREP, &data);
|
||||
if (err)
|
||||
goto exit_unlock;
|
||||
|
||||
env->dev_ops = data.verifier.ops;
|
||||
|
||||
env->prog->aux->offload->dev_state = true;
|
||||
env->prog->aux->offload->verifier_running = true;
|
||||
exit_unlock:
|
||||
rtnl_unlock();
|
||||
return err;
|
||||
}
|
||||
|
||||
static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_dev_offload *offload = prog->aux->offload;
|
||||
struct netdev_bpf data = {};
|
||||
|
||||
data.offload.prog = prog;
|
||||
|
||||
if (offload->verifier_running)
|
||||
wait_event(offload->verifier_done, !offload->verifier_running);
|
||||
|
||||
if (offload->dev_state)
|
||||
WARN_ON(__bpf_offload_ndo(prog, BPF_OFFLOAD_DESTROY, &data));
|
||||
|
||||
offload->dev_state = false;
|
||||
list_del_init(&offload->offloads);
|
||||
offload->netdev = NULL;
|
||||
}
|
||||
|
||||
void bpf_prog_offload_destroy(struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_dev_offload *offload = prog->aux->offload;
|
||||
|
||||
offload->verifier_running = false;
|
||||
wake_up(&offload->verifier_done);
|
||||
|
||||
rtnl_lock();
|
||||
__bpf_prog_offload_destroy(prog);
|
||||
rtnl_unlock();
|
||||
|
||||
kfree(offload);
|
||||
}
|
||||
|
||||
static int bpf_prog_offload_translate(struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_dev_offload *offload = prog->aux->offload;
|
||||
struct netdev_bpf data = {};
|
||||
int ret;
|
||||
|
||||
data.offload.prog = prog;
|
||||
|
||||
offload->verifier_running = false;
|
||||
wake_up(&offload->verifier_done);
|
||||
|
||||
rtnl_lock();
|
||||
ret = __bpf_offload_ndo(prog, BPF_OFFLOAD_TRANSLATE, &data);
|
||||
rtnl_unlock();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static unsigned int bpf_prog_warn_on_exec(const void *ctx,
|
||||
const struct bpf_insn *insn)
|
||||
{
|
||||
WARN(1, "attempt to execute device eBPF program on the host!");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bpf_prog_offload_compile(struct bpf_prog *prog)
|
||||
{
|
||||
prog->bpf_func = bpf_prog_warn_on_exec;
|
||||
|
||||
return bpf_prog_offload_translate(prog);
|
||||
}
|
||||
|
||||
u32 bpf_prog_offload_ifindex(struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_dev_offload *offload = prog->aux->offload;
|
||||
u32 ifindex;
|
||||
|
||||
rtnl_lock();
|
||||
ifindex = offload->netdev ? offload->netdev->ifindex : 0;
|
||||
rtnl_unlock();
|
||||
|
||||
return ifindex;
|
||||
}
|
||||
|
||||
const struct bpf_prog_ops bpf_offload_prog_ops = {
|
||||
};
|
||||
|
||||
static int bpf_offload_notification(struct notifier_block *notifier,
|
||||
ulong event, void *ptr)
|
||||
{
|
||||
struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
|
||||
struct bpf_dev_offload *offload, *tmp;
|
||||
|
||||
ASSERT_RTNL();
|
||||
|
||||
switch (event) {
|
||||
case NETDEV_UNREGISTER:
|
||||
list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs,
|
||||
offloads) {
|
||||
if (offload->netdev == netdev)
|
||||
__bpf_prog_offload_destroy(offload->prog);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return NOTIFY_OK;
|
||||
}
|
||||
|
||||
static struct notifier_block bpf_offload_notifier = {
|
||||
.notifier_call = bpf_offload_notification,
|
||||
};
|
||||
|
||||
static int __init bpf_offload_init(void)
|
||||
{
|
||||
register_netdevice_notifier(&bpf_offload_notifier);
|
||||
return 0;
|
||||
}
|
||||
|
||||
subsys_initcall(bpf_offload_init);
|
||||
|
|
@ -78,8 +78,10 @@ struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
|
|||
{
|
||||
struct pcpu_freelist_head *head;
|
||||
struct pcpu_freelist_node *node;
|
||||
unsigned long flags;
|
||||
int orig_cpu, cpu;
|
||||
|
||||
local_irq_save(flags);
|
||||
orig_cpu = cpu = raw_smp_processor_id();
|
||||
while (1) {
|
||||
head = per_cpu_ptr(s->freelist, cpu);
|
||||
|
|
@ -87,14 +89,16 @@ struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
|
|||
node = head->first;
|
||||
if (node) {
|
||||
head->first = node->next;
|
||||
raw_spin_unlock(&head->lock);
|
||||
raw_spin_unlock_irqrestore(&head->lock, flags);
|
||||
return node;
|
||||
}
|
||||
raw_spin_unlock(&head->lock);
|
||||
cpu = cpumask_next(cpu, cpu_possible_mask);
|
||||
if (cpu >= nr_cpu_ids)
|
||||
cpu = 0;
|
||||
if (cpu == orig_cpu)
|
||||
if (cpu == orig_cpu) {
|
||||
local_irq_restore(flags);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@
|
|||
#include <net/strparser.h>
|
||||
#include <net/tcp.h>
|
||||
|
||||
#define SOCK_CREATE_FLAG_MASK \
|
||||
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
struct bpf_stab {
|
||||
struct bpf_map map;
|
||||
struct sock **sock_map;
|
||||
|
|
@ -122,7 +125,7 @@ static int smap_verdict_func(struct smap_psock *psock, struct sk_buff *skb)
|
|||
*/
|
||||
TCP_SKB_CB(skb)->bpf.map = NULL;
|
||||
skb->sk = psock->sock;
|
||||
bpf_compute_data_end_sk_skb(skb);
|
||||
bpf_compute_data_pointers(skb);
|
||||
preempt_disable();
|
||||
rc = (*prog->bpf_func)(skb, prog->insnsi);
|
||||
preempt_enable();
|
||||
|
|
@ -385,7 +388,7 @@ static int smap_parse_func_strparser(struct strparser *strp,
|
|||
* any socket yet.
|
||||
*/
|
||||
skb->sk = psock->sock;
|
||||
bpf_compute_data_end_sk_skb(skb);
|
||||
bpf_compute_data_pointers(skb);
|
||||
rc = (*prog->bpf_func)(skb, prog->insnsi);
|
||||
skb->sk = NULL;
|
||||
rcu_read_unlock();
|
||||
|
|
@ -508,7 +511,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
|
|||
|
||||
/* check sanity of attributes */
|
||||
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
||||
attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
|
||||
attr->value_size != 4 || attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
if (attr->value_size > KMALLOC_MAX_SIZE)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@
|
|||
#include <linux/perf_event.h>
|
||||
#include "percpu_freelist.h"
|
||||
|
||||
#define STACK_CREATE_FLAG_MASK \
|
||||
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
struct stack_map_bucket {
|
||||
struct pcpu_freelist_node fnode;
|
||||
u32 hash;
|
||||
|
|
@ -60,7 +63,7 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
|
|||
if (!capable(CAP_SYS_ADMIN))
|
||||
return ERR_PTR(-EPERM);
|
||||
|
||||
if (attr->map_flags & ~BPF_F_NUMA_NODE)
|
||||
if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
/* check sanity of attributes */
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@
|
|||
#include <linux/version.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/cred.h>
|
||||
#include <linux/timekeeping.h>
|
||||
#include <linux/ctype.h>
|
||||
|
||||
#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
|
||||
(map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
|
||||
|
|
@ -31,6 +34,8 @@
|
|||
#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
|
||||
#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
|
||||
|
||||
#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
|
||||
|
||||
DEFINE_PER_CPU(int, bpf_prog_active);
|
||||
static DEFINE_IDR(prog_idr);
|
||||
static DEFINE_SPINLOCK(prog_idr_lock);
|
||||
|
|
@ -207,6 +212,7 @@ static void bpf_map_free_deferred(struct work_struct *work)
|
|||
struct bpf_map *map = container_of(work, struct bpf_map, work);
|
||||
|
||||
bpf_map_uncharge_memlock(map);
|
||||
security_bpf_map_free(map);
|
||||
/* implementation dependent freeing */
|
||||
map->ops->map_free(map);
|
||||
}
|
||||
|
|
@ -291,17 +297,54 @@ static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
|
|||
}
|
||||
#endif
|
||||
|
||||
static const struct file_operations bpf_map_fops = {
|
||||
static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
|
||||
loff_t *ppos)
|
||||
{
|
||||
/* We need this handler such that alloc_file() enables
|
||||
* f_mode with FMODE_CAN_READ.
|
||||
*/
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
|
||||
size_t siz, loff_t *ppos)
|
||||
{
|
||||
/* We need this handler such that alloc_file() enables
|
||||
* f_mode with FMODE_CAN_WRITE.
|
||||
*/
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
const struct file_operations bpf_map_fops = {
|
||||
#ifdef CONFIG_PROC_FS
|
||||
.show_fdinfo = bpf_map_show_fdinfo,
|
||||
#endif
|
||||
.release = bpf_map_release,
|
||||
.read = bpf_dummy_read,
|
||||
.write = bpf_dummy_write,
|
||||
};
|
||||
|
||||
int bpf_map_new_fd(struct bpf_map *map)
|
||||
int bpf_map_new_fd(struct bpf_map *map, int flags)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = security_bpf_map(map, OPEN_FMODE(flags));
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
|
||||
O_RDWR | O_CLOEXEC);
|
||||
flags | O_CLOEXEC);
|
||||
}
|
||||
|
||||
int bpf_get_file_flag(int flags)
|
||||
{
|
||||
if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
|
||||
return -EINVAL;
|
||||
if (flags & BPF_F_RDONLY)
|
||||
return O_RDONLY;
|
||||
if (flags & BPF_F_WRONLY)
|
||||
return O_WRONLY;
|
||||
return O_RDWR;
|
||||
}
|
||||
|
||||
/* helper macro to check that unused fields 'union bpf_attr' are zero */
|
||||
|
|
@ -312,18 +355,46 @@ int bpf_map_new_fd(struct bpf_map *map)
|
|||
offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
|
||||
sizeof(attr->CMD##_LAST_FIELD)) != NULL
|
||||
|
||||
#define BPF_MAP_CREATE_LAST_FIELD numa_node
|
||||
/* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
|
||||
* Return 0 on success and < 0 on error.
|
||||
*/
|
||||
static int bpf_obj_name_cpy(char *dst, const char *src)
|
||||
{
|
||||
const char *end = src + BPF_OBJ_NAME_LEN;
|
||||
|
||||
memset(dst, 0, BPF_OBJ_NAME_LEN);
|
||||
|
||||
/* Copy all isalnum() and '_' char */
|
||||
while (src < end && *src) {
|
||||
if (!isalnum(*src) && *src != '_')
|
||||
return -EINVAL;
|
||||
*dst++ = *src++;
|
||||
}
|
||||
|
||||
/* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
|
||||
if (src == end)
|
||||
return -EINVAL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define BPF_MAP_CREATE_LAST_FIELD map_name
|
||||
/* called via syscall */
|
||||
static int map_create(union bpf_attr *attr)
|
||||
{
|
||||
int numa_node = bpf_map_attr_numa_node(attr);
|
||||
struct bpf_map *map;
|
||||
int f_flags;
|
||||
int err;
|
||||
|
||||
err = CHECK_ATTR(BPF_MAP_CREATE);
|
||||
if (err)
|
||||
return -EINVAL;
|
||||
|
||||
f_flags = bpf_get_file_flag(attr->map_flags);
|
||||
if (f_flags < 0)
|
||||
return f_flags;
|
||||
|
||||
if (numa_node != NUMA_NO_NODE &&
|
||||
((unsigned int)numa_node >= nr_node_ids ||
|
||||
!node_online(numa_node)))
|
||||
|
|
@ -334,18 +405,26 @@ static int map_create(union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
err = bpf_obj_name_cpy(map->name, attr->map_name);
|
||||
if (err)
|
||||
goto free_map_nouncharge;
|
||||
|
||||
atomic_set(&map->refcnt, 1);
|
||||
atomic_set(&map->usercnt, 1);
|
||||
|
||||
err = bpf_map_charge_memlock(map);
|
||||
err = security_bpf_map_alloc(map);
|
||||
if (err)
|
||||
goto free_map_nouncharge;
|
||||
|
||||
err = bpf_map_charge_memlock(map);
|
||||
if (err)
|
||||
goto free_map_sec;
|
||||
|
||||
err = bpf_map_alloc_id(map);
|
||||
if (err)
|
||||
goto free_map;
|
||||
|
||||
err = bpf_map_new_fd(map);
|
||||
err = bpf_map_new_fd(map, f_flags);
|
||||
if (err < 0) {
|
||||
/* failed to allocate fd.
|
||||
* bpf_map_put() is needed because the above
|
||||
|
|
@ -362,6 +441,8 @@ static int map_create(union bpf_attr *attr)
|
|||
|
||||
free_map:
|
||||
bpf_map_uncharge_memlock(map);
|
||||
free_map_sec:
|
||||
security_bpf_map_free(map);
|
||||
free_map_nouncharge:
|
||||
map->ops->map_free(map);
|
||||
return err;
|
||||
|
|
@ -460,6 +541,11 @@ static int map_lookup_elem(union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
if (!(f.file->f_mode & FMODE_CAN_READ)) {
|
||||
err = -EPERM;
|
||||
goto err_put;
|
||||
}
|
||||
|
||||
key = memdup_user(ukey, map->key_size);
|
||||
if (IS_ERR(key)) {
|
||||
err = PTR_ERR(key);
|
||||
|
|
@ -540,6 +626,11 @@ static int map_update_elem(union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
|
||||
err = -EPERM;
|
||||
goto err_put;
|
||||
}
|
||||
|
||||
key = memdup_user(ukey, map->key_size);
|
||||
if (IS_ERR(key)) {
|
||||
err = PTR_ERR(key);
|
||||
|
|
@ -562,6 +653,12 @@ static int map_update_elem(union bpf_attr *attr)
|
|||
if (copy_from_user(value, uvalue, value_size) != 0)
|
||||
goto free_value;
|
||||
|
||||
/* Need to create a kthread, thus must support schedule */
|
||||
if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
|
||||
err = map->ops->map_update_elem(map, key, value, attr->flags);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
|
||||
* inside bpf map update or delete otherwise deadlocks are possible
|
||||
*/
|
||||
|
|
@ -592,7 +689,7 @@ static int map_update_elem(union bpf_attr *attr)
|
|||
}
|
||||
__this_cpu_dec(bpf_prog_active);
|
||||
preempt_enable();
|
||||
|
||||
out:
|
||||
if (!err)
|
||||
trace_bpf_map_update_elem(map, ufd, key, value);
|
||||
free_value:
|
||||
|
|
@ -623,6 +720,11 @@ static int map_delete_elem(union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
|
||||
err = -EPERM;
|
||||
goto err_put;
|
||||
}
|
||||
|
||||
key = memdup_user(ukey, map->key_size);
|
||||
if (IS_ERR(key)) {
|
||||
err = PTR_ERR(key);
|
||||
|
|
@ -666,6 +768,11 @@ static int map_get_next_key(union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
if (!(f.file->f_mode & FMODE_CAN_READ)) {
|
||||
err = -EPERM;
|
||||
goto err_put;
|
||||
}
|
||||
|
||||
if (ukey) {
|
||||
key = memdup_user(ukey, map->key_size);
|
||||
if (IS_ERR(key)) {
|
||||
|
|
@ -703,9 +810,9 @@ err_put:
|
|||
return err;
|
||||
}
|
||||
|
||||
static const struct bpf_verifier_ops * const bpf_prog_types[] = {
|
||||
#define BPF_PROG_TYPE(_id, _ops) \
|
||||
[_id] = &_ops,
|
||||
static const struct bpf_prog_ops * const bpf_prog_types[] = {
|
||||
#define BPF_PROG_TYPE(_id, _name) \
|
||||
[_id] = & _name ## _prog_ops,
|
||||
#define BPF_MAP_TYPE(_id, _ops)
|
||||
#include <linux/bpf_types.h>
|
||||
#undef BPF_PROG_TYPE
|
||||
|
|
@ -717,7 +824,10 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
|
|||
if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type])
|
||||
return -EINVAL;
|
||||
|
||||
prog->aux->ops = bpf_prog_types[type];
|
||||
if (!bpf_prog_is_dev_bound(prog->aux))
|
||||
prog->aux->ops = bpf_prog_types[type];
|
||||
else
|
||||
prog->aux->ops = &bpf_offload_prog_ops;
|
||||
prog->type = type;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -820,6 +930,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu)
|
|||
|
||||
free_used_maps(aux);
|
||||
bpf_prog_uncharge_memlock(aux->prog);
|
||||
security_bpf_prog_free(aux);
|
||||
bpf_prog_free(aux->prog);
|
||||
}
|
||||
|
||||
|
|
@ -867,15 +978,23 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
|
|||
}
|
||||
#endif
|
||||
|
||||
static const struct file_operations bpf_prog_fops = {
|
||||
const struct file_operations bpf_prog_fops = {
|
||||
#ifdef CONFIG_PROC_FS
|
||||
.show_fdinfo = bpf_prog_show_fdinfo,
|
||||
#endif
|
||||
.release = bpf_prog_release,
|
||||
.read = bpf_dummy_read,
|
||||
.write = bpf_dummy_write,
|
||||
};
|
||||
|
||||
int bpf_prog_new_fd(struct bpf_prog *prog)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = security_bpf_prog(prog);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
|
||||
O_RDWR | O_CLOEXEC);
|
||||
}
|
||||
|
|
@ -938,7 +1057,22 @@ struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
|
||||
|
||||
static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
|
||||
static bool bpf_prog_can_attach(struct bpf_prog *prog,
|
||||
enum bpf_prog_type *attach_type,
|
||||
struct net_device *netdev)
|
||||
{
|
||||
struct bpf_dev_offload *offload = prog->aux->offload;
|
||||
|
||||
if (prog->type != *attach_type)
|
||||
return false;
|
||||
if (offload && offload->netdev != netdev)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
|
||||
struct net_device *netdev)
|
||||
{
|
||||
struct fd f = fdget(ufd);
|
||||
struct bpf_prog *prog;
|
||||
|
|
@ -946,7 +1080,7 @@ static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
|
|||
prog = ____bpf_prog_get(f);
|
||||
if (IS_ERR(prog))
|
||||
return prog;
|
||||
if (type && prog->type != *type) {
|
||||
if (attach_type && !bpf_prog_can_attach(prog, attach_type, netdev)) {
|
||||
prog = ERR_PTR(-EINVAL);
|
||||
goto out;
|
||||
}
|
||||
|
|
@ -959,12 +1093,12 @@ out:
|
|||
|
||||
struct bpf_prog *bpf_prog_get(u32 ufd)
|
||||
{
|
||||
return __bpf_prog_get(ufd, NULL);
|
||||
return __bpf_prog_get(ufd, NULL, NULL);
|
||||
}
|
||||
|
||||
struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
|
||||
{
|
||||
struct bpf_prog *prog = __bpf_prog_get(ufd, &type);
|
||||
struct bpf_prog *prog = __bpf_prog_get(ufd, &type, NULL);
|
||||
|
||||
if (!IS_ERR(prog))
|
||||
trace_bpf_prog_get_type(prog);
|
||||
|
|
@ -972,8 +1106,19 @@ struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(bpf_prog_get_type);
|
||||
|
||||
struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
|
||||
struct net_device *netdev)
|
||||
{
|
||||
struct bpf_prog *prog = __bpf_prog_get(ufd, &type, netdev);
|
||||
|
||||
if (!IS_ERR(prog))
|
||||
trace_bpf_prog_get_type(prog);
|
||||
return prog;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
|
||||
|
||||
/* last field in 'union bpf_attr' used by this command */
|
||||
#define BPF_PROG_LOAD_LAST_FIELD prog_flags
|
||||
#define BPF_PROG_LOAD_LAST_FIELD prog_target_ifindex
|
||||
|
||||
static int bpf_prog_load(union bpf_attr *attr)
|
||||
{
|
||||
|
|
@ -1015,10 +1160,14 @@ static int bpf_prog_load(union bpf_attr *attr)
|
|||
if (!prog)
|
||||
return -ENOMEM;
|
||||
|
||||
err = bpf_prog_charge_memlock(prog);
|
||||
err = security_bpf_prog_alloc(prog->aux);
|
||||
if (err)
|
||||
goto free_prog_nouncharge;
|
||||
|
||||
err = bpf_prog_charge_memlock(prog);
|
||||
if (err)
|
||||
goto free_prog_sec;
|
||||
|
||||
prog->len = attr->insn_cnt;
|
||||
|
||||
err = -EFAULT;
|
||||
|
|
@ -1032,11 +1181,22 @@ static int bpf_prog_load(union bpf_attr *attr)
|
|||
atomic_set(&prog->aux->refcnt, 1);
|
||||
prog->gpl_compatible = is_gpl ? 1 : 0;
|
||||
|
||||
if (attr->prog_target_ifindex) {
|
||||
err = bpf_prog_offload_init(prog, attr);
|
||||
if (err)
|
||||
goto free_prog;
|
||||
}
|
||||
|
||||
/* find program type: socket_filter vs tracing_filter */
|
||||
err = find_prog_type(type, prog);
|
||||
if (err < 0)
|
||||
goto free_prog;
|
||||
|
||||
prog->aux->load_time = ktime_get_boot_ns();
|
||||
err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
|
||||
if (err)
|
||||
goto free_prog;
|
||||
|
||||
/* run eBPF verifier */
|
||||
err = bpf_check(&prog, attr);
|
||||
if (err < 0)
|
||||
|
|
@ -1071,16 +1231,18 @@ free_used_maps:
|
|||
free_used_maps(prog->aux);
|
||||
free_prog:
|
||||
bpf_prog_uncharge_memlock(prog);
|
||||
free_prog_sec:
|
||||
security_bpf_prog_free(prog->aux);
|
||||
free_prog_nouncharge:
|
||||
bpf_prog_free(prog);
|
||||
return err;
|
||||
}
|
||||
|
||||
#define BPF_OBJ_LAST_FIELD bpf_fd
|
||||
#define BPF_OBJ_LAST_FIELD file_flags
|
||||
|
||||
static int bpf_obj_pin(const union bpf_attr *attr)
|
||||
{
|
||||
if (CHECK_ATTR(BPF_OBJ))
|
||||
if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
|
||||
return -EINVAL;
|
||||
|
||||
return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
|
||||
|
|
@ -1088,10 +1250,12 @@ static int bpf_obj_pin(const union bpf_attr *attr)
|
|||
|
||||
static int bpf_obj_get(const union bpf_attr *attr)
|
||||
{
|
||||
if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
|
||||
if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
|
||||
attr->file_flags & ~BPF_OBJ_FLAG_MASK)
|
||||
return -EINVAL;
|
||||
|
||||
return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
|
||||
return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
|
||||
attr->file_flags);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CGROUP_BPF
|
||||
|
|
@ -1132,6 +1296,9 @@ static int sockmap_get_from_fd(const union bpf_attr *attr, bool attach)
|
|||
return 0;
|
||||
}
|
||||
|
||||
#define BPF_F_ATTACH_MASK \
|
||||
(BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
|
||||
|
||||
static int bpf_prog_attach(const union bpf_attr *attr)
|
||||
{
|
||||
enum bpf_prog_type ptype;
|
||||
|
|
@ -1145,7 +1312,7 @@ static int bpf_prog_attach(const union bpf_attr *attr)
|
|||
if (CHECK_ATTR(BPF_PROG_ATTACH))
|
||||
return -EINVAL;
|
||||
|
||||
if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
|
||||
if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
|
||||
return -EINVAL;
|
||||
|
||||
switch (attr->attach_type) {
|
||||
|
|
@ -1159,6 +1326,9 @@ static int bpf_prog_attach(const union bpf_attr *attr)
|
|||
case BPF_CGROUP_SOCK_OPS:
|
||||
ptype = BPF_PROG_TYPE_SOCK_OPS;
|
||||
break;
|
||||
case BPF_CGROUP_DEVICE:
|
||||
ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
|
||||
break;
|
||||
case BPF_SK_SKB_STREAM_PARSER:
|
||||
case BPF_SK_SKB_STREAM_VERDICT:
|
||||
return sockmap_get_from_fd(attr, true);
|
||||
|
|
@ -1176,8 +1346,8 @@ static int bpf_prog_attach(const union bpf_attr *attr)
|
|||
return PTR_ERR(cgrp);
|
||||
}
|
||||
|
||||
ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
|
||||
attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
|
||||
ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,
|
||||
attr->attach_flags);
|
||||
if (ret)
|
||||
bpf_prog_put(prog);
|
||||
cgroup_put(cgrp);
|
||||
|
|
@ -1189,6 +1359,8 @@ static int bpf_prog_attach(const union bpf_attr *attr)
|
|||
|
||||
static int bpf_prog_detach(const union bpf_attr *attr)
|
||||
{
|
||||
enum bpf_prog_type ptype;
|
||||
struct bpf_prog *prog;
|
||||
struct cgroup *cgrp;
|
||||
int ret;
|
||||
|
||||
|
|
@ -1201,26 +1373,71 @@ static int bpf_prog_detach(const union bpf_attr *attr)
|
|||
switch (attr->attach_type) {
|
||||
case BPF_CGROUP_INET_INGRESS:
|
||||
case BPF_CGROUP_INET_EGRESS:
|
||||
ptype = BPF_PROG_TYPE_CGROUP_SKB;
|
||||
break;
|
||||
case BPF_CGROUP_INET_SOCK_CREATE:
|
||||
ptype = BPF_PROG_TYPE_CGROUP_SOCK;
|
||||
break;
|
||||
case BPF_CGROUP_SOCK_OPS:
|
||||
cgrp = cgroup_get_from_fd(attr->target_fd);
|
||||
if (IS_ERR(cgrp))
|
||||
return PTR_ERR(cgrp);
|
||||
|
||||
ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
|
||||
cgroup_put(cgrp);
|
||||
ptype = BPF_PROG_TYPE_SOCK_OPS;
|
||||
break;
|
||||
case BPF_CGROUP_DEVICE:
|
||||
ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
|
||||
break;
|
||||
case BPF_SK_SKB_STREAM_PARSER:
|
||||
case BPF_SK_SKB_STREAM_VERDICT:
|
||||
ret = sockmap_get_from_fd(attr, false);
|
||||
break;
|
||||
return sockmap_get_from_fd(attr, false);
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
cgrp = cgroup_get_from_fd(attr->target_fd);
|
||||
if (IS_ERR(cgrp))
|
||||
return PTR_ERR(cgrp);
|
||||
|
||||
prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
|
||||
if (IS_ERR(prog))
|
||||
prog = NULL;
|
||||
|
||||
ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);
|
||||
if (prog)
|
||||
bpf_prog_put(prog);
|
||||
cgroup_put(cgrp);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
|
||||
|
||||
static int bpf_prog_query(const union bpf_attr *attr,
|
||||
union bpf_attr __user *uattr)
|
||||
{
|
||||
struct cgroup *cgrp;
|
||||
int ret;
|
||||
|
||||
if (!capable(CAP_NET_ADMIN))
|
||||
return -EPERM;
|
||||
if (CHECK_ATTR(BPF_PROG_QUERY))
|
||||
return -EINVAL;
|
||||
if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
|
||||
return -EINVAL;
|
||||
|
||||
switch (attr->query.attach_type) {
|
||||
case BPF_CGROUP_INET_INGRESS:
|
||||
case BPF_CGROUP_INET_EGRESS:
|
||||
case BPF_CGROUP_INET_SOCK_CREATE:
|
||||
case BPF_CGROUP_SOCK_OPS:
|
||||
case BPF_CGROUP_DEVICE:
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
cgrp = cgroup_get_from_fd(attr->query.target_fd);
|
||||
if (IS_ERR(cgrp))
|
||||
return PTR_ERR(cgrp);
|
||||
ret = cgroup_bpf_query(cgrp, attr, uattr);
|
||||
cgroup_put(cgrp);
|
||||
return ret;
|
||||
}
|
||||
#endif /* CONFIG_CGROUP_BPF */
|
||||
|
||||
#define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
|
||||
|
|
@ -1305,20 +1522,26 @@ static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
|
|||
return fd;
|
||||
}
|
||||
|
||||
#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id
|
||||
#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
|
||||
|
||||
static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
|
||||
{
|
||||
struct bpf_map *map;
|
||||
u32 id = attr->map_id;
|
||||
int f_flags;
|
||||
int fd;
|
||||
|
||||
if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID))
|
||||
if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
|
||||
attr->open_flags & ~BPF_OBJ_FLAG_MASK)
|
||||
return -EINVAL;
|
||||
|
||||
if (!capable(CAP_SYS_ADMIN))
|
||||
return -EPERM;
|
||||
|
||||
f_flags = bpf_get_file_flag(attr->open_flags);
|
||||
if (f_flags < 0)
|
||||
return f_flags;
|
||||
|
||||
spin_lock_bh(&map_idr_lock);
|
||||
map = idr_find(&map_idr, id);
|
||||
if (map)
|
||||
|
|
@ -1330,7 +1553,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
|
|||
if (IS_ERR(map))
|
||||
return PTR_ERR(map);
|
||||
|
||||
fd = bpf_map_new_fd(map);
|
||||
fd = bpf_map_new_fd(map, f_flags);
|
||||
if (fd < 0)
|
||||
bpf_map_put(map);
|
||||
|
||||
|
|
@ -1358,8 +1581,25 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
|
|||
|
||||
info.type = prog->type;
|
||||
info.id = prog->aux->id;
|
||||
info.load_time = prog->aux->load_time;
|
||||
info.created_by_uid = from_kuid_munged(current_user_ns(),
|
||||
prog->aux->user->uid);
|
||||
|
||||
memcpy(info.tag, prog->tag, sizeof(prog->tag));
|
||||
memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
|
||||
|
||||
ulen = info.nr_map_ids;
|
||||
info.nr_map_ids = prog->aux->used_map_cnt;
|
||||
ulen = min_t(u32, info.nr_map_ids, ulen);
|
||||
if (ulen) {
|
||||
u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < ulen; i++)
|
||||
if (put_user(prog->aux->used_maps[i]->id,
|
||||
&user_map_ids[i]))
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
if (!capable(CAP_SYS_ADMIN)) {
|
||||
info.jited_prog_len = 0;
|
||||
|
|
@ -1385,6 +1625,11 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
|
|||
return -EFAULT;
|
||||
}
|
||||
|
||||
if (bpf_prog_is_dev_bound(prog->aux)) {
|
||||
info.status |= BPF_PROG_STATUS_DEV_BOUND;
|
||||
info.ifindex = bpf_prog_offload_ifindex(prog);
|
||||
}
|
||||
|
||||
done:
|
||||
if (copy_to_user(uinfo, &info, info_len) ||
|
||||
put_user(info_len, &uattr->info.info_len))
|
||||
|
|
@ -1413,6 +1658,7 @@ static int bpf_map_get_info_by_fd(struct bpf_map *map,
|
|||
info.value_size = map->value_size;
|
||||
info.max_entries = map->max_entries;
|
||||
info.map_flags = map->map_flags;
|
||||
memcpy(info.name, map->name, sizeof(map->name));
|
||||
|
||||
if (copy_to_user(uinfo, &info, info_len) ||
|
||||
put_user(info_len, &uattr->info.info_len))
|
||||
|
|
@ -1467,6 +1713,10 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
|
|||
if (copy_from_user(&attr, uattr, size) != 0)
|
||||
return -EFAULT;
|
||||
|
||||
err = security_bpf(cmd, &attr, size);
|
||||
if (err < 0)
|
||||
return err;
|
||||
|
||||
switch (cmd) {
|
||||
case BPF_MAP_CREATE:
|
||||
err = map_create(&attr);
|
||||
|
|
@ -1499,6 +1749,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
|
|||
case BPF_PROG_DETACH:
|
||||
err = bpf_prog_detach(&attr);
|
||||
break;
|
||||
case BPF_PROG_QUERY:
|
||||
err = bpf_prog_query(&attr, uattr);
|
||||
break;
|
||||
#endif
|
||||
case BPF_PROG_TEST_RUN:
|
||||
err = bpf_prog_test_run(&attr, uattr);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1896,6 +1896,9 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags)
|
|||
if (ret)
|
||||
goto destroy_root;
|
||||
|
||||
ret = cgroup_bpf_inherit(root_cgrp);
|
||||
WARN_ON_ONCE(ret);
|
||||
|
||||
trace_cgroup_setup_root(root);
|
||||
|
||||
/*
|
||||
|
|
@ -4721,6 +4724,9 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
|
|||
cgrp->self.parent = &parent->self;
|
||||
cgrp->root = root;
|
||||
cgrp->level = level;
|
||||
ret = cgroup_bpf_inherit(cgrp);
|
||||
if (ret)
|
||||
goto out_idr_free;
|
||||
|
||||
for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) {
|
||||
cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
|
||||
|
|
@ -4755,13 +4761,12 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
|
|||
if (!cgroup_on_dfl(cgrp))
|
||||
cgrp->subtree_control = cgroup_control(cgrp);
|
||||
|
||||
if (parent)
|
||||
cgroup_bpf_inherit(cgrp, parent);
|
||||
|
||||
cgroup_propagate_control(cgrp);
|
||||
|
||||
return cgrp;
|
||||
|
||||
out_idr_free:
|
||||
cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
|
||||
out_cancel_ref:
|
||||
percpu_ref_exit(&cgrp->self.refcnt);
|
||||
out_free_cgrp:
|
||||
|
|
@ -5744,14 +5749,33 @@ void cgroup_sk_free(struct sock_cgroup_data *skcd)
|
|||
#endif /* CONFIG_SOCK_CGROUP_DATA */
|
||||
|
||||
#ifdef CONFIG_CGROUP_BPF
|
||||
int cgroup_bpf_update(struct cgroup *cgrp, struct bpf_prog *prog,
|
||||
enum bpf_attach_type type, bool overridable)
|
||||
int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
|
||||
enum bpf_attach_type type, u32 flags)
|
||||
{
|
||||
struct cgroup *parent = cgroup_parent(cgrp);
|
||||
int ret;
|
||||
|
||||
mutex_lock(&cgroup_mutex);
|
||||
ret = __cgroup_bpf_update(cgrp, parent, prog, type, overridable);
|
||||
ret = __cgroup_bpf_attach(cgrp, prog, type, flags);
|
||||
mutex_unlock(&cgroup_mutex);
|
||||
return ret;
|
||||
}
|
||||
int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
|
||||
enum bpf_attach_type type, u32 flags)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&cgroup_mutex);
|
||||
ret = __cgroup_bpf_detach(cgrp, prog, type, flags);
|
||||
mutex_unlock(&cgroup_mutex);
|
||||
return ret;
|
||||
}
|
||||
int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
|
||||
union bpf_attr __user *uattr)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&cgroup_mutex);
|
||||
ret = __cgroup_bpf_query(cgrp, attr, uattr);
|
||||
mutex_unlock(&cgroup_mutex);
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3601,7 +3601,6 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
|
|||
goto out;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* If the event is currently on this CPU, its either a per-task event,
|
||||
* or local to this CPU. Furthermore it means its ACTIVE (otherwise
|
||||
|
|
@ -7867,11 +7866,9 @@ void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
|
|||
struct pt_regs *regs, struct hlist_head *head,
|
||||
struct task_struct *task)
|
||||
{
|
||||
struct bpf_prog *prog = call->prog;
|
||||
|
||||
if (prog) {
|
||||
if (bpf_prog_array_valid(call)) {
|
||||
*(struct pt_regs **)raw_data = regs;
|
||||
if (!trace_call_bpf(prog, raw_data) || hlist_empty(head)) {
|
||||
if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) {
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
return;
|
||||
}
|
||||
|
|
@ -8060,13 +8057,11 @@ static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
|
|||
{
|
||||
bool is_kprobe, is_tracepoint, is_syscall_tp;
|
||||
struct bpf_prog *prog;
|
||||
int ret;
|
||||
|
||||
if (event->attr.type != PERF_TYPE_TRACEPOINT)
|
||||
return perf_event_set_bpf_handler(event, prog_fd);
|
||||
|
||||
if (event->tp_event->prog)
|
||||
return -EEXIST;
|
||||
|
||||
is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
|
||||
is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
|
||||
is_syscall_tp = is_syscall_trace_event(event->tp_event);
|
||||
|
|
@ -8094,26 +8089,20 @@ static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
|
|||
return -EACCES;
|
||||
}
|
||||
}
|
||||
event->tp_event->prog = prog;
|
||||
event->tp_event->bpf_prog_owner = event;
|
||||
|
||||
return 0;
|
||||
ret = perf_event_attach_bpf_prog(event, prog);
|
||||
if (ret)
|
||||
bpf_prog_put(prog);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void perf_event_free_bpf_prog(struct perf_event *event)
|
||||
{
|
||||
struct bpf_prog *prog;
|
||||
|
||||
perf_event_free_bpf_handler(event);
|
||||
|
||||
if (!event->tp_event)
|
||||
if (event->attr.type != PERF_TYPE_TRACEPOINT) {
|
||||
perf_event_free_bpf_handler(event);
|
||||
return;
|
||||
|
||||
prog = event->tp_event->prog;
|
||||
if (prog && event->tp_event->bpf_prog_owner == event) {
|
||||
event->tp_event->prog = NULL;
|
||||
bpf_prog_put(prog);
|
||||
}
|
||||
perf_event_detach_bpf_prog(event);
|
||||
}
|
||||
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -15,9 +15,11 @@
|
|||
#include <linux/ctype.h>
|
||||
#include "trace.h"
|
||||
|
||||
u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
|
||||
|
||||
/**
|
||||
* trace_call_bpf - invoke BPF program
|
||||
* @prog: BPF program
|
||||
* @call: tracepoint event
|
||||
* @ctx: opaque context pointer
|
||||
*
|
||||
* kprobe handlers execute BPF programs via this helper.
|
||||
|
|
@ -29,7 +31,7 @@
|
|||
* 1 - store kprobe event into ring buffer
|
||||
* Other values are reserved and currently alias to 1
|
||||
*/
|
||||
unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx)
|
||||
unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
|
||||
{
|
||||
unsigned int ret;
|
||||
|
||||
|
|
@ -49,9 +51,22 @@ unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx)
|
|||
goto out;
|
||||
}
|
||||
|
||||
rcu_read_lock();
|
||||
ret = BPF_PROG_RUN(prog, ctx);
|
||||
rcu_read_unlock();
|
||||
/*
|
||||
* Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
|
||||
* to all call sites, we did a bpf_prog_array_valid() there to check
|
||||
* whether call->prog_array is empty or not, which is
|
||||
* a heurisitc to speed up execution.
|
||||
*
|
||||
* If bpf_prog_array_valid() fetched prog_array was
|
||||
* non-NULL, we go into trace_call_bpf() and do the actual
|
||||
* proper rcu_dereference() under RCU lock.
|
||||
* If it turns out that prog_array is NULL then, we bail out.
|
||||
* For the opposite, if the bpf_prog_array_valid() fetched pointer
|
||||
* was NULL, you'll skip the prog_array with the risk of missing
|
||||
* out of events when it was updated in between this and the
|
||||
* rcu_dereference() which is accepted risk.
|
||||
*/
|
||||
ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN);
|
||||
|
||||
out:
|
||||
__this_cpu_dec(bpf_prog_active);
|
||||
|
|
@ -63,12 +78,16 @@ EXPORT_SYMBOL_GPL(trace_call_bpf);
|
|||
|
||||
BPF_CALL_3(bpf_probe_read, void *, dst, u32, size, const void *, unsafe_ptr)
|
||||
{
|
||||
int ret;
|
||||
int ret = 0;
|
||||
|
||||
if (unlikely(size == 0))
|
||||
goto out;
|
||||
|
||||
ret = probe_kernel_read(dst, unsafe_ptr, size);
|
||||
if (unlikely(ret < 0))
|
||||
memset(dst, 0, size);
|
||||
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -77,7 +96,7 @@ static const struct bpf_func_proto bpf_probe_read_proto = {
|
|||
.gpl_only = true,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
|
||||
.arg2_type = ARG_CONST_SIZE,
|
||||
.arg2_type = ARG_CONST_SIZE_OR_ZERO,
|
||||
.arg3_type = ARG_ANYTHING,
|
||||
};
|
||||
|
||||
|
|
@ -255,14 +274,14 @@ const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
|
|||
return &bpf_trace_printk_proto;
|
||||
}
|
||||
|
||||
BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
|
||||
static __always_inline int
|
||||
get_map_perf_counter(struct bpf_map *map, u64 flags,
|
||||
u64 *value, u64 *enabled, u64 *running)
|
||||
{
|
||||
struct bpf_array *array = container_of(map, struct bpf_array, map);
|
||||
unsigned int cpu = smp_processor_id();
|
||||
u64 index = flags & BPF_F_INDEX_MASK;
|
||||
struct bpf_event_entry *ee;
|
||||
u64 value = 0;
|
||||
int err;
|
||||
|
||||
if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
|
||||
return -EINVAL;
|
||||
|
|
@ -275,7 +294,15 @@ BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
|
|||
if (!ee)
|
||||
return -ENOENT;
|
||||
|
||||
err = perf_event_read_local(ee->event, &value, NULL, NULL);
|
||||
return perf_event_read_local(ee->event, value, enabled, running);
|
||||
}
|
||||
|
||||
BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
|
||||
{
|
||||
u64 value = 0;
|
||||
int err;
|
||||
|
||||
err = get_map_perf_counter(map, flags, &value, NULL, NULL);
|
||||
/*
|
||||
* this api is ugly since we miss [-22..-2] range of valid
|
||||
* counter values, but that's uapi
|
||||
|
|
@ -293,6 +320,33 @@ static const struct bpf_func_proto bpf_perf_event_read_proto = {
|
|||
.arg2_type = ARG_ANYTHING,
|
||||
};
|
||||
|
||||
BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,
|
||||
struct bpf_perf_event_value *, buf, u32, size)
|
||||
{
|
||||
int err = -EINVAL;
|
||||
|
||||
if (unlikely(size != sizeof(struct bpf_perf_event_value)))
|
||||
goto clear;
|
||||
err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,
|
||||
&buf->running);
|
||||
if (unlikely(err))
|
||||
goto clear;
|
||||
return 0;
|
||||
clear:
|
||||
memset(buf, 0, size);
|
||||
return err;
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
|
||||
.func = bpf_perf_event_read_value,
|
||||
.gpl_only = true,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_CONST_MAP_PTR,
|
||||
.arg2_type = ARG_ANYTHING,
|
||||
.arg3_type = ARG_PTR_TO_UNINIT_MEM,
|
||||
.arg4_type = ARG_CONST_SIZE,
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct perf_sample_data, bpf_sd);
|
||||
|
||||
static __always_inline u64
|
||||
|
|
@ -499,6 +553,8 @@ static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func
|
|||
return &bpf_perf_event_output_proto;
|
||||
case BPF_FUNC_get_stackid:
|
||||
return &bpf_get_stackid_proto;
|
||||
case BPF_FUNC_perf_event_read_value:
|
||||
return &bpf_perf_event_read_value_proto;
|
||||
default:
|
||||
return tracing_func_proto(func_id);
|
||||
}
|
||||
|
|
@ -524,11 +580,14 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type
|
|||
return true;
|
||||
}
|
||||
|
||||
const struct bpf_verifier_ops kprobe_prog_ops = {
|
||||
const struct bpf_verifier_ops kprobe_verifier_ops = {
|
||||
.get_func_proto = kprobe_prog_func_proto,
|
||||
.is_valid_access = kprobe_prog_is_valid_access,
|
||||
};
|
||||
|
||||
const struct bpf_prog_ops kprobe_prog_ops = {
|
||||
};
|
||||
|
||||
BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
|
||||
u64, flags, void *, data, u64, size)
|
||||
{
|
||||
|
|
@ -576,6 +635,32 @@ static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
|
|||
.arg3_type = ARG_ANYTHING,
|
||||
};
|
||||
|
||||
BPF_CALL_3(bpf_perf_prog_read_value_tp, struct bpf_perf_event_data_kern *, ctx,
|
||||
struct bpf_perf_event_value *, buf, u32, size)
|
||||
{
|
||||
int err = -EINVAL;
|
||||
|
||||
if (unlikely(size != sizeof(struct bpf_perf_event_value)))
|
||||
goto clear;
|
||||
err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled,
|
||||
&buf->running);
|
||||
if (unlikely(err))
|
||||
goto clear;
|
||||
return 0;
|
||||
clear:
|
||||
memset(buf, 0, size);
|
||||
return err;
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_perf_prog_read_value_proto_tp = {
|
||||
.func = bpf_perf_prog_read_value_tp,
|
||||
.gpl_only = true,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
||||
.arg3_type = ARG_CONST_SIZE,
|
||||
};
|
||||
|
||||
static const struct bpf_func_proto *tp_prog_func_proto(enum bpf_func_id func_id)
|
||||
{
|
||||
switch (func_id) {
|
||||
|
|
@ -583,6 +668,8 @@ static const struct bpf_func_proto *tp_prog_func_proto(enum bpf_func_id func_id)
|
|||
return &bpf_perf_event_output_proto_tp;
|
||||
case BPF_FUNC_get_stackid:
|
||||
return &bpf_get_stackid_proto_tp;
|
||||
case BPF_FUNC_perf_prog_read_value:
|
||||
return &bpf_perf_prog_read_value_proto_tp;
|
||||
default:
|
||||
return tracing_func_proto(func_id);
|
||||
}
|
||||
|
|
@ -602,11 +689,14 @@ static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type
|
|||
return true;
|
||||
}
|
||||
|
||||
const struct bpf_verifier_ops tracepoint_prog_ops = {
|
||||
const struct bpf_verifier_ops tracepoint_verifier_ops = {
|
||||
.get_func_proto = tp_prog_func_proto,
|
||||
.is_valid_access = tp_prog_is_valid_access,
|
||||
};
|
||||
|
||||
const struct bpf_prog_ops tracepoint_prog_ops = {
|
||||
};
|
||||
|
||||
static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
|
||||
struct bpf_insn_access_aux *info)
|
||||
{
|
||||
|
|
@ -662,8 +752,67 @@ static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
|
|||
return insn - insn_buf;
|
||||
}
|
||||
|
||||
const struct bpf_verifier_ops perf_event_prog_ops = {
|
||||
const struct bpf_verifier_ops perf_event_verifier_ops = {
|
||||
.get_func_proto = tp_prog_func_proto,
|
||||
.is_valid_access = pe_prog_is_valid_access,
|
||||
.convert_ctx_access = pe_prog_convert_ctx_access,
|
||||
};
|
||||
|
||||
const struct bpf_prog_ops perf_event_prog_ops = {
|
||||
};
|
||||
|
||||
static DEFINE_MUTEX(bpf_event_mutex);
|
||||
|
||||
int perf_event_attach_bpf_prog(struct perf_event *event,
|
||||
struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_prog_array __rcu *old_array;
|
||||
struct bpf_prog_array *new_array;
|
||||
int ret = -EEXIST;
|
||||
|
||||
mutex_lock(&bpf_event_mutex);
|
||||
|
||||
if (event->prog)
|
||||
goto unlock;
|
||||
|
||||
old_array = event->tp_event->prog_array;
|
||||
ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
|
||||
if (ret < 0)
|
||||
goto unlock;
|
||||
|
||||
/* set the new array to event->tp_event and set event->prog */
|
||||
event->prog = prog;
|
||||
rcu_assign_pointer(event->tp_event->prog_array, new_array);
|
||||
bpf_prog_array_free(old_array);
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&bpf_event_mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void perf_event_detach_bpf_prog(struct perf_event *event)
|
||||
{
|
||||
struct bpf_prog_array __rcu *old_array;
|
||||
struct bpf_prog_array *new_array;
|
||||
int ret;
|
||||
|
||||
mutex_lock(&bpf_event_mutex);
|
||||
|
||||
if (!event->prog)
|
||||
goto unlock;
|
||||
|
||||
old_array = event->tp_event->prog_array;
|
||||
ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array);
|
||||
if (ret < 0) {
|
||||
bpf_prog_array_delete_safe(old_array, event->prog);
|
||||
} else {
|
||||
rcu_assign_pointer(event->tp_event->prog_array, new_array);
|
||||
bpf_prog_array_free(old_array);
|
||||
}
|
||||
|
||||
bpf_prog_put(event->prog);
|
||||
event->prog = NULL;
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&bpf_event_mutex);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1174,13 +1174,12 @@ static void
|
|||
kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
|
||||
{
|
||||
struct trace_event_call *call = &tk->tp.call;
|
||||
struct bpf_prog *prog = call->prog;
|
||||
struct kprobe_trace_entry_head *entry;
|
||||
struct hlist_head *head;
|
||||
int size, __size, dsize;
|
||||
int rctx;
|
||||
|
||||
if (prog && !trace_call_bpf(prog, regs))
|
||||
if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
|
||||
return;
|
||||
|
||||
head = this_cpu_ptr(call->perf_events);
|
||||
|
|
@ -1210,13 +1209,12 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
|
|||
struct pt_regs *regs)
|
||||
{
|
||||
struct trace_event_call *call = &tk->tp.call;
|
||||
struct bpf_prog *prog = call->prog;
|
||||
struct kretprobe_trace_entry_head *entry;
|
||||
struct hlist_head *head;
|
||||
int size, __size, dsize;
|
||||
int rctx;
|
||||
|
||||
if (prog && !trace_call_bpf(prog, regs))
|
||||
if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
|
||||
return;
|
||||
|
||||
head = this_cpu_ptr(call->perf_events);
|
||||
|
|
|
|||
|
|
@ -560,9 +560,10 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
|
|||
static int sys_perf_refcount_enter;
|
||||
static int sys_perf_refcount_exit;
|
||||
|
||||
static int perf_call_bpf_enter(struct bpf_prog *prog, struct pt_regs *regs,
|
||||
struct syscall_metadata *sys_data,
|
||||
struct syscall_trace_enter *rec) {
|
||||
static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
|
||||
struct syscall_metadata *sys_data,
|
||||
struct syscall_trace_enter *rec)
|
||||
{
|
||||
struct syscall_tp_t {
|
||||
unsigned long long regs;
|
||||
unsigned long syscall_nr;
|
||||
|
|
@ -574,7 +575,7 @@ static int perf_call_bpf_enter(struct bpf_prog *prog, struct pt_regs *regs,
|
|||
param.syscall_nr = rec->nr;
|
||||
for (i = 0; i < sys_data->nb_args; i++)
|
||||
param.args[i] = rec->args[i];
|
||||
return trace_call_bpf(prog, ¶m);
|
||||
return trace_call_bpf(call, ¶m);
|
||||
}
|
||||
|
||||
static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
|
||||
|
|
@ -582,7 +583,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
|
|||
struct syscall_metadata *sys_data;
|
||||
struct syscall_trace_enter *rec;
|
||||
struct hlist_head *head;
|
||||
struct bpf_prog *prog;
|
||||
bool valid_prog_array;
|
||||
int syscall_nr;
|
||||
int rctx;
|
||||
int size;
|
||||
|
|
@ -597,9 +598,9 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
|
|||
if (!sys_data)
|
||||
return;
|
||||
|
||||
prog = READ_ONCE(sys_data->enter_event->prog);
|
||||
head = this_cpu_ptr(sys_data->enter_event->perf_events);
|
||||
if (!prog && hlist_empty(head))
|
||||
valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
|
||||
if (!valid_prog_array && hlist_empty(head))
|
||||
return;
|
||||
|
||||
/* get the size after alignment with the u32 buffer size field */
|
||||
|
|
@ -615,7 +616,8 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
|
|||
syscall_get_arguments(current, regs, 0, sys_data->nb_args,
|
||||
(unsigned long *)&rec->args);
|
||||
|
||||
if ((prog && !perf_call_bpf_enter(prog, regs, sys_data, rec)) ||
|
||||
if ((valid_prog_array &&
|
||||
!perf_call_bpf_enter(sys_data->enter_event, regs, sys_data, rec)) ||
|
||||
hlist_empty(head)) {
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
return;
|
||||
|
|
@ -660,8 +662,9 @@ static void perf_sysenter_disable(struct trace_event_call *call)
|
|||
mutex_unlock(&syscall_trace_lock);
|
||||
}
|
||||
|
||||
static int perf_call_bpf_exit(struct bpf_prog *prog, struct pt_regs *regs,
|
||||
struct syscall_trace_exit *rec) {
|
||||
static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
|
||||
struct syscall_trace_exit *rec)
|
||||
{
|
||||
struct syscall_tp_t {
|
||||
unsigned long long regs;
|
||||
unsigned long syscall_nr;
|
||||
|
|
@ -671,7 +674,7 @@ static int perf_call_bpf_exit(struct bpf_prog *prog, struct pt_regs *regs,
|
|||
*(struct pt_regs **)¶m = regs;
|
||||
param.syscall_nr = rec->nr;
|
||||
param.ret = rec->ret;
|
||||
return trace_call_bpf(prog, ¶m);
|
||||
return trace_call_bpf(call, ¶m);
|
||||
}
|
||||
|
||||
static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
|
||||
|
|
@ -679,7 +682,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
|
|||
struct syscall_metadata *sys_data;
|
||||
struct syscall_trace_exit *rec;
|
||||
struct hlist_head *head;
|
||||
struct bpf_prog *prog;
|
||||
bool valid_prog_array;
|
||||
int syscall_nr;
|
||||
int rctx;
|
||||
int size;
|
||||
|
|
@ -694,9 +697,9 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
|
|||
if (!sys_data)
|
||||
return;
|
||||
|
||||
prog = READ_ONCE(sys_data->exit_event->prog);
|
||||
head = this_cpu_ptr(sys_data->exit_event->perf_events);
|
||||
if (!prog && hlist_empty(head))
|
||||
valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
|
||||
if (!valid_prog_array && hlist_empty(head))
|
||||
return;
|
||||
|
||||
/* We can probably do that at build time */
|
||||
|
|
@ -710,7 +713,8 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
|
|||
rec->nr = syscall_nr;
|
||||
rec->ret = syscall_get_return_value(current, regs);
|
||||
|
||||
if ((prog && !perf_call_bpf_exit(prog, regs, rec)) ||
|
||||
if ((valid_prog_array &&
|
||||
!perf_call_bpf_exit(sys_data->exit_event, regs, rec)) ||
|
||||
hlist_empty(head)) {
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -1113,13 +1113,12 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
|
|||
{
|
||||
struct trace_event_call *call = &tu->tp.call;
|
||||
struct uprobe_trace_entry_head *entry;
|
||||
struct bpf_prog *prog = call->prog;
|
||||
struct hlist_head *head;
|
||||
void *data;
|
||||
int size, esize;
|
||||
int rctx;
|
||||
|
||||
if (prog && !trace_call_bpf(prog, regs))
|
||||
if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
|
||||
return;
|
||||
|
||||
esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
|
||||
|
|
|
|||
Loading…
Reference in a new issue