A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-11 15:11:48 -04:00
Code Issues Projects Releases Packages Wiki Activity

UPSTREAM: bpf, xdp: Start using the BPF dispatcher for XDP

Browse source 
This commit adds a BPF dispatcher for XDP. The dispatcher is updated
from the XDP control-path, dev_xdp_install(), and used when an XDP
program is run via bpf_prog_run_xdp().

Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20191213175112.30208-4-bjorn.topel@gmail.com
(cherry picked from commit 7e6897f95935973c3253fd756135b5ea58043dc8)
Bug: 145210207
Change-Id: Icc5baa51e9392714ff38a5b3dbce7e5dba027a11
Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
This commit is contained in:
Björn Töpel 2019-12-13 18:51:09 +01:00 • committed by basamaryan
commit 97174135f6
No known key found for this signature in database
GPG key ID: 210D5384FB2A5885
5 changed files with 83 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

15
include/linux/bpf.h
View file

@ -489,6 +489,14 @@ struct bpf_dispatcher {
u32 image_off;
};
static __always_inline unsigned int bpf_dispatcher_nopfunc(
const void *ctx,
const struct bpf_insn *insnsi,
unsigned int (*bpf_func)(const void *,
const struct bpf_insn *))
{
return bpf_func(ctx, insnsi);
}
#ifdef CONFIG_BPF_JIT
struct bpf_trampoline *bpf_trampoline_lookup(u64 key);
int bpf_trampoline_link_prog(struct bpf_prog *prog);
@ -1002,6 +1010,8 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog);
struct bpf_prog *bpf_prog_by_id(u32 id);
static inline bool unprivileged_ebpf_enabled(void)
{
return !sysctl_unprivileged_bpf_disabled;
@ -1139,6 +1149,11 @@ static inline void bpf_map_put(struct bpf_map *map)
{
}
static inline struct bpf_prog *bpf_prog_by_id(u32 id)
{
return ERR_PTR(-ENOTSUPP);
}
static inline bool unprivileged_ebpf_enabled(void)
{
return false;

40
include/linux/filter.h
View file

@ -637,23 +637,26 @@ static inline void bpf_jit_set_header_magic(struct bpf_binary_header *hdr)
}
#endif
#define BPF_PROG_RUN(prog, ctx) ({ \
u32 ret; \
cant_sleep(); \
if (static_branch_unlikely(&bpf_stats_enabled_key)) { \
struct bpf_prog_stats *stats; \
u64 start = sched_clock(); \
ret = bpf_call_func(prog, ctx); \
stats = this_cpu_ptr(prog->aux->stats); \
u64_stats_update_begin(&stats->syncp); \
stats->cnt++; \
stats->nsecs += sched_clock() - start; \
u64_stats_update_end(&stats->syncp); \
} else { \
ret = bpf_call_func(prog, ctx); \
} \
#define __BPF_PROG_RUN(prog, ctx, dfunc) ({ \
u32 ret; \
cant_sleep(); \
if (static_branch_unlikely(&bpf_stats_enabled_key)) { \
struct bpf_prog_stats *stats; \
u64 start = sched_clock(); \
ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
stats = this_cpu_ptr(prog->aux->stats); \
u64_stats_update_begin(&stats->syncp); \
stats->cnt++; \
stats->nsecs += sched_clock() - start; \
u64_stats_update_end(&stats->syncp); \
} else { \
ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
} \
ret; })
#define BPF_PROG_RUN(prog, ctx) __BPF_PROG_RUN(prog, ctx, \
bpf_dispatcher_nopfunc)
#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
struct bpf_skb_data_end {
@ -777,6 +780,8 @@ static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
return res;
}
DECLARE_BPF_DISPATCHER(bpf_dispatcher_xdp)
static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
struct xdp_buff *xdp)
{
@ -786,9 +791,12 @@ static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
* already takes rcu_read_lock() when fetching the program, so
* it's not necessary here anymore.
*/
return BPF_PROG_RUN(prog, xdp);
return __BPF_PROG_RUN(prog, xdp,
BPF_DISPATCHER_FUNC(bpf_dispatcher_xdp));
}
void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog);
static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
{
return prog->len * sizeof(struct bpf_insn);

26
kernel/bpf/syscall.c
View file

@ -2341,6 +2341,23 @@ static int bpf_obj_get_next_id(const union bpf_attr *attr,
#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
struct bpf_prog *bpf_prog_by_id(u32 id)
{
struct bpf_prog *prog;
if (!id)
return ERR_PTR(-ENOENT);
spin_lock_bh(&prog_idr_lock);
prog = idr_find(&prog_idr, id);
if (prog)
prog = bpf_prog_inc_not_zero(prog);
else
prog = ERR_PTR(-ENOENT);
spin_unlock_bh(&prog_idr_lock);
return prog;
}
static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
{
struct bpf_prog *prog;
@ -2353,14 +2370,7 @@ static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
spin_lock_bh(&prog_idr_lock);
prog = idr_find(&prog_idr, id);
if (prog)
prog = bpf_prog_inc_not_zero(prog);
else
prog = ERR_PTR(-ENOENT);
spin_unlock_bh(&prog_idr_lock);
prog = bpf_prog_by_id(id);
if (IS_ERR(prog))
return PTR_ERR(prog);

19
net/core/dev.c
View file

@ -8502,7 +8502,17 @@ static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
struct netlink_ext_ack *extack, u32 flags,
struct bpf_prog *prog)
{
bool non_hw = !(flags & XDP_FLAGS_HW_MODE);
struct bpf_prog *prev_prog = NULL;
struct netdev_bpf xdp;
int err;
if (non_hw) {
prev_prog = bpf_prog_by_id(__dev_xdp_query(dev, bpf_op,
XDP_QUERY_PROG));
if (IS_ERR(prev_prog))
prev_prog = NULL;
}
memset(&xdp, 0, sizeof(xdp));
if (flags & XDP_FLAGS_HW_MODE)
@ -8513,7 +8523,14 @@ static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
xdp.flags = flags;
xdp.prog = prog;
return bpf_op(dev, &xdp);
err = bpf_op(dev, &xdp);
if (!err && non_hw)
bpf_prog_change_xdp(prev_prog, prog);
if (prev_prog)
bpf_prog_put(prev_prog);
return err;
}
static void dev_xdp_uninstall(struct net_device *dev)

8
net/core/filter.c
View file

@ -9134,3 +9134,11 @@ const struct bpf_verifier_ops sk_reuseport_verifier_ops = {
const struct bpf_prog_ops sk_reuseport_prog_ops = {
};
#endif /* CONFIG_INET */
DEFINE_BPF_DISPATCHER(bpf_dispatcher_xdp)
void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog)
{
bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(bpf_dispatcher_xdp),
prev_prog, prog);
}