diff --git a/include/linux/bpf.h b/include/linux/bpf.h index bdadb0bb6cecd1..c6f653a5e00510 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1585,6 +1585,13 @@ struct bpf_prog { const struct bpf_insn *insn); struct bpf_prog_aux *aux; /* Auxiliary fields */ struct sock_fprog_kern *orig_prog; /* Original BPF program */ + + /* Additions to support BPF_PROG_TYPE_KTHREAD */ + + struct work_struct sched_work; + ktime_t sched_period; + struct hrtimer sched_timer; + /* Instructions for interpreter */ union { DECLARE_FLEX_ARRAY(struct sock_filter, insns); diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index fa78f49d4a9a64..6d851da9cb7202 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -83,6 +83,8 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_SYSCALL, bpf_syscall, BPF_PROG_TYPE(BPF_PROG_TYPE_NETFILTER, netfilter, struct bpf_nf_ctx, struct bpf_nf_ctx) #endif +BPF_PROG_TYPE(BPF_PROG_TYPE_KTHREAD, kthread, + void *, void *) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 4a939c90dc2e4b..78c6220fe40dbc 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1055,6 +1055,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + BPF_PROG_TYPE_KTHREAD, __MAX_BPF_PROG_TYPE }; @@ -1116,6 +1117,7 @@ enum bpf_attach_type { BPF_NETKIT_PRIMARY, BPF_NETKIT_PEER, BPF_TRACE_KPROBE_SESSION, + BPF_KTHREAD, __MAX_BPF_ATTACH_TYPE }; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 9b9c151b5c826b..29f7de68dc518d 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -25,6 +25,7 @@ obj-$(CONFIG_BPF_SYSCALL) += cpumap.o obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o +obj-$(CONFIG_BPF_SYSCALL) += bpf_kthread.o endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o diff --git a/kernel/bpf/bpf_kthread.c b/kernel/bpf/bpf_kthread.c new file mode 100644 index 00000000000000..352fd2b132ba21 --- /dev/null +++ b/kernel/bpf/bpf_kthread.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2016 Facebook + */ +#include +#include +#include +#include +#include +#include + +#include "bpf_kthread.h" + +static struct workqueue_struct *bpf_kthread_wq; + +/* Work function for schedulable BPF programs */ +void bpf_kthread_program_run(struct work_struct *work) +{ + struct bpf_prog *prog = container_of(work, struct bpf_prog, sched_work); + unsigned long dummy_ctx = 0; + int ret = bpf_prog_run(prog, &dummy_ctx); + + /* Handle the return value if necessary */ + if (ret != 0) { + pr_warn("BPF program %d failed, returned non-zero value %d\n", + prog->aux->id, ret); + } + + /* Reschedule the timer*/ + hrtimer_start(&prog->sched_timer, ktime_set(0, 100000000), HRTIMER_MODE_REL); +} + +enum hrtimer_restart bpf_kthread_timer_callback(struct hrtimer *timer) +{ + struct bpf_prog *prog = container_of(timer, struct bpf_prog, sched_timer); + + // Work is already pending, do not queue it again. + if (work_pending(&prog->sched_work)) + return HRTIMER_NORESTART; + + queue_work(bpf_kthread_wq, &prog->sched_work); + + /* The timer is restarted in the work function for periodic jobs */ + return HRTIMER_NORESTART; +} + +int bpf_kthread_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + if(!bpf_kthread_wq) { + pr_info("Init BPF workqueue\n"); + bpf_kthread_wq = alloc_workqueue("bpf_sched_wq", 0, 0); + if (!bpf_kthread_wq) { + pr_err("Failed to create bpf_sched_wq\n"); + return -ENOMEM; + } + } + + hrtimer_init(&prog->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + prog->sched_timer.function = bpf_kthread_timer_callback; + hrtimer_start(&prog->sched_timer, ktime_set(0, 100000000), HRTIMER_MODE_REL); + + INIT_WORK(&prog->sched_work, bpf_kthread_program_run); + + return 0; +} + +int bpf_kthread_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + /* Stop timer to stop periodic schedule */ + hrtimer_cancel(&prog->sched_timer); + + /* Stop any running work_struct */ + cancel_work_sync(&prog->sched_work); + + /* Clean up program reference */ + bpf_prog_put(prog); + + return 0; +} diff --git a/kernel/bpf/bpf_kthread.h b/kernel/bpf/bpf_kthread.h new file mode 100644 index 00000000000000..cdd8e087096cf5 --- /dev/null +++ b/kernel/bpf/bpf_kthread.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Isovalent */ +#ifndef __BPF_KTHREAD_H +#define __BPF_KTHREAD_H + +#include +#include +#include + +int bpf_kthread_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog); +int bpf_kthread_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog); + +void bpf_kthread_program_run(struct work_struct *work); +enum hrtimer_restart bpf_kthread_timer_callback(struct hrtimer *timer); + +#endif /* __BPF_KTHREAD_H */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c5aa127ed4cc01..d8e76f560b4b2a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -39,6 +39,7 @@ #include #include #include +#include "bpf_kthread.h" #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ @@ -3951,6 +3952,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) case BPF_NETKIT_PRIMARY: case BPF_NETKIT_PEER: return BPF_PROG_TYPE_SCHED_CLS; + case BPF_KTHREAD: + return BPF_PROG_TYPE_KTHREAD; default: return BPF_PROG_TYPE_UNSPEC; } @@ -4070,6 +4073,9 @@ static int bpf_prog_attach(const union bpf_attr *attr) } switch (ptype) { + case BPF_PROG_TYPE_KTHREAD: + ret = bpf_kthread_prog_attach(attr, prog); + break; case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: ret = sock_map_get_from_fd(attr, prog); @@ -4138,7 +4144,14 @@ static int bpf_prog_detach(const union bpf_attr *attr) return -EINVAL; } + prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); + if (IS_ERR(prog)) + return PTR_ERR(prog); + switch (ptype) { + case BPF_PROG_TYPE_KTHREAD: + ret = bpf_kthread_prog_detach(attr, prog); + break; case BPF_PROG_TYPE_SK_MSG: case BPF_PROG_TYPE_SK_SKB: ret = sock_map_prog_detach(attr, ptype); @@ -5623,8 +5636,10 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) /* copy attributes from user space, may be less than sizeof(bpf_attr) */ memset(&attr, 0, sizeof(attr)); - if (copy_from_bpfptr(&attr, uattr, size) != 0) + if (copy_from_bpfptr(&attr, uattr, size) != 0){ + pr_err("[bpf/syscall.c] L-5641: Unable to copy bpf attributes from user space\n"); return -EFAULT; + } err = security_bpf(cmd, &attr, size); if (err < 0) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 630b763e52402f..a5f92d3f099eba 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2051,6 +2051,23 @@ const struct bpf_prog_ops tracing_prog_ops = { .test_run = bpf_prog_test_run_tracing, }; +static bool kthread_prog_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + return true; +} + +const struct bpf_verifier_ops kthread_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = kthread_prog_is_valid_access, +}; + +const struct bpf_prog_ops kthread_prog_ops = { + .test_run = bpf_prog_test_run_xdp, +}; + static bool raw_tp_writable_prog_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 2ff949ea82fa66..a5bab75b139755 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -49,6 +49,7 @@ static const bool attach_types[] = { [BPF_SK_SKB_VERDICT] = true, [BPF_SK_MSG_VERDICT] = true, [BPF_FLOW_DISSECTOR] = true, + [BPF_KTHREAD] = true, [__MAX_BPF_ATTACH_TYPE] = false, }; @@ -1046,7 +1047,7 @@ static int parse_attach_detach_args(int argc, char **argv, int *progfd, return -EINVAL; } - if (*attach_type == BPF_FLOW_DISSECTOR) { + if (*attach_type == BPF_FLOW_DISSECTOR || *attach_type == BPF_KTHREAD) { *mapfd = 0; return 0; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 4a939c90dc2e4b..78c6220fe40dbc 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1055,6 +1055,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + BPF_PROG_TYPE_KTHREAD, __MAX_BPF_PROG_TYPE }; @@ -1116,6 +1117,7 @@ enum bpf_attach_type { BPF_NETKIT_PRIMARY, BPF_NETKIT_PEER, BPF_TRACE_KPROBE_SESSION, + BPF_KTHREAD, __MAX_BPF_ATTACH_TYPE }; diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 219facd0e66e8b..9a21664296bf63 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -133,6 +133,7 @@ static const char * const attach_type_name[] = { [BPF_NETKIT_PRIMARY] = "netkit_primary", [BPF_NETKIT_PEER] = "netkit_peer", [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session", + [BPF_KTHREAD] = "kthread", }; static const char * const link_type_name[] = { @@ -224,6 +225,7 @@ static const char * const prog_type_name[] = { [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", [BPF_PROG_TYPE_SYSCALL] = "syscall", [BPF_PROG_TYPE_NETFILTER] = "netfilter", + [BPF_PROG_TYPE_KTHREAD] = "kthread", }; static int __base_pr(enum libbpf_print_level level, const char *format, @@ -7518,6 +7520,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog load_attr.log_level = log_level; ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); + if (ret >= 0) { if (log_level && own_log_buf) { pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", @@ -9457,6 +9460,7 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), + SEC_DEF("kthread", KTHREAD, 0, SEC_SLEEPABLE), }; int libbpf_register_prog_handler(const char *sec,