Skip to content

Commit 269e7c9

Browse files
anakryikoanakryiko
committed
Merge branch 'bpf-add-uprobe-session-support'
Jiri Olsa says: ==================== bpf: Add uprobe session support hi, this patchset is adding support for session uprobe attachment and using it through bpf link for bpf programs. The session means that the uprobe consumer is executed on entry and return of probed function with additional control: - entry callback can control execution of the return callback - entry and return callbacks can share data/cookie Uprobe changes (on top of perf/core [1] are posted in here [2]. This patchset is based on bpf-next/master and will be merged once we pull [2] in bpf-next/master. v9 changes: - rebased on bpf-next/master with perf/core tag merged (thanks Peter!) thanks, jirka [1] git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git perf/core [2] https://lore.kernel.org/bpf/[email protected]/T/#ma43c549c4bf684ca1b17fa638aa5e7cbb46893e9 --- ==================== Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Andrii Nakryiko <[email protected]>
2 parents dcf0467 + abaec83 commit 269e7c9

File tree

17 files changed

+654
-63
lines changed

17 files changed

+654
-63
lines changed

‎include/uapi/linux/bpf.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1116,6 +1116,7 @@ enum bpf_attach_type {
11161116
BPF_NETKIT_PRIMARY,
11171117
BPF_NETKIT_PEER,
11181118
BPF_TRACE_KPROBE_SESSION,
1119+
BPF_TRACE_UPROBE_SESSION,
11191120
__MAX_BPF_ATTACH_TYPE
11201121
};
11211122

‎kernel/bpf/syscall.c‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4103,10 +4103,14 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
41034103
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
41044104
attach_type != BPF_TRACE_UPROBE_MULTI)
41054105
return -EINVAL;
4106+
if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4107+
attach_type != BPF_TRACE_UPROBE_SESSION)
4108+
return -EINVAL;
41064109
if (attach_type != BPF_PERF_EVENT &&
41074110
attach_type != BPF_TRACE_KPROBE_MULTI &&
41084111
attach_type != BPF_TRACE_KPROBE_SESSION &&
4109-
attach_type != BPF_TRACE_UPROBE_MULTI)
4112+
attach_type != BPF_TRACE_UPROBE_MULTI &&
4113+
attach_type != BPF_TRACE_UPROBE_SESSION)
41104114
return -EINVAL;
41114115
return 0;
41124116
case BPF_PROG_TYPE_SCHED_CLS:
@@ -5359,7 +5363,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
53595363
else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
53605364
attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
53615365
ret = bpf_kprobe_multi_link_attach(attr, prog);
5362-
else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)
5366+
else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5367+
attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
53635368
ret = bpf_uprobe_multi_link_attach(attr, prog);
53645369
break;
53655370
default:

‎kernel/bpf/verifier.c‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16024,6 +16024,16 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
1602416024
return -ENOTSUPP;
1602516025
}
1602616026
break;
16027+
case BPF_PROG_TYPE_KPROBE:
16028+
switch (env->prog->expected_attach_type) {
16029+
case BPF_TRACE_KPROBE_SESSION:
16030+
case BPF_TRACE_UPROBE_SESSION:
16031+
range = retval_range(0, 1);
16032+
break;
16033+
default:
16034+
return 0;
16035+
}
16036+
break;
1602716037
case BPF_PROG_TYPE_SK_LOOKUP:
1602816038
range = retval_range(SK_DROP, SK_PASS);
1602916039
break;

‎kernel/trace/bpf_trace.c‎

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1581,6 +1581,17 @@ static inline bool is_kprobe_session(const struct bpf_prog *prog)
15811581
return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
15821582
}
15831583

1584+
static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1585+
{
1586+
return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1587+
prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1588+
}
1589+
1590+
static inline bool is_uprobe_session(const struct bpf_prog *prog)
1591+
{
1592+
return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1593+
}
1594+
15841595
static const struct bpf_func_proto *
15851596
kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
15861597
{
@@ -1598,13 +1609,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
15981609
case BPF_FUNC_get_func_ip:
15991610
if (is_kprobe_multi(prog))
16001611
return &bpf_get_func_ip_proto_kprobe_multi;
1601-
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1612+
if (is_uprobe_multi(prog))
16021613
return &bpf_get_func_ip_proto_uprobe_multi;
16031614
return &bpf_get_func_ip_proto_kprobe;
16041615
case BPF_FUNC_get_attach_cookie:
16051616
if (is_kprobe_multi(prog))
16061617
return &bpf_get_attach_cookie_proto_kmulti;
1607-
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1618+
if (is_uprobe_multi(prog))
16081619
return &bpf_get_attach_cookie_proto_umulti;
16091620
return &bpf_get_attach_cookie_proto_trace;
16101621
default:
@@ -3096,6 +3107,7 @@ struct bpf_uprobe {
30963107
u64 cookie;
30973108
struct uprobe *uprobe;
30983109
struct uprobe_consumer consumer;
3110+
bool session;
30993111
};
31003112

31013113
struct bpf_uprobe_multi_link {
@@ -3108,7 +3120,7 @@ struct bpf_uprobe_multi_link {
31083120
};
31093121

31103122
struct bpf_uprobe_multi_run_ctx {
3111-
struct bpf_run_ctx run_ctx;
3123+
struct bpf_session_run_ctx session_ctx;
31123124
unsigned long entry_ip;
31133125
struct bpf_uprobe *uprobe;
31143126
};
@@ -3219,17 +3231,22 @@ static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
32193231

32203232
static int uprobe_prog_run(struct bpf_uprobe *uprobe,
32213233
unsigned long entry_ip,
3222-
struct pt_regs *regs)
3234+
struct pt_regs *regs,
3235+
bool is_return, void *data)
32233236
{
32243237
struct bpf_uprobe_multi_link *link = uprobe->link;
32253238
struct bpf_uprobe_multi_run_ctx run_ctx = {
3239+
.session_ctx = {
3240+
.is_return = is_return,
3241+
.data = data,
3242+
},
32263243
.entry_ip = entry_ip,
32273244
.uprobe = uprobe,
32283245
};
32293246
struct bpf_prog *prog = link->link.prog;
32303247
bool sleepable = prog->sleepable;
32313248
struct bpf_run_ctx *old_run_ctx;
3232-
int err = 0;
3249+
int err;
32333250

32343251
if (link->task && !same_thread_group(current, link->task))
32353252
return 0;
@@ -3241,7 +3258,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe,
32413258

32423259
migrate_disable();
32433260

3244-
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
3261+
old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
32453262
err = bpf_prog_run(link->link.prog, regs);
32463263
bpf_reset_run_ctx(old_run_ctx);
32473264

@@ -3268,9 +3285,13 @@ uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs,
32683285
__u64 *data)
32693286
{
32703287
struct bpf_uprobe *uprobe;
3288+
int ret;
32713289

32723290
uprobe = container_of(con, struct bpf_uprobe, consumer);
3273-
return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
3291+
ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3292+
if (uprobe->session)
3293+
return ret ? UPROBE_HANDLER_IGNORE : 0;
3294+
return 0;
32743295
}
32753296

32763297
static int
@@ -3280,22 +3301,25 @@ uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, s
32803301
struct bpf_uprobe *uprobe;
32813302

32823303
uprobe = container_of(con, struct bpf_uprobe, consumer);
3283-
return uprobe_prog_run(uprobe, func, regs);
3304+
uprobe_prog_run(uprobe, func, regs, true, data);
3305+
return 0;
32843306
}
32853307

32863308
static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
32873309
{
32883310
struct bpf_uprobe_multi_run_ctx *run_ctx;
32893311

3290-
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3312+
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3313+
session_ctx.run_ctx);
32913314
return run_ctx->entry_ip;
32923315
}
32933316

32943317
static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
32953318
{
32963319
struct bpf_uprobe_multi_run_ctx *run_ctx;
32973320

3298-
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3321+
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3322+
session_ctx.run_ctx);
32993323
return run_ctx->uprobe->cookie;
33003324
}
33013325

@@ -3319,7 +3343,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
33193343
if (sizeof(u64) != sizeof(void *))
33203344
return -EOPNOTSUPP;
33213345

3322-
if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
3346+
if (!is_uprobe_multi(prog))
33233347
return -EINVAL;
33243348

33253349
flags = attr->link_create.uprobe_multi.flags;
@@ -3395,11 +3419,12 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
33953419

33963420
uprobes[i].link = link;
33973421

3398-
if (flags & BPF_F_UPROBE_MULTI_RETURN)
3399-
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3400-
else
3422+
if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
34013423
uprobes[i].consumer.handler = uprobe_multi_link_handler;
3402-
3424+
if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3425+
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3426+
if (is_uprobe_session(prog))
3427+
uprobes[i].session = true;
34033428
if (pid)
34043429
uprobes[i].consumer.filter = uprobe_multi_link_filter;
34053430
}
@@ -3488,7 +3513,7 @@ static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id)
34883513
if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
34893514
return 0;
34903515

3491-
if (!is_kprobe_session(prog))
3516+
if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
34923517
return -EACCES;
34933518

34943519
return 0;

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1116,6 +1116,7 @@ enum bpf_attach_type {
11161116
BPF_NETKIT_PRIMARY,
11171117
BPF_NETKIT_PEER,
11181118
BPF_TRACE_KPROBE_SESSION,
1119+
BPF_TRACE_UPROBE_SESSION,
11191120
__MAX_BPF_ATTACH_TYPE
11201121
};
11211122

‎tools/lib/bpf/bpf.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,7 @@ int bpf_link_create(int prog_fd, int target_fd,
776776
return libbpf_err(-EINVAL);
777777
break;
778778
case BPF_TRACE_UPROBE_MULTI:
779+
case BPF_TRACE_UPROBE_SESSION:
779780
attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
780781
attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
781782
attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));

‎tools/lib/bpf/libbpf.c‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ static const char * const attach_type_name[] = {
133133
[BPF_NETKIT_PRIMARY] = "netkit_primary",
134134
[BPF_NETKIT_PEER] = "netkit_peer",
135135
[BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
136+
[BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session",
136137
};
137138

138139
static const char * const link_type_name[] = {
@@ -9441,8 +9442,10 @@ static const struct bpf_sec_def section_defs[] = {
94419442
SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
94429443
SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
94439444
SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9445+
SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi),
94449446
SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
94459447
SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9448+
SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi),
94469449
SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
94479450
SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
94489451
SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
@@ -11764,7 +11767,9 @@ static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, stru
1176411767
ret = 0;
1176511768
break;
1176611769
case 3:
11770+
opts.session = str_has_pfx(probe_type, "uprobe.session");
1176711771
opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");
11772+
1176811773
*link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
1176911774
ret = libbpf_get_error(*link);
1177011775
break;
@@ -12013,10 +12018,12 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
1201312018
const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
1201412019
LIBBPF_OPTS(bpf_link_create_opts, lopts);
1201512020
unsigned long *resolved_offsets = NULL;
12021+
enum bpf_attach_type attach_type;
1201612022
int err = 0, link_fd, prog_fd;
1201712023
struct bpf_link *link = NULL;
1201812024
char errmsg[STRERR_BUFSIZE];
1201912025
char full_path[PATH_MAX];
12026+
bool retprobe, session;
1202012027
const __u64 *cookies;
1202112028
const char **syms;
1202212029
size_t cnt;
@@ -12087,12 +12094,20 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
1208712094
offsets = resolved_offsets;
1208812095
}
1208912096

12097+
retprobe = OPTS_GET(opts, retprobe, false);
12098+
session = OPTS_GET(opts, session, false);
12099+
12100+
if (retprobe && session)
12101+
return libbpf_err_ptr(-EINVAL);
12102+
12103+
attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI;
12104+
1209012105
lopts.uprobe_multi.path = path;
1209112106
lopts.uprobe_multi.offsets = offsets;
1209212107
lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
1209312108
lopts.uprobe_multi.cookies = cookies;
1209412109
lopts.uprobe_multi.cnt = cnt;
12095-
lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;
12110+
lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0;
1209612111

1209712112
if (pid == 0)
1209812113
pid = getpid();
@@ -12106,7 +12121,7 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
1210612121
}
1210712122
link->detach = &bpf_link__detach_fd;
1210812123

12109-
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
12124+
link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
1211012125
if (link_fd < 0) {
1211112126
err = -errno;
1211212127
pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",

‎tools/lib/bpf/libbpf.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -577,10 +577,12 @@ struct bpf_uprobe_multi_opts {
577577
size_t cnt;
578578
/* create return uprobes */
579579
bool retprobe;
580+
/* create session kprobes */
581+
bool session;
580582
size_t :0;
581583
};
582584

583-
#define bpf_uprobe_multi_opts__last_field retprobe
585+
#define bpf_uprobe_multi_opts__last_field session
584586

585587
/**
586588
* @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program

‎tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "kprobe_multi_override.skel.h"
77
#include "kprobe_multi_session.skel.h"
88
#include "kprobe_multi_session_cookie.skel.h"
9+
#include "kprobe_multi_verifier.skel.h"
910
#include "bpf/libbpf_internal.h"
1011
#include "bpf/hashmap.h"
1112

@@ -764,4 +765,5 @@ void test_kprobe_multi_test(void)
764765
test_session_skel_api();
765766
if (test__start_subtest("session_cookie"))
766767
test_session_cookie_skel_api();
768+
RUN_TESTS(kprobe_multi_verifier);
767769
}

0 commit comments

Comments
 (0)