Skip to content

Commit 97e03f5

Browse files
joannekoonganakryiko
authored andcommitted
bpf: Add verifier support for dynptrs
This patch adds the bulk of the verifier work for supporting dynamic pointers (dynptrs) in bpf. A bpf_dynptr is opaque to the bpf program. It is a 16-byte structure defined internally as: struct bpf_dynptr_kern { void *data; u32 size; u32 offset; } __aligned(8); The upper 8 bits of *size* is reserved (it contains extra metadata about read-only status and dynptr type). Consequently, a dynptr only supports memory less than 16 MB. There are different types of dynptrs (eg malloc, ringbuf, ...). In this patchset, the most basic one, dynptrs to a bpf program's local memory, is added. For now only local memory that is of reg type PTR_TO_MAP_VALUE is supported. In the verifier, dynptr state information will be tracked in stack slots. When the program passes in an uninitialized dynptr (ARG_PTR_TO_DYNPTR | MEM_UNINIT), the stack slots corresponding to the frame pointer where the dynptr resides at are marked STACK_DYNPTR. For helper functions that take in initialized dynptrs (eg bpf_dynptr_read + bpf_dynptr_write which are added later in this patchset), the verifier enforces that the dynptr has been initialized properly by checking that their corresponding stack slots have been marked as STACK_DYNPTR. The 6th patch in this patchset adds test cases that the verifier should successfully reject, such as for example attempting to use a dynptr after doing a direct write into it inside the bpf program. Signed-off-by: Joanne Koong <[email protected]> Signed-off-by: Andrii Nakryiko <[email protected]> Acked-by: Andrii Nakryiko <[email protected]> Acked-by: David Vernet <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 1ec5ee8 commit 97e03f5

File tree

6 files changed

+243
-3
lines changed

6 files changed

+243
-3
lines changed

include/linux/bpf.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,10 +392,15 @@ enum bpf_type_flag {
392392

393393
MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
394394

395+
/* DYNPTR points to memory local to the bpf program. */
396+
DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS),
397+
395398
__BPF_TYPE_FLAG_MAX,
396399
__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
397400
};
398401

402+
#define DYNPTR_TYPE_FLAG_MASK DYNPTR_TYPE_LOCAL
403+
399404
/* Max number of base types. */
400405
#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
401406

@@ -438,6 +443,7 @@ enum bpf_arg_type {
438443
ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
439444
ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
440445
ARG_PTR_TO_KPTR, /* pointer to referenced kptr */
446+
ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
441447
__BPF_ARG_TYPE_MAX,
442448

443449
/* Extended arg_types. */
@@ -2376,4 +2382,26 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
23762382
u32 **bin_buf, u32 num_args);
23772383
void bpf_bprintf_cleanup(void);
23782384

2385+
/* the implementation of the opaque uapi struct bpf_dynptr */
2386+
struct bpf_dynptr_kern {
2387+
void *data;
2388+
/* Size represents the number of usable bytes of dynptr data.
2389+
* If for example the offset is at 4 for a local dynptr whose data is
2390+
* of type u64, the number of usable bytes is 4.
2391+
*
2392+
* The upper 8 bits are reserved. It is as follows:
2393+
* Bits 0 - 23 = size
2394+
* Bits 24 - 30 = dynptr type
2395+
* Bit 31 = whether dynptr is read-only
2396+
*/
2397+
u32 size;
2398+
u32 offset;
2399+
} __aligned(8);
2400+
2401+
enum bpf_dynptr_type {
2402+
BPF_DYNPTR_TYPE_INVALID,
2403+
/* Points to memory that is local to the bpf program */
2404+
BPF_DYNPTR_TYPE_LOCAL,
2405+
};
2406+
23792407
#endif /* _LINUX_BPF_H */

include/linux/bpf_verifier.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,18 @@ struct bpf_reg_state {
7272

7373
u32 mem_size; /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */
7474

75+
/* For dynptr stack slots */
76+
struct {
77+
enum bpf_dynptr_type type;
78+
/* A dynptr is 16 bytes so it takes up 2 stack slots.
79+
* We need to track which slot is the first slot
80+
* to protect against cases where the user may try to
81+
* pass in an address starting at the second slot of the
82+
* dynptr.
83+
*/
84+
bool first_slot;
85+
} dynptr;
86+
7587
/* Max size from any of the above. */
7688
struct {
7789
unsigned long raw1;
@@ -174,9 +186,15 @@ enum bpf_stack_slot_type {
174186
STACK_SPILL, /* register spilled into stack */
175187
STACK_MISC, /* BPF program wrote some data into this slot */
176188
STACK_ZERO, /* BPF program wrote constant zero */
189+
/* A dynptr is stored in this stack slot. The type of dynptr
190+
* is stored in bpf_stack_state->spilled_ptr.dynptr.type
191+
*/
192+
STACK_DYNPTR,
177193
};
178194

179195
#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */
196+
#define BPF_DYNPTR_SIZE sizeof(struct bpf_dynptr_kern)
197+
#define BPF_DYNPTR_NR_SLOTS (BPF_DYNPTR_SIZE / BPF_REG_SIZE)
180198

181199
struct bpf_stack_state {
182200
struct bpf_reg_state spilled_ptr;

include/uapi/linux/bpf.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6528,6 +6528,11 @@ struct bpf_timer {
65286528
__u64 :64;
65296529
} __attribute__((aligned(8)));
65306530

6531+
struct bpf_dynptr {
6532+
__u64 :64;
6533+
__u64 :64;
6534+
} __attribute__((aligned(8)));
6535+
65316536
struct bpf_sysctl {
65326537
__u32 write; /* Sysctl is being read (= 0) or written (= 1).
65336538
* Allows 1,2,4-byte read, but no write.

kernel/bpf/verifier.c

Lines changed: 185 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ struct bpf_call_arg_meta {
259259
u32 ret_btf_id;
260260
u32 subprogno;
261261
struct bpf_map_value_off_desc *kptr_off_desc;
262+
u8 uninit_dynptr_regno;
262263
};
263264

264265
struct btf *btf_vmlinux;
@@ -581,6 +582,7 @@ static char slot_type_char[] = {
581582
[STACK_SPILL] = 'r',
582583
[STACK_MISC] = 'm',
583584
[STACK_ZERO] = '0',
585+
[STACK_DYNPTR] = 'd',
584586
};
585587

586588
static void print_liveness(struct bpf_verifier_env *env,
@@ -596,6 +598,25 @@ static void print_liveness(struct bpf_verifier_env *env,
596598
verbose(env, "D");
597599
}
598600

601+
static int get_spi(s32 off)
602+
{
603+
return (-off - 1) / BPF_REG_SIZE;
604+
}
605+
606+
static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
607+
{
608+
int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
609+
610+
/* We need to check that slots between [spi - nr_slots + 1, spi] are
611+
* within [0, allocated_stack).
612+
*
613+
* Please note that the spi grows downwards. For example, a dynptr
614+
* takes the size of two stack slots; the first slot will be at
615+
* spi and the second slot will be at spi - 1.
616+
*/
617+
return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
618+
}
619+
599620
static struct bpf_func_state *func(struct bpf_verifier_env *env,
600621
const struct bpf_reg_state *reg)
601622
{
@@ -647,6 +668,108 @@ static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
647668
env->scratched_stack_slots = ~0ULL;
648669
}
649670

671+
static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
672+
{
673+
switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
674+
case DYNPTR_TYPE_LOCAL:
675+
return BPF_DYNPTR_TYPE_LOCAL;
676+
default:
677+
return BPF_DYNPTR_TYPE_INVALID;
678+
}
679+
}
680+
681+
static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
682+
enum bpf_arg_type arg_type, int insn_idx)
683+
{
684+
struct bpf_func_state *state = func(env, reg);
685+
enum bpf_dynptr_type type;
686+
int spi, i;
687+
688+
spi = get_spi(reg->off);
689+
690+
if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
691+
return -EINVAL;
692+
693+
for (i = 0; i < BPF_REG_SIZE; i++) {
694+
state->stack[spi].slot_type[i] = STACK_DYNPTR;
695+
state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
696+
}
697+
698+
type = arg_to_dynptr_type(arg_type);
699+
if (type == BPF_DYNPTR_TYPE_INVALID)
700+
return -EINVAL;
701+
702+
state->stack[spi].spilled_ptr.dynptr.first_slot = true;
703+
state->stack[spi].spilled_ptr.dynptr.type = type;
704+
state->stack[spi - 1].spilled_ptr.dynptr.type = type;
705+
706+
return 0;
707+
}
708+
709+
static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
710+
{
711+
struct bpf_func_state *state = func(env, reg);
712+
int spi, i;
713+
714+
spi = get_spi(reg->off);
715+
716+
if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
717+
return -EINVAL;
718+
719+
for (i = 0; i < BPF_REG_SIZE; i++) {
720+
state->stack[spi].slot_type[i] = STACK_INVALID;
721+
state->stack[spi - 1].slot_type[i] = STACK_INVALID;
722+
}
723+
724+
state->stack[spi].spilled_ptr.dynptr.first_slot = false;
725+
state->stack[spi].spilled_ptr.dynptr.type = 0;
726+
state->stack[spi - 1].spilled_ptr.dynptr.type = 0;
727+
728+
return 0;
729+
}
730+
731+
static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
732+
{
733+
struct bpf_func_state *state = func(env, reg);
734+
int spi = get_spi(reg->off);
735+
int i;
736+
737+
if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
738+
return true;
739+
740+
for (i = 0; i < BPF_REG_SIZE; i++) {
741+
if (state->stack[spi].slot_type[i] == STACK_DYNPTR ||
742+
state->stack[spi - 1].slot_type[i] == STACK_DYNPTR)
743+
return false;
744+
}
745+
746+
return true;
747+
}
748+
749+
static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
750+
enum bpf_arg_type arg_type)
751+
{
752+
struct bpf_func_state *state = func(env, reg);
753+
int spi = get_spi(reg->off);
754+
int i;
755+
756+
if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) ||
757+
!state->stack[spi].spilled_ptr.dynptr.first_slot)
758+
return false;
759+
760+
for (i = 0; i < BPF_REG_SIZE; i++) {
761+
if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
762+
state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
763+
return false;
764+
}
765+
766+
/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
767+
if (arg_type == ARG_PTR_TO_DYNPTR)
768+
return true;
769+
770+
return state->stack[spi].spilled_ptr.dynptr.type == arg_to_dynptr_type(arg_type);
771+
}
772+
650773
/* The reg state of a pointer or a bounded scalar was saved when
651774
* it was spilled to the stack.
652775
*/
@@ -5400,6 +5523,11 @@ static bool arg_type_is_release(enum bpf_arg_type type)
54005523
return type & OBJ_RELEASE;
54015524
}
54025525

5526+
static bool arg_type_is_dynptr(enum bpf_arg_type type)
5527+
{
5528+
return base_type(type) == ARG_PTR_TO_DYNPTR;
5529+
}
5530+
54035531
static int int_ptr_type_to_size(enum bpf_arg_type type)
54045532
{
54055533
if (type == ARG_PTR_TO_INT)
@@ -5539,6 +5667,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
55395667
[ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
55405668
[ARG_PTR_TO_TIMER] = &timer_types,
55415669
[ARG_PTR_TO_KPTR] = &kptr_types,
5670+
[ARG_PTR_TO_DYNPTR] = &stack_ptr_types,
55425671
};
55435672

55445673
static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -5628,8 +5757,13 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env,
56285757
bool fixed_off_ok = false;
56295758

56305759
switch ((u32)type) {
5631-
case SCALAR_VALUE:
56325760
/* Pointer types where reg offset is explicitly allowed: */
5761+
case PTR_TO_STACK:
5762+
if (arg_type_is_dynptr(arg_type) && reg->off % BPF_REG_SIZE) {
5763+
verbose(env, "cannot pass in dynptr at an offset\n");
5764+
return -EINVAL;
5765+
}
5766+
fallthrough;
56335767
case PTR_TO_PACKET:
56345768
case PTR_TO_PACKET_META:
56355769
case PTR_TO_MAP_KEY:
@@ -5639,7 +5773,7 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env,
56395773
case PTR_TO_MEM | MEM_ALLOC:
56405774
case PTR_TO_BUF:
56415775
case PTR_TO_BUF | MEM_RDONLY:
5642-
case PTR_TO_STACK:
5776+
case SCALAR_VALUE:
56435777
/* Some of the argument types nevertheless require a
56445778
* zero register offset.
56455779
*/
@@ -5837,6 +5971,36 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
58375971
bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
58385972

58395973
err = check_mem_size_reg(env, reg, regno, zero_size_allowed, meta);
5974+
} else if (arg_type_is_dynptr(arg_type)) {
5975+
if (arg_type & MEM_UNINIT) {
5976+
if (!is_dynptr_reg_valid_uninit(env, reg)) {
5977+
verbose(env, "Dynptr has to be an uninitialized dynptr\n");
5978+
return -EINVAL;
5979+
}
5980+
5981+
/* We only support one dynptr being uninitialized at the moment,
5982+
* which is sufficient for the helper functions we have right now.
5983+
*/
5984+
if (meta->uninit_dynptr_regno) {
5985+
verbose(env, "verifier internal error: multiple uninitialized dynptr args\n");
5986+
return -EFAULT;
5987+
}
5988+
5989+
meta->uninit_dynptr_regno = regno;
5990+
} else if (!is_dynptr_reg_valid_init(env, reg, arg_type)) {
5991+
const char *err_extra = "";
5992+
5993+
switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
5994+
case DYNPTR_TYPE_LOCAL:
5995+
err_extra = "local ";
5996+
break;
5997+
default:
5998+
break;
5999+
}
6000+
verbose(env, "Expected an initialized %sdynptr as arg #%d\n",
6001+
err_extra, arg + 1);
6002+
return -EINVAL;
6003+
}
58406004
} else if (arg_type_is_alloc_size(arg_type)) {
58416005
if (!tnum_is_const(reg->var_off)) {
58426006
verbose(env, "R%d is not a known constant'\n",
@@ -6970,9 +7134,27 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
69707134

69717135
regs = cur_regs(env);
69727136

7137+
if (meta.uninit_dynptr_regno) {
7138+
/* we write BPF_DW bits (8 bytes) at a time */
7139+
for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
7140+
err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno,
7141+
i, BPF_DW, BPF_WRITE, -1, false);
7142+
if (err)
7143+
return err;
7144+
}
7145+
7146+
err = mark_stack_slots_dynptr(env, &regs[meta.uninit_dynptr_regno],
7147+
fn->arg_type[meta.uninit_dynptr_regno - BPF_REG_1],
7148+
insn_idx);
7149+
if (err)
7150+
return err;
7151+
}
7152+
69737153
if (meta.release_regno) {
69747154
err = -EINVAL;
6975-
if (meta.ref_obj_id)
7155+
if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1]))
7156+
err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
7157+
else if (meta.ref_obj_id)
69767158
err = release_reference(env, meta.ref_obj_id);
69777159
/* meta.ref_obj_id can only be 0 if register that is meant to be
69787160
* released is NULL, which must be > R0.

scripts/bpf_doc.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,7 @@ def __init__(self, parser):
634634
'struct file',
635635
'struct bpf_timer',
636636
'struct mptcp_sock',
637+
'struct bpf_dynptr',
637638
]
638639
known_types = {
639640
'...',
@@ -684,6 +685,7 @@ def __init__(self, parser):
684685
'struct file',
685686
'struct bpf_timer',
686687
'struct mptcp_sock',
688+
'struct bpf_dynptr',
687689
}
688690
mapped_types = {
689691
'u8': '__u8',

tools/include/uapi/linux/bpf.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6528,6 +6528,11 @@ struct bpf_timer {
65286528
__u64 :64;
65296529
} __attribute__((aligned(8)));
65306530

6531+
struct bpf_dynptr {
6532+
__u64 :64;
6533+
__u64 :64;
6534+
} __attribute__((aligned(8)));
6535+
65316536
struct bpf_sysctl {
65326537
__u32 write; /* Sysctl is being read (= 0) or written (= 1).
65336538
* Allows 1,2,4-byte read, but no write.

0 commit comments

Comments
 (0)