Skip to content

Commit b2ff42c

Browse files
kuba-mookuba-moo
committed
Daniel Borkmann says: ==================== pull-request: bpf 2024-04-26 We've added 12 non-merge commits during the last 22 day(s) which contain a total of 14 files changed, 168 insertions(+), 72 deletions(-). The main changes are: 1) Fix BPF_PROBE_MEM in verifier and JIT to skip loads from vsyscall page, from Puranjay Mohan. 2) Fix a crash in XDP with devmap broadcast redirect when the latter map is in process of being torn down, from Toke Høiland-Jørgensen. 3) Fix arm64 and riscv64 BPF JITs to properly clear start time for BPF program runtime stats, from Xu Kuohai. 4) Fix a sockmap KCSAN-reported data race in sk_psock_skb_ingress_enqueue, from Jason Xing. 5) Fix BPF verifier error message in resolve_pseudo_ldimm64, from Anton Protopopov. 6) Fix missing DEBUG_INFO_BTF_MODULES Kconfig menu item, from Andrii Nakryiko. * tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: selftests/bpf: Test PROBE_MEM of VSYSCALL_ADDR on x86-64 bpf, x86: Fix PROBE_MEM runtime load check bpf: verifier: prevent userspace memory access xdp: use flags field to disambiguate broadcast redirect arm32, bpf: Reimplement sign-extension mov instruction riscv, bpf: Fix incorrect runtime stats bpf, arm64: Fix incorrect runtime stats bpf: Fix a verifier verbose message bpf, skmsg: Fix NULL pointer dereference in sk_psock_skb_ingress_enqueue MAINTAINERS: bpf: Add Lehui and Puranjay as riscv64 reviewers MAINTAINERS: Update email address for Puranjay Mohan bpf, kconfig: Fix DEBUG_INFO_BTF_MODULES Kconfig definition ==================== Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Jakub Kicinski <[email protected]>
2 parents 6a30653 + a86538a commit b2ff42c

File tree

14 files changed

+168
-72
lines changed

14 files changed

+168
-72
lines changed

‎.mailmap

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,7 @@ Praveen BP <[email protected]>
512512
Pradeep Kumar Chitrapu <[email protected]> <[email protected]>
513513
Prasad Sodagudi <[email protected]> <[email protected]>
514514
Punit Agrawal <[email protected]> <[email protected]>
515+
Puranjay Mohan <[email protected]> <[email protected]>
515516
Qais Yousef <[email protected]> <[email protected]>
516517
Qais Yousef <[email protected]> <[email protected]>
517518
Quentin Monnet <[email protected]> <[email protected]>

‎MAINTAINERS

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ F: Documentation/devicetree/bindings/iio/accel/adi,adxl345.yaml
553553
F: drivers/input/misc/adxl34x.c
554554

555555
ADXL355 THREE-AXIS DIGITAL ACCELEROMETER DRIVER
556-
M: Puranjay Mohan <[email protected]>
556+
M: Puranjay Mohan <[email protected]>
557557
L: [email protected]
558558
S: Supported
559559
F: Documentation/devicetree/bindings/iio/accel/adi,adxl355.yaml
@@ -3714,7 +3714,7 @@ F: drivers/iio/imu/bmi323/
37143714

37153715
BPF JIT for ARM
37163716
M: Russell King <[email protected]>
3717-
M: Puranjay Mohan <[email protected]>
3717+
M: Puranjay Mohan <[email protected]>
37183718
L: [email protected]
37193719
S: Maintained
37203720
F: arch/arm/net/
@@ -3764,6 +3764,8 @@ X: arch/riscv/net/bpf_jit_comp64.c
37643764

37653765
BPF JIT for RISC-V (64-bit)
37663766
M: Björn Töpel <[email protected]>
3767+
R: Pu Lehui <[email protected]>
3768+
R: Puranjay Mohan <[email protected]>
37673769
L: [email protected]
37683770
S: Maintained
37693771
F: arch/riscv/net/
@@ -21925,7 +21927,7 @@ F: include/linux/soc/ti/ti_sci_inta_msi.h
2192521927
F: include/linux/soc/ti/ti_sci_protocol.h
2192621928

2192721929
TEXAS INSTRUMENTS' TMP117 TEMPERATURE SENSOR DRIVER
21928-
M: Puranjay Mohan <[email protected]>
21930+
M: Puranjay Mohan <[email protected]>
2192921931
L: [email protected]
2193021932
S: Supported
2193121933
F: Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml

‎arch/arm/net/bpf_jit_32.c

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -871,16 +871,11 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
871871
}
872872

873873
/* dst = src (4 bytes)*/
874-
static inline void emit_a32_mov_r(const s8 dst, const s8 src, const u8 off,
875-
struct jit_ctx *ctx) {
874+
static inline void emit_a32_mov_r(const s8 dst, const s8 src, struct jit_ctx *ctx) {
876875
const s8 *tmp = bpf2a32[TMP_REG_1];
877876
s8 rt;
878877

879878
rt = arm_bpf_get_reg32(src, tmp[0], ctx);
880-
if (off && off != 32) {
881-
emit(ARM_LSL_I(rt, rt, 32 - off), ctx);
882-
emit(ARM_ASR_I(rt, rt, 32 - off), ctx);
883-
}
884879
arm_bpf_put_reg32(dst, rt, ctx);
885880
}
886881

@@ -889,15 +884,15 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
889884
const s8 src[],
890885
struct jit_ctx *ctx) {
891886
if (!is64) {
892-
emit_a32_mov_r(dst_lo, src_lo, 0, ctx);
887+
emit_a32_mov_r(dst_lo, src_lo, ctx);
893888
if (!ctx->prog->aux->verifier_zext)
894889
/* Zero out high 4 bytes */
895890
emit_a32_mov_i(dst_hi, 0, ctx);
896891
} else if (__LINUX_ARM_ARCH__ < 6 &&
897892
ctx->cpu_architecture < CPU_ARCH_ARMv5TE) {
898893
/* complete 8 byte move */
899-
emit_a32_mov_r(dst_lo, src_lo, 0, ctx);
900-
emit_a32_mov_r(dst_hi, src_hi, 0, ctx);
894+
emit_a32_mov_r(dst_lo, src_lo, ctx);
895+
emit_a32_mov_r(dst_hi, src_hi, ctx);
901896
} else if (is_stacked(src_lo) && is_stacked(dst_lo)) {
902897
const u8 *tmp = bpf2a32[TMP_REG_1];
903898

@@ -917,17 +912,52 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
917912
static inline void emit_a32_movsx_r64(const bool is64, const u8 off, const s8 dst[], const s8 src[],
918913
struct jit_ctx *ctx) {
919914
const s8 *tmp = bpf2a32[TMP_REG_1];
920-
const s8 *rt;
915+
s8 rs;
916+
s8 rd;
921917

922-
rt = arm_bpf_get_reg64(dst, tmp, ctx);
918+
if (is_stacked(dst_lo))
919+
rd = tmp[1];
920+
else
921+
rd = dst_lo;
922+
rs = arm_bpf_get_reg32(src_lo, rd, ctx);
923+
/* rs may be one of src[1], dst[1], or tmp[1] */
924+
925+
/* Sign extend rs if needed. If off == 32, lower 32-bits of src are moved to dst and sign
926+
* extension only happens in the upper 64 bits.
927+
*/
928+
if (off != 32) {
929+
/* Sign extend rs into rd */
930+
emit(ARM_LSL_I(rd, rs, 32 - off), ctx);
931+
emit(ARM_ASR_I(rd, rd, 32 - off), ctx);
932+
} else {
933+
rd = rs;
934+
}
935+
936+
/* Write rd to dst_lo
937+
*
938+
* Optimization:
939+
* Assume:
940+
* 1. dst == src and stacked.
941+
* 2. off == 32
942+
*
943+
* In this case src_lo was loaded into rd(tmp[1]) but rd was not sign extended as off==32.
944+
* So, we don't need to write rd back to dst_lo as they have the same value.
945+
* This saves us one str instruction.
946+
*/
947+
if (dst_lo != src_lo || off != 32)
948+
arm_bpf_put_reg32(dst_lo, rd, ctx);
923949

924-
emit_a32_mov_r(dst_lo, src_lo, off, ctx);
925950
if (!is64) {
926951
if (!ctx->prog->aux->verifier_zext)
927952
/* Zero out high 4 bytes */
928953
emit_a32_mov_i(dst_hi, 0, ctx);
929954
} else {
930-
emit(ARM_ASR_I(rt[0], rt[1], 31), ctx);
955+
if (is_stacked(dst_hi)) {
956+
emit(ARM_ASR_I(tmp[0], rd, 31), ctx);
957+
arm_bpf_put_reg32(dst_hi, tmp[0], ctx);
958+
} else {
959+
emit(ARM_ASR_I(dst_hi, rd, 31), ctx);
960+
}
931961
}
932962
}
933963

‎arch/arm64/net/bpf_jit_comp.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1844,15 +1844,15 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
18441844

18451845
emit_call(enter_prog, ctx);
18461846

1847+
/* save return value to callee saved register x20 */
1848+
emit(A64_MOV(1, A64_R(20), A64_R(0)), ctx);
1849+
18471850
/* if (__bpf_prog_enter(prog) == 0)
18481851
* goto skip_exec_of_prog;
18491852
*/
18501853
branch = ctx->image + ctx->idx;
18511854
emit(A64_NOP, ctx);
18521855

1853-
/* save return value to callee saved register x20 */
1854-
emit(A64_MOV(1, A64_R(20), A64_R(0)), ctx);
1855-
18561856
emit(A64_ADD_I(1, A64_R(0), A64_SP, args_off), ctx);
18571857
if (!p->jited)
18581858
emit_addr_mov_i64(A64_R(1), (const u64)p->insnsi, ctx);

‎arch/riscv/net/bpf_jit_comp64.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -722,16 +722,16 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of
722722
if (ret)
723723
return ret;
724724

725+
/* store prog start time */
726+
emit_mv(RV_REG_S1, RV_REG_A0, ctx);
727+
725728
/* if (__bpf_prog_enter(prog) == 0)
726729
* goto skip_exec_of_prog;
727730
*/
728731
branch_off = ctx->ninsns;
729732
/* nop reserved for conditional jump */
730733
emit(rv_nop(), ctx);
731734

732-
/* store prog start time */
733-
emit_mv(RV_REG_S1, RV_REG_A0, ctx);
734-
735735
/* arg1: &args_off */
736736
emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx);
737737
if (!p->jited)

‎arch/x86/net/bpf_jit_comp.c

Lines changed: 31 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1807,36 +1807,41 @@ st: if (is_imm8(insn->off))
18071807
if (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
18081808
BPF_MODE(insn->code) == BPF_PROBE_MEMSX) {
18091809
/* Conservatively check that src_reg + insn->off is a kernel address:
1810-
* src_reg + insn->off >= TASK_SIZE_MAX + PAGE_SIZE
1811-
* src_reg is used as scratch for src_reg += insn->off and restored
1812-
* after emit_ldx if necessary
1810+
* src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE
1811+
* and
1812+
* src_reg + insn->off < VSYSCALL_ADDR
18131813
*/
18141814

1815-
u64 limit = TASK_SIZE_MAX + PAGE_SIZE;
1815+
u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR;
18161816
u8 *end_of_jmp;
18171817

1818-
/* At end of these emitted checks, insn->off will have been added
1819-
* to src_reg, so no need to do relative load with insn->off offset
1820-
*/
1821-
insn_off = 0;
1818+
/* movabsq r10, VSYSCALL_ADDR */
1819+
emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32,
1820+
(u32)(long)VSYSCALL_ADDR);
18221821

1823-
/* movabsq r11, limit */
1824-
EMIT2(add_1mod(0x48, AUX_REG), add_1reg(0xB8, AUX_REG));
1825-
EMIT((u32)limit, 4);
1826-
EMIT(limit >> 32, 4);
1822+
/* mov src_reg, r11 */
1823+
EMIT_mov(AUX_REG, src_reg);
18271824

18281825
if (insn->off) {
1829-
/* add src_reg, insn->off */
1830-
maybe_emit_1mod(&prog, src_reg, true);
1831-
EMIT2_off32(0x81, add_1reg(0xC0, src_reg), insn->off);
1826+
/* add r11, insn->off */
1827+
maybe_emit_1mod(&prog, AUX_REG, true);
1828+
EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off);
18321829
}
18331830

1834-
/* cmp src_reg, r11 */
1835-
maybe_emit_mod(&prog, src_reg, AUX_REG, true);
1836-
EMIT2(0x39, add_2reg(0xC0, src_reg, AUX_REG));
1831+
/* sub r11, r10 */
1832+
maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true);
1833+
EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX));
1834+
1835+
/* movabsq r10, limit */
1836+
emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32,
1837+
(u32)(long)limit);
1838+
1839+
/* cmp r10, r11 */
1840+
maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true);
1841+
EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX));
18371842

1838-
/* if unsigned '>=', goto load */
1839-
EMIT2(X86_JAE, 0);
1843+
/* if unsigned '>', goto load */
1844+
EMIT2(X86_JA, 0);
18401845
end_of_jmp = prog;
18411846

18421847
/* xor dst_reg, dst_reg */
@@ -1862,18 +1867,6 @@ st: if (is_imm8(insn->off))
18621867
/* populate jmp_offset for JMP above */
18631868
start_of_ldx[-1] = prog - start_of_ldx;
18641869

1865-
if (insn->off && src_reg != dst_reg) {
1866-
/* sub src_reg, insn->off
1867-
* Restore src_reg after "add src_reg, insn->off" in prev
1868-
* if statement. But if src_reg == dst_reg, emit_ldx
1869-
* above already clobbered src_reg, so no need to restore.
1870-
* If add src_reg, insn->off was unnecessary, no need to
1871-
* restore either.
1872-
*/
1873-
maybe_emit_1mod(&prog, src_reg, true);
1874-
EMIT2_off32(0x81, add_1reg(0xE8, src_reg), insn->off);
1875-
}
1876-
18771870
if (!bpf_prog->aux->extable)
18781871
break;
18791872

@@ -3473,3 +3466,9 @@ bool bpf_jit_supports_ptr_xchg(void)
34733466
{
34743467
return true;
34753468
}
3469+
3470+
/* x86-64 JIT emits its own code to filter user addresses so return 0 here */
3471+
u64 bpf_arch_uaddress_limit(void)
3472+
{
3473+
return 0;
3474+
}

‎include/linux/filter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -963,6 +963,7 @@ bool bpf_jit_supports_far_kfunc_call(void);
963963
bool bpf_jit_supports_exceptions(void);
964964
bool bpf_jit_supports_ptr_xchg(void);
965965
bool bpf_jit_supports_arena(void);
966+
u64 bpf_arch_uaddress_limit(void);
966967
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
967968
bool bpf_helper_changes_pkt_data(void *func);
968969

‎include/linux/skmsg.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,10 +461,12 @@ static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
461461

462462
static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock)
463463
{
464+
read_lock_bh(&sk->sk_callback_lock);
464465
if (psock->saved_data_ready)
465466
psock->saved_data_ready(sk);
466467
else
467468
sk->sk_data_ready(sk);
469+
read_unlock_bh(&sk->sk_callback_lock);
468470
}
469471

470472
static inline void psock_set_prog(struct bpf_prog **pprog,

‎kernel/bpf/core.c

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2942,6 +2942,15 @@ bool __weak bpf_jit_supports_arena(void)
29422942
return false;
29432943
}
29442944

2945+
u64 __weak bpf_arch_uaddress_limit(void)
2946+
{
2947+
#if defined(CONFIG_64BIT) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE)
2948+
return TASK_SIZE;
2949+
#else
2950+
return 0;
2951+
#endif
2952+
}
2953+
29452954
/* Return TRUE if the JIT backend satisfies the following two conditions:
29462955
* 1) JIT backend supports atomic_xchg() on pointer-sized words.
29472956
* 2) Under the specific arch, the implementation of xchg() is the same

‎kernel/bpf/verifier.c

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18289,8 +18289,7 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
1828918289
f = fdget(fd);
1829018290
map = __bpf_map_get(f);
1829118291
if (IS_ERR(map)) {
18292-
verbose(env, "fd %d is not pointing to valid bpf_map\n",
18293-
insn[0].imm);
18292+
verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
1829418293
return PTR_ERR(map);
1829518294
}
1829618295

@@ -19676,6 +19675,36 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
1967619675
goto next_insn;
1967719676
}
1967819677

19678+
/* Make it impossible to de-reference a userspace address */
19679+
if (BPF_CLASS(insn->code) == BPF_LDX &&
19680+
(BPF_MODE(insn->code) == BPF_PROBE_MEM ||
19681+
BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
19682+
struct bpf_insn *patch = &insn_buf[0];
19683+
u64 uaddress_limit = bpf_arch_uaddress_limit();
19684+
19685+
if (!uaddress_limit)
19686+
goto next_insn;
19687+
19688+
*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
19689+
if (insn->off)
19690+
*patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off);
19691+
*patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32);
19692+
*patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2);
19693+
*patch++ = *insn;
19694+
*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
19695+
*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
19696+
19697+
cnt = patch - insn_buf;
19698+
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19699+
if (!new_prog)
19700+
return -ENOMEM;
19701+
19702+
delta += cnt - 1;
19703+
env->prog = prog = new_prog;
19704+
insn = new_prog->insnsi + i + delta;
19705+
goto next_insn;
19706+
}
19707+
1967919708
/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
1968019709
if (BPF_CLASS(insn->code) == BPF_LD &&
1968119710
(BPF_MODE(insn->code) == BPF_ABS ||

0 commit comments

Comments
 (0)