Skip to content

Commit 66d0d5a

Browse files
Jiong WangAlexei Starovoitov
Jiong Wang
authored and
Alexei Starovoitov
committed
riscv: bpf: eliminate zero extension code-gen
Cc: Björn Töpel <[email protected]> Acked-by: Björn Töpel <[email protected]> Tested-by: Björn Töpel <[email protected]> Signed-off-by: Jiong Wang <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]>
1 parent 836256b commit 66d0d5a

File tree

1 file changed

+30
-13
lines changed

1 file changed

+30
-13
lines changed

‎arch/riscv/net/bpf_jit_comp.c‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
731731
{
732732
bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
733733
BPF_CLASS(insn->code) == BPF_JMP;
734+
struct bpf_prog_aux *aux = ctx->prog->aux;
734735
int rvoff, i = insn - ctx->prog->insnsi;
735736
u8 rd = -1, rs = -1, code = insn->code;
736737
s16 off = insn->off;
@@ -742,8 +743,13 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
742743
/* dst = src */
743744
case BPF_ALU | BPF_MOV | BPF_X:
744745
case BPF_ALU64 | BPF_MOV | BPF_X:
746+
if (imm == 1) {
747+
/* Special mov32 for zext */
748+
emit_zext_32(rd, ctx);
749+
break;
750+
}
745751
emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);
746-
if (!is64)
752+
if (!is64 && !aux->verifier_zext)
747753
emit_zext_32(rd, ctx);
748754
break;
749755

@@ -771,19 +777,19 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
771777
case BPF_ALU | BPF_MUL | BPF_X:
772778
case BPF_ALU64 | BPF_MUL | BPF_X:
773779
emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
774-
if (!is64)
780+
if (!is64 && !aux->verifier_zext)
775781
emit_zext_32(rd, ctx);
776782
break;
777783
case BPF_ALU | BPF_DIV | BPF_X:
778784
case BPF_ALU64 | BPF_DIV | BPF_X:
779785
emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
780-
if (!is64)
786+
if (!is64 && !aux->verifier_zext)
781787
emit_zext_32(rd, ctx);
782788
break;
783789
case BPF_ALU | BPF_MOD | BPF_X:
784790
case BPF_ALU64 | BPF_MOD | BPF_X:
785791
emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
786-
if (!is64)
792+
if (!is64 && !aux->verifier_zext)
787793
emit_zext_32(rd, ctx);
788794
break;
789795
case BPF_ALU | BPF_LSH | BPF_X:
@@ -867,7 +873,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
867873
case BPF_ALU | BPF_MOV | BPF_K:
868874
case BPF_ALU64 | BPF_MOV | BPF_K:
869875
emit_imm(rd, imm, ctx);
870-
if (!is64)
876+
if (!is64 && !aux->verifier_zext)
871877
emit_zext_32(rd, ctx);
872878
break;
873879

@@ -882,7 +888,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
882888
emit(is64 ? rv_add(rd, rd, RV_REG_T1) :
883889
rv_addw(rd, rd, RV_REG_T1), ctx);
884890
}
885-
if (!is64)
891+
if (!is64 && !aux->verifier_zext)
886892
emit_zext_32(rd, ctx);
887893
break;
888894
case BPF_ALU | BPF_SUB | BPF_K:
@@ -895,7 +901,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
895901
emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :
896902
rv_subw(rd, rd, RV_REG_T1), ctx);
897903
}
898-
if (!is64)
904+
if (!is64 && !aux->verifier_zext)
899905
emit_zext_32(rd, ctx);
900906
break;
901907
case BPF_ALU | BPF_AND | BPF_K:
@@ -906,7 +912,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
906912
emit_imm(RV_REG_T1, imm, ctx);
907913
emit(rv_and(rd, rd, RV_REG_T1), ctx);
908914
}
909-
if (!is64)
915+
if (!is64 && !aux->verifier_zext)
910916
emit_zext_32(rd, ctx);
911917
break;
912918
case BPF_ALU | BPF_OR | BPF_K:
@@ -917,7 +923,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
917923
emit_imm(RV_REG_T1, imm, ctx);
918924
emit(rv_or(rd, rd, RV_REG_T1), ctx);
919925
}
920-
if (!is64)
926+
if (!is64 && !aux->verifier_zext)
921927
emit_zext_32(rd, ctx);
922928
break;
923929
case BPF_ALU | BPF_XOR | BPF_K:
@@ -928,31 +934,31 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
928934
emit_imm(RV_REG_T1, imm, ctx);
929935
emit(rv_xor(rd, rd, RV_REG_T1), ctx);
930936
}
931-
if (!is64)
937+
if (!is64 && !aux->verifier_zext)
932938
emit_zext_32(rd, ctx);
933939
break;
934940
case BPF_ALU | BPF_MUL | BPF_K:
935941
case BPF_ALU64 | BPF_MUL | BPF_K:
936942
emit_imm(RV_REG_T1, imm, ctx);
937943
emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
938944
rv_mulw(rd, rd, RV_REG_T1), ctx);
939-
if (!is64)
945+
if (!is64 && !aux->verifier_zext)
940946
emit_zext_32(rd, ctx);
941947
break;
942948
case BPF_ALU | BPF_DIV | BPF_K:
943949
case BPF_ALU64 | BPF_DIV | BPF_K:
944950
emit_imm(RV_REG_T1, imm, ctx);
945951
emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
946952
rv_divuw(rd, rd, RV_REG_T1), ctx);
947-
if (!is64)
953+
if (!is64 && !aux->verifier_zext)
948954
emit_zext_32(rd, ctx);
949955
break;
950956
case BPF_ALU | BPF_MOD | BPF_K:
951957
case BPF_ALU64 | BPF_MOD | BPF_K:
952958
emit_imm(RV_REG_T1, imm, ctx);
953959
emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
954960
rv_remuw(rd, rd, RV_REG_T1), ctx);
955-
if (!is64)
961+
if (!is64 && !aux->verifier_zext)
956962
emit_zext_32(rd, ctx);
957963
break;
958964
case BPF_ALU | BPF_LSH | BPF_K:
@@ -1239,6 +1245,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
12391245
emit_imm(RV_REG_T1, off, ctx);
12401246
emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
12411247
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1248+
if (insn_is_zext(&insn[1]))
1249+
return 1;
12421250
break;
12431251
case BPF_LDX | BPF_MEM | BPF_H:
12441252
if (is_12b_int(off)) {
@@ -1249,6 +1257,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
12491257
emit_imm(RV_REG_T1, off, ctx);
12501258
emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
12511259
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1260+
if (insn_is_zext(&insn[1]))
1261+
return 1;
12521262
break;
12531263
case BPF_LDX | BPF_MEM | BPF_W:
12541264
if (is_12b_int(off)) {
@@ -1259,6 +1269,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
12591269
emit_imm(RV_REG_T1, off, ctx);
12601270
emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
12611271
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1272+
if (insn_is_zext(&insn[1]))
1273+
return 1;
12621274
break;
12631275
case BPF_LDX | BPF_MEM | BPF_DW:
12641276
if (is_12b_int(off)) {
@@ -1503,6 +1515,11 @@ static void bpf_flush_icache(void *start, void *end)
15031515
flush_icache_range((unsigned long)start, (unsigned long)end);
15041516
}
15051517

1518+
bool bpf_jit_needs_zext(void)
1519+
{
1520+
return true;
1521+
}
1522+
15061523
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
15071524
{
15081525
bool tmp_blinded = false, extra_pass = false;

0 commit comments

Comments
 (0)