@@ -117,6 +117,8 @@ static bool is_simm32(s64 value)
117117#define IA32_JLE 0x7E
118118#define IA32_JG 0x7F
119119
120+ #define COND_JMP_OPCODE_INVALID (0xFF)
121+
120122/*
121123 * Map eBPF registers to IA32 32bit registers or stack scratch space.
122124 *
@@ -698,19 +700,12 @@ static inline void emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog)
698700 STACK_VAR (dst_hi ));
699701 }
700702
701- /* xor ecx,ecx */
702- EMIT2 (0x31 , add_2reg (0xC0 , IA32_ECX , IA32_ECX ));
703- /* sub dreg_lo,ecx */
704- EMIT2 (0x2B , add_2reg (0xC0 , dreg_lo , IA32_ECX ));
705- /* mov dreg_lo,ecx */
706- EMIT2 (0x89 , add_2reg (0xC0 , dreg_lo , IA32_ECX ));
707-
708- /* xor ecx,ecx */
709- EMIT2 (0x31 , add_2reg (0xC0 , IA32_ECX , IA32_ECX ));
710- /* sbb dreg_hi,ecx */
711- EMIT2 (0x19 , add_2reg (0xC0 , dreg_hi , IA32_ECX ));
712- /* mov dreg_hi,ecx */
713- EMIT2 (0x89 , add_2reg (0xC0 , dreg_hi , IA32_ECX ));
703+ /* neg dreg_lo */
704+ EMIT2 (0xF7 , add_1reg (0xD8 , dreg_lo ));
705+ /* adc dreg_hi,0x0 */
706+ EMIT3 (0x83 , add_1reg (0xD0 , dreg_hi ), 0x00 );
707+ /* neg dreg_hi */
708+ EMIT2 (0xF7 , add_1reg (0xD8 , dreg_hi ));
714709
715710 if (dstk ) {
716711 /* mov dword ptr [ebp+off],dreg_lo */
@@ -1613,6 +1608,75 @@ static inline void emit_push_r64(const u8 src[], u8 **pprog)
16131608 * pprog = prog ;
16141609}
16151610
1611+ static u8 get_cond_jmp_opcode (const u8 op , bool is_cmp_lo )
1612+ {
1613+ u8 jmp_cond ;
1614+
1615+ /* Convert BPF opcode to x86 */
1616+ switch (op ) {
1617+ case BPF_JEQ :
1618+ jmp_cond = IA32_JE ;
1619+ break ;
1620+ case BPF_JSET :
1621+ case BPF_JNE :
1622+ jmp_cond = IA32_JNE ;
1623+ break ;
1624+ case BPF_JGT :
1625+ /* GT is unsigned '>', JA in x86 */
1626+ jmp_cond = IA32_JA ;
1627+ break ;
1628+ case BPF_JLT :
1629+ /* LT is unsigned '<', JB in x86 */
1630+ jmp_cond = IA32_JB ;
1631+ break ;
1632+ case BPF_JGE :
1633+ /* GE is unsigned '>=', JAE in x86 */
1634+ jmp_cond = IA32_JAE ;
1635+ break ;
1636+ case BPF_JLE :
1637+ /* LE is unsigned '<=', JBE in x86 */
1638+ jmp_cond = IA32_JBE ;
1639+ break ;
1640+ case BPF_JSGT :
1641+ if (!is_cmp_lo )
1642+ /* Signed '>', GT in x86 */
1643+ jmp_cond = IA32_JG ;
1644+ else
1645+ /* GT is unsigned '>', JA in x86 */
1646+ jmp_cond = IA32_JA ;
1647+ break ;
1648+ case BPF_JSLT :
1649+ if (!is_cmp_lo )
1650+ /* Signed '<', LT in x86 */
1651+ jmp_cond = IA32_JL ;
1652+ else
1653+ /* LT is unsigned '<', JB in x86 */
1654+ jmp_cond = IA32_JB ;
1655+ break ;
1656+ case BPF_JSGE :
1657+ if (!is_cmp_lo )
1658+ /* Signed '>=', GE in x86 */
1659+ jmp_cond = IA32_JGE ;
1660+ else
1661+ /* GE is unsigned '>=', JAE in x86 */
1662+ jmp_cond = IA32_JAE ;
1663+ break ;
1664+ case BPF_JSLE :
1665+ if (!is_cmp_lo )
1666+ /* Signed '<=', LE in x86 */
1667+ jmp_cond = IA32_JLE ;
1668+ else
1669+ /* LE is unsigned '<=', JBE in x86 */
1670+ jmp_cond = IA32_JBE ;
1671+ break ;
1672+ default : /* to silence GCC warning */
1673+ jmp_cond = COND_JMP_OPCODE_INVALID ;
1674+ break ;
1675+ }
1676+
1677+ return jmp_cond ;
1678+ }
1679+
16161680static int do_jit (struct bpf_prog * bpf_prog , int * addrs , u8 * image ,
16171681 int oldproglen , struct jit_context * ctx )
16181682{
@@ -2069,10 +2133,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
20692133 case BPF_JMP | BPF_JLT | BPF_X :
20702134 case BPF_JMP | BPF_JGE | BPF_X :
20712135 case BPF_JMP | BPF_JLE | BPF_X :
2072- case BPF_JMP | BPF_JSGT | BPF_X :
2073- case BPF_JMP | BPF_JSLE | BPF_X :
2074- case BPF_JMP | BPF_JSLT | BPF_X :
2075- case BPF_JMP | BPF_JSGE | BPF_X :
20762136 case BPF_JMP32 | BPF_JEQ | BPF_X :
20772137 case BPF_JMP32 | BPF_JNE | BPF_X :
20782138 case BPF_JMP32 | BPF_JGT | BPF_X :
@@ -2118,6 +2178,40 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
21182178 EMIT2 (0x39 , add_2reg (0xC0 , dreg_lo , sreg_lo ));
21192179 goto emit_cond_jmp ;
21202180 }
2181+ case BPF_JMP | BPF_JSGT | BPF_X :
2182+ case BPF_JMP | BPF_JSLE | BPF_X :
2183+ case BPF_JMP | BPF_JSLT | BPF_X :
2184+ case BPF_JMP | BPF_JSGE | BPF_X : {
2185+ u8 dreg_lo = dstk ? IA32_EAX : dst_lo ;
2186+ u8 dreg_hi = dstk ? IA32_EDX : dst_hi ;
2187+ u8 sreg_lo = sstk ? IA32_ECX : src_lo ;
2188+ u8 sreg_hi = sstk ? IA32_EBX : src_hi ;
2189+
2190+ if (dstk ) {
2191+ EMIT3 (0x8B , add_2reg (0x40 , IA32_EBP , IA32_EAX ),
2192+ STACK_VAR (dst_lo ));
2193+ EMIT3 (0x8B ,
2194+ add_2reg (0x40 , IA32_EBP ,
2195+ IA32_EDX ),
2196+ STACK_VAR (dst_hi ));
2197+ }
2198+
2199+ if (sstk ) {
2200+ EMIT3 (0x8B , add_2reg (0x40 , IA32_EBP , IA32_ECX ),
2201+ STACK_VAR (src_lo ));
2202+ EMIT3 (0x8B ,
2203+ add_2reg (0x40 , IA32_EBP ,
2204+ IA32_EBX ),
2205+ STACK_VAR (src_hi ));
2206+ }
2207+
2208+ /* cmp dreg_hi,sreg_hi */
2209+ EMIT2 (0x39 , add_2reg (0xC0 , dreg_hi , sreg_hi ));
2210+ EMIT2 (IA32_JNE , 10 );
2211+ /* cmp dreg_lo,sreg_lo */
2212+ EMIT2 (0x39 , add_2reg (0xC0 , dreg_lo , sreg_lo ));
2213+ goto emit_cond_jmp_signed ;
2214+ }
21212215 case BPF_JMP | BPF_JSET | BPF_X :
21222216 case BPF_JMP32 | BPF_JSET | BPF_X : {
21232217 bool is_jmp64 = BPF_CLASS (insn -> code ) == BPF_JMP ;
@@ -2194,10 +2288,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
21942288 case BPF_JMP | BPF_JLT | BPF_K :
21952289 case BPF_JMP | BPF_JGE | BPF_K :
21962290 case BPF_JMP | BPF_JLE | BPF_K :
2197- case BPF_JMP | BPF_JSGT | BPF_K :
2198- case BPF_JMP | BPF_JSLE | BPF_K :
2199- case BPF_JMP | BPF_JSLT | BPF_K :
2200- case BPF_JMP | BPF_JSGE | BPF_K :
22012291 case BPF_JMP32 | BPF_JEQ | BPF_K :
22022292 case BPF_JMP32 | BPF_JNE | BPF_K :
22032293 case BPF_JMP32 | BPF_JGT | BPF_K :
@@ -2238,50 +2328,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
22382328 /* cmp dreg_lo,sreg_lo */
22392329 EMIT2 (0x39 , add_2reg (0xC0 , dreg_lo , sreg_lo ));
22402330
2241- emit_cond_jmp : /* Convert BPF opcode to x86 */
2242- switch (BPF_OP (code )) {
2243- case BPF_JEQ :
2244- jmp_cond = IA32_JE ;
2245- break ;
2246- case BPF_JSET :
2247- case BPF_JNE :
2248- jmp_cond = IA32_JNE ;
2249- break ;
2250- case BPF_JGT :
2251- /* GT is unsigned '>', JA in x86 */
2252- jmp_cond = IA32_JA ;
2253- break ;
2254- case BPF_JLT :
2255- /* LT is unsigned '<', JB in x86 */
2256- jmp_cond = IA32_JB ;
2257- break ;
2258- case BPF_JGE :
2259- /* GE is unsigned '>=', JAE in x86 */
2260- jmp_cond = IA32_JAE ;
2261- break ;
2262- case BPF_JLE :
2263- /* LE is unsigned '<=', JBE in x86 */
2264- jmp_cond = IA32_JBE ;
2265- break ;
2266- case BPF_JSGT :
2267- /* Signed '>', GT in x86 */
2268- jmp_cond = IA32_JG ;
2269- break ;
2270- case BPF_JSLT :
2271- /* Signed '<', LT in x86 */
2272- jmp_cond = IA32_JL ;
2273- break ;
2274- case BPF_JSGE :
2275- /* Signed '>=', GE in x86 */
2276- jmp_cond = IA32_JGE ;
2277- break ;
2278- case BPF_JSLE :
2279- /* Signed '<=', LE in x86 */
2280- jmp_cond = IA32_JLE ;
2281- break ;
2282- default : /* to silence GCC warning */
2331+ emit_cond_jmp : jmp_cond = get_cond_jmp_opcode (BPF_OP (code ), false);
2332+ if (jmp_cond == COND_JMP_OPCODE_INVALID )
22832333 return - EFAULT ;
2284- }
22852334 jmp_offset = addrs [i + insn -> off ] - addrs [i ];
22862335 if (is_imm8 (jmp_offset )) {
22872336 EMIT2 (jmp_cond , jmp_offset );
@@ -2291,7 +2340,66 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
22912340 pr_err ("cond_jmp gen bug %llx\n" , jmp_offset );
22922341 return - EFAULT ;
22932342 }
2343+ break ;
2344+ }
2345+ case BPF_JMP | BPF_JSGT | BPF_K :
2346+ case BPF_JMP | BPF_JSLE | BPF_K :
2347+ case BPF_JMP | BPF_JSLT | BPF_K :
2348+ case BPF_JMP | BPF_JSGE | BPF_K : {
2349+ u8 dreg_lo = dstk ? IA32_EAX : dst_lo ;
2350+ u8 dreg_hi = dstk ? IA32_EDX : dst_hi ;
2351+ u8 sreg_lo = IA32_ECX ;
2352+ u8 sreg_hi = IA32_EBX ;
2353+ u32 hi ;
22942354
2355+ if (dstk ) {
2356+ EMIT3 (0x8B , add_2reg (0x40 , IA32_EBP , IA32_EAX ),
2357+ STACK_VAR (dst_lo ));
2358+ EMIT3 (0x8B ,
2359+ add_2reg (0x40 , IA32_EBP ,
2360+ IA32_EDX ),
2361+ STACK_VAR (dst_hi ));
2362+ }
2363+
2364+ /* mov ecx,imm32 */
2365+ EMIT2_off32 (0xC7 , add_1reg (0xC0 , IA32_ECX ), imm32 );
2366+ hi = imm32 & (1 << 31 ) ? (u32 )~0 : 0 ;
2367+ /* mov ebx,imm32 */
2368+ EMIT2_off32 (0xC7 , add_1reg (0xC0 , IA32_EBX ), hi );
2369+ /* cmp dreg_hi,sreg_hi */
2370+ EMIT2 (0x39 , add_2reg (0xC0 , dreg_hi , sreg_hi ));
2371+ EMIT2 (IA32_JNE , 10 );
2372+ /* cmp dreg_lo,sreg_lo */
2373+ EMIT2 (0x39 , add_2reg (0xC0 , dreg_lo , sreg_lo ));
2374+
2375+ /*
2376+ * For simplicity of branch offset computation,
2377+ * let's use fixed jump coding here.
2378+ */
2379+ emit_cond_jmp_signed : /* Check the condition for low 32-bit comparison */
2380+ jmp_cond = get_cond_jmp_opcode (BPF_OP (code ), true);
2381+ if (jmp_cond == COND_JMP_OPCODE_INVALID )
2382+ return - EFAULT ;
2383+ jmp_offset = addrs [i + insn -> off ] - addrs [i ] + 8 ;
2384+ if (is_simm32 (jmp_offset )) {
2385+ EMIT2_off32 (0x0F , jmp_cond + 0x10 , jmp_offset );
2386+ } else {
2387+ pr_err ("cond_jmp gen bug %llx\n" , jmp_offset );
2388+ return - EFAULT ;
2389+ }
2390+ EMIT2 (0xEB , 6 );
2391+
2392+ /* Check the condition for high 32-bit comparison */
2393+ jmp_cond = get_cond_jmp_opcode (BPF_OP (code ), false);
2394+ if (jmp_cond == COND_JMP_OPCODE_INVALID )
2395+ return - EFAULT ;
2396+ jmp_offset = addrs [i + insn -> off ] - addrs [i ];
2397+ if (is_simm32 (jmp_offset )) {
2398+ EMIT2_off32 (0x0F , jmp_cond + 0x10 , jmp_offset );
2399+ } else {
2400+ pr_err ("cond_jmp gen bug %llx\n" , jmp_offset );
2401+ return - EFAULT ;
2402+ }
22952403 break ;
22962404 }
22972405 case BPF_JMP | BPF_JA :
0 commit comments