From 39084a201d11f684ca4c324e1cfe7c542655253f Mon Sep 17 00:00:00 2001 From: jkim Date: Wed, 13 Aug 2008 19:25:09 +0000 Subject: - Remove unnecessary jump instruction(s) when offset(s) is/are zero(s). - Constantly use conditional jumps for unsigned integers. --- sys/amd64/amd64/bpf_jit_machdep.c | 79 ++++++++++++++++--------------------- sys/amd64/amd64/bpf_jit_machdep.h | 83 +++++++++++++++++++++++---------------- sys/i386/i386/bpf_jit_machdep.c | 79 ++++++++++++++++--------------------- sys/i386/i386/bpf_jit_machdep.h | 83 +++++++++++++++++++++++---------------- 4 files changed, 168 insertions(+), 156 deletions(-) (limited to 'sys') diff --git a/sys/amd64/amd64/bpf_jit_machdep.c b/sys/amd64/amd64/bpf_jit_machdep.c index b20c433..eb8e924 100644 --- a/sys/amd64/amd64/bpf_jit_machdep.c +++ b/sys/amd64/amd64/bpf_jit_machdep.c @@ -159,7 +159,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(int), ECX); CMPrd(EDI, ECX); - JLEb(6); + JBEb(6); ZEROrd(EAX); MOVrq3(R8, RBX); RET(); @@ -173,7 +173,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(short), ECX); CMPrd(EDI, ECX); - JLEb(4); + JBEb(4); MOVrq3(R8, RBX); RET(); MOVobw(RBX, RSI, AX); @@ -184,7 +184,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) ZEROrd(EAX); MOVid(ins->k, ECX); CMPrd(EDI, ECX); - JLEb(4); + JBEb(4); MOVrq3(R8, RBX); RET(); MOVobb(RBX, RCX, AL); @@ -204,7 +204,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(int), ECX); CMPrd(EDI, ECX); - JLEb(6); + JBEb(6); ZEROrd(EAX); MOVrq3(R8, RBX); RET(); @@ -219,7 +219,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(short), ECX); CMPrd(EDI, ECX); - JLEb(4); + JBEb(4); MOVrq3(R8, RBX); RET(); MOVobw(RBX, RSI, AX); @@ -231,7 +231,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVid(ins->k, ECX); ADDrd(EDX, ECX); CMPrd(EDI, ECX); - JLEb(4); + JBEb(4); MOVrq3(R8, RBX); RET(); MOVobb(RBX, RCX, AL); @@ -240,7 +240,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ECX); CMPrd(EDI, ECX); - JLEb(6); + JBEb(6); ZEROrd(EAX); MOVrq3(R8, RBX); RET(); @@ -293,70 +293,59 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) break; case BPF_JMP|BPF_JGT|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - /* 5 is the size of the following JMP */ - JG(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5 ); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - JGE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - JE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_K: - MOVrd(EAX, ECX); - ANDid(ins->k, ECX); - JE(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc]); + if (ins->jt == 0 && ins->jf == 0) + break; + TESTid(ins->k, EAX); + JCC(JNE, JE); break; case BPF_JMP|BPF_JGT|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JA(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JAE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_X: - MOVrd(EAX, ECX); - ANDrd(EDX, ECX); - JE(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc]); + if (ins->jt == 0 && ins->jf == 0) + break; + TESTrd(EDX, EAX); + JCC(JNE, JE); break; case BPF_ALU|BPF_ADD|BPF_X: @@ -374,7 +363,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) break; case BPF_ALU|BPF_DIV|BPF_X: - CMPid(0, EDX); + TESTrd(EDX, EDX); JNEb(6); ZEROrd(EAX); MOVrq3(R8, RBX); diff --git a/sys/amd64/amd64/bpf_jit_machdep.h b/sys/amd64/amd64/bpf_jit_machdep.h index ac2ae2b..0fc83be 100644 --- a/sys/amd64/amd64/bpf_jit_machdep.h +++ b/sys/amd64/amd64/bpf_jit_machdep.h @@ -294,6 +294,24 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ } while (0) +/* testl i32,r32 */ +#define TESTid(i32, r32) do { \ + if (r32 == EAX) { \ + emitm(&stream, 0xa9, 1); \ + } else { \ + emitm(&stream, 0xf7, 1); \ + emitm(&stream, (3 << 6) | r32, 1); \ + } \ + emitm(&stream, i32, 4); \ +} while (0) + +/* testl sr32,dr32 */ +#define TESTrd(sr32, dr32) do { \ + emitm(&stream, 0x85, 1); \ + emitm(&stream, \ + (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ +} while (0) + /* orl sr32,dr32 */ #define ORrd(sr32, dr32) do { \ emitm(&stream, 0x09, 1); \ @@ -369,42 +387,12 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, off8, 1); \ } while (0) -/* je off32 */ -#define JE(off32) do { \ - emitm(&stream, 0x840f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jle off8 */ -#define JLEb(off8) do { \ - emitm(&stream, 0x7e, 1); \ +/* jbe off8 */ +#define JBEb(off8) do { \ + emitm(&stream, 0x76, 1); \ emitm(&stream, off8, 1); \ } while (0) -/* ja off32 */ -#define JA(off32) do { \ - emitm(&stream, 0x870f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jae off32 */ -#define JAE(off32) do { \ - emitm(&stream, 0x830f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jg off32 */ -#define JG(off32) do { \ - emitm(&stream, 0x8f0f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jge off32 */ -#define JGE(off32) do { \ - emitm(&stream, 0x8d0f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - /* jmp off32 */ #define JMP(off32) do { \ emitm(&stream, 0xe9, 1); \ @@ -417,4 +405,33 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \ } while (0) +/* + * Conditional long jumps + */ +#define JB 0x82 +#define JAE 0x83 +#define JE 0x84 +#define JNE 0x85 +#define JBE 0x86 +#define JA 0x87 + +#define JCC(t, f) do { \ + if (ins->jt != 0 && ins->jf != 0) { \ + /* 5 is the size of the following jmp */ \ + emitm(&stream, ((t) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ + stream.refs[stream.bpf_pc] + 5, 4); \ + JMP(stream.refs[stream.bpf_pc + ins->jf] - \ + stream.refs[stream.bpf_pc]); \ + } else if (ins->jt != 0) { \ + emitm(&stream, ((t) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ + stream.refs[stream.bpf_pc], 4); \ + } else { \ + emitm(&stream, ((f) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] - \ + stream.refs[stream.bpf_pc], 4); \ + } \ +} while (0) + #endif /* _BPF_JIT_MACHDEP_H_ */ diff --git a/sys/i386/i386/bpf_jit_machdep.c b/sys/i386/i386/bpf_jit_machdep.c index 5e3161b..b5b966b 100644 --- a/sys/i386/i386/bpf_jit_machdep.c +++ b/sys/i386/i386/bpf_jit_machdep.c @@ -166,7 +166,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(int), ECX); CMPrd(EDI, ECX); - JLEb(7); + JBEb(7); ZEROrd(EAX); POP(EBX); POP(ESI); @@ -182,7 +182,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(short), ECX); CMPrd(EDI, ECX); - JLEb(5); + JBEb(5); POP(EBX); POP(ESI); POP(EDI); @@ -195,7 +195,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) ZEROrd(EAX); MOVid(ins->k, ECX); CMPrd(EDI, ECX); - JLEb(5); + JBEb(5); POP(EBX); POP(ESI); POP(EDI); @@ -217,7 +217,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(int), ECX); CMPrd(EDI, ECX); - JLEb(7); + JBEb(7); ZEROrd(EAX); POP(EBX); POP(ESI); @@ -234,7 +234,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVrd(ECX, ESI); ADDib(sizeof(short), ECX); CMPrd(EDI, ECX); - JLEb(5); + JBEb(5); POP(EBX); POP(ESI); POP(EDI); @@ -248,7 +248,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) MOVid(ins->k, ECX); ADDrd(EDX, ECX); CMPrd(EDI, ECX); - JLEb(5); + JBEb(5); POP(EBX); POP(ESI); POP(EDI); @@ -259,7 +259,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ECX); CMPrd(EDI, ECX); - JLEb(7); + JBEb(7); ZEROrd(EAX); POP(EBX); POP(ESI); @@ -314,70 +314,59 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) break; case BPF_JMP|BPF_JGT|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - /* 5 is the size of the following JMP */ - JG(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5 ); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - JGE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_K: + if (ins->jt == 0 && ins->jf == 0) + break; CMPid(ins->k, EAX); - JE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_K: - MOVrd(EAX, ECX); - ANDid(ins->k, ECX); - JE(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc]); + if (ins->jt == 0 && ins->jf == 0) + break; + TESTid(ins->k, EAX); + JCC(JNE, JE); break; case BPF_JMP|BPF_JGT|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JA(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JAE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_X: + if (ins->jt == 0 && ins->jf == 0) + break; CMPrd(EDX, EAX); - JE(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc]); + JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_X: - MOVrd(EAX, ECX); - ANDrd(EDX, ECX); - JE(stream.refs[stream.bpf_pc + ins->jf] - - stream.refs[stream.bpf_pc] + 5); - JMP(stream.refs[stream.bpf_pc + ins->jt] - - stream.refs[stream.bpf_pc]); + if (ins->jt == 0 && ins->jf == 0) + break; + TESTrd(EDX, EAX); + JCC(JNE, JE); break; case BPF_ALU|BPF_ADD|BPF_X: @@ -395,7 +384,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) break; case BPF_ALU|BPF_DIV|BPF_X: - CMPid(0, EDX); + TESTrd(EDX, EDX); JNEb(7); ZEROrd(EAX); POP(EBX); diff --git a/sys/i386/i386/bpf_jit_machdep.h b/sys/i386/i386/bpf_jit_machdep.h index d2eca52..c3c06cf 100644 --- a/sys/i386/i386/bpf_jit_machdep.h +++ b/sys/i386/i386/bpf_jit_machdep.h @@ -244,6 +244,24 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ } while (0) +/* testl i32,r32 */ +#define TESTid(i32, r32) do { \ + if (r32 == EAX) { \ + emitm(&stream, 0xa9, 1); \ + } else { \ + emitm(&stream, 0xf7, 1); \ + emitm(&stream, (3 << 6) | r32, 1); \ + } \ + emitm(&stream, i32, 4); \ +} while (0) + +/* testl sr32,dr32 */ +#define TESTrd(sr32, dr32) do { \ + emitm(&stream, 0x85, 1); \ + emitm(&stream, \ + (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ +} while (0) + /* orl sr32,dr32 */ #define ORrd(sr32, dr32) do { \ emitm(&stream, 0x09, 1); \ @@ -319,42 +337,12 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, off8, 1); \ } while (0) -/* je off32 */ -#define JE(off32) do { \ - emitm(&stream, 0x840f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jle off8 */ -#define JLEb(off8) do { \ - emitm(&stream, 0x7e, 1); \ +/* jbe off8 */ +#define JBEb(off8) do { \ + emitm(&stream, 0x76, 1); \ emitm(&stream, off8, 1); \ } while (0) -/* ja off32 */ -#define JA(off32) do { \ - emitm(&stream, 0x870f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jae off32 */ -#define JAE(off32) do { \ - emitm(&stream, 0x830f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jg off32 */ -#define JG(off32) do { \ - emitm(&stream, 0x8f0f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - -/* jge off32 */ -#define JGE(off32) do { \ - emitm(&stream, 0x8d0f, 2); \ - emitm(&stream, off32, 4); \ -} while (0) - /* jmp off32 */ #define JMP(off32) do { \ emitm(&stream, 0xe9, 1); \ @@ -367,4 +355,33 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \ } while (0) +/* + * Conditional long jumps + */ +#define JB 0x82 +#define JAE 0x83 +#define JE 0x84 +#define JNE 0x85 +#define JBE 0x86 +#define JA 0x87 + +#define JCC(t, f) do { \ + if (ins->jt != 0 && ins->jf != 0) { \ + /* 5 is the size of the following jmp */ \ + emitm(&stream, ((t) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ + stream.refs[stream.bpf_pc] + 5, 4); \ + JMP(stream.refs[stream.bpf_pc + ins->jf] - \ + stream.refs[stream.bpf_pc]); \ + } else if (ins->jt != 0) { \ + emitm(&stream, ((t) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ + stream.refs[stream.bpf_pc], 4); \ + } else { \ + emitm(&stream, ((f) << 8) | 0x0f, 2); \ + emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] - \ + stream.refs[stream.bpf_pc], 4); \ + } \ +} while (0) + #endif /* _BPF_JIT_MACHDEP_H_ */ -- cgit v1.1