- Remove unnecessary jump instruction(s) when offset(s) is/are zero(s).

- Constantly use conditional jumps for unsigned integers.

4 files changed, 168 insertions, 156 deletions

diff --git a/sys/amd64/amd64/bpf_jit_machdep.c b/sys/amd64/amd64/bpf_jit_machdep.c
index b20c433..eb8e924 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.c
+++ b/sys/amd64/amd64/bpf_jit_machdep.c
@@ -159,7 +159,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(int), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(6);
+				JBEb(6);
 				ZEROrd(EAX);
 				MOVrq3(R8, RBX);
 				RET();
@@ -173,7 +173,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(short), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(4);
+				JBEb(4);
 				MOVrq3(R8, RBX);
 				RET();
 				MOVobw(RBX, RSI, AX);
@@ -184,7 +184,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				ZEROrd(EAX);
 				MOVid(ins->k, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(4);
+				JBEb(4);
 				MOVrq3(R8, RBX);
 				RET();
 				MOVobb(RBX, RCX, AL);
@@ -204,7 +204,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(int), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(6);
+				JBEb(6);
 				ZEROrd(EAX);
 				MOVrq3(R8, RBX);
 				RET();
@@ -219,7 +219,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(short), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(4);
+				JBEb(4);
 				MOVrq3(R8, RBX);
 				RET();
 				MOVobw(RBX, RSI, AX);
@@ -231,7 +231,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVid(ins->k, ECX);
 				ADDrd(EDX, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(4);
+				JBEb(4);
 				MOVrq3(R8, RBX);
 				RET();
 				MOVobb(RBX, RCX, AL);
@@ -240,7 +240,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 			case BPF_LDX|BPF_MSH|BPF_B:
 				MOVid(ins->k, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(6);
+				JBEb(6);
 				ZEROrd(EAX);
 				MOVrq3(R8, RBX);
 				RET();
@@ -293,70 +293,59 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				/* 5 is the size of the following JMP */
-				JG(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5 );
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				JGE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_K:
-				MOVrd(EAX, ECX);
-				ANDid(ins->k, ECX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTid(ins->k, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JA(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JAE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_X:
-				MOVrd(EAX, ECX);
-				ANDrd(EDX, ECX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTrd(EDX, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_ALU|BPF_ADD|BPF_X:
@@ -374,7 +363,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_X:
-				CMPid(0, EDX);
+				TESTrd(EDX, EDX);
 				JNEb(6);
 				ZEROrd(EAX);
 				MOVrq3(R8, RBX);
diff --git a/sys/amd64/amd64/bpf_jit_machdep.h b/sys/amd64/amd64/bpf_jit_machdep.h
index ac2ae2b..0fc83be 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.h
+++ b/sys/amd64/amd64/bpf_jit_machdep.h
@@ -294,6 +294,24 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
+/* testl i32,r32 */
+#define TESTid(i32, r32) do {						\
+	if (r32 == EAX) {						\
+		emitm(&stream, 0xa9, 1);				\
+	} else {							\
+		emitm(&stream, 0xf7, 1);				\
+		emitm(&stream, (3 << 6) | r32, 1);			\
+	}								\
+	emitm(&stream, i32, 4);						\
+} while (0)
+
+/* testl sr32,dr32 */
+#define TESTrd(sr32, dr32) do {						\
+	emitm(&stream, 0x85, 1);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
 /* orl sr32,dr32 */
 #define ORrd(sr32, dr32) do {						\
 	emitm(&stream, 0x09, 1);					\
@@ -369,42 +387,12 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* je off32 */
-#define JE(off32) do {							\
-	emitm(&stream, 0x840f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off8 */
-#define JLEb(off8) do {							\
-	emitm(&stream, 0x7e, 1);					\
+/* jbe off8 */
+#define JBEb(off8) do {							\
+	emitm(&stream, 0x76, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* ja off32 */
-#define JA(off32) do {							\
-	emitm(&stream, 0x870f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jae off32 */
-#define JAE(off32) do {							\
-	emitm(&stream, 0x830f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jg off32 */
-#define JG(off32) do {							\
-	emitm(&stream, 0x8f0f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jge off32 */
-#define JGE(off32) do {							\
-	emitm(&stream, 0x8d0f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
 /* jmp off32 */
 #define JMP(off32) do {							\
 	emitm(&stream, 0xe9, 1);					\
@@ -417,4 +405,33 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
 } while (0)
 
+/*
+ * Conditional long jumps
+ */
+#define	JB	0x82
+#define	JAE	0x83
+#define	JE	0x84
+#define	JNE	0x85
+#define	JBE	0x86
+#define	JA	0x87
+
+#define	JCC(t, f) do {							\
+	if (ins->jt != 0 && ins->jf != 0) {				\
+		/* 5 is the size of the following jmp */		\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc] + 5, 4);			\
+		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
+		    stream.refs[stream.bpf_pc]);			\
+	} else if (ins->jt != 0) {					\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	} else {							\
+		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	}								\
+} while (0)
+
 #endif	/* _BPF_JIT_MACHDEP_H_ */
diff --git a/sys/i386/i386/bpf_jit_machdep.c b/sys/i386/i386/bpf_jit_machdep.c
index 5e3161b..b5b966b 100644
--- a/sys/i386/i386/bpf_jit_machdep.c
+++ b/sys/i386/i386/bpf_jit_machdep.c
@@ -166,7 +166,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(int), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(7);
+				JBEb(7);
 				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
@@ -182,7 +182,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(short), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(5);
+				JBEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
@@ -195,7 +195,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				ZEROrd(EAX);
 				MOVid(ins->k, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(5);
+				JBEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
@@ -217,7 +217,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(int), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(7);
+				JBEb(7);
 				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
@@ -234,7 +234,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVrd(ECX, ESI);
 				ADDib(sizeof(short), ECX);
 				CMPrd(EDI, ECX);
-				JLEb(5);
+				JBEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
@@ -248,7 +248,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				MOVid(ins->k, ECX);
 				ADDrd(EDX, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(5);
+				JBEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
@@ -259,7 +259,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 			case BPF_LDX|BPF_MSH|BPF_B:
 				MOVid(ins->k, ECX);
 				CMPrd(EDI, ECX);
-				JLEb(7);
+				JBEb(7);
 				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
@@ -314,70 +314,59 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				/* 5 is the size of the following JMP */
-				JG(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5 );
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				JGE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_K:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPid(ins->k, EAX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_K:
-				MOVrd(EAX, ECX);
-				ANDid(ins->k, ECX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTid(ins->k, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JA(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JAE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_X:
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
 				CMPrd(EDX, EAX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_X:
-				MOVrd(EAX, ECX);
-				ANDrd(EDX, ECX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTrd(EDX, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_ALU|BPF_ADD|BPF_X:
@@ -395,7 +384,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_X:
-				CMPid(0, EDX);
+				TESTrd(EDX, EDX);
 				JNEb(7);
 				ZEROrd(EAX);
 				POP(EBX);
diff --git a/sys/i386/i386/bpf_jit_machdep.h b/sys/i386/i386/bpf_jit_machdep.h
index d2eca52..c3c06cf 100644
--- a/sys/i386/i386/bpf_jit_machdep.h
+++ b/sys/i386/i386/bpf_jit_machdep.h
@@ -244,6 +244,24 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
+/* testl i32,r32 */
+#define TESTid(i32, r32) do {						\
+	if (r32 == EAX) {						\
+		emitm(&stream, 0xa9, 1);				\
+	} else {							\
+		emitm(&stream, 0xf7, 1);				\
+		emitm(&stream, (3 << 6) | r32, 1);			\
+	}								\
+	emitm(&stream, i32, 4);						\
+} while (0)
+
+/* testl sr32,dr32 */
+#define TESTrd(sr32, dr32) do {						\
+	emitm(&stream, 0x85, 1);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
 /* orl sr32,dr32 */
 #define ORrd(sr32, dr32) do {						\
 	emitm(&stream, 0x09, 1);					\
@@ -319,42 +337,12 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* je off32 */
-#define JE(off32) do {							\
-	emitm(&stream, 0x840f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off8 */
-#define JLEb(off8) do {							\
-	emitm(&stream, 0x7e, 1);					\
+/* jbe off8 */
+#define JBEb(off8) do {							\
+	emitm(&stream, 0x76, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* ja off32 */
-#define JA(off32) do {							\
-	emitm(&stream, 0x870f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jae off32 */
-#define JAE(off32) do {							\
-	emitm(&stream, 0x830f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jg off32 */
-#define JG(off32) do {							\
-	emitm(&stream, 0x8f0f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jge off32 */
-#define JGE(off32) do {							\
-	emitm(&stream, 0x8d0f, 2);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
 /* jmp off32 */
 #define JMP(off32) do {							\
 	emitm(&stream, 0xe9, 1);					\
@@ -367,4 +355,33 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
 } while (0)
 
+/*
+ * Conditional long jumps
+ */
+#define	JB	0x82
+#define	JAE	0x83
+#define	JE	0x84
+#define	JNE	0x85
+#define	JBE	0x86
+#define	JA	0x87
+
+#define	JCC(t, f) do {							\
+	if (ins->jt != 0 && ins->jf != 0) {				\
+		/* 5 is the size of the following jmp */		\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc] + 5, 4);			\
+		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
+		    stream.refs[stream.bpf_pc]);			\
+	} else if (ins->jt != 0) {					\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	} else {							\
+		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	}								\
+} while (0)
+
 #endif	/* _BPF_JIT_MACHDEP_H_ */