- Allocate scratch memory on stack instead of pre-allocating it with

the filter as we do from bpf_filter()[1].
- Revert experimental use of contigmalloc(9)/contigfree(9).  It has no
performance benefit over malloc(9)/free(9)[2].

Requested by:	rwatson[1]
Pointed out by:	rwatson, jhb, alc[2]

6 files changed, 98 insertions, 87 deletions

diff --git a/sys/amd64/amd64/bpf_jit_machdep.c b/sys/amd64/amd64/bpf_jit_machdep.c
index da2b0f1..bbaebed 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.c
+++ b/sys/amd64/amd64/bpf_jit_machdep.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 
 #include <amd64/amd64/bpf_jit_machdep.h>
 
-bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
 
 /*
  * emit routine to update the jump table
@@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
  * Function that does the real stuff
  */
 bpf_filter_func
-bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
+bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
 {
 	bpf_bin_stream stream;
 	struct bpf_insn *ins;
@@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 
 	/* Allocate the reference table for the jumps */
 #ifdef _KERNEL
-	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
-	    M_BPFJIT, M_NOWAIT);
+	stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
 #else
-	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+	stream.refs = malloc((nins + 1) * sizeof(u_int));
 #endif
 	if (stream.refs == NULL)
 		return (NULL);
@@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 		ins = prog;
 
 		/* create the procedure header */
-		MOVrq2(RBX, R8);
-		MOVrq(RDI, RBX);
+		PUSH(RBP);
+		MOVrq(RSP, RBP);
+		SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
+		MOVrq2(RDI, R8);
 		MOVrd2(ESI, R9D);
 		MOVrd(EDX, EDI);
 
@@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 
 			case BPF_RET|BPF_K:
 				MOVid(ins->k, EAX);
-				MOVrq3(R8, RBX);
-				RET();
+				LEAVE_RET();
 				break;
 
 			case BPF_RET|BPF_A:
-				MOVrq3(R8, RBX);
-				RET();
+				LEAVE_RET();
 				break;
 
 			case BPF_LD|BPF_W|BPF_ABS:
@@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				MOVrd(EDI, ECX);
 				SUBrd(ESI, ECX);
 				CMPid(sizeof(int32_t), ECX);
-				JAEb(6);
+				JAEb(4);
 				ZEROrd(EAX);
-				MOVrq3(R8, RBX);
-				RET();
-				MOVobd(RBX, RSI, EAX);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
+				MOVobd(RCX, RSI, EAX);
 				BSWAP(EAX);
 				break;
 
@@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				MOVrd(EDI, ECX);
 				SUBrd(ESI, ECX);
 				CMPid(sizeof(int16_t), ECX);
-				JAEb(4);
-				MOVrq3(R8, RBX);
-				RET();
-				MOVobw(RBX, RSI, AX);
+				JAEb(2);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
+				MOVobw(RCX, RSI, AX);
 				SWAP_AX();
 				break;
 
@@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				ZEROrd(EAX);
 				MOVid(ins->k, ESI);
 				CMPrd(EDI, ESI);
-				JBb(4);
-				MOVrq3(R8, RBX);
-				RET();
-				MOVobb(RBX, RSI, AL);
+				JBb(2);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
+				MOVobb(RCX, RSI, AL);
 				break;
 
 			case BPF_LD|BPF_W|BPF_LEN:
@@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				MOVrd(EDI, ECX);
 				SUBrd(ESI, ECX);
 				CMPid(sizeof(int32_t), ECX);
-				JAEb(6);
+				JAEb(4);
 				ZEROrd(EAX);
-				MOVrq3(R8, RBX);
-				RET();
-				MOVobd(RBX, RSI, EAX);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
+				MOVobd(RCX, RSI, EAX);
 				BSWAP(EAX);
 				break;
 
@@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				MOVrd(EDI, ECX);
 				SUBrd(ESI, ECX);
 				CMPid(sizeof(int16_t), ECX);
-				JAEb(4);
-				MOVrq3(R8, RBX);
-				RET();
-				MOVobw(RBX, RSI, AX);
+				JAEb(2);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
+				MOVobw(RCX, RSI, AX);
 				SWAP_AX();
 				break;
 
@@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				MOVrd(EDI, ECX);
 				SUBrd(EDX, ECX);
 				CMPrd(ESI, ECX);
-				JAb(4);
-				MOVrq3(R8, RBX);
-				RET();
+				JAb(2);
+				LEAVE_RET();
+				MOVrq3(R8, RCX);
 				ADDrd(EDX, ESI);
-				MOVobb(RBX, RSI, AL);
+				MOVobb(RCX, RSI, AL);
 				break;
 
 			case BPF_LDX|BPF_MSH|BPF_B:
 				MOVid(ins->k, ESI);
 				CMPrd(EDI, ESI);
-				JBb(6);
+				JBb(4);
 				ZEROrd(EAX);
-				MOVrq3(R8, RBX);
-				RET();
+				LEAVE_RET();
 				ZEROrd(EDX);
-				MOVobb(RBX, RSI, DL);
+				MOVrq3(R8, RCX);
+				MOVobb(RCX, RSI, DL);
 				ANDib(0x0f, DL);
 				SHLib(2, EDX);
 				break;
@@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				break;
 
 			case BPF_LD|BPF_MEM:
-				MOViq((uintptr_t)mem, RCX);
-				MOVid(ins->k * 4, ESI);
-				MOVobd(RCX, RSI, EAX);
+				MOVid(ins->k * sizeof(uint32_t), ESI);
+				MOVobd(RSP, RSI, EAX);
 				break;
 
 			case BPF_LDX|BPF_MEM:
-				MOViq((uintptr_t)mem, RCX);
-				MOVid(ins->k * 4, ESI);
-				MOVobd(RCX, RSI, EDX);
+				MOVid(ins->k * sizeof(uint32_t), ESI);
+				MOVobd(RSP, RSI, EDX);
 				break;
 
 			case BPF_ST:
@@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				 * be optimized if the previous instruction
 				 * was already of this type
 				 */
-				MOViq((uintptr_t)mem, RCX);
-				MOVid(ins->k * 4, ESI);
-				MOVomd(EAX, RCX, RSI);
+				MOVid(ins->k * sizeof(uint32_t), ESI);
+				MOVomd(EAX, RSP, RSI);
 				break;
 
 			case BPF_STX:
-				MOViq((uintptr_t)mem, RCX);
-				MOVid(ins->k * 4, ESI);
-				MOVomd(EDX, RCX, RSI);
+				MOVid(ins->k * sizeof(uint32_t), ESI);
+				MOVomd(EDX, RSP, RSI);
 				break;
 
 			case BPF_JMP|BPF_JA:
@@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 
 			case BPF_ALU|BPF_DIV|BPF_X:
 				TESTrd(EDX, EDX);
-				JNEb(6);
+				JNEb(4);
 				ZEROrd(EAX);
-				MOVrq3(R8, RBX);
-				RET();
+				LEAVE_RET();
 				MOVrd(EDX, ECX);
 				ZEROrd(EDX);
 				DIVrd(ECX);
@@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 		}
 
 #ifdef _KERNEL
-		stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
-		    M_NOWAIT, 0, ~0UL, 16, 0);
+		stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
 		if (stream.ibuf == NULL)
 			break;
 #else
-		stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
-		    PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+		stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
+		    MAP_ANON, -1, 0);
 		if (stream.ibuf == MAP_FAILED) {
 			stream.ibuf = NULL;
 			break;
diff --git a/sys/amd64/amd64/bpf_jit_machdep.h b/sys/amd64/amd64/bpf_jit_machdep.h
index 154bbae..4634187 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.h
+++ b/sys/amd64/amd64/bpf_jit_machdep.h
@@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, 0xc486, 2);					\
 } while (0)
 
-/* ret */
-#define RET() do {						\
-	emitm(&stream, 0xc3, 1);					\
+/* pushq r64 */
+#define PUSH(r64) do {							\
+	emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);		\
+} while (0)
+
+/* leave/ret */
+#define LEAVE_RET() do {						\
+	emitm(&stream, 0xc3c9, 2);					\
 } while (0)
 
 /* addl sr32,dr32 */
@@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, i32, 4);						\
 } while (0)
 
+/* subq i8,r64 */
+#define SUBib(i8, r64) do {						\
+	emitm(&stream, 0x8348, 2);					\
+	emitm(&stream, (29 << 3) | (r64 & 0x7), 1);			\
+	emitm(&stream, i8, 1);						\
+} while (0)
+
 /* mull r32 */
 #define MULrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
diff --git a/sys/i386/i386/bpf_jit_machdep.c b/sys/i386/i386/bpf_jit_machdep.c
index 2b40b19..8e51f7b 100644
--- a/sys/i386/i386/bpf_jit_machdep.c
+++ b/sys/i386/i386/bpf_jit_machdep.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 
 #include <i386/i386/bpf_jit_machdep.h>
 
-bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
 
 /*
  * emit routine to update the jump table
@@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
  * Function that does the real stuff
  */
 bpf_filter_func
-bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
+bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
 {
 	bpf_bin_stream stream;
 	struct bpf_insn *ins;
@@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 
 	/* Allocate the reference table for the jumps */
 #ifdef _KERNEL
-	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
-	    M_BPFJIT, M_NOWAIT);
+	stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
 #else
-	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+	stream.refs = malloc((nins + 1) * sizeof(u_int));
 #endif
 	if (stream.refs == NULL)
 		return (NULL);
@@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 		/* create the procedure header */
 		PUSH(EBP);
 		MOVrd(ESP, EBP);
+		SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
 		PUSH(EDI);
 		PUSH(ESI);
 		PUSH(EBX);
@@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				break;
 
 			case BPF_LD|BPF_MEM:
-				MOVid((uintptr_t)mem, ECX);
-				MOVid(ins->k * 4, ESI);
+				MOVrd(EBP, ECX);
+				MOVid(((int)ins->k - BPF_MEMWORDS) *
+				    sizeof(uint32_t), ESI);
 				MOVobd(ECX, ESI, EAX);
 				break;
 
 			case BPF_LDX|BPF_MEM:
-				MOVid((uintptr_t)mem, ECX);
-				MOVid(ins->k * 4, ESI);
+				MOVrd(EBP, ECX);
+				MOVid(((int)ins->k - BPF_MEMWORDS) *
+				    sizeof(uint32_t), ESI);
 				MOVobd(ECX, ESI, EDX);
 				break;
 
@@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 				 * be optimized if the previous instruction
 				 * was already of this type
 				 */
-				MOVid((uintptr_t)mem, ECX);
-				MOVid(ins->k * 4, ESI);
+				MOVrd(EBP, ECX);
+				MOVid(((int)ins->k - BPF_MEMWORDS) *
+				    sizeof(uint32_t), ESI);
 				MOVomd(EAX, ECX, ESI);
 				break;
 
 			case BPF_STX:
-				MOVid((uintptr_t)mem, ECX);
-				MOVid(ins->k * 4, ESI);
+				MOVrd(EBP, ECX);
+				MOVid(((int)ins->k - BPF_MEMWORDS) *
+				    sizeof(uint32_t), ESI);
 				MOVomd(EDX, ECX, ESI);
 				break;
 
@@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
 		}
 
 #ifdef _KERNEL
-		stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
-		    M_NOWAIT, 0, ~0UL, 16, 0);
+		stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
 		if (stream.ibuf == NULL)
 			break;
 #else
-		stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
-		    PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+		stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
+		    MAP_ANON, -1, 0);
 		if (stream.ibuf == MAP_FAILED) {
 			stream.ibuf = NULL;
 			break;
diff --git a/sys/i386/i386/bpf_jit_machdep.h b/sys/i386/i386/bpf_jit_machdep.h
index 0ec94d2..7e6f2b0 100644
--- a/sys/i386/i386/bpf_jit_machdep.h
+++ b/sys/i386/i386/bpf_jit_machdep.h
@@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, i32, 4);						\
 } while (0)
 
+/* subl i8,r32 */
+#define SUBib(i8, r32) do {						\
+	emitm(&stream, 0x83, 1);					\
+	emitm(&stream, (29 << 3) | (r32 & 0x7), 1);			\
+	emitm(&stream, i8, 1);						\
+} while (0)
+
 /* mull r32 */
 #define MULrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
diff --git a/sys/net/bpf_jitter.c b/sys/net/bpf_jitter.c
index 112b873..a72d2d4 100644
--- a/sys/net/bpf_jitter.c
+++ b/sys/net/bpf_jitter.c
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <net/bpf.h>
 #include <net/bpf_jitter.h>
 
-bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
 
 static u_int	bpf_jit_accept_all(u_char *, u_int, u_int);
 
@@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
 
 	/* Allocate the filter structure */
 	filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
-	    M_BPFJIT, M_NOWAIT | M_ZERO);
+	    M_BPFJIT, M_NOWAIT);
 	if (filter == NULL)
 		return (NULL);
 
@@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
 	}
 
 	/* Create the binary */
-	if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
-	    filter->mem)) == NULL) {
+	if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
 		free(filter, M_BPFJIT);
 		return (NULL);
 	}
@@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter)
 {
 
 	if (filter->func != bpf_jit_accept_all)
-		contigfree(filter->func, filter->size, M_BPFJIT);
+		free(filter->func, M_BPFJIT);
 	free(filter, M_BPFJIT);
 }
 #else
@@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins)
 	filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
 	if (filter == NULL)
 		return (NULL);
-	memset(filter, 0, sizeof(*filter));
 
 	/* No filter means accept all */
 	if (fp == NULL || nins == 0) {
@@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
 	}
 
 	/* Create the binary */
-	if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
-	    filter->mem)) == NULL) {
+	if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
 		free(filter);
 		return (NULL);
 	}
diff --git a/sys/net/bpf_jitter.h b/sys/net/bpf_jitter.h
index fa99868..90a1ff5 100644
--- a/sys/net/bpf_jitter.h
+++ b/sys/net/bpf_jitter.h
@@ -54,7 +54,6 @@ typedef struct bpf_jit_filter {
 	/* The native filtering binary, in the form of a bpf_filter_func. */
 	bpf_filter_func	func;
 	size_t		size;
-	int		mem[BPF_MEMWORDS];	/* Scratch memory */
 } bpf_jit_filter;
 
 /*