Add experimental BPF Just-In-Time compiler for amd64 and i386.

Use the following kernel configuration option to enable:

	options BPF_JITTER

If you want to use bpf_filter() instead (e. g., debugging), do:

	sysctl net.bpf.jitter.enable=0

to turn it off.

Currently BIOCSETWF and bpf_mtap2() are unsupported, and bpf_mtap() is
partially supported because 1) no need, 2) avoid expensive m_copydata(9).

Obtained from:	WinPcap 3.1 (for i386)

4 files changed, 222 insertions, 3 deletions

diff --git a/sys/net/bpf.c b/sys/net/bpf.c
index d2c2d51..5ffcb00 100644
--- a/sys/net/bpf.c
+++ b/sys/net/bpf.c
@@ -65,6 +65,9 @@
 
 #include <net/if.h>
 #include <net/bpf.h>
+#ifdef BPF_JITTER
+#include <net/bpf_jitter.h>
+#endif
 #include <net/bpfdesc.h>
 
 #include <netinet/in.h>
@@ -126,6 +129,12 @@ SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
     &bpf_maxinsns, 0, "Maximum bpf program instructions");
 SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW,
     bpf_stats_sysctl, "bpf statistics portal");
+#ifdef BPF_JITTER
+SYSCTL_NODE(_net_bpf, OID_AUTO, jitter, CTLFLAG_RW, 0, "bpf jitter sysctl");
+static int bpf_jitter_enable = 1;
+SYSCTL_INT(_net_bpf_jitter, OID_AUTO, enable, CTLFLAG_RW,
+    &bpf_jitter_enable, 0, "bpf JIT compiler");
+#endif
 
 static	d_open_t	bpfopen;
 static	d_close_t	bpfclose;
@@ -1017,13 +1026,22 @@ bpf_setf(d, fp, cmd)
 {
 	struct bpf_insn *fcode, *old;
 	u_int wfilter, flen, size;
+#if BPF_JITTER
+	bpf_jit_filter *ofunc;
+#endif
 
 	if (cmd == BIOCSETWF) {
 		old = d->bd_wfilter;
 		wfilter = 1;
+#if BPF_JITTER
+		ofunc = NULL;
+#endif
 	} else {
 		wfilter = 0;
 		old = d->bd_rfilter;
+#if BPF_JITTER
+		ofunc = d->bd_bfilter;
+#endif
 	}
 	if (fp->bf_insns == NULL) {
 		if (fp->bf_len != 0)
@@ -1031,12 +1049,20 @@ bpf_setf(d, fp, cmd)
 		BPFD_LOCK(d);
 		if (wfilter)
 			d->bd_wfilter = NULL;
-		else
+		else {
 			d->bd_rfilter = NULL;
+#if BPF_JITTER
+			d->bd_bfilter = NULL;
+#endif
+		}
 		reset_d(d);
 		BPFD_UNLOCK(d);
 		if (old != NULL)
 			free((caddr_t)old, M_BPF);
+#if BPF_JITTER
+		if (ofunc != NULL)
+			bpf_destroy_jit_filter(ofunc);
+#endif
 		return (0);
 	}
 	flen = fp->bf_len;
@@ -1050,12 +1076,20 @@ bpf_setf(d, fp, cmd)
 		BPFD_LOCK(d);
 		if (wfilter)
 			d->bd_wfilter = fcode;
-		else
+		else {
 			d->bd_rfilter = fcode;
+#if BPF_JITTER
+			d->bd_bfilter = bpf_jitter(fcode, flen);
+#endif
+		}
 		reset_d(d);
 		BPFD_UNLOCK(d);
 		if (old != NULL)
 			free((caddr_t)old, M_BPF);
+#if BPF_JITTER
+		if (ofunc != NULL)
+			bpf_destroy_jit_filter(ofunc);
+#endif
 
 		return (0);
 	}
@@ -1255,6 +1289,11 @@ bpf_tap(bp, pkt, pktlen)
 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
 		BPFD_LOCK(d);
 		++d->bd_rcount;
+#ifdef BPF_JITTER
+		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL)
+			slen = (*(d->bd_bfilter->func))(pkt, pktlen, pktlen);
+		else
+#endif
 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
 		if (slen != 0) {
 			d->bd_fcount++;
@@ -1321,6 +1360,14 @@ bpf_mtap(bp, m)
 			continue;
 		BPFD_LOCK(d);
 		++d->bd_rcount;
+#ifdef BPF_JITTER
+		/* XXX We cannot handle multiple mbufs. */
+		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL &&
+		    m->m_next == NULL)
+			slen = (*(d->bd_bfilter->func))(mtod(m, u_char *),
+			    pktlen, pktlen);
+		else
+#endif
 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
 		if (slen != 0) {
 			d->bd_fcount++;
@@ -1506,8 +1553,12 @@ bpf_freed(d)
 		if (d->bd_fbuf != NULL)
 			free(d->bd_fbuf, M_BPF);
 	}
-	if (d->bd_rfilter)
+	if (d->bd_rfilter) {
 		free((caddr_t)d->bd_rfilter, M_BPF);
+#ifdef BPF_JITTER
+		bpf_destroy_jit_filter(d->bd_bfilter);
+#endif
+	}
 	if (d->bd_wfilter)
 		free((caddr_t)d->bd_wfilter, M_BPF);
 	mtx_destroy(&d->bd_mtx);
diff --git a/sys/net/bpf_jitter.c b/sys/net/bpf_jitter.c
new file mode 100644
index 0000000..26a57fc
--- /dev/null
+++ b/sys/net/bpf_jitter.c
@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the Politecnico di Torino nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS intERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_bpf.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+
+#include <net/bpf.h>
+#include <net/bpf_jitter.h>
+
+MALLOC_DEFINE(M_BPFJIT, "BPF_JIT", "BPF JIT compiler");
+
+bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, int *);
+
+bpf_jit_filter *
+bpf_jitter(struct bpf_insn *fp, int nins)
+{
+	bpf_jit_filter *filter;
+
+	/* Allocate the filter structure */
+	filter = (struct bpf_jit_filter *)malloc(sizeof(struct bpf_jit_filter),
+	    M_BPFJIT, M_WAITOK);
+	if (filter == NULL)
+		return NULL;
+
+	/* Allocate the filter's memory */
+	filter->mem = (int *)malloc(BPF_MEMWORDS * sizeof(int),
+	    M_BPFJIT, M_WAITOK);
+	if (filter->mem == NULL) {
+		free(filter, M_BPFJIT);
+		return NULL;
+	}
+
+	/* Create the binary */
+	if ((filter->func = bpf_jit_compile(fp, nins, filter->mem)) == NULL) {
+		free(filter->mem, M_BPFJIT);
+		free(filter, M_BPFJIT);
+		return NULL;
+	}
+
+	return filter;
+}
+
+void
+bpf_destroy_jit_filter(bpf_jit_filter *filter)
+{
+
+	free(filter->mem, M_BPFJIT);
+	free(filter->func, M_BPFJIT);
+	free(filter, M_BPFJIT);
+}
diff --git a/sys/net/bpf_jitter.h b/sys/net/bpf_jitter.h
new file mode 100644
index 0000000..0ae64f9
--- /dev/null
+++ b/sys/net/bpf_jitter.h
@@ -0,0 +1,80 @@
+/*-
+ * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the Politecnico di Torino nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS intERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NET_BPF_JITTER_H_
+#define _NET_BPF_JITTER_H_
+
+MALLOC_DECLARE(M_BPFJIT);
+
+/*
+ * Prototype of a filtering function created by the jitter.
+ *
+ * The syntax and the meaning of the parameters is analogous to the one of
+ * bpf_filter(). Notice that the filter is not among the parameters because
+ * it is hardwired in the function.
+ */
+typedef u_int (*bpf_filter_func)(u_char *, u_int, u_int);
+
+/* Structure describing a native filtering program created by the jitter. */
+typedef struct bpf_jit_filter {
+	/* The native filtering binary, in the form of a bpf_filter_func. */
+	bpf_filter_func	func;
+
+	int		*mem;
+} bpf_jit_filter;
+
+/*
+ * BPF jitter, builds a machine function from a BPF program.
+ *
+ * param fp	The BPF pseudo-assembly filter that will be translated
+ *		into native code.
+ * param nins	Number of instructions of the input filter.
+ * return	The bpf_jit_filter structure containing the native filtering
+ *		binary.
+ *
+ * bpf_jitter allocates the buffers for the new native filter and
+ * then translates the program pointed by fp calling bpf_jit_compile().
+ */
+bpf_jit_filter	*bpf_jitter(struct bpf_insn *fp, int nins);
+
+/*
+ * Deletes a filtering function that was previously created by bpf_jitter().
+ *
+ * param filter	The filter to destroy.
+ *
+ * This function frees the variuos buffers (code, memory, etc.) associated
+ * with a filtering function.
+ */
+void		bpf_destroy_jit_filter(bpf_jit_filter *filter);
+
+#endif	/* _NET_BPF_JITTER_H_ */
diff --git a/sys/net/bpfdesc.h b/sys/net/bpfdesc.h
index d2cb985..3de11d8 100644
--- a/sys/net/bpfdesc.h
+++ b/sys/net/bpfdesc.h
@@ -71,6 +71,9 @@ struct bpf_d {
 	u_long		bd_rtout;	/* Read timeout in 'ticks' */
 	struct bpf_insn *bd_rfilter; 	/* read filter code */
 	struct bpf_insn *bd_wfilter;	/* write filter code */
+#ifdef BPF_JITTER
+	bpf_jit_filter	*bd_bfilter;	/* binary filter code */
+#endif
 	u_long		bd_rcount;	/* number of packets received */
 	u_long		bd_dcount;	/* number of packets dropped */