MFhead @ r282076

10 files changed, 988 insertions, 393 deletions

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index 8695499..755b504 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -366,8 +366,6 @@
         ..
         kqueue
         ..
-        mmap
-        ..
         mqueue
         ..
         netinet
diff --git a/sys/netinet/ip_fw.h b/sys/netinet/ip_fw.h
index 9c53793..d1764bb 100644
--- a/sys/netinet/ip_fw.h
+++ b/sys/netinet/ip_fw.h
@@ -40,10 +40,12 @@
 #define	IPFW_MAX_SETS		32	/* Number of sets supported by ipfw*/
 
 /*
- * Default number of ipfw tables.
+ * Compat values for old clients
  */
+#ifndef	_KERNEL
 #define	IPFW_TABLES_MAX		65535
 #define	IPFW_TABLES_DEFAULT	128
+#endif
 
 /*
  * Most commands (queue, pipe, tag, untag, limit...) can have a 16-bit
@@ -963,7 +965,6 @@ typedef struct _ipfw_ta_info {
 	uint64_t	spare1;
 } ipfw_ta_info;
 
-#define	IPFW_OBJTYPE_TABLE	1
 typedef struct _ipfw_obj_header {
 	ip_fw3_opheader	opheader;	/* IP_FW3 opcode		*/
 	uint32_t	spare;
diff --git a/sys/netpfil/ipfw/ip_fw2.c b/sys/netpfil/ipfw/ip_fw2.c
index a30f6bf..4da0ee0 100644
--- a/sys/netpfil/ipfw/ip_fw2.c
+++ b/sys/netpfil/ipfw/ip_fw2.c
@@ -2762,6 +2762,10 @@ vnet_ipfw_init(const void *unused)
 	LIST_INIT(&chain->nat);
 #endif
 
+	/* Init shared services hash table */
+	ipfw_init_srv(chain);
+
+	ipfw_init_obj_rewriter();
 	ipfw_init_counters();
 	/* insert the default rule and create the initial map */
 	chain->n_rules = 1;
@@ -2860,9 +2864,11 @@ vnet_ipfw_uninit(const void *unused)
 	if (reap != NULL)
 		ipfw_reap_rules(reap);
 	vnet_ipfw_iface_destroy(chain);
+	ipfw_destroy_srv(chain);
 	IPFW_LOCK_DESTROY(chain);
 	ipfw_dyn_uninit(1);	/* free the remaining parts */
 	ipfw_destroy_counters();
+	ipfw_destroy_obj_rewriter();
 	return (0);
 }
 
diff --git a/sys/netpfil/ipfw/ip_fw_private.h b/sys/netpfil/ipfw/ip_fw_private.h
index bb5b3ba..504093b 100644
--- a/sys/netpfil/ipfw/ip_fw_private.h
+++ b/sys/netpfil/ipfw/ip_fw_private.h
@@ -264,10 +264,10 @@ struct ip_fw_chain {
 	struct ip_fw	**map;		/* array of rule ptrs to ease lookup */
 	uint32_t	id;		/* ruleset id */
 	int		n_rules;	/* number of static rules */
-	LIST_HEAD(nat_list, cfg_nat) nat;       /* list of nat entries */
 	void		*tablestate;	/* runtime table info */
 	void		*valuestate;	/* runtime table value info */
 	int		*idxmap;	/* skipto array of rules */
+	void		**srvstate;	/* runtime service mappings */
 #if defined( __linux__ ) || defined( _WIN32 )
 	spinlock_t rwmtx;
 #else
@@ -275,10 +275,12 @@ struct ip_fw_chain {
 #endif
 	int		static_len;	/* total len of static rules (v0) */
 	uint32_t	gencnt;		/* NAT generation count */
+	LIST_HEAD(nat_list, cfg_nat) nat;       /* list of nat entries */
 	struct ip_fw	*default_rule;
 	struct tables_config *tblcfg;	/* tables module data */
 	void		*ifcfg;		/* interface module data */
 	int		*idxmap_back;	/* standby skipto array of rules */
+	struct namedobj_instance	*srvmap; /* cfg name->number mappings */
 #if defined( __linux__ ) || defined( _WIN32 )
 	spinlock_t uh_lock;
 #else
@@ -306,16 +308,15 @@ struct table_value {
 	uint64_t	refcnt;		/* Number of references */
 };
 
-struct namedobj_instance;
 
 struct named_object {
 	TAILQ_ENTRY(named_object)	nn_next;	/* namehash */
 	TAILQ_ENTRY(named_object)	nv_next;	/* valuehash */
 	char			*name;	/* object name */
-	uint8_t			type;	/* object type */
-	uint8_t			compat;	/* Object name is number */
+	uint8_t			subtype;	/* object subtype within class */
+	uint8_t			etlv;	/* Export TLV id */
+	uint16_t		spare[2];
 	uint16_t		kidx;	/* object kernel index */
-	uint16_t		uidx;	/* userland idx for compat records */
 	uint32_t		set;	/* set object belongs to */
 	uint32_t		refcnt;	/* number of references */
 };
@@ -450,7 +451,7 @@ struct obj_idx {
 
 struct rule_check_info {
 	uint16_t	flags;		/* rule-specific check flags */
-	uint16_t	table_opcodes;	/* count of opcodes referencing table */
+	uint16_t	object_opcodes;	/* num of opcodes referencing objects */
 	uint16_t	urule_numoff;	/* offset of rulenum in bytes */
 	uint8_t		version;	/* rule version */
 	uint8_t		spare;
@@ -507,6 +508,84 @@ struct ip_fw_bcounter0 {
     (r)->cmd_len * 4 - 4, 8))
 #define	RULEKSIZE1(r)	roundup2((sizeof(struct ip_fw) + (r)->cmd_len*4 - 4), 8)
 
+/*
+ * Tables/Objects index rewriting code
+ */
+
+/* Default and maximum number of ipfw tables/objects. */
+#define	IPFW_TABLES_MAX		65536
+#define	IPFW_TABLES_DEFAULT	128
+#define	IPFW_OBJECTS_MAX	65536
+#define	IPFW_OBJECTS_DEFAULT	128
+
+#define	CHAIN_TO_SRV(ch)	((ch)->srvmap)
+
+struct tid_info {
+	uint32_t	set;	/* table set */
+	uint16_t	uidx;	/* table index */
+	uint8_t		type;	/* table type */
+	uint8_t		atype;
+	uint8_t		spare;
+	int		tlen;	/* Total TLV size block */
+	void		*tlvs;	/* Pointer to first TLV */
+};
+
+/*
+ * Classifier callback. Checks if @cmd opcode contains kernel object reference.
+ * If true, returns its index and type.
+ * Returns 0 if match is found, 1 overwise.
+ */
+typedef int (ipfw_obj_rw_cl)(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype);
+/*
+ * Updater callback. Sets kernel object reference index to @puidx
+ */
+typedef void (ipfw_obj_rw_upd)(ipfw_insn *cmd, uint16_t puidx);
+/*
+ * Finder callback. Tries to find named object by name (specified via @ti).
+ * Stores found named object pointer in @pno.
+ * If object was not found, NULL is stored.
+ *
+ * Return 0 if input data was valid.
+ */
+typedef int (ipfw_obj_fname_cb)(struct ip_fw_chain *ch,
+    struct tid_info *ti, struct named_object **pno);
+/*
+ * Another finder callback. Tries to findex named object by kernel index.
+ *
+ * Returns pointer to named object or NULL.
+ */
+typedef struct named_object *(ipfw_obj_fidx_cb)(struct ip_fw_chain *ch,
+    uint16_t kidx);
+/*
+ * Object creator callback. Tries to create object specified by @ti.
+ * Stores newly-allocated object index in @pkidx.
+ *
+ * Returns 0 on success.
+ */
+typedef int (ipfw_obj_create_cb)(struct ip_fw_chain *ch, struct tid_info *ti,
+    uint16_t *pkidx);
+
+
+struct opcode_obj_rewrite {
+	uint32_t		opcode;		/* Opcode to act upon */
+	uint32_t		etlv;		/* Relevant export TLV id  */
+	ipfw_obj_rw_cl		*classifier;	/* Check if rewrite is needed */
+	ipfw_obj_rw_upd		*update;	/* update cmd with new value */
+	ipfw_obj_fname_cb	*find_byname;	/* Find named object by name */
+	ipfw_obj_fidx_cb	*find_bykidx;	/* Find named object by kidx */
+	ipfw_obj_create_cb	*create_object;	/* Create named object */
+};
+
+#define	IPFW_ADD_OBJ_REWRITER(f, c)	do {	\
+	if ((f) != 0) 				\
+		ipfw_add_obj_rewriter(c,	\
+		    sizeof(c) / sizeof(c[0]));	\
+	} while(0)
+#define	IPFW_DEL_OBJ_REWRITER(l, c)	do {	\
+	if ((l) != 0) 				\
+		ipfw_del_obj_rewriter(c,	\
+		    sizeof(c) / sizeof(c[0]));	\
+	} while(0)
 
 /* In ip_fw_iface.c */
 int ipfw_iface_init(void);
@@ -562,6 +641,7 @@ caddr_t ipfw_get_sopt_header(struct sockopt_data *sd, size_t needed);
 		    sizeof(c) / sizeof(c[0]));	\
 	} while(0)
 
+struct namedobj_instance;
 typedef void (objhash_cb_t)(struct namedobj_instance *ni, struct named_object *,
     void *arg);
 typedef uint32_t (objhash_hash_f)(struct namedobj_instance *ni, void *key,
@@ -578,6 +658,8 @@ void ipfw_objhash_bitmap_free(void *idx, int blocks);
 void ipfw_objhash_set_hashf(struct namedobj_instance *ni, objhash_hash_f *f);
 struct named_object *ipfw_objhash_lookup_name(struct namedobj_instance *ni,
     uint32_t set, char *name);
+struct named_object *ipfw_objhash_lookup_name_type(struct namedobj_instance *ni,
+    uint32_t set, uint32_t type, char *name);
 struct named_object *ipfw_objhash_lookup_kidx(struct namedobj_instance *ni,
     uint16_t idx);
 int ipfw_objhash_same_name(struct namedobj_instance *ni, struct named_object *a,
@@ -591,6 +673,25 @@ int ipfw_objhash_free_idx(struct namedobj_instance *ni, uint16_t idx);
 int ipfw_objhash_alloc_idx(void *n, uint16_t *pidx);
 void ipfw_objhash_set_funcs(struct namedobj_instance *ni,
     objhash_hash_f *hash_f, objhash_cmp_f *cmp_f);
+void ipfw_init_obj_rewriter(void);
+void ipfw_destroy_obj_rewriter(void);
+void ipfw_add_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count);
+int ipfw_del_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count);
+
+int ipfw_rewrite_rule_uidx(struct ip_fw_chain *chain,
+    struct rule_check_info *ci);
+int ipfw_mark_object_kidx(struct ip_fw_chain *chain, struct ip_fw *rule,
+    uint32_t *bmask);
+int ref_opcode_object(struct ip_fw_chain *ch, ipfw_insn *cmd, struct tid_info *ti,
+    struct obj_idx *pidx, int *found, int *unresolved);
+void unref_oib_objects(struct ip_fw_chain *ch, ipfw_insn *cmd,
+    struct obj_idx *oib, struct obj_idx *end);
+int create_objects_compat(struct ip_fw_chain *ch, ipfw_insn *cmd,
+    struct obj_idx *oib, struct obj_idx *pidx, struct tid_info *ti);
+void update_opcode_kidx(ipfw_insn *cmd, uint16_t idx);
+int classify_opcode_kidx(ipfw_insn *cmd, uint16_t *puidx);
+void ipfw_init_srv(struct ip_fw_chain *ch);
+void ipfw_destroy_srv(struct ip_fw_chain *ch);
 
 /* In ip_fw_table.c */
 struct table_info;
diff --git a/sys/netpfil/ipfw/ip_fw_sockopt.c b/sys/netpfil/ipfw/ip_fw_sockopt.c
index ef1ff6c..6f55511 100644
--- a/sys/netpfil/ipfw/ip_fw_sockopt.c
+++ b/sys/netpfil/ipfw/ip_fw_sockopt.c
@@ -148,6 +148,21 @@ static struct ipfw_sopt_handler	scodes[] = {
 	{ IP_FW_DUMP_SOPTCODES,	0,	HDIR_GET,	dump_soptcodes },
 };
 
+static int
+set_legacy_obj_kidx(struct ip_fw_chain *ch, struct ip_fw_rule0 *rule);
+struct opcode_obj_rewrite *ipfw_find_op_rw(uint16_t opcode);
+static int mark_object_kidx(struct ip_fw_chain *ch, struct ip_fw *rule,
+    uint32_t *bmask);
+static void unref_rule_objects(struct ip_fw_chain *chain, struct ip_fw *rule);
+static int export_objhash_ntlv(struct namedobj_instance *ni, uint16_t kidx,
+    struct sockopt_data *sd);
+
+/*
+ * Opcode object rewriter variables
+ */
+struct opcode_obj_rewrite *ctl3_rewriters;
+static size_t ctl3_rsize;
+
 /*
  * static variables followed by global ones
  */
@@ -646,17 +661,18 @@ commit_rules(struct ip_fw_chain *chain, struct rule_check_info *rci, int count)
 	struct ip_fw *krule;
 	struct ip_fw **map;	/* the new array of pointers */
 
-	/* Check if we need to do table remap */
+	/* Check if we need to do table/obj index remap */
 	tcount = 0;
 	for (ci = rci, i = 0; i < count; ci++, i++) {
-		if (ci->table_opcodes == 0)
+		if (ci->object_opcodes == 0)
 			continue;
 
 		/*
-		 * Rule has some table opcodes.
-		 * Reference & allocate needed tables/
+		 * Rule has some object opcodes.
+		 * We need to find (and create non-existing)
+		 * kernel objects, and reference existing ones.
 		 */
-		error = ipfw_rewrite_table_uidx(chain, ci);
+		error = ipfw_rewrite_rule_uidx(chain, ci);
 		if (error != 0) {
 
 			/*
@@ -674,9 +690,9 @@ commit_rules(struct ip_fw_chain *chain, struct rule_check_info *rci, int count)
 				IPFW_UH_WLOCK(chain);
 				while (ci != rci) {
 					ci--;
-					if (ci->table_opcodes == 0)
+					if (ci->object_opcodes == 0)
 						continue;
-					ipfw_unref_rule_tables(chain,ci->krule);
+					unref_rule_objects(chain,ci->krule);
 
 				}
 				IPFW_UH_WUNLOCK(chain);
@@ -696,10 +712,10 @@ commit_rules(struct ip_fw_chain *chain, struct rule_check_info *rci, int count)
 			/* Unbind tables */
 			IPFW_UH_WLOCK(chain);
 			for (ci = rci, i = 0; i < count; ci++, i++) {
-				if (ci->table_opcodes == 0)
+				if (ci->object_opcodes == 0)
 					continue;
 
-				ipfw_unref_rule_tables(chain, ci->krule);
+				unref_rule_objects(chain, ci->krule);
 			}
 			IPFW_UH_WUNLOCK(chain);
 		}
@@ -759,7 +775,7 @@ ipfw_reap_add(struct ip_fw_chain *chain, struct ip_fw **head,
 	IPFW_UH_WLOCK_ASSERT(chain);
 
 	/* Unlink rule from everywhere */
-	ipfw_unref_rule_tables(chain, rule);
+	unref_rule_objects(chain, rule);
 
 	*((struct ip_fw **)rule) = *head;
 	*head = rule;
@@ -1542,7 +1558,7 @@ check_ipfw_rule_body(ipfw_insn *cmd, int cmd_len, struct rule_check_info *ci)
 			    cmdlen != F_INSN_SIZE(ipfw_insn_u32) + 1 &&
 			    cmdlen != F_INSN_SIZE(ipfw_insn_u32))
 				goto bad_size;
-			ci->table_opcodes++;
+			ci->object_opcodes++;
 			break;
 		case O_IP_FLOW_LOOKUP:
 			if (cmd->arg1 >= V_fw_tables_max) {
@@ -1553,7 +1569,7 @@ check_ipfw_rule_body(ipfw_insn *cmd, int cmd_len, struct rule_check_info *ci)
 			if (cmdlen != F_INSN_SIZE(ipfw_insn) &&
 			    cmdlen != F_INSN_SIZE(ipfw_insn_u32))
 				goto bad_size;
-			ci->table_opcodes++;
+			ci->object_opcodes++;
 			break;
 		case O_MACADDR2:
 			if (cmdlen != F_INSN_SIZE(ipfw_insn_mac))
@@ -1587,7 +1603,7 @@ check_ipfw_rule_body(ipfw_insn *cmd, int cmd_len, struct rule_check_info *ci)
 		case O_XMIT:
 		case O_VIA:
 			if (((ipfw_insn_if *)cmd)->name[0] == '\1')
-				ci->table_opcodes++;
+				ci->object_opcodes++;
 			if (cmdlen != F_INSN_SIZE(ipfw_insn_if))
 				goto bad_size;
 			break;
@@ -1631,6 +1647,7 @@ check_ipfw_rule_body(ipfw_insn *cmd, int cmd_len, struct rule_check_info *ci)
 				return EINVAL;
 			if (cmdlen != F_INSN_SIZE(ipfw_insn_nat))
  				goto bad_size;		
+			ci->object_opcodes++;
  			goto check_action;
 		case O_FORWARD_MAC: /* XXX not implemented yet */
 		case O_CHECK_STATE:
@@ -1788,7 +1805,7 @@ ipfw_getrules(struct ip_fw_chain *chain, void *buf, size_t space)
 		    l = RULESIZE7(rule);
 		    if (bp + l + sizeof(uint32_t) <= ep) {
 			bcopy(rule, bp, l + sizeof(uint32_t));
-			error = ipfw_rewrite_table_kidx(chain,
+			error = set_legacy_obj_kidx(chain,
 			    (struct ip_fw_rule0 *)bp);
 			if (error != 0)
 				return (0);
@@ -1817,7 +1834,7 @@ ipfw_getrules(struct ip_fw_chain *chain, void *buf, size_t space)
 		}
 		dst = (struct ip_fw_rule0 *)bp;
 		export_rule0(rule, dst, l);
-		error = ipfw_rewrite_table_kidx(chain, dst);
+		error = set_legacy_obj_kidx(chain, dst);
 
 		/*
 		 * XXX HACK. Store the disable mask in the "next"
@@ -1861,6 +1878,34 @@ struct dump_args {
 };
 
 /*
+ * Export named object info in instance @ni, identified by @kidx
+ * to ipfw_obj_ntlv. TLV is allocated from @sd space.
+ *
+ * Returns 0 on success.
+ */
+static int
+export_objhash_ntlv(struct namedobj_instance *ni, uint16_t kidx,
+    struct sockopt_data *sd)
+{
+	struct named_object *no;
+	ipfw_obj_ntlv *ntlv;
+
+	no = ipfw_objhash_lookup_kidx(ni, kidx);
+	KASSERT(no != NULL, ("invalid object kernel index passed"));
+
+	ntlv = (ipfw_obj_ntlv *)ipfw_get_sopt_space(sd, sizeof(*ntlv));
+	if (ntlv == NULL)
+		return (ENOMEM);
+
+	ntlv->head.type = no->etlv;
+	ntlv->head.length = sizeof(*ntlv);
+	ntlv->idx = no->kidx;
+	strlcpy(ntlv->name, no->name, sizeof(ntlv->name));
+
+	return (0);
+}
+
+/*
  * Dumps static rules with table TLVs in buffer @sd.
  *
  * Returns 0 on success.
@@ -1874,6 +1919,7 @@ dump_static_rules(struct ip_fw_chain *chain, struct dump_args *da,
 	uint32_t tcount;
 	ipfw_obj_ctlv *ctlv;
 	struct ip_fw *krule;
+	struct namedobj_instance *ni;
 	caddr_t dst;
 
 	/* Dump table names first (if any) */
@@ -1891,13 +1937,21 @@ dump_static_rules(struct ip_fw_chain *chain, struct dump_args *da,
 
 	i = 0;
 	tcount = da->tcount;
+	ni = ipfw_get_table_objhash(chain);
 	while (tcount > 0) {
 		if ((bmask[i / 32] & (1 << (i % 32))) == 0) {
 			i++;
 			continue;
 		}
 
-		if ((error = ipfw_export_table_ntlv(chain, i, sd)) != 0)
+		/* Jump to shared named object bitmask */
+		if (i >= IPFW_TABLES_MAX) {
+			ni = CHAIN_TO_SRV(chain);
+			i -= IPFW_TABLES_MAX;
+			bmask += IPFW_TABLES_MAX / 32;
+		}
+
+		if ((error = export_objhash_ntlv(ni, i, sd)) != 0)
 			return (error);
 
 		i++;
@@ -1929,6 +1983,52 @@ dump_static_rules(struct ip_fw_chain *chain, struct dump_args *da,
 }
 
 /*
+ * Marks every object index used in @rule with bit in @bmask.
+ * Used to generate bitmask of referenced tables/objects for given ruleset
+ * or its part.
+ *
+ * Returns number of newly-referenced objects.
+ */
+static int
+mark_object_kidx(struct ip_fw_chain *ch, struct ip_fw *rule,
+    uint32_t *bmask)
+{
+	int cmdlen, l, count;
+	ipfw_insn *cmd;
+	uint16_t kidx;
+	struct opcode_obj_rewrite *rw;
+	int bidx;
+	uint8_t subtype;
+
+	l = rule->cmd_len;
+	cmd = rule->cmd;
+	cmdlen = 0;
+	count = 0;
+	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
+		cmdlen = F_LEN(cmd);
+
+		rw = ipfw_find_op_rw(cmd->opcode);
+		if (rw == NULL)
+			continue;
+
+		if (rw->classifier(cmd, &kidx, &subtype) != 0)
+			continue;
+
+		bidx = kidx / 32;
+		/* Maintain separate bitmasks for table and non-table objects */
+		if (rw->etlv != IPFW_TLV_TBL_NAME)
+			bidx += IPFW_TABLES_MAX / 32;
+
+		if ((bmask[bidx] & (1 << (kidx % 32))) == 0)
+			count++;
+
+		bmask[bidx] |= 1 << (kidx % 32);
+	}
+
+	return (count);
+}
+
+/*
  * Dumps requested objects data
  * Data layout (version 0)(current):
  * Request: [ ipfw_cfg_lheader ] + IPFW_CFG_GET_* flags
@@ -1963,9 +2063,9 @@ dump_config(struct ip_fw_chain *chain, ip_fw3_opheader *op3,
 
 	error = 0;
 	bmask = NULL;
-	/* Allocate needed state */
+	/* Allocate needed state. Note we allocate 2xspace mask, for table&srv  */
 	if (hdr->flags & IPFW_CFG_GET_STATIC)
-		bmask = malloc(IPFW_TABLES_MAX / 8, M_TEMP, M_WAITOK | M_ZERO);
+		bmask = malloc(IPFW_TABLES_MAX / 4, M_TEMP, M_WAITOK | M_ZERO);
 
 	IPFW_UH_RLOCK(chain);
 
@@ -1994,7 +2094,8 @@ dump_config(struct ip_fw_chain *chain, ip_fw3_opheader *op3,
 			rule = chain->map[i];
 			da.rsize += RULEUSIZE1(rule) + sizeof(ipfw_obj_tlv);
 			da.rcount++;
-			da.tcount += ipfw_mark_table_kidx(chain, rule, bmask);
+			/* Update bitmask of used objects for given range */
+			da.tcount += mark_object_kidx(chain, rule, bmask);
 		}
 		/* Add counters if requested */
 		if (hdr->flags & IPFW_CFG_GET_COUNTERS) {
@@ -2064,6 +2165,241 @@ check_object_name(ipfw_obj_ntlv *ntlv)
 }
 
 /*
+ * Creates non-existent objects referenced by rule.
+ *
+ * Return 0 on success.
+ */
+int
+create_objects_compat(struct ip_fw_chain *ch, ipfw_insn *cmd,
+    struct obj_idx *oib, struct obj_idx *pidx, struct tid_info *ti)
+{
+	struct opcode_obj_rewrite *rw;
+	struct obj_idx *p;
+	uint16_t kidx;
+	int error;
+
+	/*
+	 * Compatibility stuff: do actual creation for non-existing,
+	 * but referenced objects.
+	 */
+	for (p = oib; p < pidx; p++) {
+		if (p->kidx != 0)
+			continue;
+
+		ti->uidx = p->uidx;
+		ti->type = p->type;
+		ti->atype = 0;
+
+		rw = ipfw_find_op_rw((cmd + p->off)->opcode);
+		KASSERT(rw != NULL, ("Unable to find handler for op %d",
+		    (cmd + p->off)->opcode));
+
+		error = rw->create_object(ch, ti, &kidx);
+		if (error == 0) {
+			p->kidx = kidx;
+			continue;
+		}
+
+		/*
+		 * Error happened. We have to rollback everything.
+		 * Drop all already acquired references.
+		 */
+		IPFW_UH_WLOCK(ch);
+		unref_oib_objects(ch, cmd, oib, pidx);
+		IPFW_UH_WUNLOCK(ch);
+
+		return (error);
+	}
+
+	return (error);
+}
+
+/*
+ * Compatibility function for old ipfw(8) binaries.
+ * Rewrites table/nat kernel indices with userland ones.
+ * Convert tables matching '/^\d+$/' to their atoi() value.
+ * Use number 65535 for other tables.
+ *
+ * Returns 0 on success.
+ */
+static int
+set_legacy_obj_kidx(struct ip_fw_chain *ch, struct ip_fw_rule0 *rule)
+{
+	int cmdlen, error, l;
+	ipfw_insn *cmd;
+	uint16_t kidx, uidx;
+	struct named_object *no;
+	struct opcode_obj_rewrite *rw;
+	uint8_t subtype;
+	char *end;
+	long val;
+
+	error = 0;
+
+	l = rule->cmd_len;
+	cmd = rule->cmd;
+	cmdlen = 0;
+	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
+		cmdlen = F_LEN(cmd);
+
+		rw = ipfw_find_op_rw(cmd->opcode);
+		if (rw == NULL)
+			continue;
+
+		/* Check if is index in given opcode */
+		if (rw->classifier(cmd, &kidx, &subtype) != 0)
+			continue;
+
+		/* Try to find referenced kernel object */
+		no = rw->find_bykidx(ch, kidx);
+		if (no == NULL)
+			continue;
+
+		val = strtol(no->name, &end, 10);
+		if (*end == '\0' && val < 65535) {
+			uidx = val;
+		} else {
+
+			/*
+			 * We are called via legacy opcode.
+			 * Save error and show table as fake number
+			 * not to make ipfw(8) hang.
+			 */
+			uidx = 65535;
+			error = 2;
+		}
+
+		rw->update(cmd, uidx);
+	}
+
+	return (error);
+}
+
+
+/*
+ * Unreferences all already-referenced objects in given @cmd rule,
+ * using information in @oib.
+ *
+ * Used to rollback partially converted rule on error.
+ */
+void
+unref_oib_objects(struct ip_fw_chain *ch, ipfw_insn *cmd, struct obj_idx *oib,
+    struct obj_idx *end)
+{
+	struct opcode_obj_rewrite *rw;
+	struct named_object *no;
+	struct obj_idx *p;
+
+	IPFW_UH_WLOCK_ASSERT(ch);
+
+	for (p = oib; p < end; p++) {
+		if (p->kidx == 0)
+			continue;
+
+		rw = ipfw_find_op_rw((cmd + p->off)->opcode);
+		KASSERT(rw != NULL, ("Unable to find handler for op %d",
+		    (cmd + p->off)->opcode));
+
+		/* Find & unref by existing idx */
+		no = rw->find_bykidx(ch, p->kidx);
+		KASSERT(no != NULL, ("Ref'd object %d disappeared", p->kidx));
+		no->refcnt--;
+	}
+}
+
+/*
+ * Remove references from every object used in @rule.
+ * Used at rule removal code.
+ */
+static void
+unref_rule_objects(struct ip_fw_chain *ch, struct ip_fw *rule)
+{
+	int cmdlen, l;
+	ipfw_insn *cmd;
+	struct named_object *no;
+	uint16_t kidx;
+	struct opcode_obj_rewrite *rw;
+	uint8_t subtype;
+
+	IPFW_UH_WLOCK_ASSERT(ch);
+
+	l = rule->cmd_len;
+	cmd = rule->cmd;
+	cmdlen = 0;
+	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
+		cmdlen = F_LEN(cmd);
+
+		rw = ipfw_find_op_rw(cmd->opcode);
+		if (rw == NULL)
+			continue;
+		if (rw->classifier(cmd, &kidx, &subtype) != 0)
+			continue;
+
+		no = rw->find_bykidx(ch, kidx);
+
+		KASSERT(no != NULL, ("table id %d not found", kidx));
+		KASSERT(no->subtype == subtype,
+		    ("wrong type %d (%d) for table id %d",
+		    no->subtype, subtype, kidx));
+		KASSERT(no->refcnt > 0, ("refcount for table %d is %d",
+		    kidx, no->refcnt));
+
+		no->refcnt--;
+	}
+}
+
+
+/*
+ * Find and reference object (if any) stored in instruction @cmd.
+ *
+ * Saves object info in @pidx, sets
+ *  - @found to 1 if object was found and references
+ *  - @unresolved to 1 if object should exists but not found
+ *
+ * Returns non-zero value in case of error.
+ */
+int
+ref_opcode_object(struct ip_fw_chain *ch, ipfw_insn *cmd, struct tid_info *ti,
+    struct obj_idx *pidx, int *found, int *unresolved)
+{
+	struct named_object *no;
+	struct opcode_obj_rewrite *rw;
+	int error;
+
+	*found = 0;
+	*unresolved = 0;
+
+	/* Check if this opcode is candidate for rewrite */
+	rw = ipfw_find_op_rw(cmd->opcode);
+	if (rw == NULL)
+		return (0);
+
+	/* Check if we need to rewrite this opcode */
+	if (rw->classifier(cmd, &ti->uidx, &ti->type) != 0)
+		return (0);
+
+	/* Need to rewrite. Save necessary fields */
+	pidx->uidx = ti->uidx;
+	pidx->type = ti->type;
+
+	/* Try to find referenced kernel object */
+	error = rw->find_byname(ch, ti, &no);
+	if (error != 0)
+		return (error);
+	if (no == NULL) {
+		*unresolved = 1;
+		return (0);
+	}
+
+	/* Found. bump refcount */
+	*found = 1;
+	no->refcnt++;
+	pidx->kidx = no->kidx;
+
+	return (0);
+}
+
+/*
  * Adds one or more rules to ipfw @chain.
  * Data layout (version 0)(current):
  * Request:
@@ -2315,6 +2651,160 @@ dump_soptcodes(struct ip_fw_chain *chain, ip_fw3_opheader *op3,
 }
 
 /*
+ * Compares two opcodes.
+ * Used both in qsort() and bsearch().
+ *
+ * Returns 0 if match is found.
+ */
+static int
+compare_opcodes(const void *_a, const void *_b)
+{
+	const struct opcode_obj_rewrite *a, *b;
+
+	a = (const struct opcode_obj_rewrite *)_a;
+	b = (const struct opcode_obj_rewrite *)_b;
+
+	if (a->opcode < b->opcode)
+		return (-1);
+	else if (a->opcode > b->opcode)
+		return (1);
+
+	return (0);
+}
+
+/*
+ * Finds opcode object rewriter based on @code.
+ *
+ * Returns pointer to handler or NULL.
+ */
+struct opcode_obj_rewrite *
+ipfw_find_op_rw(uint16_t opcode)
+{
+	struct opcode_obj_rewrite *rw, h;
+
+	memset(&h, 0, sizeof(h));
+	h.opcode = opcode;
+
+	rw = (struct opcode_obj_rewrite *)bsearch(&h, ctl3_rewriters,
+	    ctl3_rsize, sizeof(h), compare_opcodes);
+
+	return (rw);
+}
+
+int
+classify_opcode_kidx(ipfw_insn *cmd, uint16_t *puidx)
+{
+	struct opcode_obj_rewrite *rw;
+	uint8_t subtype;
+
+	rw = ipfw_find_op_rw(cmd->opcode);
+	if (rw == NULL)
+		return (1);
+
+	return (rw->classifier(cmd, puidx, &subtype));
+}
+
+void
+update_opcode_kidx(ipfw_insn *cmd, uint16_t idx)
+{
+	struct opcode_obj_rewrite *rw;
+
+	rw = ipfw_find_op_rw(cmd->opcode);
+	KASSERT(rw != NULL, ("No handler to update opcode %d", cmd->opcode));
+	rw->update(cmd, idx);
+}
+
+void
+ipfw_init_obj_rewriter()
+{
+
+	ctl3_rewriters = NULL;
+	ctl3_rsize = 0;
+}
+
+void
+ipfw_destroy_obj_rewriter()
+{
+
+	if (ctl3_rewriters != NULL)
+		free(ctl3_rewriters, M_IPFW);
+	ctl3_rewriters = NULL;
+	ctl3_rsize = 0;
+}
+
+/*
+ * Adds one or more opcode object rewrite handlers to the global array.
+ * Function may sleep.
+ */
+void
+ipfw_add_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count)
+{
+	size_t sz;
+	struct opcode_obj_rewrite *tmp;
+
+	CTL3_LOCK();
+
+	for (;;) {
+		sz = ctl3_rsize + count;
+		CTL3_UNLOCK();
+		tmp = malloc(sizeof(*rw) * sz, M_IPFW, M_WAITOK | M_ZERO);
+		CTL3_LOCK();
+		if (ctl3_rsize + count <= sz)
+			break;
+
+		/* Retry */
+		free(tmp, M_IPFW);
+	}
+
+	/* Merge old & new arrays */
+	sz = ctl3_rsize + count;
+	memcpy(tmp, ctl3_rewriters, ctl3_rsize * sizeof(*rw));
+	memcpy(&tmp[ctl3_rsize], rw, count * sizeof(*rw));
+	qsort(tmp, sz, sizeof(*rw), compare_opcodes);
+	/* Switch new and free old */
+	if (ctl3_rewriters != NULL)
+		free(ctl3_rewriters, M_IPFW);
+	ctl3_rewriters = tmp;
+	ctl3_rsize = sz;
+
+	CTL3_UNLOCK();
+}
+
+/*
+ * Removes one or more object rewrite handlers from the global array.
+ */
+int
+ipfw_del_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count)
+{
+	size_t sz;
+	struct opcode_obj_rewrite *tmp, *h;
+	int i;
+
+	CTL3_LOCK();
+
+	for (i = 0; i < count; i++) {
+		tmp = &rw[i];
+		h = ipfw_find_op_rw(tmp->opcode);
+		if (h == NULL)
+			continue;
+
+		sz = (ctl3_rewriters + ctl3_rsize - (h + 1)) * sizeof(*h);
+		memmove(h, h + 1, sz);
+		ctl3_rsize--;
+	}
+
+	if (ctl3_rsize == 0) {
+		if (ctl3_rewriters != NULL)
+			free(ctl3_rewriters, M_IPFW);
+		ctl3_rewriters = NULL;
+	}
+
+	CTL3_UNLOCK();
+
+	return (0);
+}
+
+/*
  * Compares two sopt handlers (code, version and handler ptr).
  * Used both as qsort() and bsearch().
  * Does not compare handler for latter case.
@@ -3150,6 +3640,23 @@ convert_rule_to_8(struct ip_fw_rule0 *rule)
  *
  */
 
+void
+ipfw_init_srv(struct ip_fw_chain *ch)
+{
+
+	ch->srvmap = ipfw_objhash_create(IPFW_OBJECTS_DEFAULT);
+	ch->srvstate = malloc(sizeof(void *) * IPFW_OBJECTS_DEFAULT,
+	    M_IPFW, M_WAITOK | M_ZERO);
+}
+
+void
+ipfw_destroy_srv(struct ip_fw_chain *ch)
+{
+
+	free(ch->srvstate, M_IPFW);
+	ipfw_objhash_destroy(ch->srvmap);
+}
+
 /*
  * Allocate new bitmask which can be used to enlarge/shrink
  * named instance index.
@@ -3323,6 +3830,26 @@ ipfw_objhash_lookup_name(struct namedobj_instance *ni, uint32_t set, char *name)
 	return (NULL);
 }
 
+/*
+ * Find named object by name, considering also its TLV type.
+ */
+struct named_object *
+ipfw_objhash_lookup_name_type(struct namedobj_instance *ni, uint32_t set,
+    uint32_t type, char *name)
+{
+	struct named_object *no;
+	uint32_t hash;
+
+	hash = ni->hash_f(ni, name, set) % ni->nn_size;
+
+	TAILQ_FOREACH(no, &ni->names[hash], nn_next) {
+		if (ni->cmp_f(no, name, set) == 0 && no->etlv == type)
+			return (no);
+	}
+
+	return (NULL);
+}
+
 struct named_object *
 ipfw_objhash_lookup_kidx(struct namedobj_instance *ni, uint16_t kidx)
 {
diff --git a/sys/netpfil/ipfw/ip_fw_table.c b/sys/netpfil/ipfw/ip_fw_table.c
index 4498ace..7858c78 100644
--- a/sys/netpfil/ipfw/ip_fw_table.c
+++ b/sys/netpfil/ipfw/ip_fw_table.c
@@ -89,6 +89,8 @@ struct table_config {
 	struct namedobj_instance	*vi;
 };
 
+static int find_table_err(struct namedobj_instance *ni, struct tid_info *ti,
+    struct table_config **tc);
 static struct table_config *find_table(struct namedobj_instance *ni,
     struct tid_info *ti);
 static struct table_config *alloc_table_config(struct ip_fw_chain *ch,
@@ -122,7 +124,6 @@ static struct table_algo *find_table_algo(struct tables_config *tableconf,
 
 static void objheader_to_ti(struct _ipfw_obj_header *oh, struct tid_info *ti);
 static void ntlv_to_ti(struct _ipfw_obj_ntlv *ntlv, struct tid_info *ti);
-static int classify_table_opcode(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype);
 
 #define	CHAIN_TO_NI(chain)	(CHAIN_TO_TCFG(chain)->namehash)
 #define	KIDX_TO_TI(ch, k)	(&(((struct table_info *)(ch)->tablestate)[k]))
@@ -297,7 +298,7 @@ find_ref_table(struct ip_fw_chain *ch, struct tid_info *ti,
 	tc = NULL;
 	if ((tc = find_table(ni, ti)) != NULL) {
 		/* check table type */
-		if (tc->no.type != ti->type)
+		if (tc->no.subtype != ti->type)
 			return (EINVAL);
 
 		if (tc->locked != 0)
@@ -1116,7 +1117,7 @@ find_table_entry(struct ip_fw_chain *ch, ip_fw3_opheader *op3,
 	}
 
 	/* check table type */
-	if (tc->no.type != ti.type) {
+	if (tc->no.subtype != ti.type) {
 		IPFW_UH_RUNLOCK(ch);
 		return (EINVAL);
 	}
@@ -1399,7 +1400,7 @@ swap_tables(struct ip_fw_chain *ch, struct tid_info *a,
 	}
 
 	/* Check type and value are the same */
-	if (tc_a->no.type != tc_b->no.type || tc_a->tflags != tc_b->tflags) {
+	if (tc_a->no.subtype!=tc_b->no.subtype || tc_a->tflags!=tc_b->tflags) {
 		IPFW_UH_WUNLOCK(ch);
 		return (EINVAL);
 	}
@@ -1613,7 +1614,6 @@ ipfw_switch_tables_namespace(struct ip_fw_chain *ch, unsigned int sets)
 	ipfw_insn *cmd;
 	int cmdlen, i, l;
 	uint16_t kidx;
-	uint8_t type;
 
 	IPFW_UH_WLOCK(ch);
 
@@ -1636,7 +1636,7 @@ ipfw_switch_tables_namespace(struct ip_fw_chain *ch, unsigned int sets)
 		for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
 			cmdlen = F_LEN(cmd);
 
-			if (classify_table_opcode(cmd, &kidx, &type) != 0)
+			if (classify_opcode_kidx(cmd, &kidx) != 0)
 				continue;
 
 			no = ipfw_objhash_lookup_kidx(ni, kidx);
@@ -1920,7 +1920,7 @@ create_table_internal(struct ip_fw_chain *ch, struct tid_info *ti,
 		 * requesting to create existing table
 		 * which has the same type
 		 */
-		if (compat == 0 || tc_new->no.type != tc->no.type) {
+		if (compat == 0 || tc_new->no.subtype != tc->no.subtype) {
 			IPFW_UH_WUNLOCK(ch);
 			free_table_config(ni, tc);
 			return (EEXIST);
@@ -1940,6 +1940,7 @@ create_table_internal(struct ip_fw_chain *ch, struct tid_info *ti,
 			return (EBUSY);
 		}
 		tc->no.kidx = kidx;
+		tc->no.etlv = IPFW_TLV_TBL_NAME;
 
 		IPFW_WLOCK(ch);
 		link_table(ch, tc);
@@ -1977,6 +1978,13 @@ objheader_to_ti(struct _ipfw_obj_header *oh, struct tid_info *ti)
 	ntlv_to_ti(&oh->ntlv, ti);
 }
 
+struct namedobj_instance *
+ipfw_get_table_objhash(struct ip_fw_chain *ch)
+{
+
+	return (CHAIN_TO_NI(ch));
+}
+
 /*
  * Exports basic table info as name TLV.
  * Used inside dump_static_rules() to provide info
@@ -2009,40 +2017,6 @@ ipfw_export_table_ntlv(struct ip_fw_chain *ch, uint16_t kidx,
 	return (0);
 }
 
-/*
- * Marks every table kidx used in @rule with bit in @bmask.
- * Used to generate bitmask of referenced tables for given ruleset.
- * 
- * Returns number of newly-referenced tables.
- */
-int
-ipfw_mark_table_kidx(struct ip_fw_chain *chain, struct ip_fw *rule,
-    uint32_t *bmask)
-{
-	int cmdlen, l, count;
-	ipfw_insn *cmd;
-	uint16_t kidx;
-	uint8_t type;
-
-	l = rule->cmd_len;
-	cmd = rule->cmd;
-	cmdlen = 0;
-	count = 0;
-	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
-		cmdlen = F_LEN(cmd);
-
-		if (classify_table_opcode(cmd, &kidx, &type) != 0)
-			continue;
-
-		if ((bmask[kidx / 32] & (1 << (kidx % 32))) == 0)
-			count++;
-
-		bmask[kidx / 32] |= 1 << (kidx % 32);
-	}
-
-	return (count);
-}
-
 struct dump_args {
 	struct ip_fw_chain *ch;
 	struct table_info *ti;
@@ -2111,7 +2085,7 @@ export_table_info(struct ip_fw_chain *ch, struct table_config *tc,
 	struct table_info *ti;
 	struct table_algo *ta;
 	
-	i->type = tc->no.type;
+	i->type = tc->no.subtype;
 	i->tflags = tc->tflags;
 	i->vmask = tc->vmask;
 	i->set = tc->no.set;
@@ -2296,7 +2270,7 @@ dump_table_v0(struct ip_fw_chain *ch, ip_fw3_opheader *op3,
 
 	xtbl->cnt = count;
 	xtbl->size = sz;
-	xtbl->type = tc->no.type;
+	xtbl->type = tc->no.subtype;
 	xtbl->tbl = ti.uidx;
 
 	if (sd->valsize < sz) {
@@ -2440,7 +2414,7 @@ ipfw_dump_table_legacy(struct ip_fw_chain *ch, struct tid_info *ti,
 	ta = tc->ta;
 
 	/* This dump format supports IPv4 only */
-	if (tc->no.type != IPFW_TABLE_ADDR)
+	if (tc->no.subtype != IPFW_TABLE_ADDR)
 		return (0);
 
 	memset(&da, 0, sizeof(da));
@@ -2531,7 +2505,7 @@ dump_table_xentry(void *e, void *arg)
 	pval = get_table_value(da->ch, da->tc, da->tent.v.kidx);
 	xent->value = ipfw_export_table_value_legacy(pval);
 	/* Apply some hacks */
-	if (tc->no.type == IPFW_TABLE_ADDR && tent->subtype == AF_INET) {
+	if (tc->no.subtype == IPFW_TABLE_ADDR && tent->subtype == AF_INET) {
 		xent->k.addr6.s6_addr32[3] = tent->k.addr.s_addr;
 		xent->flags = IPFW_TCF_INET;
 	} else
@@ -2781,114 +2755,157 @@ list_table_algo(struct ip_fw_chain *ch, ip_fw3_opheader *op3,
 	return (0);
 }
 
-/*
- * Tables rewriting code 
- */
-
-/*
- * Determine table number and lookup type for @cmd.
- * Fill @tbl and @type with appropriate values.
- * Returns 0 for relevant opcodes, 1 otherwise.
- */
 static int
-classify_table_opcode(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
+classify_srcdst(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
 {
-	ipfw_insn_if *cmdif;
-	int skip;
-	uint16_t v;
-
-	skip = 1;
-
-	switch (cmd->opcode) {
-	case O_IP_SRC_LOOKUP:
-	case O_IP_DST_LOOKUP:
-		/* Basic IPv4/IPv6 or u32 lookups */
-		*puidx = cmd->arg1;
-		/* Assume ADDR by default */
-		*ptype = IPFW_TABLE_ADDR;
-		skip = 0;
+	/* Basic IPv4/IPv6 or u32 lookups */
+	*puidx = cmd->arg1;
+	/* Assume ADDR by default */
+	*ptype = IPFW_TABLE_ADDR;
+	int v;
 		
-		if (F_LEN(cmd) > F_INSN_SIZE(ipfw_insn_u32)) {
-			/*
-			 * generic lookup. The key must be
-			 * in 32bit big-endian format.
-			 */
-			v = ((ipfw_insn_u32 *)cmd)->d[1];
-			switch (v) {
-			case 0:
-			case 1:
-				/* IPv4 src/dst */
-				break;
-			case 2:
-			case 3:
-				/* src/dst port */
-				*ptype = IPFW_TABLE_NUMBER;
-				break;
-			case 4:
-				/* uid/gid */
-				*ptype = IPFW_TABLE_NUMBER;
-				break;
-			case 5:
-				/* jid */
-				*ptype = IPFW_TABLE_NUMBER;
-				break;
-			case 6:
-				/* dscp */
-				*ptype = IPFW_TABLE_NUMBER;
-				break;
-			}
-		}
-		break;
-	case O_XMIT:
-	case O_RECV:
-	case O_VIA:
-		/* Interface table, possibly */
-		cmdif = (ipfw_insn_if *)cmd;
-		if (cmdif->name[0] != '\1')
+	if (F_LEN(cmd) > F_INSN_SIZE(ipfw_insn_u32)) {
+		/*
+		 * generic lookup. The key must be
+		 * in 32bit big-endian format.
+		 */
+		v = ((ipfw_insn_u32 *)cmd)->d[1];
+		switch (v) {
+		case 0:
+		case 1:
+			/* IPv4 src/dst */
 			break;
-
-		*ptype = IPFW_TABLE_INTERFACE;
-		*puidx = cmdif->p.kidx;
-		skip = 0;
-		break;
-	case O_IP_FLOW_LOOKUP:
-		*puidx = cmd->arg1;
-		*ptype = IPFW_TABLE_FLOW;
-		skip = 0;
-		break;
+		case 2:
+		case 3:
+			/* src/dst port */
+			*ptype = IPFW_TABLE_NUMBER;
+			break;
+		case 4:
+			/* uid/gid */
+			*ptype = IPFW_TABLE_NUMBER;
+			break;
+		case 5:
+			/* jid */
+			*ptype = IPFW_TABLE_NUMBER;
+			break;
+		case 6:
+			/* dscp */
+			*ptype = IPFW_TABLE_NUMBER;
+			break;
+		}
 	}
 
-	return (skip);
+	return (0);
+}
+
+static int
+classify_via(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
+{
+	ipfw_insn_if *cmdif;
+
+	/* Interface table, possibly */
+	cmdif = (ipfw_insn_if *)cmd;
+	if (cmdif->name[0] != '\1')
+		return (1);
+
+	*ptype = IPFW_TABLE_INTERFACE;
+	*puidx = cmdif->p.kidx;
+
+	return (0);
+}
+
+static int
+classify_flow(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
+{
+
+	*puidx = cmd->arg1;
+	*ptype = IPFW_TABLE_FLOW;
+
+	return (0);
+}
+
+static void
+update_arg1(ipfw_insn *cmd, uint16_t idx)
+{
+
+	cmd->arg1 = idx;
 }
 
-/*
- * Sets new table value for given opcode.
- * Assume the same opcodes as classify_table_opcode()
- */
 static void
-update_table_opcode(ipfw_insn *cmd, uint16_t idx)
+update_via(ipfw_insn *cmd, uint16_t idx)
 {
 	ipfw_insn_if *cmdif;
 
-	switch (cmd->opcode) {
-	case O_IP_SRC_LOOKUP:
-	case O_IP_DST_LOOKUP:
-		/* Basic IPv4/IPv6 or u32 lookups */
-		cmd->arg1 = idx;
-		break;
-	case O_XMIT:
-	case O_RECV:
-	case O_VIA:
-		/* Interface table, possibly */
-		cmdif = (ipfw_insn_if *)cmd;
-		cmdif->p.kidx = idx;
-		break;
-	case O_IP_FLOW_LOOKUP:
-		cmd->arg1 = idx;
-		break;
-	}
+	cmdif = (ipfw_insn_if *)cmd;
+	cmdif->p.kidx = idx;
 }
 
+static int
+table_findbyname(struct ip_fw_chain *ch, struct tid_info *ti,
+    struct named_object **pno)
+{
+	struct table_config *tc;
+	int error;
+
+	IPFW_UH_WLOCK_ASSERT(ch);
+
+	error = find_table_err(CHAIN_TO_NI(ch), ti, &tc);
+	if (error != 0)
+		return (error);
+
+	*pno = &tc->no;
+	return (0);
+}
+
+/* XXX: sets-sets! */
+static struct named_object *
+table_findbykidx(struct ip_fw_chain *ch, uint16_t idx)
+{
+	struct namedobj_instance *ni;
+	struct table_config *tc;
+
+	IPFW_UH_WLOCK_ASSERT(ch);
+	ni = CHAIN_TO_NI(ch);
+	tc = (struct table_config *)ipfw_objhash_lookup_kidx(ni, idx);
+	KASSERT(tc != NULL, ("Table with index %d not found", idx));
+
+	return (&tc->no);
+}
+
+static struct opcode_obj_rewrite opcodes[] = {
+	{
+		O_IP_SRC_LOOKUP, IPFW_TLV_TBL_NAME,
+		classify_srcdst, update_arg1,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+	{
+		O_IP_DST_LOOKUP, IPFW_TLV_TBL_NAME,
+		classify_srcdst, update_arg1,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+	{
+		O_IP_FLOW_LOOKUP, IPFW_TLV_TBL_NAME,
+		classify_flow, update_arg1,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+	{
+		O_XMIT, IPFW_TLV_TBL_NAME,
+		classify_via, update_via,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+	{
+		O_RECV, IPFW_TLV_TBL_NAME,
+		classify_via, update_via,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+	{
+		O_VIA, IPFW_TLV_TBL_NAME,
+		classify_via, update_via,
+		table_findbyname, table_findbykidx, create_table_compat
+	},
+};
+
+
 /*
  * Checks table name for validity.
  * Enforce basic length checks, the rest
@@ -2960,10 +2977,11 @@ find_name_tlv(void *tlvs, int len, uint16_t uidx)
  * or name in ntlv.
  * Note @ti structure contains unchecked data from userland.
  *
- * Returns pointer to table_config or NULL.
+ * Returns 0 in success and fills in @tc with found config
  */
-static struct table_config *
-find_table(struct namedobj_instance *ni, struct tid_info *ti)
+static int
+find_table_err(struct namedobj_instance *ni, struct tid_info *ti,
+    struct table_config **tc)
 {
 	char *name, bname[16];
 	struct named_object *no;
@@ -2973,7 +2991,7 @@ find_table(struct namedobj_instance *ni, struct tid_info *ti)
 	if (ti->tlvs != NULL) {
 		ntlv = find_name_tlv(ti->tlvs, ti->tlen, ti->uidx);
 		if (ntlv == NULL)
-			return (NULL);
+			return (EINVAL);
 		name = ntlv->name;
 
 		/*
@@ -2989,8 +3007,27 @@ find_table(struct namedobj_instance *ni, struct tid_info *ti)
 	}
 
 	no = ipfw_objhash_lookup_name(ni, set, name);
+	*tc = (struct table_config *)no;
 
-	return ((struct table_config *)no);
+	return (0);
+}
+
+/*
+ * Finds table config based on either legacy index
+ * or name in ntlv.
+ * Note @ti structure contains unchecked data from userland.
+ *
+ * Returns pointer to table_config or NULL.
+ */
+static struct table_config *
+find_table(struct namedobj_instance *ni, struct tid_info *ti)
+{
+	struct table_config *tc;
+
+	if (find_table_err(ni, ti, &tc) != 0)
+		return (NULL);
+
+	return (tc);
 }
 
 /*
@@ -3016,6 +3053,7 @@ alloc_table_config(struct ip_fw_chain *ch, struct tid_info *ti,
 		name = ntlv->name;
 		set = ntlv->set;
 	} else {
+		/* Compat part: convert number to string representation */
 		snprintf(bname, sizeof(bname), "%d", ti->uidx);
 		name = bname;
 		set = 0;
@@ -3023,7 +3061,7 @@ alloc_table_config(struct ip_fw_chain *ch, struct tid_info *ti,
 
 	tc = malloc(sizeof(struct table_config), M_IPFW, M_WAITOK | M_ZERO);
 	tc->no.name = tc->tablename;
-	tc->no.type = ta->type;
+	tc->no.subtype = ta->type;
 	tc->no.set = set;
 	tc->tflags = tflags;
 	tc->ta = ta;
@@ -3031,11 +3069,6 @@ alloc_table_config(struct ip_fw_chain *ch, struct tid_info *ti,
 	/* Set "shared" value type by default */
 	tc->vshared = 1;
 
-	if (ti->tlvs == NULL) {
-		tc->no.compat = 1;
-		tc->no.uidx = ti->uidx;
-	}
-
 	/* Preallocate data structures for new tables */
 	error = ta->init(ch, &tc->astate, &tc->ti_copy, aname, tflags);
 	if (error != 0) {
@@ -3211,7 +3244,6 @@ ipfw_move_tables_sets(struct ip_fw_chain *ch, ipfw_range_tlv *rt,
 	struct namedobj_instance *ni;
 	int bad, i, l, cmdlen;
 	uint16_t kidx;
-	uint8_t type;
 	ipfw_insn *cmd;
 
 	IPFW_UH_WLOCK_ASSERT(ch);
@@ -3229,7 +3261,7 @@ ipfw_move_tables_sets(struct ip_fw_chain *ch, ipfw_range_tlv *rt,
 		cmdlen = 0;
 		for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
 			cmdlen = F_LEN(cmd);
-			if (classify_table_opcode(cmd, &kidx, &type) != 0)
+			if (classify_opcode_kidx(cmd, &kidx) != 0)
 				continue;
 			no = ipfw_objhash_lookup_kidx(ni, kidx);
 			KASSERT(no != NULL, 
@@ -3252,7 +3284,7 @@ ipfw_move_tables_sets(struct ip_fw_chain *ch, ipfw_range_tlv *rt,
 		cmdlen = 0;
 		for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
 			cmdlen = F_LEN(cmd);
-			if (classify_table_opcode(cmd, &kidx, &type) != 0)
+			if (classify_opcode_kidx(cmd, &kidx) != 0)
 				continue;
 			no = ipfw_objhash_lookup_kidx(ni, kidx);
 			KASSERT(no != NULL, 
@@ -3291,7 +3323,7 @@ ipfw_move_tables_sets(struct ip_fw_chain *ch, ipfw_range_tlv *rt,
 		cmdlen = 0;
 		for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
 			cmdlen = F_LEN(cmd);
-			if (classify_table_opcode(cmd, &kidx, &type) != 0)
+			if (classify_opcode_kidx(cmd, &kidx) != 0)
 				continue;
 			no = ipfw_objhash_lookup_kidx(ni, kidx);
 			KASSERT(no != NULL, 
@@ -3313,215 +3345,63 @@ ipfw_move_tables_sets(struct ip_fw_chain *ch, ipfw_range_tlv *rt,
 }
 
 /*
- * Finds and bumps refcount for tables referenced by given @rule.
+ * Finds and bumps refcount for objects referenced by given @rule.
  * Auto-creates non-existing tables.
  * Fills in @oib array with userland/kernel indexes.
- * First free oidx pointer is saved back in @oib.
  *
  * Returns 0 on success.
  */
 static int
-find_ref_rule_tables(struct ip_fw_chain *ch, struct ip_fw *rule,
-    struct rule_check_info *ci, struct obj_idx **oib, struct tid_info *ti)
+ref_rule_objects(struct ip_fw_chain *ch, struct ip_fw *rule,
+    struct rule_check_info *ci, struct obj_idx *oib, struct tid_info *ti)
 {
-	struct table_config *tc;
-	struct namedobj_instance *ni;
-	struct named_object *no;
 	int cmdlen, error, l, numnew;
-	uint16_t kidx;
 	ipfw_insn *cmd;
-	struct obj_idx *pidx, *pidx_first, *p;
+	struct obj_idx *pidx;
+	int found, unresolved;
 
-	pidx_first = *oib;
-	pidx = pidx_first;
+	pidx = oib;
 	l = rule->cmd_len;
 	cmd = rule->cmd;
 	cmdlen = 0;
 	error = 0;
 	numnew = 0;
+	found = 0;
+	unresolved = 0;
 
 	IPFW_UH_WLOCK(ch);
-	ni = CHAIN_TO_NI(ch);
 
 	/* Increase refcount on each existing referenced table. */
 	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
 		cmdlen = F_LEN(cmd);
 
-		if (classify_table_opcode(cmd, &ti->uidx, &ti->type) != 0)
-			continue;
-
-		pidx->uidx = ti->uidx;
-		pidx->type = ti->type;
-
-		if ((tc = find_table(ni, ti)) != NULL) {
-			if (tc->no.type != ti->type) {
-				/* Incompatible types */
-				error = EINVAL;
-				break;
-			}
-
-			/* Reference found table and save kidx */
-			tc->no.refcnt++;
-			pidx->kidx = tc->no.kidx;
+		error = ref_opcode_object(ch, cmd, ti, pidx, &found, &unresolved);
+		if (error != 0)
+			break;
+		if (found || unresolved) {
+			pidx->off = rule->cmd_len - l;
 			pidx++;
-			continue;
 		}
-
 		/*
 		 * Compability stuff for old clients:
-		 * prepare to manually create non-existing tables.
+		 * prepare to manually create non-existing objects.
 		 */
-		pidx++;
-		numnew++;
+		if (unresolved)
+			numnew++;
 	}
 
 	if (error != 0) {
 		/* Unref everything we have already done */
-		for (p = *oib; p < pidx; p++) {
-			if (p->kidx == 0)
-				continue;
-
-			/* Find & unref by existing idx */
-			no = ipfw_objhash_lookup_kidx(ni, p->kidx);
-			KASSERT(no != NULL, ("Ref'd table %d disappeared",
-			    p->kidx));
-
-			no->refcnt--;
-		}
-	}
-
-	IPFW_UH_WUNLOCK(ch);
-
-	if (numnew == 0) {
-		*oib = pidx;
-		return (error);
-	}
-
-	/*
-	 * Compatibility stuff: do actual creation for non-existing,
-	 * but referenced tables.
-	 */
-	for (p = pidx_first; p < pidx; p++) {
-		if (p->kidx != 0)
-			continue;
-
-		ti->uidx = p->uidx;
-		ti->type = p->type;
-		ti->atype = 0;
-
-		error = create_table_compat(ch, ti, &kidx);
-		if (error == 0) {
-			p->kidx = kidx;
-			continue;
-		}
-
-		/* Error. We have to drop references */
-		IPFW_UH_WLOCK(ch);
-		for (p = pidx_first; p < pidx; p++) {
-			if (p->kidx == 0)
-				continue;
-
-			/* Find & unref by existing idx */
-			no = ipfw_objhash_lookup_kidx(ni, p->kidx);
-			KASSERT(no != NULL, ("Ref'd table %d disappeared",
-			    p->kidx));
-
-			no->refcnt--;
-		}
+		unref_oib_objects(ch, rule->cmd, oib, pidx);
 		IPFW_UH_WUNLOCK(ch);
-
 		return (error);
 	}
 
-	*oib = pidx;
-
-	return (error);
-}
-
-/*
- * Remove references from every table used in @rule.
- */
-void
-ipfw_unref_rule_tables(struct ip_fw_chain *chain, struct ip_fw *rule)
-{
-	int cmdlen, l;
-	ipfw_insn *cmd;
-	struct namedobj_instance *ni;
-	struct named_object *no;
-	uint16_t kidx;
-	uint8_t type;
-
-	IPFW_UH_WLOCK_ASSERT(chain);
-	ni = CHAIN_TO_NI(chain);
-
-	l = rule->cmd_len;
-	cmd = rule->cmd;
-	cmdlen = 0;
-	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
-		cmdlen = F_LEN(cmd);
-
-		if (classify_table_opcode(cmd, &kidx, &type) != 0)
-			continue;
-
-		no = ipfw_objhash_lookup_kidx(ni, kidx); 
-
-		KASSERT(no != NULL, ("table id %d not found", kidx));
-		KASSERT(no->type == type, ("wrong type %d (%d) for table id %d",
-		    no->type, type, kidx));
-		KASSERT(no->refcnt > 0, ("refcount for table %d is %d",
-		    kidx, no->refcnt));
-
-		no->refcnt--;
-	}
-}
-
-/*
- * Compatibility function for old ipfw(8) binaries.
- * Rewrites table kernel indices with userland ones.
- * Convert tables matching '/^\d+$/' to their atoi() value.
- * Use number 65535 for other tables.
- *
- * Returns 0 on success.
- */
-int
-ipfw_rewrite_table_kidx(struct ip_fw_chain *chain, struct ip_fw_rule0 *rule)
-{
-	int cmdlen, error, l;
-	ipfw_insn *cmd;
-	uint16_t kidx, uidx;
-	uint8_t type;
-	struct named_object *no;
-	struct namedobj_instance *ni;
-
-	ni = CHAIN_TO_NI(chain);
-	error = 0;
-
-	l = rule->cmd_len;
-	cmd = rule->cmd;
-	cmdlen = 0;
-	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
-		cmdlen = F_LEN(cmd);
-
-		if (classify_table_opcode(cmd, &kidx, &type) != 0)
-			continue;
-
-		if ((no = ipfw_objhash_lookup_kidx(ni, kidx)) == NULL)
-			return (1);
-
-		uidx = no->uidx;
-		if (no->compat == 0) {
-
-			/*
-			 * We are called via legacy opcode.
-			 * Save error and show table as fake number
-			 * not to make ipfw(8) hang.
-			 */
-			uidx = 65535;
-			error = 2;
-		}
+	IPFW_UH_WUNLOCK(ch);
 
-		update_table_opcode(cmd, uidx);
-	}
+	/* Perform auto-creation for non-existing objects */
+	if (numnew != 0)
+		error = create_objects_compat(ch, rule->cmd, oib, pidx, ti);
 
 	return (error);
 }
@@ -3534,31 +3414,27 @@ ipfw_rewrite_table_kidx(struct ip_fw_chain *chain, struct ip_fw_rule0 *rule)
  * Returns 0 on success and appropriate error code otherwise.
  */
 int
-ipfw_rewrite_table_uidx(struct ip_fw_chain *chain,
+ipfw_rewrite_rule_uidx(struct ip_fw_chain *chain,
     struct rule_check_info *ci)
 {
-	int cmdlen, error, l;
+	int error;
 	ipfw_insn *cmd;
-	uint16_t uidx;
 	uint8_t type;
-	struct namedobj_instance *ni;
 	struct obj_idx *p, *pidx_first, *pidx_last;
 	struct tid_info ti;
 
-	ni = CHAIN_TO_NI(chain);
-
 	/*
 	 * Prepare an array for storing opcode indices.
 	 * Use stack allocation by default.
 	 */
-	if (ci->table_opcodes <= (sizeof(ci->obuf)/sizeof(ci->obuf[0]))) {
+	if (ci->object_opcodes <= (sizeof(ci->obuf)/sizeof(ci->obuf[0]))) {
 		/* Stack */
 		pidx_first = ci->obuf;
 	} else
-		pidx_first = malloc(ci->table_opcodes * sizeof(struct obj_idx),
+		pidx_first = malloc(ci->object_opcodes * sizeof(struct obj_idx),
 		    M_IPFW, M_WAITOK | M_ZERO);
 
-	pidx_last = pidx_first;
+	pidx_last = pidx_first + ci->object_opcodes;
 	error = 0;
 	type = 0;
 	memset(&ti, 0, sizeof(ti));
@@ -3573,28 +3449,18 @@ ipfw_rewrite_table_uidx(struct ip_fw_chain *chain,
 		ti.tlen = ci->ctlv->head.length - sizeof(ipfw_obj_ctlv);
 	}
 
-	/* Reference all used tables */
-	error = find_ref_rule_tables(chain, ci->krule, ci, &pidx_last, &ti);
+	/* Reference all used tables and other objects */
+	error = ref_rule_objects(chain, ci->krule, ci, pidx_first, &ti);
 	if (error != 0)
 		goto free;
 
-	IPFW_UH_WLOCK(chain);
-
 	/* Perform rule rewrite */
-	l = ci->krule->cmd_len;
-	cmd = ci->krule->cmd;
-	cmdlen = 0;
 	p = pidx_first;
-	for ( ;	l > 0 ; l -= cmdlen, cmd += cmdlen) {
-		cmdlen = F_LEN(cmd);
-		if (classify_table_opcode(cmd, &uidx, &type) != 0)
-			continue;
-		update_table_opcode(cmd, p->kidx);
-		p++;
+	for (p = pidx_first; p < pidx_last; p++) {
+		cmd = ci->krule->cmd + p->off;
+		update_opcode_kidx(cmd, p->kidx);
 	}
 
-	IPFW_UH_WUNLOCK(chain);
-
 free:
 	if (pidx_first != ci->obuf)
 		free(pidx_first, M_IPFW);
@@ -3641,6 +3507,7 @@ ipfw_destroy_tables(struct ip_fw_chain *ch, int last)
 {
 
 	IPFW_DEL_SOPT_HANDLER(last, scodes);
+	IPFW_DEL_OBJ_REWRITER(last, opcodes);
 
 	/* Remove all tables from working set */
 	IPFW_UH_WLOCK(ch);
@@ -3678,6 +3545,7 @@ ipfw_init_tables(struct ip_fw_chain *ch, int first)
 	ipfw_table_value_init(ch, first);
 	ipfw_table_algo_init(ch);
 
+	IPFW_ADD_OBJ_REWRITER(first, opcodes);
 	IPFW_ADD_SOPT_HANDLER(first, scodes);
 	return (0);
 }
diff --git a/sys/netpfil/ipfw/ip_fw_table.h b/sys/netpfil/ipfw/ip_fw_table.h
index 028e450..ca49fd4 100644
--- a/sys/netpfil/ipfw/ip_fw_table.h
+++ b/sys/netpfil/ipfw/ip_fw_table.h
@@ -53,16 +53,6 @@ struct table_info {
 	u_long		data;		/* Hints for given func */
 };
 
-/* Internal structures for handling sockopt data */
-struct tid_info {
-	uint32_t	set;	/* table set */
-	uint16_t	uidx;	/* table index */
-	uint8_t		type;	/* table type */
-	uint8_t		atype;
-	void		*tlvs;	/* Pointer to first TLV */
-	int		tlen;	/* Total TLV size block */
-};
-
 struct table_value;
 struct tentry_info {
 	void		*paddr;
@@ -189,13 +179,12 @@ void rollback_table_values(struct tableop_state *ts);
 
 int ipfw_rewrite_table_uidx(struct ip_fw_chain *chain,
     struct rule_check_info *ci);
-int ipfw_rewrite_table_kidx(struct ip_fw_chain *chain,
-    struct ip_fw_rule0 *rule);
 int ipfw_mark_table_kidx(struct ip_fw_chain *chain, struct ip_fw *rule,
     uint32_t *bmask);
 int ipfw_export_table_ntlv(struct ip_fw_chain *ch, uint16_t kidx,
     struct sockopt_data *sd);
 void ipfw_unref_rule_tables(struct ip_fw_chain *chain, struct ip_fw *rule);
+struct namedobj_instance *ipfw_get_table_objhash(struct ip_fw_chain *ch);
 
 /* utility functions  */
 int ipfw_check_table_name(char *name);
diff --git a/tests/sys/Makefile b/tests/sys/Makefile
index 084f069..b54feca 100644
--- a/tests/sys/Makefile
+++ b/tests/sys/Makefile
@@ -10,7 +10,6 @@ TESTS_SUBDIRS+=		file
 TESTS_SUBDIRS+=		kern
 TESTS_SUBDIRS+=		kqueue
 TESTS_SUBDIRS+=		mqueue
-TESTS_SUBDIRS+=		mmap
 TESTS_SUBDIRS+=		netinet
 TESTS_SUBDIRS+=		opencrypto
 TESTS_SUBDIRS+=		posixshm
diff --git a/tests/sys/kern/Makefile b/tests/sys/kern/Makefile
index d366aa1..3d2127f 100644
--- a/tests/sys/kern/Makefile
+++ b/tests/sys/kern/Makefile
@@ -3,6 +3,7 @@
 TESTSDIR=	${TESTSBASE}/sys/kern
 
 ATF_TESTS_C+=	kern_descrip_test
+TAP_TESTS_C+=	mmap_test
 ATF_TESTS_C+=	unix_seqpacket_test
 TEST_METADATA.unix_seqpacket_test+=	timeout="15"
 
diff --git a/tests/sys/kern/mmap_test.c b/tests/sys/kern/mmap_test.c
new file mode 100644
index 0000000..7591a09
--- /dev/null
+++ b/tests/sys/kern/mmap_test.c
@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2009	Simon L. Nielsen <simon@FreeBSD.org>,
+ * 			Bjoern A. Zeeb <bz@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+static const struct {
+	void	*addr;
+	int	ok[2];	/* Depending on security.bsd.map_at_zero {0, !=0}. */
+} tests[] = {
+	{ (void *)0,			{ 0, 1 } }, /* Test sysctl. */
+	{ (void *)1,			{ 0, 0 } },
+	{ (void *)(PAGE_SIZE - 1),	{ 0, 0 } },
+	{ (void *)PAGE_SIZE,		{ 1, 1 } },
+	{ (void *)-1,			{ 0, 0 } },
+	{ (void *)(-PAGE_SIZE),		{ 0, 0 } },
+	{ (void *)(-1 - PAGE_SIZE),	{ 0, 0 } },
+	{ (void *)(-1 - PAGE_SIZE - 1),	{ 0, 0 } },
+	{ (void *)(0x1000 * PAGE_SIZE),	{ 1, 1 } },
+};
+
+#define	MAP_AT_ZERO	"security.bsd.map_at_zero"
+
+int
+main(void)
+{
+	void *p;
+	size_t len;
+	int i, error, mib[3], map_at_zero;
+
+	error = 0;
+
+	/* Get the current sysctl value of security.bsd.map_at_zero. */
+	len = sizeof(mib) / sizeof(*mib);
+	if (sysctlnametomib(MAP_AT_ZERO, mib, &len) == -1) {
+		printf("1..0 # SKIP: sysctlnametomib(\"%s\") failed: %s\n",
+		    MAP_AT_ZERO, strerror(errno));
+		return (0);
+	}
+
+	len = sizeof(map_at_zero);
+	if (sysctl(mib, 3, &map_at_zero, &len, NULL, 0) == -1) {
+		printf("1..0 # SKIP: sysctl for %s failed: %s\n", MAP_AT_ZERO,
+		    strerror(errno));
+		return (0);
+	}
+
+	/* Normalize to 0 or 1 for array access. */
+	map_at_zero = !!map_at_zero;
+
+	printf("1..%zu\n", nitems(tests));
+	for (i = 0; i < (int)nitems(tests); i++) {
+		p = mmap((void *)tests[i].addr, PAGE_SIZE,
+		    PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_FIXED,
+		    -1, 0);
+		if (p == MAP_FAILED) {
+			if (tests[i].ok[map_at_zero] != 0)
+				error++;
+			printf("%sok %d # mmap(%p, ...) failed\n",
+			    tests[i].ok[map_at_zero] == 0 ? "" : "not ",
+			    i + 1,
+			    tests[i].addr);
+		} else {
+			if (tests[i].ok[map_at_zero] != 1)
+				error++;
+			printf("%sok %d # mmap(%p, ...) succeeded: p=%p\n",
+			    tests[i].ok[map_at_zero] == 1 ? "" : "not ",
+			    i + 1,
+			    tests[i].addr, p);
+		}
+	}
+
+	return (error != 0);
+}